From 7f216baa426068957c2c71bd7ca40984c58d486c Mon Sep 17 00:00:00 2001 From: Jeffrey Quesnelle Date: Mon, 9 Jan 2023 16:03:30 +0000 Subject: [PATCH 01/59] add soda_synethetic_dialogue dataset --- openassistant/datasets/__init__.py | 0 .../soda_synthetic_dialogue/README.md | 94 +++++++ .../soda_synthetic_dialogue/__init__.py | 0 .../datasets/soda_synthetic_dialogue/hub.py | 21 ++ .../soda_synthetic_dialogue/prepare.py | 245 ++++++++++++++++++ .../soda_synthetic_dialogue.py | 123 +++++++++ 6 files changed, 483 insertions(+) create mode 100644 openassistant/datasets/__init__.py create mode 100644 openassistant/datasets/soda_synthetic_dialogue/README.md create mode 100644 openassistant/datasets/soda_synthetic_dialogue/__init__.py create mode 100644 openassistant/datasets/soda_synthetic_dialogue/hub.py create mode 100644 openassistant/datasets/soda_synthetic_dialogue/prepare.py create mode 100644 openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py diff --git a/openassistant/datasets/__init__.py b/openassistant/datasets/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openassistant/datasets/soda_synthetic_dialogue/README.md b/openassistant/datasets/soda_synthetic_dialogue/README.md new file mode 100644 index 00000000..ef5dfdf8 --- /dev/null +++ b/openassistant/datasets/soda_synthetic_dialogue/README.md @@ -0,0 +1,94 @@ +--- +annotations_creators: +- no-annotation +language: +- en +language_creators: +- machine-generated +license: +- mit +multilinguality: +- monolingual +pretty_name: "SODA Synthetic Dialogue" +size_categories: +- 1M 1 and sys.argv[1] == "--print" + + +def main(output_dir: str = "data"): + """Download and prepare the dataset for use.""" + + random.seed(42) + dataset = load_dataset("allenai/soda") + os.makedirs(output_dir, exist_ok=True) + + for split in ["train", "test", "validation"]: + with open(f"{output_dir}/{split}.jsonl", "w", encoding="utf8") as output: + + for i in tqdm(range(len(dataset[split])), desc=split): + dat = dataset["train"][i] + title = dat["literal"] + story = dat["narrative"] + + if dat["relation"] == "xWant": + theme = "wanting " + dat["tail"] + elif dat["relation"] == "xNeed": + theme = "needing " + dat["tail"] + elif not dat["tail"].startswith("to ") and not dat["tail"].startswith("and "): + theme = "being " + dat["tail"] + elif dat["tail"].startswith("and "): + theme = "people are " + dat["tail"].replace("and PersonY ", "") + else: + theme = dat["tail"] + theme = theme.replace("PersonY", "another person") + theme = theme.replace("being is", "being") + + dialogue = [s2 + ": " + s1 for s1, s2 in zip(dat["dialogue"], dat["speakers"])] + + if random.randint(0, 6) == 0: + # print("##") + # print(f"User: Can you give me a short story description for this dialog?") + # print(" " + "\n ".join(dialog)) + # print(f"Assistant: Sure, a short story description for this dialog could be: \n {story}") + # print("User: And a title?") + # print(f"Assistant: Sure, a title for this dialog could be: \n {title}") + # if theme: + # print("User: What would be one theme of this story?") + # print(f'Assistant: One theme of this story could be: "{theme}"') + conversation = SUMMARY_TEMPLATE.format(dialogue="\n ".join(dialogue), story=story, title=title) + if theme: + conversation = conversation + THEME_TEMPLATE.format(theme=theme) + elif random.randint(0, 6) == 0: + # print("##") + # print(f"User: Can you write a short dialog based on this story:\n {story}") + # print(f"Assistant: Sure, a dialog for this story could be:") + # print(" " + "\n ".join(dialog)) + # print("User: And a title?") + # print(f"Assistant: Sure, a title for this dialog could be: \n {title}") + # if theme: + # print("User: What would be one theme of this story?") + # print(f'Assistant: One theme of this story could be: "{theme}"') + conversation = NEW_DIALOGUE_TEMPLATE.format( + story=story, dialogue="\n ".join(dialogue), title=title + ) + if theme: + conversation = conversation + THEME_TEMPLATE.format(theme=theme) + elif random.randint(0, 3) == 0: + # print("##") + # print(f"User: Can you write the next few lines of dialog for this scene:") + # if random.randint(0, 1) == 0: + # print(" " + "\n ".join(dialog[:-5])) + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-5:])) + # elif random.randint(0, 1) == 0: + # print(" " + "\n ".join(dialog[:-3])) + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-3:])) + # else: + # print(" " + "\n ".join(dialog[:-4])) + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-4:])) + # print("User: And a title?") + # print(f"Assistant: Sure, a title for this dialog could be: \n {title}") + # print("User: How about a short description?") + # print(f"Assistant: Sure, a short description for this dialog could be: \n {story}") + # if theme: + # print("User: What would be one theme of this story?") + # print(f'Assistant: One theme of this story could be: "{theme}"') + if random.randint(0, 1) == 0: + depth = -5 + elif random.randint(0, 1) == 0: + depth = -3 + else: + depth = -4 + conversation = NEXT_LINES_TEMPLATE.format( + scene="\n ".join(dialogue[:depth]), + dialogue="\n ".join(dialogue[depth:]), + title=title, + story=story, + ) + if theme: + conversation = conversation + THEME_TEMPLATE.format(theme=theme) + elif random.randint(0, 3) == 0: + # print("##") + # title1 = title.split(".")[0] + # title2 = title.split(".")[1] + # print(f"User: Can you write short story and dialog about: {title1}") + # print(f'Assistant: Sure, a short story and dialog about: "{title1}" could be:') + # print(f" {story}") + # if random.randint(0, 1) == 0: + # print(" " + "\n ".join(dialog)) + # elif random.randint(0, 1) == 0 and len(dialog) > 5: + # print(" " + "\n ".join(dialog[:-5])) + # print(f'User: Can you provide more dialog assuming "{title2}"?') + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-5:])) + # elif random.randint(0, 1) == 0: + # print(" " + "\n ".join(dialog[:-3])) + # print("User: more please.") + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-3:])) + # else: + # print(" " + "\n ".join(dialog[:-4])) + # print(f'User: Can you provide more dialog assuming "{title2}"?') + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-4:])) + # if theme: + # print("User: What would be one theme of this story?") + # print(f'Assistant: One theme of this story could be: "{theme}"') + title1 = title.split(".")[0] + title2 = title.split(".")[1] + conversation = NEW_STORY_AND_DIALOGUE_TEMPLATE.format(title1=title1, story=story) + if random.randint(0, 1) == 0: + conversation = FULL_DIALOGUE_TEMPLATE.format( + conversation=conversation, dialogue="\n ".join(dialogue) + ) + elif random.randint(0, 1) == 0 and len(dialogue) > 5: + conversation = MORE_DIALOGUE_TEMPLATE.format( + conversation=conversation, + dialogue1="\n ".join(dialogue[:-5]), + title2=title2, + dialogue2="\n ".join(dialogue[-5:]), + ) + elif random.randint(0, 1) == 0: + conversation = NEXT_DIALOGUE_TEMPLATE.format( + conversation=conversation, + dialogue1="\n ".join(dialogue[:-3]), + dialogue2="\n ".join(dialogue[-3:]), + ) + else: + conversation = MORE_DIALOGUE_TEMPLATE.format( + conversation=conversation, + dialogue1="\n ".join(dialogue[:-4]), + title2=title2, + dialogue2="\n ".join(dialogue[-4:]), + ) + if theme: + conversation = conversation + THEME_TEMPLATE.format(theme=theme) + else: + # print("##") + # print(f"User: Can you write short story and dialog based on the theme:\n {theme}") + # print(f'Assistant: Sure, a short story and dialog based on the theme "{theme}" could be:') + # print(f" {story}") + # print(" " + "\n ".join(dialog)) + # print("User: And a title?") + # print(f"Assistant: Sure, a title for this dialog could be: \n {title}") + conversation = NEW_STORY_AND_DIALOGUE_FROM_THEME_TEMPLATE.format( + theme=theme, story=story, dialogue="\n ".join(dialogue), title=title + ) + if PRINT: + print("##") + print(conversation) + + output.write(f"{json.dumps({'conversation': conversation})}\n") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py b/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py new file mode 100644 index 00000000..4b588394 --- /dev/null +++ b/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py @@ -0,0 +1,123 @@ +# Copyright 2023 The OpenAssistant Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This dataset is a set of dialogues synthesized from the SODA dataset. +In each dialogue, User and Assistant have a conversation about a story. + +The original collab notebook for this dataset can be found at: +https://colab.research.google.com/drive/1Sw3px5dP8whdqT7QMNoqwmqIasZkMbJi?usp=sharing +""" + +import json +from typing import Dict, List, Tuple + +import datasets + +from .hub import OpenAssistantConfig + +from .hub import features + +_CITATION = """\ +@article{ontocard2023sodasynth, + author = {ontocard and Jeffrey Quesnelle}, + title = {SODA Synthetic Dialogue}, + year = {2023} +} +""" +_DATASETNAME = "soda_synthetic_dialogue" +_DISPLAYNAME = "🥤SODA Synthetic Dialogue" +_DESCRIPTION = "A set of dialogues synthesized from the SODA dataset." +_HOMEPAGE = "" +_LICENSE = "mit" +_URLS = { + _DATASETNAME: { + 'train': './data/train.jsonl', + 'test': './data/test.jsonl', + 'validation': './data/validation.jsonl' + } +} +_SUPPORTED_TASKS = ["dialogue-modeling"] +_VERSION = "1.0.0" + + +class SODASyntheticDialogueDataset(datasets.GeneratorBasedBuilder): + """A set of dialogues synthesized from the SODA dataset.""" + + VERSION = datasets.Version(_VERSION) + + BUILDER_CONFIGS = [ + OpenAssistantConfig( + name=f"{_DATASETNAME}_dialogue_modeling", + version=VERSION, + description=f"OpenAssistant dataset config for {_DATASETNAME}", + schema="dialogue_modeling", + subset_id=_DATASETNAME, + ) + ] + + DEFAULT_CONFIG_NAME = f"{_DATASETNAME}_dialogue_modeling" + + def _info(self) -> datasets.DatasetInfo: + + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]: + """Returns SplitGenerators.""" + + urls = _URLS[_DATASETNAME] + data_dir = dl_manager.download_and_extract(urls) + + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={ + "filepath": data_dir, + "split": "train" + }, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + gen_kwargs={ + "filepath": data_dir, + "split": "test" + }, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + gen_kwargs={ + "filepath": data_dir, + "split": "validation" + }, + ), + ] + + def _generate_examples(self, filepath, split: str) -> Tuple[int, Dict]: + """Yields examples as (key, example) tuples.""" + + if self.config.schema == "dialogue_modeling": + key = 0 + with open(filepath[split], 'r', encoding='utf8') as data: + while True: + line = data.readline() + if not line: + return + yield key, json.loads(line) + key += 1 From c04abe0259047694b0d231e7afc3970dad26ae63 Mon Sep 17 00:00:00 2001 From: Jeffrey Quesnelle Date: Tue, 10 Jan 2023 02:30:36 +0000 Subject: [PATCH 02/59] soda_synthetic_dialogue lint fixes --- .../soda_synthetic_dialogue/README.md | 69 +++++++++++-------- .../soda_synthetic_dialogue/prepare.py | 3 +- .../soda_synthetic_dialogue.py | 27 ++------ 3 files changed, 49 insertions(+), 50 deletions(-) diff --git a/openassistant/datasets/soda_synthetic_dialogue/README.md b/openassistant/datasets/soda_synthetic_dialogue/README.md index ef5dfdf8..595089a7 100644 --- a/openassistant/datasets/soda_synthetic_dialogue/README.md +++ b/openassistant/datasets/soda_synthetic_dialogue/README.md @@ -1,41 +1,41 @@ --- annotations_creators: -- no-annotation + - no-annotation language: -- en + - en language_creators: -- machine-generated + - machine-generated license: -- mit + - mit multilinguality: -- monolingual + - monolingual pretty_name: "SODA Synthetic Dialogue" size_categories: -- 1M Date: Tue, 10 Jan 2023 14:26:04 +0000 Subject: [PATCH 03/59] fix soda_synthetic_dialogue author --- openassistant/datasets/soda_synthetic_dialogue/README.md | 6 +++--- .../soda_synthetic_dialogue/soda_synthetic_dialogue.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openassistant/datasets/soda_synthetic_dialogue/README.md b/openassistant/datasets/soda_synthetic_dialogue/README.md index 595089a7..b645dbfa 100644 --- a/openassistant/datasets/soda_synthetic_dialogue/README.md +++ b/openassistant/datasets/soda_synthetic_dialogue/README.md @@ -89,7 +89,7 @@ text `conversation` feature. ## Source data The script to synthesize this dataset was originally created by -[ontocard](https://github.com/ontocord) in +[ontocord](https://github.com/ontocord) in [this Colab notebook](https://colab.research.google.com/drive/1Sw3px5dP8whdqT7QMNoqwmqIasZkMbJi?usp=sharing) and prepared for Hugging Face by [Jeffrey Quesnelle](https://github.com/jquesnelle/). @@ -99,8 +99,8 @@ and prepared for Hugging Face by Please cite our work if you find the resources in this repository useful: ``` -@article{ontocard2023sodasynth, - author = {ontocard and Jeffrey Quesnelle}, +@article{ontocord2023sodasynth, + author = {ontocord and Jeffrey Quesnelle}, title = {SODA Synthetic Dialogue}, year = {2023} } diff --git a/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py b/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py index 856215ec..ddc7c883 100644 --- a/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py +++ b/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py @@ -28,8 +28,8 @@ import datasets from .hub import OpenAssistantConfig, features _CITATION = """\ -@article{ontocard2023sodasynth, - author = {ontocard and Jeffrey Quesnelle}, +@article{ontocord2023sodasynth, + author = {ontocord and Jeffrey Quesnelle}, title = {SODA Synthetic Dialogue}, year = {2023} } From 4dd329f5089355bcebe22ea24318cb29daa29beb Mon Sep 17 00:00:00 2001 From: Jeffrey Quesnelle Date: Mon, 16 Jan 2023 22:45:28 -0500 Subject: [PATCH 04/59] add link to soda paper --- openassistant/datasets/soda_synthetic_dialogue/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openassistant/datasets/soda_synthetic_dialogue/README.md b/openassistant/datasets/soda_synthetic_dialogue/README.md index b645dbfa..08fde59f 100644 --- a/openassistant/datasets/soda_synthetic_dialogue/README.md +++ b/openassistant/datasets/soda_synthetic_dialogue/README.md @@ -56,7 +56,8 @@ from a title or theme. This data was created by synthesizing the dialogues in [🥤Soda](https://huggingface.co/datasets/allenai/soda) and applying a set of -templates to generate the conversation. +templates to generate the conversation. The original research paper can be +found [here](https://arxiv.org/pdf/2212.10465v1.pdf). Example: From 3222485d6fb23c932745ab929c7883f367a81024 Mon Sep 17 00:00:00 2001 From: Jeffrey Quesnelle Date: Mon, 16 Jan 2023 22:57:47 -0500 Subject: [PATCH 05/59] lint fix --- openassistant/datasets/soda_synthetic_dialogue/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openassistant/datasets/soda_synthetic_dialogue/README.md b/openassistant/datasets/soda_synthetic_dialogue/README.md index 08fde59f..c4866e16 100644 --- a/openassistant/datasets/soda_synthetic_dialogue/README.md +++ b/openassistant/datasets/soda_synthetic_dialogue/README.md @@ -56,8 +56,8 @@ from a title or theme. This data was created by synthesizing the dialogues in [🥤Soda](https://huggingface.co/datasets/allenai/soda) and applying a set of -templates to generate the conversation. The original research paper can be -found [here](https://arxiv.org/pdf/2212.10465v1.pdf). +templates to generate the conversation. The original research paper can be found +[here](https://arxiv.org/pdf/2212.10465v1.pdf). Example: From 801ad553b846c96843d83c0fcdf3fd4ec5af9753 Mon Sep 17 00:00:00 2001 From: notmd Date: Wed, 18 Jan 2023 15:19:00 +0700 Subject: [PATCH 06/59] Allow to filter `user` by `display_name` --- backend/oasst_backend/user_repository.py | 4 +- website/package-lock.json | 45 +++++++ website/package.json | 1 + website/src/components/DataTable.tsx | 158 +++++++++++++++++++++++ website/src/components/UserTable.tsx | 133 +++++++++++++++++++ website/src/components/UsersCell.tsx | 137 -------------------- website/src/lib/oasst_api_client.ts | 10 ++ website/src/pages/admin/index.tsx | 5 +- website/src/pages/api/admin/users.ts | 13 +- 9 files changed, 361 insertions(+), 145 deletions(-) create mode 100644 website/src/components/DataTable.tsx create mode 100644 website/src/components/UserTable.tsx delete mode 100644 website/src/components/UsersCell.tsx diff --git a/backend/oasst_backend/user_repository.py b/backend/oasst_backend/user_repository.py index 578dc5f1..c244d67f 100644 --- a/backend/oasst_backend/user_repository.py +++ b/backend/oasst_backend/user_repository.py @@ -161,10 +161,10 @@ class UserRepository: users = users.order_by(User.display_name) if gt: - users = users.filter(User.display_name > gt) + users = users.filter(User.id > gt) if lt: - users = users.filter(User.display_name < lt) + users = users.filter(User.id < lt).order_by(None).order_by(User.id.desc()) if limit is not None: users = users.limit(limit) diff --git a/website/package-lock.json b/website/package-lock.json index 1fa3d14d..29cd0326 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -21,6 +21,7 @@ "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", "@tailwindcss/forms": "^0.5.3", + "@tanstack/react-table": "^8.7.6", "autoprefixer": "^10.4.13", "axios": "^1.2.1", "boolean": "^3.2.0", @@ -12294,6 +12295,37 @@ "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1" } }, + "node_modules/@tanstack/react-table": { + "version": "8.7.6", + "resolved": "https://registry.npmjs.org/@tanstack/react-table/-/react-table-8.7.6.tgz", + "integrity": "sha512-/QijmMFeP7wDLBnr0MQ/5MlbXePbIL/1nOtkxBC9zvmBu4gDKJEDBqipUyM7Wc/iBpSd0IFyqBlvZvTPD9FYDA==", + "dependencies": { + "@tanstack/table-core": "8.7.6" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": ">=16", + "react-dom": ">=16" + } + }, + "node_modules/@tanstack/table-core": { + "version": "8.7.6", + "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.7.6.tgz", + "integrity": "sha512-sqiNTMzB6cpyL8DFH6/VqW48SwiflLqxQqYpo2wNock7rdVGvlm0BLNI8vZUJbr1+fmmWmHwBvi5OMgZw8n1DA==", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@testing-library/dom": { "version": "8.19.1", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-8.19.1.tgz", @@ -46565,6 +46597,19 @@ "mini-svg-data-uri": "^1.2.3" } }, + "@tanstack/react-table": { + "version": "8.7.6", + "resolved": "https://registry.npmjs.org/@tanstack/react-table/-/react-table-8.7.6.tgz", + "integrity": "sha512-/QijmMFeP7wDLBnr0MQ/5MlbXePbIL/1nOtkxBC9zvmBu4gDKJEDBqipUyM7Wc/iBpSd0IFyqBlvZvTPD9FYDA==", + "requires": { + "@tanstack/table-core": "8.7.6" + } + }, + "@tanstack/table-core": { + "version": "8.7.6", + "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.7.6.tgz", + "integrity": "sha512-sqiNTMzB6cpyL8DFH6/VqW48SwiflLqxQqYpo2wNock7rdVGvlm0BLNI8vZUJbr1+fmmWmHwBvi5OMgZw8n1DA==" + }, "@testing-library/dom": { "version": "8.19.1", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-8.19.1.tgz", diff --git a/website/package.json b/website/package.json index 580d0be3..6dcbb26a 100644 --- a/website/package.json +++ b/website/package.json @@ -38,6 +38,7 @@ "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", "@tailwindcss/forms": "^0.5.3", + "@tanstack/react-table": "^8.7.6", "autoprefixer": "^10.4.13", "axios": "^1.2.1", "boolean": "^3.2.0", diff --git a/website/src/components/DataTable.tsx b/website/src/components/DataTable.tsx new file mode 100644 index 00000000..eafca6d1 --- /dev/null +++ b/website/src/components/DataTable.tsx @@ -0,0 +1,158 @@ +import { + Box, + Button, + Card, + CardBody, + Flex, + FormControl, + FormLabel, + Input, + Popover, + PopoverArrow, + PopoverBody, + PopoverCloseButton, + PopoverContent, + PopoverTrigger, + Spacer, + Table, + TableCaption, + TableContainer, + Tbody, + Td, + Th, + Thead, + Tr, + useDisclosure, +} from "@chakra-ui/react"; +import { ColumnDef, flexRender, getCoreRowModel, useReactTable } from "@tanstack/react-table"; +import { ChangeEvent, ReactNode } from "react"; +import { FaFilter } from "react-icons/fa"; +import { useDebouncedCallback } from "use-debounce"; + +export type DataTableColumnDef = ColumnDef & { + filterable?: boolean; +}; + +// TODO: stricter type +export type FilterItem = { + id: string; + value: string; +}; + +export type DataTableProps = { + data: T[]; + columns: DataTableColumnDef[]; + caption?: string; + filterValues?: FilterItem[]; + onNextClick?: () => void; + onPreviousClick?: () => void; + onFilterChange?: (items: FilterItem[]) => void; +}; + +export const DataTable = ({ + data, + columns, + caption, + filterValues = [], + onNextClick, + onPreviousClick, + onFilterChange, +}: DataTableProps) => { + const { getHeaderGroups, getRowModel } = useReactTable({ + data, + columns, + getCoreRowModel: getCoreRowModel(), + }); + + const handleFilterChange = (value: FilterItem) => { + const idx = filterValues.findIndex((oldValue) => oldValue.id === value.id); + let newValues: FilterItem[] = []; + if (idx === -1) { + newValues = [...filterValues, value]; + } else { + newValues = filterValues.map((oldValue) => (oldValue.id === value.id ? value : oldValue)); + } + onFilterChange(newValues); + }; + return ( + + + + + + + + + + {caption} + + {getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + + {getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
+ + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + {(header.column.columnDef as DataTableColumnDef).filterable && ( + value.id === header.id)?.value ?? ""} + onChange={(value) => handleFilterChange({ id: header.id, value })} + label={flexRender(header.column.columnDef.header, header.getContext())} + > + )} + +
{flexRender(cell.column.columnDef.cell, cell.getContext())}
+ + + + ); +}; + +const FilterModal = ({ + label, + onChange, + value, +}: { + label: ReactNode; + onChange: (val: string) => void; + value: string; +}) => { + const { isOpen, onOpen, onClose } = useDisclosure(); + + const handleInputChange = useDebouncedCallback((e: ChangeEvent) => { + onChange(e.target.value); + }, 500); + + return ( + + + + + + + + + + {label} + + + + + + ); +}; diff --git a/website/src/components/UserTable.tsx b/website/src/components/UserTable.tsx new file mode 100644 index 00000000..3b63255a --- /dev/null +++ b/website/src/components/UserTable.tsx @@ -0,0 +1,133 @@ +import { IconButton, useToast } from "@chakra-ui/react"; +import { createColumnHelper } from "@tanstack/react-table"; +import Link from "next/link"; +import { memo, useState } from "react"; +import { FaPen } from "react-icons/fa"; +import { get } from "src/lib/api"; +import type { User } from "src/types/Users"; +import useSWR from "swr"; + +import { DataTable, DataTableColumnDef, FilterItem } from "./DataTable"; + +interface Pagination { + /** + * The user's `display_name` used for pagination. + */ + cursor: string; + + /** + * The pagination direction. + */ + direction: "forward" | "back"; +} + +const columnHelper = createColumnHelper(); + +const columns: DataTableColumnDef[] = [ + columnHelper.accessor("user_id", { + header: "ID", + }), + columnHelper.accessor("id", { + header: "Auth ID", + }), + columnHelper.accessor("auth_method", { + header: "Auth Method", + }), + { + ...columnHelper.accessor("display_name", { + header: "Name", + }), + filterable: true, + }, + columnHelper.accessor("role", { + header: "Role", + }), + columnHelper.accessor((user) => user.user_id, { + cell: ({ getValue }) => ( + } + > + ), + header: "Update", + }), +]; + +export const UserTable = memo(function UserTable() { + const toast = useToast(); + const [pagination, setPagination] = useState({ cursor: "", direction: "forward" }); + const [users, setUsers] = useState([]); + const [filterValues, setFilterValues] = useState([]); + // Fetch and save the users. + // This follows useSWR's recommendation for simple pagination: + // https://swr.vercel.app/docs/pagination#when-to-use-useswr + const display_name = filterValues.find((value) => value.id === "display_name")?.value ?? ""; + useSWR( + `/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}&display_name=${display_name}`, + get, + { + onSuccess: (data) => { + // When no more users can be found, trigger a toast to indicate why no + // changes have taken place. We have to maintain a non-empty set of + // users otherwise we can't paginate using a cursor (since we've lost the + // cursor). + if (data.length === 0) { + toast({ + title: "No more users", + status: "warning", + duration: 1000, + isClosable: true, + }); + return; + } + setUsers(data); + }, + } + ); + + const toPreviousPage = () => { + if (users.length >= 0) { + setPagination({ + cursor: users[0].user_id, + direction: "back", + }); + } else { + toast({ + title: "Can not paginate when no users are found", + status: "warning", + duration: 1000, + isClosable: true, + }); + } + }; + + const toNextPage = () => { + if (users.length >= 0) { + setPagination({ + cursor: users[users.length - 1].user_id, + direction: "forward", + }); + } else { + toast({ + title: "Can not paginate when no users are found", + status: "warning", + duration: 1000, + isClosable: true, + }); + } + }; + + return ( + + ); +}); diff --git a/website/src/components/UsersCell.tsx b/website/src/components/UsersCell.tsx deleted file mode 100644 index 99824090..00000000 --- a/website/src/components/UsersCell.tsx +++ /dev/null @@ -1,137 +0,0 @@ -import { - Button, - Flex, - Spacer, - Stack, - Table, - TableCaption, - TableContainer, - Tbody, - Td, - Th, - Thead, - Tr, - useToast, -} from "@chakra-ui/react"; -import Link from "next/link"; -import { useState } from "react"; -import { get } from "src/lib/api"; -import type { User } from "src/types/Users"; -import useSWR from "swr"; - -interface Pagination { - /** - * The user's `display_name` used for pagination. - */ - cursor: string; - - /** - * The pagination direction. - */ - direction: "forward" | "back"; -} - -/** - * Fetches users from the users api route and then presents them in a simple Chakra table. - */ -const UsersCell = () => { - const toast = useToast(); - const [pagination, setPagination] = useState({ cursor: "", direction: "forward" }); - const [users, setUsers] = useState([]); - - // Fetch and save the users. - // This follows useSWR's recommendation for simple pagination: - // https://swr.vercel.app/docs/pagination#when-to-use-useswr - useSWR(`/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}`, get, { - onSuccess: (data) => { - // When no more users can be found, trigger a toast to indicate why no - // changes have taken place. We have to maintain a non-empty set of - // users otherwise we can't paginate using a cursor (since we've lost the - // cursor). - if (data.length === 0) { - toast({ - title: "No more users", - status: "warning", - duration: 1000, - isClosable: true, - }); - return; - } - setUsers(data); - }, - }); - - const toPreviousPage = () => { - if (users.length >= 0) { - setPagination({ - cursor: users[0].display_name, - direction: "back", - }); - } else { - toast({ - title: "Can not paginate when no users are found", - status: "warning", - duration: 1000, - isClosable: true, - }); - } - }; - - const toNextPage = () => { - if (users.length >= 0) { - setPagination({ - cursor: users[users.length - 1].display_name, - direction: "forward", - }); - } else { - toast({ - title: "Can not paginate when no users are found", - status: "warning", - duration: 1000, - isClosable: true, - }); - } - }; - - // Present users in a naive table. - return ( - - - - - - - - - Users - - - - - - - - - - - - {users.map(({ id, user_id, auth_method, display_name, role }) => ( - - - - - - - - - ))} - -
IdAuth IdAuth MethodNameRoleUpdate
{user_id}{id}{auth_method}{display_name}{role} - Manage -
- - - ); -}; - -export default UsersCell; diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index fb11adec..866b2907 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -187,6 +187,16 @@ export class OasstApiClient { return this.get(url); } + async fetch_user_by_display_name(name: string): Promise { + const params = new URLSearchParams({ + search_text: name, + }); + + const endpoint = `/api/v1/frontend_users/by_display_name`; + + return this.get(`${endpoint}?${params.toString()}`); + } + /** * Returns the `Message`s associated with `user_id` in the backend. */ diff --git a/website/src/pages/admin/index.tsx b/website/src/pages/admin/index.tsx index 9cbea222..397230bd 100644 --- a/website/src/pages/admin/index.tsx +++ b/website/src/pages/admin/index.tsx @@ -3,7 +3,7 @@ import { useRouter } from "next/router"; import { useSession } from "next-auth/react"; import { useEffect } from "react"; import { getAdminLayout } from "src/components/Layout"; -import UsersCell from "src/components/UsersCell"; +import { UserTable } from "src/components/UserTable"; /** * Provides the admin index page that will display a list of users and give @@ -27,7 +27,6 @@ const AdminIndex = () => { } router.push("/"); }, [router, session, status]); - return ( <> @@ -37,7 +36,7 @@ const AdminIndex = () => { content="Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world." /> -
{status === "loading" ? "loading..." : }
+
{status === "loading" ? "loading..." : }
); }; diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts index e600650d..5cc41354 100644 --- a/website/src/pages/api/admin/users.ts +++ b/website/src/pages/api/admin/users.ts @@ -1,11 +1,12 @@ import { withRole } from "src/lib/auth"; import { oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; +import { BackendUser } from "src/types/Users"; /** * The number of users to fetch in a single request. Could later be a query parameter. */ -const PAGE_SIZE = 20; +const PAGE_SIZE = 1; /** * Returns a list of user results from the database when the requesting user is @@ -17,10 +18,16 @@ const PAGE_SIZE = 20; * direction. */ const handler = withRole("admin", async (req, res) => { - const { cursor, direction } = req.query; + const { cursor, direction, display_name = "" } = req.query; // First, get all the users according to the backend. - const all_users = await oasstApiClient.fetch_users(PAGE_SIZE, cursor as string, direction === "forward"); + let all_users: BackendUser[] = []; + + if (typeof display_name === "string" && display_name) { + all_users = await oasstApiClient.fetch_user_by_display_name(display_name); + } else { + all_users = await oasstApiClient.fetch_users(PAGE_SIZE, cursor as string, direction === "forward"); + } // Next, get all the users stored in the web's auth database to fetch their role. const local_user_ids = all_users.map(({ id }) => id); From 8eff6932d6867e85b53de314e744f46d3f36953c Mon Sep 17 00:00:00 2001 From: notmd Date: Wed, 18 Jan 2023 15:40:20 +0700 Subject: [PATCH 07/59] switch PAGE_SIZE back to 20 --- website/src/pages/api/admin/users.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts index 5cc41354..52921213 100644 --- a/website/src/pages/api/admin/users.ts +++ b/website/src/pages/api/admin/users.ts @@ -6,7 +6,7 @@ import { BackendUser } from "src/types/Users"; /** * The number of users to fetch in a single request. Could later be a query parameter. */ -const PAGE_SIZE = 1; +const PAGE_SIZE = 20; /** * Returns a list of user results from the database when the requesting user is From 622a4768f63415ca2b5ff1b9efa31f7e6550e2d4 Mon Sep 17 00:00:00 2001 From: notmd Date: Wed, 18 Jan 2023 16:07:54 +0700 Subject: [PATCH 08/59] fix default column --- backend/oasst_backend/user_repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/oasst_backend/user_repository.py b/backend/oasst_backend/user_repository.py index c244d67f..7c46a026 100644 --- a/backend/oasst_backend/user_repository.py +++ b/backend/oasst_backend/user_repository.py @@ -158,7 +158,7 @@ class UserRepository: if auth_method: users = users.filter(User.auth_method == auth_method) - users = users.order_by(User.display_name) + users = users.order_by(User.id) if gt: users = users.filter(User.id > gt) From 0c5e2fc45deb0f8491d34229b728428e4ef1f964 Mon Sep 17 00:00:00 2001 From: AbdBarho Date: Fri, 20 Jan 2023 13:00:59 +0100 Subject: [PATCH 09/59] Show last updated on leaderboard --- website/public/locales/en/common.json | 3 ++- .../LeaderboardGridCell.tsx | 27 ++++++++++++++----- website/src/pages/api/leaderboard.ts | 6 ++--- website/src/types/Leaderboard.ts | 1 + 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/website/public/locales/en/common.json b/website/public/locales/en/common.json index e18eb8ec..de764cc2 100644 --- a/website/public/locales/en/common.json +++ b/website/public/locales/en/common.json @@ -13,5 +13,6 @@ "sign_in": "Sign In", "sign_out": "Sign Out", "terms_of_service": "Terms of Service", - "title": "Open Assistant" + "title": "Open Assistant", + "last_updated_at": "Last updated at: {{val, datetime}}" } diff --git a/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx b/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx index df18735d..9750a851 100644 --- a/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx +++ b/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx @@ -1,8 +1,9 @@ -import { Table, TableContainer, Tbody, Td, Th, Thead, Tr, useColorModeValue } from "@chakra-ui/react"; -import React from "react"; +import { Table, TableContainer, Tbody, Td, Text, Th, Thead, Tr, useColorModeValue } from "@chakra-ui/react"; +import { useTranslation } from "next-i18next"; +import React, { useMemo } from "react"; import { useTable } from "react-table"; import { get } from "src/lib/api"; -import { LeaderboardEntity, LeaderboardTimeFrame } from "src/types/Leaderboard"; +import { LeaderboardReply, LeaderboardTimeFrame } from "src/types/Leaderboard"; import useSWRImmutable from "swr/immutable"; const columns = [ @@ -26,13 +27,26 @@ const columns = [ * Presents a grid of leaderboard entries with more detailed information. */ const LeaderboardGridCell = ({ timeFrame }: { timeFrame: LeaderboardTimeFrame }) => { - const { data } = useSWRImmutable(`/api/leaderboard?time_frame=${timeFrame}`, get, { - fallbackData: [], + const { t } = useTranslation(); + const { data: reply } = useSWRImmutable(`/api/leaderboard?time_frame=${timeFrame}`, get, { revalidateOnMount: true, }); + + const { getTableProps, getTableBodyProps, headerGroups, rows, prepareRow } = useTable({ + columns, + data: reply?.leaderboard ?? [], + }); + const backgroundColor = useColorModeValue("white", "gray.800"); - const { getTableProps, getTableBodyProps, headerGroups, rows, prepareRow } = useTable({ columns, data }); + const lastUpdated = useMemo(() => { + const val = new Date(reply?.last_updated); + return t("last_updated_at", { val, formatParams: { val: { dateStyle: "full", timeStyle: "short" } } }); + }, [t, reply?.last_updated]); + + if (!reply) { + return null; + } return ( @@ -66,6 +80,7 @@ const LeaderboardGridCell = ({ timeFrame }: { timeFrame: LeaderboardTimeFrame }) })} + {lastUpdated} ); }; diff --git a/website/src/pages/api/leaderboard.ts b/website/src/pages/api/leaderboard.ts index 592f3da5..1ddf947e 100644 --- a/website/src/pages/api/leaderboard.ts +++ b/website/src/pages/api/leaderboard.ts @@ -6,9 +6,9 @@ import { LeaderboardTimeFrame } from "src/types/Leaderboard"; * Returns the set of valid labels that can be applied to messages. */ const handler = withoutRole("banned", async (req, res) => { - const time_frame = (req.query.time_frame as LeaderboardTimeFrame) || LeaderboardTimeFrame.day; - const { leaderboard } = await oasstApiClient.fetch_leaderboard(time_frame); - res.status(200).json(leaderboard); + const time_frame = (req.query.time_frame as LeaderboardTimeFrame) ?? LeaderboardTimeFrame.day; + const info = await oasstApiClient.fetch_leaderboard(time_frame); + res.status(200).json(info); }); export default handler; diff --git a/website/src/types/Leaderboard.ts b/website/src/types/Leaderboard.ts index 21c91766..5c0acfc3 100644 --- a/website/src/types/Leaderboard.ts +++ b/website/src/types/Leaderboard.ts @@ -12,6 +12,7 @@ export const enum LeaderboardTimeFrame { } export interface LeaderboardReply { time_frame: LeaderboardTimeFrame; + last_updated: string; // date time iso string leaderboard: LeaderboardEntity[]; } From 2802bc15812fbc75d84405e8c6e3d45871b00145 Mon Sep 17 00:00:00 2001 From: AbdBarho Date: Fri, 20 Jan 2023 13:51:47 +0100 Subject: [PATCH 10/59] Render whitespace in messages --- website/src/components/Messages/MessageTableEntry.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/website/src/components/Messages/MessageTableEntry.tsx b/website/src/components/Messages/MessageTableEntry.tsx index d18bd910..1205991e 100644 --- a/website/src/components/Messages/MessageTableEntry.tsx +++ b/website/src/components/Messages/MessageTableEntry.tsx @@ -50,6 +50,7 @@ export function MessageTableEntry(props: MessageTableEntryProps) { bg={item.is_assistant ? backgroundColor : backgroundColor2} onClick={props.enabled && goToMessage} _hover={props.enabled && { cursor: "pointer", opacity: 0.9 }} + whiteSpace="pre-wrap" > {inlineAvatar && avatar} {item.text} From 7eb5023e8222f0977ad0d56f45107342a9f3df14 Mon Sep 17 00:00:00 2001 From: notmd Date: Sat, 21 Jan 2023 14:34:22 +0700 Subject: [PATCH 11/59] use cursor endpoint --- website/src/components/UserTable.tsx | 63 +++++++++++++++------------- website/src/lib/oasst_api_client.ts | 51 ++++++++++++++++------ website/src/pages/api/admin/users.ts | 26 ++++++------ 3 files changed, 86 insertions(+), 54 deletions(-) diff --git a/website/src/components/UserTable.tsx b/website/src/components/UserTable.tsx index 3b63255a..68285bfa 100644 --- a/website/src/components/UserTable.tsx +++ b/website/src/components/UserTable.tsx @@ -4,6 +4,7 @@ import Link from "next/link"; import { memo, useState } from "react"; import { FaPen } from "react-icons/fa"; import { get } from "src/lib/api"; +import { FetchUsersResponse } from "src/lib/oasst_api_client"; import type { User } from "src/types/Users"; import useSWR from "swr"; @@ -58,39 +59,43 @@ const columns: DataTableColumnDef[] = [ export const UserTable = memo(function UserTable() { const toast = useToast(); const [pagination, setPagination] = useState({ cursor: "", direction: "forward" }); - const [users, setUsers] = useState([]); + const [response, setResponse] = useState, "sort_key" | "order">>({ + items: [], + }); const [filterValues, setFilterValues] = useState([]); + const handleFilterValuesChange = (values: FilterItem[]) => { + setFilterValues(values); + setPagination((old) => ({ ...old, cursor: "" })); + }; // Fetch and save the users. // This follows useSWR's recommendation for simple pagination: // https://swr.vercel.app/docs/pagination#when-to-use-useswr const display_name = filterValues.find((value) => value.id === "display_name")?.value ?? ""; - useSWR( - `/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}&display_name=${display_name}`, - get, - { - onSuccess: (data) => { - // When no more users can be found, trigger a toast to indicate why no - // changes have taken place. We have to maintain a non-empty set of - // users otherwise we can't paginate using a cursor (since we've lost the - // cursor). - if (data.length === 0) { - toast({ - title: "No more users", - status: "warning", - duration: 1000, - isClosable: true, - }); - return; - } - setUsers(data); - }, - } - ); + useSWR< + FetchUsersResponse + >(`/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}&searchDisplayName=${display_name}&sortKey=display_name`, get, { + onSuccess: (data) => { + // When no more users can be found, trigger a toast to indicate why no + // changes have taken place. We have to maintain a non-empty set of + // users otherwise we can't paginate using a cursor (since we've lost the + // cursor). + if (data.items.length === 0) { + toast({ + title: "No more users", + status: "warning", + duration: 1000, + isClosable: true, + }); + return; + } + setResponse(data); + }, + }); const toPreviousPage = () => { - if (users.length >= 0) { + if (response.items.length >= 0) { setPagination({ - cursor: users[0].user_id, + cursor: response.prev, direction: "back", }); } else { @@ -104,9 +109,9 @@ export const UserTable = memo(function UserTable() { }; const toNextPage = () => { - if (users.length >= 0) { + if (response.items.length >= 0) { setPagination({ - cursor: users[users.length - 1].user_id, + cursor: response.next, direction: "forward", }); } else { @@ -121,13 +126,13 @@ export const UserTable = memo(function UserTable() { return ( ); }); diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index 7db6e3c2..50adf267 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -1,7 +1,7 @@ import type { Message } from "src/types/Conversation"; import { LeaderboardReply, LeaderboardTimeFrame } from "src/types/Leaderboard"; import type { AvailableTasks } from "src/types/Task"; -import type { BackendUser, BackendUserCore } from "src/types/Users"; +import type { BackendUser, BackendUserCore, User } from "src/types/Users"; export class OasstError { message: string; @@ -15,6 +15,22 @@ export class OasstError { } } +export type FetchUsersParams = { + limit: number; + cursor?: string; + direction: "forward" | "back"; + searchDisplayName?: string; + sortKey?: "username" | "display_name"; +}; + +export type FetchUsersResponse = { + items: T[]; + next?: string; + prev?: string; + sort_key: "username" | "display_name"; + order: "asc" | "desc"; +}; + export class OasstApiClient { oasstApiUrl: string; oasstApiKey: string; @@ -164,30 +180,39 @@ export class OasstApiClient { * forward. If false and `cursor` is not empty, pages backwards. * @returns {Promise} A Promise that returns an array of `BackendUser` objects. */ - async fetch_users(max_count: number, cursor: string, isForward: boolean): Promise { - const params = new URLSearchParams(); - params.append("max_count", max_count.toString()); + async fetch_users({ + direction, + limit, + cursor, + searchDisplayName, + sortKey = "display_name", + }: FetchUsersParams): Promise { + const params = new URLSearchParams({ + search_text: searchDisplayName, + sort_key: sortKey, + max_count: limit.toString(), + }); // The backend API uses different query parameters depending on the // pagination direction but they both take the same cursor value. // Depending on direction, pick the right query param. if (cursor !== "") { - params.append(isForward ? "gt" : "lt", cursor); + params.append(direction === "forward" ? "gt" : "lt", cursor); } - const BASE_URL = `/api/v1/frontend_users`; + const BASE_URL = `/api/v1/users/cursor`; const url = `${BASE_URL}/?${params.toString()}`; return this.get(url); } - async fetch_user_by_display_name(name: string): Promise { - const params = new URLSearchParams({ - search_text: name, - }); + // async fetch_user_by_display_name(name: string): Promise { + // const params = new URLSearchParams({ + // search_text: name, + // }); - const endpoint = `/api/v1/frontend_users/by_display_name`; + // const endpoint = `/api/v1/frontend_users/by_display_name`; - return this.get(`${endpoint}?${params.toString()}`); - } + // return this.get(`${endpoint}?${params.toString()}`); + // } /** * Returns the `Message`s associated with `user_id` in the backend. diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts index 52921213..f43af305 100644 --- a/website/src/pages/api/admin/users.ts +++ b/website/src/pages/api/admin/users.ts @@ -1,12 +1,11 @@ import { withRole } from "src/lib/auth"; -import { oasstApiClient } from "src/lib/oasst_api_client"; +import { FetchUsersParams, oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; -import { BackendUser } from "src/types/Users"; /** * The number of users to fetch in a single request. Could later be a query parameter. */ -const PAGE_SIZE = 20; +const PAGE_SIZE = 2; /** * Returns a list of user results from the database when the requesting user is @@ -18,16 +17,16 @@ const PAGE_SIZE = 20; * direction. */ const handler = withRole("admin", async (req, res) => { - const { cursor, direction, display_name = "" } = req.query; + const { cursor, direction, searchDisplayName = "", sortKey = "username" } = req.query; // First, get all the users according to the backend. - let all_users: BackendUser[] = []; - - if (typeof display_name === "string" && display_name) { - all_users = await oasstApiClient.fetch_user_by_display_name(display_name); - } else { - all_users = await oasstApiClient.fetch_users(PAGE_SIZE, cursor as string, direction === "forward"); - } + const { items: all_users, ...rest } = await oasstApiClient.fetch_users({ + searchDisplayName: searchDisplayName as FetchUsersParams["searchDisplayName"], + direction: direction as FetchUsersParams["direction"], + limit: PAGE_SIZE, + cursor: cursor as FetchUsersParams["cursor"], + sortKey: sortKey === "username" || sortKey === "display_name" ? sortKey : undefined, + }); // Next, get all the users stored in the web's auth database to fetch their role. const local_user_ids = all_users.map(({ id }) => id); @@ -58,7 +57,10 @@ const handler = withRole("admin", async (req, res) => { }; }); - res.status(200).json(users); + res.status(200).json({ + items: users, + ...rest, + }); }); export default handler; From 77210ee6d41100a912498dbe898901f6d605ffa5 Mon Sep 17 00:00:00 2001 From: notmd Date: Sat, 21 Jan 2023 14:37:05 +0700 Subject: [PATCH 12/59] remove debug code --- website/src/pages/api/admin/users.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts index f43af305..57944cff 100644 --- a/website/src/pages/api/admin/users.ts +++ b/website/src/pages/api/admin/users.ts @@ -5,7 +5,7 @@ import prisma from "src/lib/prismadb"; /** * The number of users to fetch in a single request. Could later be a query parameter. */ -const PAGE_SIZE = 2; +const PAGE_SIZE = 20; /** * Returns a list of user results from the database when the requesting user is From b9ce2fb1406c3a5a79f2680bc6b19c6adbeaceb8 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 21 Jan 2023 17:56:29 +0900 Subject: [PATCH 13/59] Ensuring localization works across all pages and fix user data fetching in admin view --- website/src/lib/oasst_api_client.ts | 2 +- website/src/pages/auth/verify.tsx | 5 +++-- website/src/pages/tasks/random.tsx | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index b1639462..1e74b020 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -152,7 +152,7 @@ export class OasstApiClient { * Returns the `BackendUser` associated with `user_id` */ async fetch_user(user_id: string): Promise { - return this.get(`/api/v1/users/users/${user_id}`); + return this.get(`/api/v1/users/${user_id}`); } /** diff --git a/website/src/pages/auth/verify.tsx b/website/src/pages/auth/verify.tsx index 876aa677..d7a64a63 100644 --- a/website/src/pages/auth/verify.tsx +++ b/website/src/pages/auth/verify.tsx @@ -1,6 +1,7 @@ import { useColorMode } from "@chakra-ui/react"; import Head from "next/head"; import { getCsrfToken, getProviders } from "next-auth/react"; +import { serverSideTranslations } from "next-i18next/serverSideTranslations"; export default function Verify() { const { colorMode } = useColorMode(); @@ -21,14 +22,14 @@ export default function Verify() { ); } -// eslint-disable-next-line @typescript-eslint/no-unused-vars -export async function getServerSideProps(context) { +export async function getServerSideProps({ locale }) { const csrfToken = await getCsrfToken(); const providers = await getProviders(); return { props: { csrfToken, providers, + ...(await serverSideTranslations(locale, ["common"])), }, }; } diff --git a/website/src/pages/tasks/random.tsx b/website/src/pages/tasks/random.tsx index be1809c3..f1c04d2c 100644 --- a/website/src/pages/tasks/random.tsx +++ b/website/src/pages/tasks/random.tsx @@ -4,6 +4,7 @@ import { getDashboardLayout } from "src/components/Layout"; import { LoadingScreen } from "src/components/Loading/LoadingScreen"; import { Task } from "src/components/Tasks/Task"; import { useGenericTaskAPI } from "src/hooks/tasks/useGenericTaskAPI"; +export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props"; import { TaskType } from "src/types/Task"; const RandomTask = () => { From e756e5d9926fa14a2a454cafa046f58d5fdd9e22 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 21 Jan 2023 19:03:12 +0900 Subject: [PATCH 14/59] Fixing another page with locale data --- website/src/pages/tasks/all.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/website/src/pages/tasks/all.tsx b/website/src/pages/tasks/all.tsx index ed0659c8..3ccfd4e8 100644 --- a/website/src/pages/tasks/all.tsx +++ b/website/src/pages/tasks/all.tsx @@ -2,6 +2,7 @@ import Head from "next/head"; import { TaskOption } from "src/components/Dashboard"; import { getDashboardLayout } from "src/components/Layout"; import { TaskCategory } from "src/components/Tasks/TaskTypes"; +export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props"; const AllTasks = () => { return ( From 186aabe3a547b9c04f9cce57a6ff5d51c9576a72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Sat, 21 Jan 2023 12:19:45 +0100 Subject: [PATCH 15/59] improve prev,next cursor values --- backend/oasst_backend/api/v1/messages.py | 11 ++++++++-- backend/oasst_backend/api/v1/users.py | 27 ++++++++++++++++++------ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/backend/oasst_backend/api/v1/messages.py b/backend/oasst_backend/api/v1/messages.py index 2dcca64b..1ef1e929 100644 --- a/backend/oasst_backend/api/v1/messages.py +++ b/backend/oasst_backend/api/v1/messages.py @@ -96,8 +96,15 @@ def get_messages_cursor( items = utils.prepare_message_list(messages) n, p = None, None if len(items) > 0: - p = str(items[0].id) + "$" + items[0].created_date.isoformat() - n = str(items[-1].id) + "$" + items[-1].created_date.isoformat() + if len(items) == max_count or gte_created_date: + p = str(items[0].id) + "$" + items[0].created_date.isoformat() + if len(items) == max_count or lte_created_date: + n = str(items[-1].id) + "$" + items[-1].created_date.isoformat() + else: + if gte_created_date: + p = gte_created_date.isoformat() + if lte_created_date: + n = lte_created_date.isoformat() order = "desc" if desc else "asc" return protocol.MessagePage(prev=p, next=n, sort_key="created_date", order=order, items=items) diff --git a/backend/oasst_backend/api/v1/users.py b/backend/oasst_backend/api/v1/users.py index 63a55691..2697d6fe 100644 --- a/backend/oasst_backend/api/v1/users.py +++ b/backend/oasst_backend/api/v1/users.py @@ -1,5 +1,5 @@ import datetime -from typing import Optional +from typing import Callable, Optional from uuid import UUID from fastapi import APIRouter, Depends, Query @@ -93,6 +93,21 @@ def get_users_cursor( return x, None items: list[protocol.FrontEndUser] + + def get_next_prev(lte: str | None, gte: str | None, key_fn: Callable[[protocol.FrontEndUser], str]): + p, n = None, None + if len(items) > 0: + if len(items) == max_count or gte: + p = str(items[0].user_id) + "$" + key_fn(items[0]) + if len(items) == max_count or lte: + n = str(items[-1].user_id) + "$" + key_fn(items[-1]) + else: + if gte: + p = gte + if lte: + n = lte + return p, n + n, p = None, None if sort_key == "username": lte_username, lt_id = split_cursor(lt) @@ -109,9 +124,8 @@ def get_users_cursor( api_client=api_client, db=db, ) - if len(items) > 0: - p = str(items[0].user_id) + "$" + items[0].id - n = str(items[-1].user_id) + "$" + items[-1].id + p, n = get_next_prev(lte_username, gte_username, lambda x: x.id) + elif sort_key == "display_name": lte_display_name, lt_id = split_cursor(lt) gte_display_name, gt_id = split_cursor(gt) @@ -127,9 +141,8 @@ def get_users_cursor( api_client=api_client, db=db, ) - if len(items) > 0: - p = str(items[0].user_id) + "$" + items[0].display_name - n = str(items[-1].user_id) + "$" + items[-1].display_name + p, n = get_next_prev(lte_display_name, gte_display_name, lambda x: x.display_name) + else: raise OasstError(f"Unsupported sort key: '{sort_key}'", OasstErrorCode.SORT_KEY_UNSUPPORTED) From 7274512c2f408355370d7b36779bd78690b6bbfb Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Sat, 21 Jan 2023 21:36:37 +0900 Subject: [PATCH 16/59] Implementing core of setting user languages and fetching language specific tasks --- website/next-i18next.config.js | 2 +- website/package-lock.json | 45 +++++++++++++++++++ website/package.json | 2 + website/src/components/Header/Header.tsx | 2 + .../LanguageSelector/LanguageSelector.tsx | 40 +++++++++++++++++ .../src/components/LanguageSelector/index.tsx | 1 + website/src/lib/oasst_api_client.ts | 7 ++- website/src/lib/users.ts | 16 ++++++- website/src/pages/api/new_task/[task_type].ts | 5 ++- website/src/pages/api/update_task.ts | 13 +++++- 10 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 website/src/components/LanguageSelector/LanguageSelector.tsx create mode 100644 website/src/components/LanguageSelector/index.tsx diff --git a/website/next-i18next.config.js b/website/next-i18next.config.js index 7c87a7a4..40c4b14d 100644 --- a/website/next-i18next.config.js +++ b/website/next-i18next.config.js @@ -1,6 +1,6 @@ module.exports = { i18n: { defaultLocale: "en", - locales: ["en"], + locales: ["de", "en", "fr"], }, }; diff --git a/website/package-lock.json b/website/package-lock.json index 06f3c98d..5c5dc795 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -21,6 +21,7 @@ "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", "@tailwindcss/forms": "^0.5.3", + "accept-language-parser": "^1.5.0", "autoprefixer": "^10.4.13", "axios": "^1.2.1", "boolean": "^3.2.0", @@ -38,6 +39,7 @@ "npm": "^9.2.0", "postcss-focus-visible": "^7.1.0", "react": "18.2.0", + "react-cookies": "^0.1.1", "react-dom": "18.2.0", "react-feature-flags": "^1.0.0", "react-hook-form": "^7.42.1", @@ -13616,6 +13618,11 @@ "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", "dev": true }, + "node_modules/accept-language-parser": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/accept-language-parser/-/accept-language-parser-1.5.0.tgz", + "integrity": "sha512-QhyTbMLYo0BBGg1aWbeMG4ekWtds/31BrEU+DONOg/7ax23vxpL03Pb7/zBmha2v7vdD3AyzZVWBVGEZxKOXWw==" + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -32466,6 +32473,23 @@ "react": "^15.3.0 || ^16.0.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/react-cookies": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/react-cookies/-/react-cookies-0.1.1.tgz", + "integrity": "sha512-PP75kJ4vtoHuuTdq0TAD3RmlAv7vuDQh9fkC4oDlhntgs9vX1DmREomO0Y1mcQKR9nMZ6/zxoflaMJ3MAmF5KQ==", + "dependencies": { + "cookie": "^0.3.1", + "object-assign": "^4.1.1" + } + }, + "node_modules/react-cookies/node_modules/cookie": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/react-docgen": { "version": "5.4.3", "resolved": "https://registry.npmjs.org/react-docgen/-/react-docgen-5.4.3.tgz", @@ -47817,6 +47841,11 @@ "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", "dev": true }, + "accept-language-parser": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/accept-language-parser/-/accept-language-parser-1.5.0.tgz", + "integrity": "sha512-QhyTbMLYo0BBGg1aWbeMG4ekWtds/31BrEU+DONOg/7ax23vxpL03Pb7/zBmha2v7vdD3AyzZVWBVGEZxKOXWw==" + }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -61962,6 +61991,22 @@ "@babel/runtime": "^7.12.13" } }, + "react-cookies": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/react-cookies/-/react-cookies-0.1.1.tgz", + "integrity": "sha512-PP75kJ4vtoHuuTdq0TAD3RmlAv7vuDQh9fkC4oDlhntgs9vX1DmREomO0Y1mcQKR9nMZ6/zxoflaMJ3MAmF5KQ==", + "requires": { + "cookie": "^0.3.1", + "object-assign": "^4.1.1" + }, + "dependencies": { + "cookie": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw==" + } + } + }, "react-docgen": { "version": "5.4.3", "resolved": "https://registry.npmjs.org/react-docgen/-/react-docgen-5.4.3.tgz", diff --git a/website/package.json b/website/package.json index 4ae762e4..8866a9e2 100644 --- a/website/package.json +++ b/website/package.json @@ -38,6 +38,7 @@ "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", "@tailwindcss/forms": "^0.5.3", + "accept-language-parser": "^1.5.0", "autoprefixer": "^10.4.13", "axios": "^1.2.1", "boolean": "^3.2.0", @@ -55,6 +56,7 @@ "npm": "^9.2.0", "postcss-focus-visible": "^7.1.0", "react": "18.2.0", + "react-cookies": "^0.1.1", "react-dom": "18.2.0", "react-feature-flags": "^1.0.0", "react-hook-form": "^7.42.1", diff --git a/website/src/components/Header/Header.tsx b/website/src/components/Header/Header.tsx index a1b36123..64614578 100644 --- a/website/src/components/Header/Header.tsx +++ b/website/src/components/Header/Header.tsx @@ -5,6 +5,7 @@ import { useSession } from "next-auth/react"; import { useTranslation } from "next-i18next"; import { Flags } from "react-feature-flags"; import { FaUser } from "react-icons/fa"; +import { LanguageSelector } from "src/components/LanguageSelector"; import { UserMenu } from "./UserMenu"; @@ -45,6 +46,7 @@ export function Header() { FlagTest + diff --git a/website/src/components/LanguageSelector/LanguageSelector.tsx b/website/src/components/LanguageSelector/LanguageSelector.tsx new file mode 100644 index 00000000..e611bf0f --- /dev/null +++ b/website/src/components/LanguageSelector/LanguageSelector.tsx @@ -0,0 +1,40 @@ +import { Select } from "@chakra-ui/react"; +import { useRouter } from "next/router"; +import { useTranslation } from "next-i18next"; +import { useCallback, useMemo, useState } from "react"; +import cookie from "react-cookies"; + +const LanguageSelector = () => { + const router = useRouter(); + const { i18n } = useTranslation(); + + const { language: currentLanguage } = i18n; + const languageNames = useMemo(() => { + return new Intl.DisplayNames([currentLanguage], { + type: "language", + }); + }, [currentLanguage]); + + const languageChanged = useCallback( + async (option) => { + const locale = option.target.value; + cookie.save("NEXT_LOCALE", locale, { path: "/" }); + const path = router.asPath; + return router.push(path, path, { locale }); + }, + [router] + ); + + const locales = router.locales; + return ( + + ); +}; + +export { LanguageSelector }; diff --git a/website/src/components/LanguageSelector/index.tsx b/website/src/components/LanguageSelector/index.tsx new file mode 100644 index 00000000..feb9f322 --- /dev/null +++ b/website/src/components/LanguageSelector/index.tsx @@ -0,0 +1 @@ +export * from "./LanguageSelector"; diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index 1e74b020..36e3ae33 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -108,10 +108,11 @@ export class OasstApiClient { // TODO return a strongly typed Task? // This method is used to store a task in RegisteredTask.task. // This is a raw Json type, so we can't use it to strongly type the task. - async fetchTask(taskType: string, user: BackendUserCore): Promise { + async fetchTask(taskType: string, user: BackendUserCore, lang: string): Promise { return this.post("/api/v1/tasks/", { type: taskType, user, + lang, }); } @@ -136,7 +137,8 @@ export class OasstApiClient { messageId: string, userMessageId: string, content: object, - user: BackendUserCore + user: BackendUserCore, + lang: string ): Promise { return this.post("/api/v1/tasks/interaction", { type: updateType, @@ -144,6 +146,7 @@ export class OasstApiClient { task_id: taskId, message_id: messageId, user_message_id: userMessageId, + lang, ...content, }); } diff --git a/website/src/lib/users.ts b/website/src/lib/users.ts index 2aa8c708..3dbe5a08 100644 --- a/website/src/lib/users.ts +++ b/website/src/lib/users.ts @@ -1,6 +1,20 @@ +import parser from "accept-language-parser"; +import type { NextApiRequest } from "next"; import prisma from "src/lib/prismadb"; import type { BackendUserCore } from "src/types/Users"; +const getUserLanguage = (req: NextApiRequest) => { + const cookieLanguage = req.cookies["NEXT_LOCALE"]; + if (cookieLanguage) { + return cookieLanguage; + } + const headerLanguages = parser.parse(req.headers["accept-language"]); + if (headerLanguages.length > 0) { + return headerLanguages[0].code; + } + return "en"; +}; + /** * Returns a `BackendUserCore` that can be used for interacting with the Backend service. * @@ -35,4 +49,4 @@ const getBackendUserCore = async (id: string) => { } as BackendUserCore; }; -export { getBackendUserCore }; +export { getBackendUserCore, getUserLanguage }; diff --git a/website/src/pages/api/new_task/[task_type].ts b/website/src/pages/api/new_task/[task_type].ts index c8255b18..360b8faa 100644 --- a/website/src/pages/api/new_task/[task_type].ts +++ b/website/src/pages/api/new_task/[task_type].ts @@ -1,7 +1,7 @@ import { withoutRole } from "src/lib/auth"; import { oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; -import { getBackendUserCore } from "src/lib/users"; +import { getBackendUserCore, getUserLanguage } from "src/lib/users"; /** * Returns a new task created from the Task Backend. We do a few things here: @@ -14,11 +14,12 @@ import { getBackendUserCore } from "src/lib/users"; const handler = withoutRole("banned", async (req, res, token) => { // Fetch the new task. const { task_type } = req.query; + const userLanguage = getUserLanguage(req); const user = await getBackendUserCore(token.sub); let task; try { - task = await oasstApiClient.fetchTask(task_type as string, user); + task = await oasstApiClient.fetchTask(task_type as string, user, userLanguage); } catch (err) { console.error(err); res.status(500).json(err); diff --git a/website/src/pages/api/update_task.ts b/website/src/pages/api/update_task.ts index c547503a..6f08d640 100644 --- a/website/src/pages/api/update_task.ts +++ b/website/src/pages/api/update_task.ts @@ -2,7 +2,7 @@ import { Prisma } from "@prisma/client"; import { withoutRole } from "src/lib/auth"; import { oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; -import { getBackendUserCore } from "src/lib/users"; +import { getBackendUserCore, getUserLanguage } from "src/lib/users"; /** * Stores the task interaction with the Task Backend and then returns the next task generated. @@ -41,9 +41,18 @@ const handler = withoutRole("banned", async (req, res, token) => { }); const user = await getBackendUserCore(token.sub); + const userLanguage = getUserLanguage(req); let newTask; try { - newTask = await oasstApiClient.interactTask(update_type, taskId, frontendId, interaction.id, content, user); + newTask = await oasstApiClient.interactTask( + update_type, + taskId, + frontendId, + interaction.id, + content, + user, + userLanguage + ); } catch (err) { console.error(JSON.stringify(err)); return res.status(500).json(err); From d15d8357a1d2bf6dd1d442aa43fc207cc136066d Mon Sep 17 00:00:00 2001 From: James Mete Date: Sat, 21 Jan 2023 15:54:21 +0300 Subject: [PATCH 17/59] 126 twitter data (#620) * Added a script file to process json archive files into more unified parquet files focused on tweet reply rows for further processing. * Added README file for Twitter data collection. * Re did code for processing json into standardized parquet files. * Added file to process parquet files into a conversation tree jsonl file. * Added requirements and ran pre-commit. --- scripts/data-collection/twitter/README.md | 80 ++++++ .../data-collection/twitter/requirements.txt | 3 + .../twitter/twitter_create_convs.py | 141 +++++++++++ .../twitter/twitter_process_json.py | 233 ++++++++++++++++++ 4 files changed, 457 insertions(+) create mode 100644 scripts/data-collection/twitter/README.md create mode 100644 scripts/data-collection/twitter/requirements.txt create mode 100644 scripts/data-collection/twitter/twitter_create_convs.py create mode 100644 scripts/data-collection/twitter/twitter_process_json.py diff --git a/scripts/data-collection/twitter/README.md b/scripts/data-collection/twitter/README.md new file mode 100644 index 00000000..b8c7bfc9 --- /dev/null +++ b/scripts/data-collection/twitter/README.md @@ -0,0 +1,80 @@ +# Twitter data collection for Open Assistant + +Conversations on Twitter can be an interesting and useful source of data for our +model to learn from. Certain twitter threads may contain helpful prompts and +replies, in a similar fashion to how we want our model to be able to respond to +prompts in a useful way. + +Thus, these scripts are intended to process twitter data from a variety of +sources, process them into cleaner and more useful formats, and then combine the +various outputs into a unified training set that can be fed to our model as a +conversation, or at least as a prompt with replies. + +**Note: Based on issue #126** + +## Possible Data Paths + +- Twitterstream archive: https://archive.org/details/twitterstream These are + large .tar files with compressed json files inside. However, some data points + such as reply counts seem to always be 0 due to limitations when scraping the + Twitter API. +- Alternative APIs such as snscrape, twint, etc. These alternative APIs often + are harder to use than the official Twitter API but can often bypass API + limits which can make it useful for larger scale data collection. The downside + is potentially slower speed, and less features. +- The official Twitter API + +## Currently Completed Items + +- Downloaded various archive files (both are .tar, but each have a different + format of json compression. One used .gz, and the other.bz2). Each json file + is roughly 2000 rows of tweets. There are thousands of these compressed json + files. Managing the IO of opening lots of small files is one of the + challenges, which is why future steps will consolidate data into larger easier + to process files. +- Wrote script that can loop through the compressed json files, cleans them up a + bit by removing truncated tweets or tweets that aren't replies. The script + then standardizes the columns, and exports the polars dataframes into parquet + files for future processing. Note: Using polars instead of pandas due to + performance reasons. +- Wrote scripts that process the large dump of tweets into conversation threads + using the tree and node architecture. This results in aroun 17K conversation + threads bassed on a dump of 90M tweets. +- Script can output the conversation threads into a jsonl file for further + filtering or use in models. + +## Main Issue + +- The issue is that we can easily scrape replies, but there is no guarantee the + original tweet is in the archive file. Furthermore, the archives are large so + they would need to be kept completely in-memory or in a db to reference. We + still need to decide if we want to try to mine the archive to piece together + the conversations, or we can take the list of replied tweets and loop through + those and use alternative apis to fetch the original tweet text, and then + match it with the confirmed replies already in our archive to generate the + prompt/replies data. Currently, my script can extract conversations based on + the dump, but it is a small percentage of the overall dump, and there is no + guarantee of the quality of the tweets. +- The tweet quality is the other major issue. We can get conversations through + the currently made scripts, but they most likely don't match a useful + instruction -> fulfilment. We are trying to filter the tweets through various + means such as matching useful hashtags, or by using cosine similarity against + known instructions. +- The modern Twitter API has conversation_id as a field which can be a way to + gather all tweets in a thread sort of automatically although there is + pagination limits. The main issue with this is it seems hard to search for it + using alternative APIs. + +## TODO + +- Write scripts to filter existing conversations into useful instructions -> + fulfilment with hashtags or cosine similarity. +- Train model to detect if text is a suitable instruction. This could then be + run through the conversations (or full tweet dump) to simplify the process. + Related to issue #143. +- Write script that matches the original tweets and their text with the archive + data to create the prompt/reply dataset. (Optional) +- Decide on final output format and storage options for the dataset. Currently + in JSONL with tree / node architecture as python dicts which is acceptable I + believe. +- Alternatively: Store processed tweets into DB or alternative option.(Optional) diff --git a/scripts/data-collection/twitter/requirements.txt b/scripts/data-collection/twitter/requirements.txt new file mode 100644 index 00000000..2b084674 --- /dev/null +++ b/scripts/data-collection/twitter/requirements.txt @@ -0,0 +1,3 @@ +numpy==1.21.5 +polars==0.15.14 +tqdm==4.64.0 diff --git a/scripts/data-collection/twitter/twitter_create_convs.py b/scripts/data-collection/twitter/twitter_create_convs.py new file mode 100644 index 00000000..b1fd709a --- /dev/null +++ b/scripts/data-collection/twitter/twitter_create_convs.py @@ -0,0 +1,141 @@ +import json +from pathlib import Path + +import polars as pl +from tqdm import tqdm + +# Sets up paths +# TODO: Source paths from env file +path_string = "PUT THE PATH HERE TO WHERE YOU STORED THE PARQUET FILES" +folder_path = Path(path_string) +processed_folder_path = folder_path / "processed" +output_path = folder_path / "twitter-conv-trees.jsonl" + +# Get parq files +parq_files = sorted(processed_folder_path.rglob("*.parquet")) + +wanted_cols = [ + "timestamp_ms", + "id", + "text", + "truncated", + "in_reply_to_status_id", + "in_reply_to_user_id", + "is_quote_status", + "quote_count", + "reply_count", + "retweet_count", + "favorite_count", + "filter_level", + "lang", + "possibly_sensitive", + "hashtags", + "user_id", + "user_verified", + "user_followers_count", + "user_statuses_count", +] + +# Load parqs into list. Using Polars for performance reasons. +df_list = [] +for p in parq_files: + df_list.append(pl.read_parquet(p, columns=wanted_cols)) + +# Create major dataframe. +# This can be done incrementally if RAM is constrained by modifying the above code. +p_df = pl.concat(df_list) + +# Clean up the reference just in case to help with memory if needed. +del df_list + +# Get tweets that are replies to other tweets +p_df_replies_only = p_df.filter(pl.col("in_reply_to_status_id").is_null().is_not()) + +# Group by replied to status id to see the most replied to statuses. This can take some time. +p_df_group_reply_to_status = p_df_replies_only.groupby("in_reply_to_status_id").count().sort("count", reverse=True) + +# Save output of grouping the top replied to statuses +group_reply_parq = folder_path / "group_reply_parq.parquet" +p_df_group_reply_to_status.write_parquet(group_reply_parq) + +# Join the main dataframe with the top replies to find tweets that have replies. +p_join = p_df.join(p_df_group_reply_to_status, left_on="id", right_on="in_reply_to_status_id", how="inner") + +# Save output of tweets that have replies +tweets_that_have_replies_path = folder_path / "tweets_that_have_replies.parquet" +p_join.write_parquet(tweets_that_have_replies_path) + +# Save output of tweets that are replies to other tweets +tweets_that_are_replies_path = folder_path / "tweets_that_are_replies.parquet" +p_df_replies_only.write_parquet(tweets_that_are_replies_path) + +# Filter the tweets that have replies to ones that aren't replies to others. +# Also filter for only english for now. +# This gives the root tweets that have replies but are the start of a conversation. +origin_tweets = p_join.filter((pl.col("in_reply_to_status_id").is_null()) & (pl.col("lang") == "en")) + + +# Helper functions and classes below for the next steps + + +def role_decide(user_id, prompt_user): + if user_id == prompt_user: + return "prompter" + else: + return "assistant" + + +class ConversationTreeNode: + def __init__(self, tweet_id, prompt_user, from_df, children_df, metadata=None): + + if metadata: + self.metadata = metadata + else: + self.metadata = from_df.filter(pl.col("id") == tweet_id).to_dicts()[0] + + self.metadata["prompt_user"] = prompt_user + self.role = role_decide(self.metadata["user_id"], prompt_user) + self.children = None + self.text = self.metadata["text"] + del self.metadata["text"] + self.get_children(tweet_id=tweet_id, children_df=children_df) + + def get_children(self, tweet_id, children_df): + children_dicts = children_df.filter(pl.col("in_reply_to_status_id") == tweet_id).to_dicts() + if len(children_dicts) > 0: + children = [ + ConversationTreeNode( + tweet_id=c["id"], + prompt_user=self.metadata["prompt_user"], + from_df=children_df, + children_df=children_df, + metadata=c, + ) + for c in children_dicts + ] + self.children = children + + +class ConversationTree: + def __init__(self, tweet_id, prompt_user, from_df, children_df, r_metadata=None): + + self.root = ConversationTreeNode( + tweet_id=tweet_id, prompt_user=prompt_user, from_df=from_df, children_df=children_df, metadata=r_metadata + ) + self.metadata = None + + +# Create conversation trees +conv_tree_list = [ + ConversationTree( + tweet_id=r["id"], prompt_user=r["user_id"], from_df=origin_tweets, children_df=p_df_replies_only, r_metadata=r + ) + for r in tqdm(origin_tweets.to_dicts()) +] + +# Write conversation trees to jsonl file. +# Might need to clean up the last newline. +with open(output_path, "w") as output: + for t in tqdm(conv_tree_list): + json.dump(obj=t, fp=output, default=lambda x: x.__dict__) + output.write("\n") diff --git a/scripts/data-collection/twitter/twitter_process_json.py b/scripts/data-collection/twitter/twitter_process_json.py new file mode 100644 index 00000000..a1e47d7d --- /dev/null +++ b/scripts/data-collection/twitter/twitter_process_json.py @@ -0,0 +1,233 @@ +# This file loops through compressed json tweet data, pre-processes them, +# and then extracts them into more unified parquet files that can be handed +# off for further processing. The main focus is on producing viable replies. + +# Initial data exploration seems that there is no guarantee that the original +# tweets are in the archive, so we might need to extract suitable replies +# then get the original tweets separately, and then combine them into a +# suitable thread format that can be used by our instruction model. + +# This assumes data downloaded from https://archive.org/details/twitterstream +# and that the internal .tar files are extracted locally. +# They are large files so using something like 7Zip or WinRar migth be easier +# than putting all of it in scripts, but it is a possibility. + +# I often work in notebooks. If you encounter any issue, please reach out to let me know. + +import bz2 +import gzip +import json +import pickle +from pathlib import Path + +import numpy as np +import polars as pl +from tqdm import tqdm + +# TODO: OPTIONAL - Put the Untar process in a script instead of doing that part externally. Twitterstream archives are .tar with folders and json.gz files inside. +# TODO: Set up list of important hashtags & keywords. This might have to be done after we get the original tweets in a separate file. +# TODO: Process data and filter based on hashtags & keywords + +# Sets up paths +# TODO: Source paths from env file +path_string = "PUT THE PATH HERE TO WHERE YOU DOWNLOADED AND EXTRACTED THE ARCHIVE .TAR" +folder_path = Path(path_string) +file_list_pkl = folder_path / "file_list.pkl" +processed_file_list_pkl = folder_path / "processed_file_list.pkl" + +# For the processed folder to save inside, we can create the directory if it doesn't exist +processed_folder_path = folder_path / "processed" +processed_folder_path.mkdir(parents=True, exist_ok=True) + +# Set max buffer to store temporary dataframes for processing +# Change this depending on the memory of your computer +processed_max_buffer = 5000 + +# Set up list of wanted column names. +# Note: User columns are prefixed with user_ +wanted_cols = [ + "timestamp_ms", + "id", + "text", + "truncated", + "in_reply_to_status_id", + "in_reply_to_user_id", + "is_quote_status", + "quote_count", + "reply_count", + "retweet_count", + "favorite_count", + "filter_level", + "lang", + "possibly_sensitive", + "hashtags", + "user_id", + "user_verified", + "user_followers_count", + "user_statuses_count", +] + + +def main(file_list_pkl, folder_path, processed_max_buffer): + """ + Runs the main processing script to get files, loop through them, and process them. + Outputs larger json.gz files made by concat the pre-filtered dataframes from + the original json.gz files. + """ + + file_list = get_file_paths(file_list_pkl, folder_path) + + process_json(file_list, processed_max_buffer) + + print("Done") + + +def get_file_paths(file_list_pkl, folder_path): + """ + Gets the file paths by recursively checking the folder structure. + # Based on code from stackoverflow https://stackoverflow.com/questions/26835477/pickle-load-variable-if-exists-or-create-and-save-it + """ + try: + allpaths = pickle.load(open(file_list_pkl, "rb")) + except (OSError, IOError) as e: + print(e) + allpaths = sorted(list(folder_path.rglob("*.[gz bz2]*"))) + pickle.dump(allpaths, open(file_list_pkl, "wb")) + print("Got file paths.") + return allpaths + + +def get_processed_list(processed_file_list_pkl): + # Gets processed file list if stored, if not, creates it. + try: + processed_list = pickle.load(open(processed_file_list_pkl, "rb")) + except (OSError, IOError) as e: + print(e) + processed_list = [] + pickle.dump(processed_list, open(processed_file_list_pkl, "wb")) + return processed_list + + +def modify_dict_cols(j_dict): + # Extracting some nested json + j_dict["user_id"] = np.int64(j_dict["user"]["id"]) + j_dict["user_followers_count"] = np.int64(j_dict["user"]["followers_count"]) + j_dict["user_statuses_count"] = np.int64(j_dict["user"]["statuses_count"]) + + # Get hashtags as a list of strings + j_dict["hashtags"] = [h["text"] for h in j_dict["entities"]["hashtags"]] + + j_dict["id"] = np.int64(j_dict["id"]) + + try: + j_dict["in_reply_to_status_id"] = np.int64(j_dict["in_reply_to_status_id"]) + except Exception as e: + print(e) + j_dict["in_reply_to_status_id"] = j_dict["in_reply_to_status_id"] + + try: + j_dict["in_reply_to_user_id"] = np.int64(j_dict["in_reply_to_user_id"]) + except Exception as e: + print(e) + j_dict["in_reply_to_user_id"] = j_dict["in_reply_to_user_id"] + + # Make sure relevant columns are available or none. + for key in wanted_cols: + if key not in j_dict: + j_dict[key] = None + + # Ordering keys and taking wanted columns + j_dict = {key: j_dict[key] for key in wanted_cols} + + return j_dict + + +def process_single_file(f, processed_list): + + j_dict_list = [] + if f not in processed_list: + # Check for compression type + if f.suffix == ".bz2": + with bz2.BZ2File(f) as file: + for line in file: + # Load JSON + j_dict = json.loads(line) + # Check if user key exists + if "delete" not in j_dict: + if j_dict["truncated"] is False: + + j_dict = modify_dict_cols(j_dict) + + j_dict_list.append(j_dict) + + else: + with gzip.open(f, "r") as file: + for line in file: + # Load JSON + j_dict = json.loads(line) + # Check if user key exists + if "delete" not in j_dict: + if j_dict["truncated"] is False: + + j_dict = modify_dict_cols(j_dict) + + j_dict_list.append(j_dict) + + return j_dict_list + + +def process_json(file_list, processed_max_buffer): + """ + Loops through file list and loads the compressed + json into a list of dicts after some pre-processing. + + Makes sure dicts are ordered in a specific + way to make sure polars can read them. + """ + + # Gets processed file list if stored, if not, creates it. + processed_list = get_processed_list(processed_file_list_pkl) + + j_list = [] + temp_processed_files = [] + + for i, f in enumerate(tqdm(file_list)): + + j_dict_list = process_single_file(f, processed_list) + + j_list.extend(j_dict_list) + + temp_processed_files.append(f) + + if len(temp_processed_files) == processed_max_buffer: + # If we reach our buffer, + # combine into polars dataframe + # and write to parquet as + # a checkpoint + processed_file_name = f"processed_json_{i}.parquet" + processed_file_path = processed_folder_path / processed_file_name + + pl.DataFrame(j_list, columns=wanted_cols).write_parquet(processed_file_path) + + # Make note of which files have been processed + processed_list.extend(temp_processed_files) + pickle.dump(processed_list, open(processed_file_list_pkl, "wb")) + + # Reset buffer lists + j_list = [] + temp_processed_files = [] + + # Process remaining files + processed_file_name = f"processed_json_{i}.parquet" + processed_file_path = processed_folder_path / processed_file_name + pl.from_dicts(j_dict_list).write_parquet(processed_file_path) + processed_list.extend(temp_processed_files) + pickle.dump(processed_list, open(processed_file_list_pkl, "wb")) + j_dict_list = [] + temp_processed_files = [] + + print("Processing completed") + + +if __name__ == "__main__": + main(file_list_pkl, folder_path, processed_max_buffer) From 27e1e549c42e830f6b560a52c3b595eddf7cc6d3 Mon Sep 17 00:00:00 2001 From: notmd Date: Sat, 21 Jan 2023 20:10:19 +0700 Subject: [PATCH 18/59] fix query in backward direction --- backend/oasst_backend/user_repository.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/backend/oasst_backend/user_repository.py b/backend/oasst_backend/user_repository.py index 1e9ac78f..e960d944 100644 --- a/backend/oasst_backend/user_repository.py +++ b/backend/oasst_backend/user_repository.py @@ -200,6 +200,7 @@ class UserRepository: search_text: Optional[str] = None, limit: Optional[int] = 100, ) -> list[User]: + if not self.api_client.trusted: if not api_client_id: # Let unprivileged api clients query their own users without api_client_id being set @@ -226,11 +227,15 @@ class UserRepository: if lte_display_name is not None: if lt_id: - qry = qry.filter( - or_( - User.display_name < lte_display_name, - and_(User.display_name == lte_display_name, User.id < lt_id), + qry = ( + qry.filter( + or_( + User.display_name < lte_display_name, + and_(User.display_name == lte_display_name, User.id < lt_id), + ) ) + .order_by(None) + .order_by(User.display_name.desc(), User.id.desc()) ) else: qry = qry.filter(User.display_name <= lte_display_name) @@ -252,4 +257,9 @@ class UserRepository: if limit is not None: qry = qry.limit(limit) - return qry.all() + users = qry.all() + + if lte_display_name and lt_id: + users.reverse() + + return users From e08b36e6757d5ea1c9f980edb681af2485503302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Sat, 21 Jan 2023 16:04:43 +0100 Subject: [PATCH 19/59] imporve prev/next status for user cursor --- backend/oasst_backend/api/v1/users.py | 37 +++++++++++++++++++-------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/backend/oasst_backend/api/v1/users.py b/backend/oasst_backend/api/v1/users.py index 2697d6fe..04b3cc99 100644 --- a/backend/oasst_backend/api/v1/users.py +++ b/backend/oasst_backend/api/v1/users.py @@ -84,6 +84,8 @@ def get_users_cursor( api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db), ): + assert max_count is not None + def split_cursor(x: str | None) -> tuple[str, UUID]: if not x: return None, None @@ -93,21 +95,32 @@ def get_users_cursor( return x, None items: list[protocol.FrontEndUser] + qry_max_count = max_count + 1 if lt is None or gt is None else max_count - def get_next_prev(lte: str | None, gte: str | None, key_fn: Callable[[protocol.FrontEndUser], str]): + def get_next_prev(num_rows: int, lt: str | None, gt: str | None, key_fn: Callable[[protocol.FrontEndUser], str]): p, n = None, None if len(items) > 0: - if len(items) == max_count or gte: + if (num_rows > max_count and lt) or gt: p = str(items[0].user_id) + "$" + key_fn(items[0]) - if len(items) == max_count or lte: + if num_rows > max_count or lt: n = str(items[-1].user_id) + "$" + key_fn(items[-1]) else: - if gte: - p = gte - if lte: - n = lte + if gt: + p = gt + if lt: + n = lt return p, n + def remove_extra_item(items: list[protocol.FrontEndUser], lt: str | None, gt: str): + num_rows = len(items) + if qry_max_count > max_count and num_rows == qry_max_count: + assert not (lt and gt) + if lt: + items = items[1:] + else: + items = items[:-1] + return items, num_rows + n, p = None, None if sort_key == "username": lte_username, lt_id = split_cursor(lt) @@ -120,11 +133,12 @@ def get_users_cursor( lt_id=lt_id, auth_method=auth_method, search_text=search_text, - max_count=max_count, + max_count=qry_max_count, api_client=api_client, db=db, ) - p, n = get_next_prev(lte_username, gte_username, lambda x: x.id) + items, num_rows = remove_extra_item(items, lte_username, gte_username) + p, n = get_next_prev(num_rows, lte_username, gte_username, lambda x: x.id) elif sort_key == "display_name": lte_display_name, lt_id = split_cursor(lt) @@ -137,11 +151,12 @@ def get_users_cursor( lt_id=lt_id, auth_method=auth_method, search_text=search_text, - max_count=max_count, + max_count=qry_max_count, api_client=api_client, db=db, ) - p, n = get_next_prev(lte_display_name, gte_display_name, lambda x: x.display_name) + items, num_rows = remove_extra_item(items, lte_display_name, gte_display_name) + p, n = get_next_prev(num_rows, lte_display_name, gte_display_name, lambda x: x.display_name) else: raise OasstError(f"Unsupported sort key: '{sort_key}'", OasstErrorCode.SORT_KEY_UNSUPPORTED) From c1dd188cbea8e88a945b8dc190c5e32dc1872c90 Mon Sep 17 00:00:00 2001 From: notmd Date: Sat, 21 Jan 2023 23:12:19 +0700 Subject: [PATCH 20/59] use pre and next cursor check from the server --- website/src/components/DataTable.tsx | 94 +++++++++++++++------------- website/src/components/UserTable.tsx | 94 ++++++++++------------------ website/src/pages/api/admin/users.ts | 2 +- 3 files changed, 84 insertions(+), 106 deletions(-) diff --git a/website/src/components/DataTable.tsx b/website/src/components/DataTable.tsx index eafca6d1..1784650a 100644 --- a/website/src/components/DataTable.tsx +++ b/website/src/components/DataTable.tsx @@ -1,8 +1,6 @@ import { Box, Button, - Card, - CardBody, Flex, FormControl, FormLabel, @@ -47,6 +45,8 @@ export type DataTableProps = { onNextClick?: () => void; onPreviousClick?: () => void; onFilterChange?: (items: FilterItem[]) => void; + disableNext?: boolean; + disablePrevious?: boolean; }; export const DataTable = ({ @@ -57,6 +57,8 @@ export const DataTable = ({ onNextClick, onPreviousClick, onFilterChange, + disableNext, + disablePrevious, }: DataTableProps) => { const { getHeaderGroups, getRowModel } = useReactTable({ data, @@ -75,49 +77,51 @@ export const DataTable = ({ onFilterChange(newValues); }; return ( - - - - - - - - - - {caption} - - {getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - - ))} - - ))} - - - {getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - - ))} - - ))} - -
- - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - {(header.column.columnDef as DataTableColumnDef).filterable && ( - value.id === header.id)?.value ?? ""} - onChange={(value) => handleFilterChange({ id: header.id, value })} - label={flexRender(header.column.columnDef.header, header.getContext())} - > - )} - -
{flexRender(cell.column.columnDef.cell, cell.getContext())}
- - - + <> + + + + + + + + {caption} + + {getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + + {getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
+ + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + {(header.column.columnDef as DataTableColumnDef).filterable && ( + value.id === header.id)?.value ?? ""} + onChange={(value) => handleFilterChange({ id: header.id, value })} + label={flexRender(header.column.columnDef.header, header.getContext())} + > + )} + +
{flexRender(cell.column.columnDef.cell, cell.getContext())}
+ + ); }; diff --git a/website/src/components/UserTable.tsx b/website/src/components/UserTable.tsx index 68285bfa..1f4ccfca 100644 --- a/website/src/components/UserTable.tsx +++ b/website/src/components/UserTable.tsx @@ -1,4 +1,4 @@ -import { IconButton, useToast } from "@chakra-ui/react"; +import { Card, CardBody, IconButton } from "@chakra-ui/react"; import { createColumnHelper } from "@tanstack/react-table"; import Link from "next/link"; import { memo, useState } from "react"; @@ -57,11 +57,7 @@ const columns: DataTableColumnDef[] = [ ]; export const UserTable = memo(function UserTable() { - const toast = useToast(); const [pagination, setPagination] = useState({ cursor: "", direction: "forward" }); - const [response, setResponse] = useState, "sort_key" | "order">>({ - items: [], - }); const [filterValues, setFilterValues] = useState([]); const handleFilterValuesChange = (values: FilterItem[]) => { setFilterValues(values); @@ -71,68 +67,46 @@ export const UserTable = memo(function UserTable() { // This follows useSWR's recommendation for simple pagination: // https://swr.vercel.app/docs/pagination#when-to-use-useswr const display_name = filterValues.find((value) => value.id === "display_name")?.value ?? ""; - useSWR< - FetchUsersResponse - >(`/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}&searchDisplayName=${display_name}&sortKey=display_name`, get, { - onSuccess: (data) => { - // When no more users can be found, trigger a toast to indicate why no - // changes have taken place. We have to maintain a non-empty set of - // users otherwise we can't paginate using a cursor (since we've lost the - // cursor). - if (data.items.length === 0) { - toast({ - title: "No more users", - status: "warning", - duration: 1000, - isClosable: true, - }); - return; - } - setResponse(data); - }, - }); + const { data, error } = useSWR>( + `/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}&searchDisplayName=${display_name}&sortKey=display_name`, + get, + { + keepPreviousData: true, + } + ); const toPreviousPage = () => { - if (response.items.length >= 0) { - setPagination({ - cursor: response.prev, - direction: "back", - }); - } else { - toast({ - title: "Can not paginate when no users are found", - status: "warning", - duration: 1000, - isClosable: true, - }); - } + setPagination({ + cursor: data.prev, + direction: "back", + }); }; const toNextPage = () => { - if (response.items.length >= 0) { - setPagination({ - cursor: response.next, - direction: "forward", - }); - } else { - toast({ - title: "Can not paginate when no users are found", - status: "warning", - duration: 1000, - isClosable: true, - }); - } + setPagination({ + cursor: data.next, + direction: "forward", + }); }; return ( - + + + {data && ( + + )} + {error && "Unable to load users."} + + ); }); diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts index 57944cff..f43af305 100644 --- a/website/src/pages/api/admin/users.ts +++ b/website/src/pages/api/admin/users.ts @@ -5,7 +5,7 @@ import prisma from "src/lib/prismadb"; /** * The number of users to fetch in a single request. Could later be a query parameter. */ -const PAGE_SIZE = 20; +const PAGE_SIZE = 2; /** * Returns a list of user results from the database when the requesting user is From aebfaacac8e5abb8a5b5097ea6f30d68afd9763c Mon Sep 17 00:00:00 2001 From: notmd Date: Sat, 21 Jan 2023 23:21:10 +0700 Subject: [PATCH 21/59] remove debug code --- website/src/pages/api/admin/users.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts index f43af305..57944cff 100644 --- a/website/src/pages/api/admin/users.ts +++ b/website/src/pages/api/admin/users.ts @@ -5,7 +5,7 @@ import prisma from "src/lib/prismadb"; /** * The number of users to fetch in a single request. Could later be a query parameter. */ -const PAGE_SIZE = 2; +const PAGE_SIZE = 20; /** * Returns a list of user results from the database when the requesting user is From 15acd1c64e917910e68b32c800f23ff516655d43 Mon Sep 17 00:00:00 2001 From: notmd Date: Sat, 21 Jan 2023 23:56:21 +0700 Subject: [PATCH 22/59] handle error --- website/src/components/DataTable.tsx | 94 +++++++++++++++------------- website/src/components/UserTable.tsx | 34 +++++----- 2 files changed, 64 insertions(+), 64 deletions(-) diff --git a/website/src/components/DataTable.tsx b/website/src/components/DataTable.tsx index 1784650a..f9ef4e49 100644 --- a/website/src/components/DataTable.tsx +++ b/website/src/components/DataTable.tsx @@ -1,6 +1,8 @@ import { Box, Button, + Card, + CardBody, Flex, FormControl, FormLabel, @@ -77,51 +79,53 @@ export const DataTable = ({ onFilterChange(newValues); }; return ( - <> - - - - - - - - {caption} - - {getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => ( - - ))} - - ))} - - - {getRowModel().rows.map((row) => ( - - {row.getVisibleCells().map((cell) => ( - - ))} - - ))} - -
- - {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} - {(header.column.columnDef as DataTableColumnDef).filterable && ( - value.id === header.id)?.value ?? ""} - onChange={(value) => handleFilterChange({ id: header.id, value })} - label={flexRender(header.column.columnDef.header, header.getContext())} - > - )} - -
{flexRender(cell.column.columnDef.cell, cell.getContext())}
- - + + + + + + + + + + {caption} + + {getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + + {getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
+ + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + {(header.column.columnDef as DataTableColumnDef).filterable && ( + value.id === header.id)?.value ?? ""} + onChange={(value) => handleFilterChange({ id: header.id, value })} + label={flexRender(header.column.columnDef.header, header.getContext())} + > + )} + +
{flexRender(cell.column.columnDef.cell, cell.getContext())}
+ + + ); }; diff --git a/website/src/components/UserTable.tsx b/website/src/components/UserTable.tsx index 1f4ccfca..df412bbc 100644 --- a/website/src/components/UserTable.tsx +++ b/website/src/components/UserTable.tsx @@ -1,4 +1,4 @@ -import { Card, CardBody, IconButton } from "@chakra-ui/react"; +import { IconButton } from "@chakra-ui/react"; import { createColumnHelper } from "@tanstack/react-table"; import Link from "next/link"; import { memo, useState } from "react"; @@ -90,23 +90,19 @@ export const UserTable = memo(function UserTable() { }; return ( - - - {data && ( - - )} - {error && "Unable to load users."} - - + <> + + {error && "Unable to load users."} + ); }); From ea7e320a7853ded66f6e2c6ff5286523b4b7b60a Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Thu, 19 Jan 2023 21:06:37 -0500 Subject: [PATCH 23/59] add status option in admin page left bar --- website/src/components/Layout.tsx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/website/src/components/Layout.tsx b/website/src/components/Layout.tsx index 484d16ec..55085550 100644 --- a/website/src/components/Layout.tsx +++ b/website/src/components/Layout.tsx @@ -2,7 +2,7 @@ import { Box, Grid } from "@chakra-ui/react"; import type { NextPage } from "next"; -import { FiBarChart2, FiLayout, FiMessageSquare, FiUsers } from "react-icons/fi"; +import { FiBarChart2, FiLayout, FiMessageSquare, FiUsers, FiActivity } from "react-icons/fi"; import { Header } from "src/components/Header"; import { SlimFooter } from "./Dashboard/SlimFooter"; @@ -75,6 +75,12 @@ export const getAdminLayout = (page: React.ReactElement) => ( desc: "Users Dashboard", icon: FiUsers, }, + { + label: "Status", + pathname: "/admin/status", + desc: "Status Dashboard", + icon: FiActivity, + }, ]} > {page} From 812c8cb209f9c4d76350b08c63305767e655ab09 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Fri, 20 Jan 2023 01:35:48 -0500 Subject: [PATCH 24/59] create status page --- website/src/pages/admin/status/index.tsx | 168 ++++++++++++++++++ website/src/pages/api/admin/stats.ts | 18 ++ .../src/pages/api/admin/tasks_availability.ts | 28 +++ website/src/pages/api/admin/tree_manager.ts | 18 ++ 4 files changed, 232 insertions(+) create mode 100644 website/src/pages/admin/status/index.tsx create mode 100644 website/src/pages/api/admin/stats.ts create mode 100644 website/src/pages/api/admin/tasks_availability.ts create mode 100644 website/src/pages/api/admin/tree_manager.ts diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx new file mode 100644 index 00000000..6af65b61 --- /dev/null +++ b/website/src/pages/admin/status/index.tsx @@ -0,0 +1,168 @@ +import { Box, Button, Container, Flex, FormControl, FormLabel, Input, Select, SimpleGrid, Stack, Text, useToast } from "@chakra-ui/react"; +import { + Spacer, + Table, + TableCaption, + TableContainer, + Tbody, + Td, + Th, + Thead, + Tr, + useColorMode +} from "@chakra-ui/react"; +/* import { Field, Form, Formik } from "formik"; */ +import Head from "next/head"; +import { useRouter } from "next/router"; +import { useSession } from "next-auth/react"; +import { useEffect } from "react"; +import { getAdminLayout } from "src/components/Layout"; +import poster from "src/lib/poster"; +import prisma from "src/lib/prismadb"; +import useSWR from "swr/mutation"; + +import { useState } from "react"; +import useSWRMutation from "swr/mutation"; +import useSWRImmutable from "swr/immutable"; +import { get, post } from "src/lib/api"; + +/* import TreeManagerCell from "src/components/TreeManagerCell"; */ + +/** + * Provides the admin status page that shows result of calls to several backend API endpoints, + * namely /api/v1/tasks/availability, /api/v1/stats/, /api/v1/stats/tree_manager + */ + +const StatusIndex = ({ user }) => { + const toast = useToast(); + const router = useRouter(); + const { data: session, status } = useSession(); + + const { colorMode } = useColorMode(); + const backgroundColor = colorMode === "light" ? "white" : "gray.700"; + const dataBackgroundColor = colorMode === "light" ? "gray.100" : "gray.700"; + // Check when the user session is loaded and re-route if the user is not an + // admin. This follows the suggestion by NextJS for handling private pages: + // https://nextjs.org/docs/api-reference/next/router#usage + // + // All admin pages should use the same check and routing steps. + useEffect(() => { + if (status === "loading") { + return; + } + if (session?.user?.role === "admin") { + return; + } + router.push("/"); + }, [router, session, status]); + + const [availability, setAvailability] = useState(0); + const [stats, setStats] = useState(0); + const [treeManager, setTreeManager] = useState(0); + + const {data: dataAvailability, error: errorAvailability, isLoading: isLoadingAvailability} = useSWRImmutable("/api/admin/tasks_availability", get, { + onSuccess: (data) => { + setAvailability(data); } + }); + const {data: dataStats, error: errorStats, isLoading: isLoadingStats} = useSWRImmutable("/api/admin/stats", get, { + onSuccess: (data) => { setStats(data); } + }); + const {data: dataTreeManager, error: errorTreeManager, isLoading: isLoadingTreeManager} = useSWRImmutable("/api/admin/tree_manager", get, { + onSuccess: (data) => { + setTreeManager(data); } + }); + + return ( + <> + + Status - Open Assistant + + + + + + + /api/v1/tasks/availability + + + 
+                            {availability ? JSON.stringify(availability, null, 2) : "None"}
+
+ + + + + + /api/v1/stats/ + + + 
+                            {stats ? JSON.stringify(stats, null, 2) : "None"}
+
+ + + + +
+ + + + + /api/v1/stats/tree_manager + + + {treeManager ? + + + state_counts + + + 
+                                 {JSON.stringify(treeManager.state_counts, null, 2)}
+
+ + +
+ + message_counts + + + Tree Manager + + + + + + + + + + + + + {treeManager.message_counts.map(({ message_tree_id, state, depth, oldest, youngest, count, goal_tree_size }) => ( + + + + + + + + + + ))} + +
Message Tree IDStateDepthOldestYoungestCountGoal Tree Size
{message_tree_id}{state}{depth}{oldest}{youngest}{count}{goal_tree_size}
+ + : "None"} + + + ); +}; + +StatusIndex.getLayout = getAdminLayout; + +export default StatusIndex; diff --git a/website/src/pages/api/admin/stats.ts b/website/src/pages/api/admin/stats.ts new file mode 100644 index 00000000..1dedbd30 --- /dev/null +++ b/website/src/pages/api/admin/stats.ts @@ -0,0 +1,18 @@ +import { withRole } from "src/lib/auth"; + +/** + * Returns the messages recorded by the backend for a user. + */ +const handler = withRole("admin", async (req, res) => { + const statsRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/stats/`, { + method: "GET", + headers: { + "X-API-Key": process.env.FASTAPI_KEY, + }, + }); + const stats = await statsRes.json(); + + res.status(statsRes.status).json(stats); +}); + +export default handler; diff --git a/website/src/pages/api/admin/tasks_availability.ts b/website/src/pages/api/admin/tasks_availability.ts new file mode 100644 index 00000000..6d64d88b --- /dev/null +++ b/website/src/pages/api/admin/tasks_availability.ts @@ -0,0 +1,28 @@ +import { withRole } from "src/lib/auth"; +import { oasstApiClient } from "src/lib/oasst_api_client"; +import type { Message } from "src/types/Conversation"; + +/** + * Returns the messages recorded by the backend for a user. + */ +const handler = withRole("admin", async (req, res) => { + + const tasksAvailabilityRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/tasks/availability`, { + method: "POST", + headers: { + "X-API-Key": process.env.FASTAPI_KEY, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + id: "__dummy_user__", + display_name: "Dummy User", + auth_method: "local", + }), + + }); + const tasksAvailability = await tasksAvailabilityRes.json(); + + res.status(tasksAvailabilityRes.status).json(tasksAvailability); +}); + +export default handler; diff --git a/website/src/pages/api/admin/tree_manager.ts b/website/src/pages/api/admin/tree_manager.ts new file mode 100644 index 00000000..b47562bf --- /dev/null +++ b/website/src/pages/api/admin/tree_manager.ts @@ -0,0 +1,18 @@ +import { withRole } from "src/lib/auth"; + +/** + * Returns the messages recorded by the backend for a user. + */ +const handler = withRole("admin", async (req, res) => { + const treeManagerRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/stats/tree_manager`, { + method: "GET", + headers: { + "X-API-Key": process.env.FASTAPI_KEY, + }, + }); + const treeManager = await treeManagerRes.json(); + + res.status(treeManagerRes.status).json(treeManager); +}); + +export default handler; From 91bb5603833da5b021202586d0488163de445cf1 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Fri, 20 Jan 2023 12:16:06 -0500 Subject: [PATCH 25/59] clean up status page code --- website/src/pages/admin/status/index.tsx | 300 ++++++++++++----------- website/src/pages/api/admin/stats.ts | 1 + 2 files changed, 162 insertions(+), 139 deletions(-) diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx index 6af65b61..4eaa5a7e 100644 --- a/website/src/pages/admin/status/index.tsx +++ b/website/src/pages/admin/status/index.tsx @@ -1,32 +1,31 @@ -import { Box, Button, Container, Flex, FormControl, FormLabel, Input, Select, SimpleGrid, Stack, Text, useToast } from "@chakra-ui/react"; import { - Spacer, - Table, - TableCaption, - TableContainer, - Tbody, - Td, - Th, - Thead, - Tr, - useColorMode + Box, + Button, + CircularProgress, + Container, + SimpleGrid, + Text, + Table, + TableCaption, + TableContainer, + Tbody, + Td, + Th, + Thead, + Tr, + View, + useColorMode, + useToast, } from "@chakra-ui/react"; -/* import { Field, Form, Formik } from "formik"; */ import Head from "next/head"; import { useRouter } from "next/router"; import { useSession } from "next-auth/react"; -import { useEffect } from "react"; +import { useEffect, useState } from "react"; +import useSWRImmutable from "swr/immutable"; import { getAdminLayout } from "src/components/Layout"; import poster from "src/lib/poster"; import prisma from "src/lib/prismadb"; -import useSWR from "swr/mutation"; - -import { useState } from "react"; -import useSWRMutation from "swr/mutation"; -import useSWRImmutable from "swr/immutable"; -import { get, post } from "src/lib/api"; - -/* import TreeManagerCell from "src/components/TreeManagerCell"; */ +import { get } from "src/lib/api"; /** * Provides the admin status page that shows result of calls to several backend API endpoints, @@ -34,133 +33,156 @@ import { get, post } from "src/lib/api"; */ const StatusIndex = ({ user }) => { - const toast = useToast(); - const router = useRouter(); - const { data: session, status } = useSession(); + const toast = useToast(); + const router = useRouter(); + const { data: session, status } = useSession(); - const { colorMode } = useColorMode(); - const backgroundColor = colorMode === "light" ? "white" : "gray.700"; - const dataBackgroundColor = colorMode === "light" ? "gray.100" : "gray.700"; - // Check when the user session is loaded and re-route if the user is not an - // admin. This follows the suggestion by NextJS for handling private pages: - // https://nextjs.org/docs/api-reference/next/router#usage - // - // All admin pages should use the same check and routing steps. - useEffect(() => { - if (status === "loading") { - return; - } - if (session?.user?.role === "admin") { - return; - } - router.push("/"); - }, [router, session, status]); + const { colorMode } = useColorMode(); + const backgroundColor = colorMode === "light" ? "white" : "gray.700"; + const dataBackgroundColor = colorMode === "light" ? "gray.100" : "gray.800"; + // Check when the user session is loaded and re-route if the user is not an + // admin. This follows the suggestion by NextJS for handling private pages: + // https://nextjs.org/docs/api-reference/next/router#usage + // + // All admin pages should use the same check and routing steps. + useEffect(() => { + if (status === "loading") { + return; + } + if (session?.user?.role === "admin") { + return; + } + router.push("/"); + }, [router, session, status]); - const [availability, setAvailability] = useState(0); - const [stats, setStats] = useState(0); - const [treeManager, setTreeManager] = useState(0); + const [availability, setAvailability] = useState(0); + const [stats, setStats] = useState(0); + const [treeManager, setTreeManager] = useState(0); - const {data: dataAvailability, error: errorAvailability, isLoading: isLoadingAvailability} = useSWRImmutable("/api/admin/tasks_availability", get, { - onSuccess: (data) => { - setAvailability(data); } - }); - const {data: dataStats, error: errorStats, isLoading: isLoadingStats} = useSWRImmutable("/api/admin/stats", get, { - onSuccess: (data) => { setStats(data); } - }); - const {data: dataTreeManager, error: errorTreeManager, isLoading: isLoadingTreeManager} = useSWRImmutable("/api/admin/tree_manager", get, { - onSuccess: (data) => { - setTreeManager(data); } - }); + const { + data: dataAvailability, + error: errorAvailability, + isLoading: isLoadingAvailability, + } = useSWRImmutable("/api/admin/tasks_availability", get, { + onSuccess: (data) => { + setAvailability(data); + }, + onError: (error) => { + setAvailability(error); + }, + }); + const { + data: dataStats, + error: errorStats, + isLoading: isLoadingStats, + } = useSWRImmutable("/api/admin/stats", get, { + onSuccess: (data) => { + setStats(data); + }, + onError: (error) => { + setStats(error); + }, + }); + const { + data: dataTreeManager, + error: errorTreeManager, + isLoading: isLoadingTreeManager, + } = useSWRImmutable("/api/admin/tree_manager", get, { + onSuccess: (data) => { + setTreeManager(data); + }, + onError: (error) => { + setTreeManager(error); + }, + }); - return ( - <> - - Status - Open Assistant - - + return ( + <> + + Status - Open Assistant + + - - - /api/v1/tasks/availability - - - 
-                            {availability ? JSON.stringify(availability, null, 2) : "None"}
-
- - - - - - /api/v1/stats/ - - - 
-                            {stats ? JSON.stringify(stats, null, 2) : "None"}
-
- - + + + /api/v1/tasks/availability + + + 
+              {availability ? JSON.stringify(availability, null, 2) : }
+
+ + + + + /api/v1/stats/ + + + 
{stats ? JSON.stringify(stats, null, 2) : }
+ + -
+
+ + + /api/v1/stats/tree_manager + - - - - /api/v1/stats/tree_manager - - - {treeManager ? - - - state_counts - - - 
-                                 {JSON.stringify(treeManager.state_counts, null, 2)}
-
- - -
- - message_counts - - - Tree Manager - - - - - - - - - - - - - {treeManager.message_counts.map(({ message_tree_id, state, depth, oldest, youngest, count, goal_tree_size }) => ( - - - - - - - - - - ))} - -
Message Tree IDStateDepthOldestYoungestCountGoal Tree Size
{message_tree_id}{state}{depth}{oldest}{youngest}{count}{goal_tree_size}
- - : "None"} - - - ); + {treeManager ? ( + + + state_counts + + + 
{JSON.stringify(treeManager.state_counts, null, 2)}
+ + +
+ + message_counts + + + Tree Manager + + + + + + + + + + + + + {treeManager.message_counts.map( + ({ message_tree_id, state, depth, oldest, youngest, count, goal_tree_size }) => ( + + + + + + + + + + ) + )} + +
Message Tree IDStateDepthOldestYoungestCountGoal Tree Size
{message_tree_id}{state}{depth}{oldest}{youngest}{count}{goal_tree_size}
+ + + ) : ( + + )} + + + ); }; StatusIndex.getLayout = getAdminLayout; diff --git a/website/src/pages/api/admin/stats.ts b/website/src/pages/api/admin/stats.ts index 1dedbd30..dfc4768b 100644 --- a/website/src/pages/api/admin/stats.ts +++ b/website/src/pages/api/admin/stats.ts @@ -10,6 +10,7 @@ const handler = withRole("admin", async (req, res) => { "X-API-Key": process.env.FASTAPI_KEY, }, }); + const stats = await statsRes.json(); res.status(statsRes.status).json(stats); From 0f2a65ea3c211649019ddb948cf237a5d92a8d7e Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Fri, 20 Jan 2023 12:37:58 -0500 Subject: [PATCH 26/59] remove unnecessary imports --- website/src/pages/api/admin/tasks_availability.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/website/src/pages/api/admin/tasks_availability.ts b/website/src/pages/api/admin/tasks_availability.ts index 6d64d88b..687daa36 100644 --- a/website/src/pages/api/admin/tasks_availability.ts +++ b/website/src/pages/api/admin/tasks_availability.ts @@ -1,12 +1,9 @@ import { withRole } from "src/lib/auth"; -import { oasstApiClient } from "src/lib/oasst_api_client"; -import type { Message } from "src/types/Conversation"; /** * Returns the messages recorded by the backend for a user. */ const handler = withRole("admin", async (req, res) => { - const tasksAvailabilityRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/tasks/availability`, { method: "POST", headers: { @@ -18,7 +15,6 @@ const handler = withRole("admin", async (req, res) => { display_name: "Dummy User", auth_method: "local", }), - }); const tasksAvailability = await tasksAvailabilityRes.json(); From 4ccc88e0bb4ac679ceff130be5d8490caf1646b4 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Fri, 20 Jan 2023 12:59:24 -0500 Subject: [PATCH 27/59] fix admin status api handler comments --- website/src/pages/api/admin/stats.ts | 2 +- website/src/pages/api/admin/tasks_availability.ts | 2 +- website/src/pages/api/admin/tree_manager.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/website/src/pages/api/admin/stats.ts b/website/src/pages/api/admin/stats.ts index dfc4768b..7f8a43a0 100644 --- a/website/src/pages/api/admin/stats.ts +++ b/website/src/pages/api/admin/stats.ts @@ -1,7 +1,7 @@ import { withRole } from "src/lib/auth"; /** - * Returns the messages recorded by the backend for a user. + * Returns the message stats. */ const handler = withRole("admin", async (req, res) => { const statsRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/stats/`, { diff --git a/website/src/pages/api/admin/tasks_availability.ts b/website/src/pages/api/admin/tasks_availability.ts index 687daa36..c7f1d1c5 100644 --- a/website/src/pages/api/admin/tasks_availability.ts +++ b/website/src/pages/api/admin/tasks_availability.ts @@ -1,7 +1,7 @@ import { withRole } from "src/lib/auth"; /** - * Returns the messages recorded by the backend for a user. + * Returns result of tasks availability query using a dummy user. */ const handler = withRole("admin", async (req, res) => { const tasksAvailabilityRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/tasks/availability`, { diff --git a/website/src/pages/api/admin/tree_manager.ts b/website/src/pages/api/admin/tree_manager.ts index b47562bf..c127f635 100644 --- a/website/src/pages/api/admin/tree_manager.ts +++ b/website/src/pages/api/admin/tree_manager.ts @@ -1,7 +1,7 @@ import { withRole } from "src/lib/auth"; /** - * Returns the messages recorded by the backend for a user. + * Returns tree manager stats. */ const handler = withRole("admin", async (req, res) => { const treeManagerRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/stats/tree_manager`, { From b8ef87fb548fbae63f67755f2caf591cc60a2061 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Fri, 20 Jan 2023 21:42:56 -0500 Subject: [PATCH 28/59] use Card component for status page --- website/src/pages/admin/status/index.tsx | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx index 4eaa5a7e..5e1aba13 100644 --- a/website/src/pages/admin/status/index.tsx +++ b/website/src/pages/admin/status/index.tsx @@ -1,5 +1,6 @@ import { Box, + Card, Button, CircularProgress, Container, @@ -107,7 +108,7 @@ const StatusIndex = ({ user }) => { - + /api/v1/tasks/availability @@ -116,19 +117,19 @@ const StatusIndex = ({ user }) => { {availability ? JSON.stringify(availability, null, 2) : } - + - + /api/v1/stats/ 
{stats ? JSON.stringify(stats, null, 2) : }
 - +
- + /api/v1/stats/tree_manager @@ -180,7 +181,7 @@ const StatusIndex = ({ user }) => { ) : ( )} - + ); }; From a2932d1613e5aa9be240f9ff8dc824bfe9e3c55a Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Fri, 20 Jan 2023 23:44:11 -0500 Subject: [PATCH 29/59] unify 3 status endpoints and move status fetches to oasst_api_client --- website/src/lib/oasst_api_client.ts | 21 +++++++++ website/src/pages/admin/status/index.tsx | 47 +++---------------- website/src/pages/api/admin/stats.ts | 19 -------- website/src/pages/api/admin/status.ts | 26 ++++++++++ .../src/pages/api/admin/tasks_availability.ts | 24 ---------- website/src/pages/api/admin/tree_manager.ts | 18 ------- 6 files changed, 54 insertions(+), 101 deletions(-) delete mode 100644 website/src/pages/api/admin/stats.ts create mode 100644 website/src/pages/api/admin/status.ts delete mode 100644 website/src/pages/api/admin/tasks_availability.ts delete mode 100644 website/src/pages/api/admin/tree_manager.ts diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index 1e74b020..44d8ae7c 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -148,6 +148,27 @@ export class OasstApiClient { }); } + /** + * Returns the tasks availability information for given `user`. + */ + async fetch_tasks_availability(user: object): Promise { + return this.post("/api/v1/tasks/availability", user); + } + + /** + * Returns the message stats from the backend. + */ + async fetch_stats(): Promise { + return this.get("/api/v1/stats/"); + } + + /** + * Returns the tree manager stats from the backend. + */ + async fetch_tree_manager(): Promise { + return this.get("/api/v1/stats/tree_manager"); + } + /** * Returns the `BackendUser` associated with `user_id` */ diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx index 5e1aba13..5d5d9ba4 100644 --- a/website/src/pages/admin/status/index.tsx +++ b/website/src/pages/admin/status/index.tsx @@ -56,46 +56,13 @@ const StatusIndex = ({ user }) => { router.push("/"); }, [router, session, status]); - const [availability, setAvailability] = useState(0); - const [stats, setStats] = useState(0); - const [treeManager, setTreeManager] = useState(0); + const { + data: dataStatus, + error: errorStatus, + isLoading: isLoadingStatus, + } = useSWRImmutable("/api/admin/status", get); - const { - data: dataAvailability, - error: errorAvailability, - isLoading: isLoadingAvailability, - } = useSWRImmutable("/api/admin/tasks_availability", get, { - onSuccess: (data) => { - setAvailability(data); - }, - onError: (error) => { - setAvailability(error); - }, - }); - const { - data: dataStats, - error: errorStats, - isLoading: isLoadingStats, - } = useSWRImmutable("/api/admin/stats", get, { - onSuccess: (data) => { - setStats(data); - }, - onError: (error) => { - setStats(error); - }, - }); - const { - data: dataTreeManager, - error: errorTreeManager, - isLoading: isLoadingTreeManager, - } = useSWRImmutable("/api/admin/tree_manager", get, { - onSuccess: (data) => { - setTreeManager(data); - }, - onError: (error) => { - setTreeManager(error); - }, - }); + const { tasksAvailability, stats, treeManager } = dataStatus || {}; return ( <> @@ -114,7 +81,7 @@ const StatusIndex = ({ user }) => { 
-              {availability ? JSON.stringify(availability, null, 2) : }
+              {tasksAvailability ? JSON.stringify(tasksAvailability, null, 2) : }
diff --git a/website/src/pages/api/admin/stats.ts b/website/src/pages/api/admin/stats.ts deleted file mode 100644 index 7f8a43a0..00000000 --- a/website/src/pages/api/admin/stats.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { withRole } from "src/lib/auth"; - -/** - * Returns the message stats. - */ -const handler = withRole("admin", async (req, res) => { - const statsRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/stats/`, { - method: "GET", - headers: { - "X-API-Key": process.env.FASTAPI_KEY, - }, - }); - - const stats = await statsRes.json(); - - res.status(statsRes.status).json(stats); -}); - -export default handler; diff --git a/website/src/pages/api/admin/status.ts b/website/src/pages/api/admin/status.ts new file mode 100644 index 00000000..4fb91857 --- /dev/null +++ b/website/src/pages/api/admin/status.ts @@ -0,0 +1,26 @@ +import { withRole } from "src/lib/auth"; +import { oasstApiClient } from "src/lib/oasst_api_client"; + +/** + * Returns tasks availability, stats, and tree manager stats. + */ +const handler = withRole("admin", async (req, res) => { + const dummy_user = { + id: "__dummy_user__", + display_name: "Dummy User", + auth_method: "local", + }; + const tasksAvailabilityData = await oasstApiClient.fetch_tasks_availability(dummy_user); + const statsData = await oasstApiClient.fetch_stats(); + const treeManagerData = await oasstApiClient.fetch_tree_manager(); + + const status = { + tasksAvailability: tasksAvailabilityData, + stats: statsData, + treeManager: treeManagerData, + }; + + res.status(200).json(status); +}); + +export default handler; diff --git a/website/src/pages/api/admin/tasks_availability.ts b/website/src/pages/api/admin/tasks_availability.ts deleted file mode 100644 index c7f1d1c5..00000000 --- a/website/src/pages/api/admin/tasks_availability.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { withRole } from "src/lib/auth"; - -/** - * Returns result of tasks availability query using a dummy user. - */ -const handler = withRole("admin", async (req, res) => { - const tasksAvailabilityRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/tasks/availability`, { - method: "POST", - headers: { - "X-API-Key": process.env.FASTAPI_KEY, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - id: "__dummy_user__", - display_name: "Dummy User", - auth_method: "local", - }), - }); - const tasksAvailability = await tasksAvailabilityRes.json(); - - res.status(tasksAvailabilityRes.status).json(tasksAvailability); -}); - -export default handler; diff --git a/website/src/pages/api/admin/tree_manager.ts b/website/src/pages/api/admin/tree_manager.ts deleted file mode 100644 index c127f635..00000000 --- a/website/src/pages/api/admin/tree_manager.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { withRole } from "src/lib/auth"; - -/** - * Returns tree manager stats. - */ -const handler = withRole("admin", async (req, res) => { - const treeManagerRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/stats/tree_manager`, { - method: "GET", - headers: { - "X-API-Key": process.env.FASTAPI_KEY, - }, - }); - const treeManager = await treeManagerRes.json(); - - res.status(treeManagerRes.status).json(treeManager); -}); - -export default handler; From 93969bd49d22a2ee7786cab190705dbdae8f6916 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Sat, 21 Jan 2023 00:11:02 -0500 Subject: [PATCH 30/59] status use current admin user --- website/src/pages/api/admin/status.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/website/src/pages/api/admin/status.ts b/website/src/pages/api/admin/status.ts index 4fb91857..3046bb27 100644 --- a/website/src/pages/api/admin/status.ts +++ b/website/src/pages/api/admin/status.ts @@ -1,3 +1,4 @@ +import { getToken } from "next-auth/jwt"; import { withRole } from "src/lib/auth"; import { oasstApiClient } from "src/lib/oasst_api_client"; @@ -5,12 +6,13 @@ import { oasstApiClient } from "src/lib/oasst_api_client"; * Returns tasks availability, stats, and tree manager stats. */ const handler = withRole("admin", async (req, res) => { - const dummy_user = { - id: "__dummy_user__", - display_name: "Dummy User", + const token = await getToken({ req }); + const currentUser = { + id: token.sub, + display_name: token.name, auth_method: "local", }; - const tasksAvailabilityData = await oasstApiClient.fetch_tasks_availability(dummy_user); + const tasksAvailabilityData = await oasstApiClient.fetch_tasks_availability(currentUser); const statsData = await oasstApiClient.fetch_stats(); const treeManagerData = await oasstApiClient.fetch_tree_manager(); From 2d7d14b4c91703892e912271059aa27832a853b5 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Sat, 21 Jan 2023 11:44:49 -0500 Subject: [PATCH 31/59] Update website/src/pages/admin/status/index.tsx Co-authored-by: notmd <33456881+notmd@users.noreply.github.com> --- website/src/pages/admin/status/index.tsx | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx index 5d5d9ba4..e4f510d7 100644 --- a/website/src/pages/admin/status/index.tsx +++ b/website/src/pages/admin/status/index.tsx @@ -75,15 +75,17 @@ const StatusIndex = ({ user }) => { - - - /api/v1/tasks/availability - - - 
-              {tasksAvailability ? JSON.stringify(tasksAvailability, null, 2) : }
-
- + + + + /api/v1/tasks/availability + + + 
+                {tasksAvailability ? JSON.stringify(tasksAvailability, null, 2) : }
+
+ + From 56fb28f953b35f3e7263e9411a110f89942ec904 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Sat, 21 Jan 2023 11:45:56 -0500 Subject: [PATCH 32/59] use CardBody in website/src/pages/admin/status/index.tsx Co-authored-by: notmd <33456881+notmd@users.noreply.github.com> --- website/src/pages/admin/status/index.tsx | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx index e4f510d7..0abe1404 100644 --- a/website/src/pages/admin/status/index.tsx +++ b/website/src/pages/admin/status/index.tsx @@ -88,13 +88,15 @@ const StatusIndex = ({ user }) => { - - - /api/v1/stats/ - - - 
{stats ? JSON.stringify(stats, null, 2) : }
- + + + + /api/v1/stats/ + + + 
{stats ? JSON.stringify(stats, null, 2) : }
+ +
From 05dce5b4a95807c42147f3817b7c43919bab9da6 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Sat, 21 Jan 2023 11:46:16 -0500 Subject: [PATCH 33/59] Use CardBody website/src/pages/admin/status/index.tsx Co-authored-by: notmd <33456881+notmd@users.noreply.github.com> --- website/src/pages/admin/status/index.tsx | 101 ++++++++++++----------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx index 0abe1404..30c55c9a 100644 --- a/website/src/pages/admin/status/index.tsx +++ b/website/src/pages/admin/status/index.tsx @@ -100,58 +100,59 @@ const StatusIndex = ({ user }) => {
- - - /api/v1/stats/tree_manager - - - {treeManager ? ( - - - state_counts - - - 
{JSON.stringify(treeManager.state_counts, null, 2)}
- - -
+ + + + /api/v1/stats/tree_manager + + {treeManager ? ( + - message_counts + state_counts - - Tree Manager - - - - - - - - - - - - - {treeManager.message_counts.map( - ({ message_tree_id, state, depth, oldest, youngest, count, goal_tree_size }) => ( - - - - - - - - - - ) - )} - -
Message Tree IDStateDepthOldestYoungestCountGoal Tree Size
{message_tree_id}{state}{depth}{oldest}{youngest}{count}{goal_tree_size}
- - - ) : ( - - )} + + 
{JSON.stringify(treeManager.state_counts, null, 2)}
+ + +
+ + message_counts + + + Tree Manager + + + + + + + + + + + + + {treeManager.message_counts.map( + ({ message_tree_id, state, depth, oldest, youngest, count, goal_tree_size }) => ( + + + + + + + + + + ) + )} + +
Message Tree IDStateDepthOldestYoungestCountGoal Tree Size
{message_tree_id}{state}{depth}{oldest}{youngest}{count}{goal_tree_size}
+ + + ) : ( + + )} + ); From 72d61cc44c9f152647db284ee595a72073afe228 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Sat, 21 Jan 2023 11:46:40 -0500 Subject: [PATCH 34/59] use Promise.all website/src/pages/api/admin/status.ts Co-authored-by: notmd <33456881+notmd@users.noreply.github.com> --- website/src/pages/api/admin/status.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/website/src/pages/api/admin/status.ts b/website/src/pages/api/admin/status.ts index 3046bb27..fb93740f 100644 --- a/website/src/pages/api/admin/status.ts +++ b/website/src/pages/api/admin/status.ts @@ -12,9 +12,11 @@ const handler = withRole("admin", async (req, res) => { display_name: token.name, auth_method: "local", }; - const tasksAvailabilityData = await oasstApiClient.fetch_tasks_availability(currentUser); - const statsData = await oasstApiClient.fetch_stats(); - const treeManagerData = await oasstApiClient.fetch_tree_manager(); + const [tasksAvailabilityData, statsData, treeManagerData] = await Promise.all([ + oasstApiClient.fetch_tasks_availability(currentUser), + oasstApiClient.fetch_stats(), + oasstApiClient.fetch_tree_manager(), + ]); const status = { tasksAvailability: tasksAvailabilityData, From fbc2ba468b39ff39428a20806b0033678496bf79 Mon Sep 17 00:00:00 2001 From: ml729 <85370083+ml729@users.noreply.github.com> Date: Sat, 21 Jan 2023 11:48:30 -0500 Subject: [PATCH 35/59] removed unnecessary element id --- website/src/pages/admin/status/index.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx index 30c55c9a..8894888f 100644 --- a/website/src/pages/admin/status/index.tsx +++ b/website/src/pages/admin/status/index.tsx @@ -81,7 +81,7 @@ const StatusIndex = ({ user }) => { /api/v1/tasks/availability - 
+              
                 {tasksAvailability ? JSON.stringify(tasksAvailability, null, 2) : }
               

             
@@ -94,7 +94,7 @@ const StatusIndex = ({ user }) => {
               /api/v1/stats/
             
             
-              
{stats ? JSON.stringify(stats, null, 2) : }

+              
{stats ? JSON.stringify(stats, null, 2) : }

             
           
         
@@ -111,7 +111,7 @@ const StatusIndex = ({ user }) => {
                 state_counts
               
               
-                
{JSON.stringify(treeManager.state_counts, null, 2)}

+                
{JSON.stringify(treeManager.state_counts, null, 2)}

               
               
                 


From 4dd809585e77da4ac0205a906b98143a2b2e5ed8 Mon Sep 17 00:00:00 2001
From: ml729 <85370083+ml729@users.noreply.github.com>
Date: Sat, 21 Jan 2023 12:24:15 -0500
Subject: [PATCH 36/59] handle error/loading case correctly in status page

---
 website/src/pages/admin/status/index.tsx | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx
index 8894888f..efb734b2 100644
--- a/website/src/pages/admin/status/index.tsx
+++ b/website/src/pages/admin/status/index.tsx
@@ -1,9 +1,8 @@
 import {
   Box,
   Card,
-  Button,
+  CardBody,
   CircularProgress,
-  Container,
   SimpleGrid,
   Text,
   Table,
@@ -14,7 +13,6 @@ import {
   Th,
   Thead,
   Tr,
-  View,
   useColorMode,
   useToast,
 } from "@chakra-ui/react";
@@ -81,9 +79,13 @@ const StatusIndex = ({ user }) => {
               /api/v1/tasks/availability
             
             
-              
-                {tasksAvailability ? JSON.stringify(tasksAvailability, null, 2) : }
-              

+              {tasksAvailability ? (
+                
{JSON.stringify(tasksAvailability, null, 2)}

+              ) : errorStatus ? (
+                
{JSON.stringify(errorStatus, null, 2)}

+              ) : (
+                
+              )}
             
           
         
@@ -94,7 +96,13 @@ const StatusIndex = ({ user }) => {
               /api/v1/stats/
             
             
-              
{stats ? JSON.stringify(stats, null, 2) : }

+              {stats ? (
+                
{JSON.stringify(stats, null, 2)}

+              ) : errorStatus ? (
+                
{JSON.stringify(errorStatus, null, 2)}

+              ) : (
+                
+              )}
             
           
         
@@ -149,6 +157,8 @@ const StatusIndex = ({ user }) => {
                 
               
             
+          ) : errorStatus ? (
+            
{JSON.stringify(errorStatus, null, 2)}

           ) : (
             
           )}

From a75c13ef423d99ea210284d767c63de158635fea Mon Sep 17 00:00:00 2001
From: ml729 <85370083+ml729@users.noreply.github.com>
Date: Sat, 21 Jan 2023 13:13:15 -0500
Subject: [PATCH 37/59] use getBackendUserCore for taskAvailability

---
 website/src/pages/api/admin/status.ts | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/website/src/pages/api/admin/status.ts b/website/src/pages/api/admin/status.ts
index fb93740f..64299db8 100644
--- a/website/src/pages/api/admin/status.ts
+++ b/website/src/pages/api/admin/status.ts
@@ -1,17 +1,14 @@
 import { getToken } from "next-auth/jwt";
 import { withRole } from "src/lib/auth";
 import { oasstApiClient } from "src/lib/oasst_api_client";
+import { getBackendUserCore } from "src/lib/users";
 
 /**
  * Returns tasks availability, stats, and tree manager stats.
  */
 const handler = withRole("admin", async (req, res) => {
   const token = await getToken({ req });
-  const currentUser = {
-    id: token.sub,
-    display_name: token.name,
-    auth_method: "local",
-  };
+  const currentUser = await getBackendUserCore(token.sub);
   const [tasksAvailabilityData, statsData, treeManagerData] = await Promise.all([
     oasstApiClient.fetch_tasks_availability(currentUser),
     oasstApiClient.fetch_stats(),

From 323f5a19a2599ed1a72701c2196d610a191e9c45 Mon Sep 17 00:00:00 2001
From: ml729 <85370083+ml729@users.noreply.github.com>
Date: Sat, 21 Jan 2023 14:01:58 -0500
Subject: [PATCH 38/59] use Promise.allSettled for admin status

---
 website/src/pages/admin/status/index.tsx | 20 +++++++++++++-------
 website/src/pages/api/admin/status.ts    |  8 ++++----
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx
index efb734b2..ae0f2f1d 100644
--- a/website/src/pages/admin/status/index.tsx
+++ b/website/src/pages/admin/status/index.tsx
@@ -79,8 +79,10 @@ const StatusIndex = ({ user }) => {
               /api/v1/tasks/availability
             
             
-              {tasksAvailability ? (
-                
{JSON.stringify(tasksAvailability, null, 2)}

+              {tasksAvailability?.status === "fulfilled" ? (
+                
{JSON.stringify(tasksAvailability.value, null, 2)}

+              ) : tasksAvailability?.status === "rejected" ? (
+                
{JSON.stringify(tasksAvailability.reason, null, 2)}

               ) : errorStatus ? (
                 
{JSON.stringify(errorStatus, null, 2)}

               ) : (
@@ -96,8 +98,10 @@ const StatusIndex = ({ user }) => {
               /api/v1/stats/
             
             
-              {stats ? (
-                
{JSON.stringify(stats, null, 2)}

+              {stats?.status === "fulfilled" ? (
+                
{JSON.stringify(stats.value, null, 2)}

+              ) : stats?.status === "rejected" ? (
+                
{JSON.stringify(stats.reason, null, 2)}

               ) : errorStatus ? (
                 
{JSON.stringify(errorStatus, null, 2)}

               ) : (
@@ -113,13 +117,13 @@ const StatusIndex = ({ user }) => {
           
             /api/v1/stats/tree_manager
           
-          {treeManager ? (
+          {treeManager?.status === "fulfilled" ? (
             
               
                 state_counts
               
               
-                
{JSON.stringify(treeManager.state_counts, null, 2)}

+                
{JSON.stringify(treeManager.value.state_counts, null, 2)}

               
               
                 

@@ -140,7 +144,7 @@ const StatusIndex = ({ user }) => {
                     
                   
                   
-                    {treeManager.message_counts.map(
+                    {treeManager.value.message_counts.map(
                       ({ message_tree_id, state, depth, oldest, youngest, count, goal_tree_size }) => (
                         
                           {message_tree_id}
@@ -157,6 +161,8 @@ const StatusIndex = ({ user }) => {
                 
               
             
+          ) : treeManager?.status === "rejected" ? (
+            
{JSON.stringify(treeManager.reason, null, 2)}

           ) : errorStatus ? (
             
{JSON.stringify(errorStatus, null, 2)}

           ) : (
diff --git a/website/src/pages/api/admin/status.ts b/website/src/pages/api/admin/status.ts
index 64299db8..1f0ba915 100644
--- a/website/src/pages/api/admin/status.ts
+++ b/website/src/pages/api/admin/status.ts
@@ -9,16 +9,16 @@ import { getBackendUserCore } from "src/lib/users";
 const handler = withRole("admin", async (req, res) => {
   const token = await getToken({ req });
   const currentUser = await getBackendUserCore(token.sub);
-  const [tasksAvailabilityData, statsData, treeManagerData] = await Promise.all([
+  const [tasksAvailabilityOutcome, statsOutcome, treeManagerOutcome] = await Promise.allSettled([
     oasstApiClient.fetch_tasks_availability(currentUser),
     oasstApiClient.fetch_stats(),
     oasstApiClient.fetch_tree_manager(),
   ]);
 
   const status = {
-    tasksAvailability: tasksAvailabilityData,
-    stats: statsData,
-    treeManager: treeManagerData,
+    tasksAvailability: tasksAvailabilityOutcome,
+    stats: statsOutcome,
+    treeManager: treeManagerOutcome,
   };
 
   res.status(200).json(status);

From 81a392232e8d9dde49d340986958ca321fcbbb1e Mon Sep 17 00:00:00 2001
From: rjmacarthy 
Date: Fri, 20 Jan 2023 19:20:31 +0000
Subject: [PATCH 39/59] Add locale to leaderboard and organize locale json

---
 website/public/locales/en/index.json               | 13 ++++++-------
 website/public/locales/en/leaderboard.json         | 10 ++++++++++
 website/src/components/Hero.tsx                    |  4 ++--
 .../LeaderboardGridCell/LeaderboardGridCell.tsx    |  2 +-
 website/src/pages/index.tsx                        | 10 ++--------
 website/src/pages/leaderboard.tsx                  | 14 ++++++++------
 6 files changed, 29 insertions(+), 24 deletions(-)
 create mode 100644 website/public/locales/en/leaderboard.json

diff --git a/website/public/locales/en/index.json b/website/public/locales/en/index.json
index 3443e444..7d8c9152 100644
--- a/website/public/locales/en/index.json
+++ b/website/public/locales/en/index.json
@@ -1,16 +1,15 @@
 {
-  "title": "Open Assistant",
-  "subtitle": "Conversational AI for everyone.",
-  "description": "Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world.",
   "blurb": "We believe we can create a revolution.",
   "blurb1": "In the same way that Stable Diffusion helped the world make art and images in new ways, we want to improve the world by providing amazing conversational AI.",
-  "join_us_title": "Join us",
-  "join_us_description": "All open source projects begin with people like you. Open source is the belief that if we collaborate we can together gift our knowledge and technology to the world for the benefit of humanity. Are you in? Find us here:",
-  "faq_title": "Frequently Asked Questions",
+  "description": "Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world.",
   "faq_items": {
     "q0": "How far along is this project?",
     "a0": "We are in the early stages of development, working from established research in applying RLHF to large language models.",
     "q1": "Who is behind Open Assistant?",
     "a1": "Open Assistant is a project organized by LAION and individuals around the world interested in bringing this technology to everyone."
-  }
+  },
+  "faq_title": "Frequently Asked Questions",
+  "join_us_description": "All open source projects begin with people like you. Open source is the belief that if we collaborate we can together gift our knowledge and technology to the world for the benefit of humanity. Are you in? Find us here:",
+  "join_us_title": "Join us",
+  "subtitle": "Conversational AI for everyone."
 }
diff --git a/website/public/locales/en/leaderboard.json b/website/public/locales/en/leaderboard.json
new file mode 100644
index 00000000..78600248
--- /dev/null
+++ b/website/public/locales/en/leaderboard.json
@@ -0,0 +1,10 @@
+{
+  "daily": "Daily",
+  "leaderboard": "Leaderboard",
+  "monthly": "Monthly",
+  "overall": "Overall",
+  "rank": "Rank",
+  "score": "Score",
+  "user": "User",
+  "weekly": "Weekly"
+}
diff --git a/website/src/components/Hero.tsx b/website/src/components/Hero.tsx
index d401e47e..d9cab0c2 100644
--- a/website/src/components/Hero.tsx
+++ b/website/src/components/Hero.tsx
@@ -6,7 +6,7 @@ import { AnimatedCircles } from "./AnimatedCircles";
 import { Container } from "./Container";
 
 export function Hero() {
-  const { t } = useTranslation("index");
+  const { t } = useTranslation(["index", "common"]);
   const { colorMode } = useColorMode();
   const pTextColor = colorMode === "light" ? "text-gray-600" : "text-white";
   const fancyTextGradientClasses =
@@ -17,7 +17,7 @@ export function Hero() {
         
           
             
-              {t("title")}
+              {t("common:title")}
             
              {
   const router = useRouter();
   const { status } = useSession();
-  const { t } = useTranslation("index");
+  const { t } = useTranslation();
   useEffect(() => {
     if (status === "authenticated") {
       router.push("/dashboard");
@@ -37,10 +37,4 @@ const Home = () => {
 
 Home.getLayout = getTransparentHeaderLayout;
 
-export const getStaticProps = async ({ locale }) => ({
-  props: {
-    ...(await serverSideTranslations(locale, ["index", "common"])),
-  },
-});
-
 export default Home;
diff --git a/website/src/pages/leaderboard.tsx b/website/src/pages/leaderboard.tsx
index f79dac52..e413366f 100644
--- a/website/src/pages/leaderboard.tsx
+++ b/website/src/pages/leaderboard.tsx
@@ -1,27 +1,29 @@
 import { Box, Heading, Tab, TabList, TabPanel, TabPanels, Tabs } from "@chakra-ui/react";
 import Head from "next/head";
+import { useTranslation } from "next-i18next";
 import { getDashboardLayout } from "src/components/Layout";
 import { LeaderboardGridCell } from "src/components/LeaderboardGridCell";
 export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props";
 import { LeaderboardTimeFrame } from "src/types/Leaderboard";
 
 const Leaderboard = () => {
+  const { t } = useTranslation(["leaderboard", "common"]);
   return (
     <>
       
-        Leaderboard - Open Assistant
+        {`${t("leaderboard")} - ${t("common:title")}`}
         
       
       
         
-          Leaderboard
+          {t("leaderboard")}
         
         
           
-            Daily
-            Weekly
-            Monthly
-            Overall
+            {t("daily")}
+            {t("weekly")}
+            {t("monthly")}
+            {t("overall")}
           
 
           

From edeae8cc84e42d5cc456c023a30e5d32561cca04 Mon Sep 17 00:00:00 2001
From: rjmacarthy 
Date: Sat, 21 Jan 2023 20:33:27 +0000
Subject: [PATCH 40/59] Fix locale leaderboard after rebase

---
 website/public/locales/en/common.json                |  3 +--
 website/public/locales/en/leaderboard.json           |  1 +
 .../LeaderboardGridCell/LeaderboardGridCell.tsx      | 12 +++++++-----
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/website/public/locales/en/common.json b/website/public/locales/en/common.json
index 82f378c9..99edf6c3 100644
--- a/website/public/locales/en/common.json
+++ b/website/public/locales/en/common.json
@@ -13,6 +13,5 @@
   "sign_in": "Sign In",
   "sign_out": "Sign Out",
   "terms_of_service": "Terms of Service",
-  "title": "Open Assistant",
-  "last_updated_at": "Last updated at: {{val, datetime}}"
+  "title": "Open Assistant"
 }
diff --git a/website/public/locales/en/leaderboard.json b/website/public/locales/en/leaderboard.json
index 78600248..c2dd0832 100644
--- a/website/public/locales/en/leaderboard.json
+++ b/website/public/locales/en/leaderboard.json
@@ -1,5 +1,6 @@
 {
   "daily": "Daily",
+  "last_updated_at": "Last updated at: {{val, datetime}}",
   "leaderboard": "Leaderboard",
   "monthly": "Monthly",
   "overall": "Overall",
diff --git a/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx b/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx
index 887c9122..2c576e51 100644
--- a/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx
+++ b/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx
@@ -6,19 +6,19 @@ import { get } from "src/lib/api";
 import { LeaderboardReply, LeaderboardTimeFrame } from "src/types/Leaderboard";
 import useSWRImmutable from "swr/immutable";
 
-const columns = [
+const getColumns = (t) => [
   {
-    Header: "Rank",
+    Header: t("rank"),
     accessor: "rank",
     style: { width: "90px" },
   },
   {
-    Header: "Score",
+    Header: t("score"),
     accessor: "leader_score",
     style: { width: "90px" },
   },
   {
-    Header: "User",
+    Header: t("user"),
     accessor: "display_name",
   },
 ];
@@ -27,11 +27,13 @@ const columns = [
  * Presents a grid of leaderboard entries with more detailed information.
  */
 const LeaderboardGridCell = ({ timeFrame }: { timeFrame: LeaderboardTimeFrame }) => {
-  const { t } = useTranslation();
+  const { t } = useTranslation(["leaderboard", "common"]);
   const { data: reply } = useSWRImmutable(`/api/leaderboard?time_frame=${timeFrame}`, get, {
     revalidateOnMount: true,
   });
 
+  const columns = useMemo(() => getColumns(t), [t]);
+
   const { getTableProps, getTableBodyProps, headerGroups, rows, prepareRow } = useTable({
     columns,
     data: reply?.leaderboard ?? [],

From 3f50cf108032274b7c06da02ffb9b18bf73b4663 Mon Sep 17 00:00:00 2001
From: rjmacarthy 
Date: Sat, 21 Jan 2023 20:35:02 +0000
Subject: [PATCH 41/59] Pre-commit

---
 .../src/components/LeaderboardGridCell/LeaderboardGridCell.tsx  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx b/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx
index 2c576e51..7886784a 100644
--- a/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx
+++ b/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx
@@ -87,4 +87,4 @@ const LeaderboardGridCell = ({ timeFrame }: { timeFrame: LeaderboardTimeFrame })
   );
 };
 
-export { LeaderboardGridCell };
\ No newline at end of file
+export { LeaderboardGridCell };

From cec49614c21d74e5ef19062e8c572d3051974aad Mon Sep 17 00:00:00 2001
From: Ori Yonay 
Date: Sat, 21 Jan 2023 15:37:42 -0600
Subject: [PATCH 42/59] added message size limit of 2000 characters (issue 822)
 (#880)

* added message size limit (issue 822)
---
 backend/oasst_backend/api/v1/admin.py                   | 1 +
 backend/oasst_backend/config.py                         | 1 +
 backend/oasst_backend/tree_manager.py                   | 7 +++++++
 oasst-shared/oasst_shared/exceptions/oasst_api_error.py | 1 +
 4 files changed, 10 insertions(+)

diff --git a/backend/oasst_backend/api/v1/admin.py b/backend/oasst_backend/api/v1/admin.py
index 59c5efda..584c5fb7 100644
--- a/backend/oasst_backend/api/v1/admin.py
+++ b/backend/oasst_backend/api/v1/admin.py
@@ -54,6 +54,7 @@ class PublicSettings(pydantic.BaseModel):
 
     PROJECT_NAME: str
     API_V1_STR: str
+    MESSAGE_SIZE_LIMIT: int
     DEBUG_USE_SEED_DATA: bool
     DEBUG_ALLOW_SELF_LABELING: bool
     DEBUG_SKIP_EMBEDDING_COMPUTATION: bool
diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py
index 99b10cb4..8ca2a413 100644
--- a/backend/oasst_backend/config.py
+++ b/backend/oasst_backend/config.py
@@ -72,6 +72,7 @@ class Settings(BaseSettings):
     DATABASE_MAX_TX_RETRY_COUNT: int = 3
 
     RATE_LIMIT: bool = True
+    MESSAGE_SIZE_LIMIT: int = 2000
     REDIS_HOST: str = "localhost"
     REDIS_PORT: str = "6379"
 
diff --git a/backend/oasst_backend/tree_manager.py b/backend/oasst_backend/tree_manager.py
index 026bb564..1224af4a 100644
--- a/backend/oasst_backend/tree_manager.py
+++ b/backend/oasst_backend/tree_manager.py
@@ -465,6 +465,13 @@ class TreeManager:
                     f"Frontend reports text reply to {interaction.message_id=} with {interaction.text=} by {interaction.user=}."
                 )
 
+                # ensure message size is below the predefined limit
+                if len(interaction.text) > settings.MESSAGE_SIZE_LIMIT:
+                    logger.error(
+                        f"Message size {len(interaction.text)=} exceeds size limit of {settings.MESSAGE_SIZE_LIMIT=}."
+                    )
+                    raise OasstError("Message size too long.", OasstErrorCode.TASK_MESSAGE_TOO_LONG)
+
                 # here we store the text reply in the database
                 message = pr.store_text_reply(
                     text=interaction.text,
diff --git a/oasst-shared/oasst_shared/exceptions/oasst_api_error.py b/oasst-shared/oasst_shared/exceptions/oasst_api_error.py
index bed6f942..0a548ebb 100644
--- a/oasst-shared/oasst_shared/exceptions/oasst_api_error.py
+++ b/oasst-shared/oasst_shared/exceptions/oasst_api_error.py
@@ -37,6 +37,7 @@ class OasstErrorCode(IntEnum):
     TASK_GENERATION_FAILED = 1005
     TASK_REQUESTED_TYPE_NOT_AVAILABLE = 1006
     TASK_AVAILABILITY_QUERY_FAILED = 1007
+    TASK_MESSAGE_TOO_LONG = 1008
 
     # 2000-3000: prompt_repository
     INVALID_FRONTEND_MESSAGE_ID = 2000

From 1709dc03247fb72202b76703d5b8e18c837d7488 Mon Sep 17 00:00:00 2001
From: Yannic Kilcher 
Date: Sat, 21 Jan 2023 22:38:18 +0100
Subject: [PATCH 43/59] Initial implementation of the inference system (#869)

* very primitive implementation of inference

* re-worked with security in mind

* removed polling from clients

* switched workers to websockets

* implemented back and forth chats
---
 inference/README.md                           |  35 ++++
 inference/server/README.md                    |  10 +
 inference/server/main.py                      | 193 ++++++++++++++++++
 inference/server/requirements.txt             |   6 +
 inference/text-client/__main__.py             |  40 ++++
 inference/text-client/requirements.txt        |   3 +
 inference/worker/__main__.py                  |  79 +++++++
 inference/worker/requirements.txt             |   6 +
 .../oasst_shared/schemas/inference.py         |  21 ++
 oasst-shared/oasst_shared/schemas/protocol.py |  12 ++
 10 files changed, 405 insertions(+)
 create mode 100644 inference/README.md
 create mode 100644 inference/server/README.md
 create mode 100644 inference/server/main.py
 create mode 100644 inference/server/requirements.txt
 create mode 100644 inference/text-client/__main__.py
 create mode 100644 inference/text-client/requirements.txt
 create mode 100644 inference/worker/__main__.py
 create mode 100644 inference/worker/requirements.txt
 create mode 100644 oasst-shared/oasst_shared/schemas/inference.py

diff --git a/inference/README.md b/inference/README.md
new file mode 100644
index 00000000..3dee94f9
--- /dev/null
+++ b/inference/README.md
@@ -0,0 +1,35 @@
+# OpenAssitant Inference
+
+Preliminary implementation of the inference engine for OpenAssistant.
+
+## Development (you'll need multiple terminals)
+
+Run a redis container (or use the one of the general docker compose file):
+
+```bash
+docker run --rm -it -p 6379:6379 redis
+```
+
+Run the inference server:
+
+```bash
+cd server
+pip install -r requirements.txt
+uvicorn main:app --reload
+```
+
+Run one (or more) workers:
+
+```bash
+cd worker
+pip install -r requirements.txt
+python __main__.py
+```
+
+Run the client:
+
+```bash
+cd text-client
+pip install -r requirements.txt
+python __main__.py
+```
diff --git a/inference/server/README.md b/inference/server/README.md
new file mode 100644
index 00000000..a235a7e6
--- /dev/null
+++ b/inference/server/README.md
@@ -0,0 +1,10 @@
+# OpenAssistant Inference Server
+
+Workers communicate with the `/work` endpoint via Websocket. They provide their
+configuration and if a task is available, the server returns it. The worker then
+performs the task and returns the result in a streaming fashion to the server,
+also via websocket.
+
+Clients first call `/chat` to make a new chat, then add to that via
+`/chat//message`. The response is a SSE event source, which will send tokens
+as they are available.
diff --git a/inference/server/main.py b/inference/server/main.py
new file mode 100644
index 00000000..f3ec02b1
--- /dev/null
+++ b/inference/server/main.py
@@ -0,0 +1,193 @@
+import asyncio
+import enum
+import uuid
+
+import fastapi
+import pydantic
+import redis.asyncio as redis
+from fastapi.middleware.cors import CORSMiddleware
+from loguru import logger
+from oasst_shared.schemas import inference, protocol
+from sse_starlette.sse import EventSourceResponse
+
+app = fastapi.FastAPI()
+
+# Allow CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+class Settings(pydantic.BaseSettings):
+    redis_host: str = "localhost"
+    redis_port: int = 6379
+    redis_db: int = 0
+
+    sse_retry_timeout: int = 15000
+
+
+settings = Settings()
+
+# create async redis client
+redisClient = redis.Redis(
+    host=settings.redis_host, port=settings.redis_port, db=settings.redis_db, decode_responses=True
+)
+
+
+class CreateChatRequest(pydantic.BaseModel):
+    pass
+
+
+class CreateChatResponse(pydantic.BaseModel):
+    id: str
+
+
+class MessageRequest(pydantic.BaseModel):
+    message: str = pydantic.Field(..., repr=False)
+    model_name: str = "distilgpt2"
+    max_new_tokens: int = 100
+
+    def compatible_with(self, worker_config: inference.WorkerConfig) -> bool:
+        return self.model_name == worker_config.model_name
+
+
+class TokenResponseEvent(pydantic.BaseModel):
+    token: str
+
+
+class MessageRequestState(str, enum.Enum):
+    pending = "pending"
+    in_progress = "in_progress"
+    complete = "complete"
+
+
+class DbChatEntry(pydantic.BaseModel):
+    id: str = pydantic.Field(default_factory=lambda: str(uuid.uuid4()))
+    conversation: protocol.Conversation = pydantic.Field(default_factory=protocol.Conversation)
+    pending_message_request: MessageRequest | None = None
+    message_request_state: MessageRequestState | None = None
+
+
+# TODO: make real database
+CHATS: dict[str, DbChatEntry] = {}
+
+
+@app.post("/chat")
+async def create_chat(request: CreateChatRequest) -> CreateChatResponse:
+    """Allows a client to create a new chat."""
+    logger.info(f"Received {request}")
+    chat = DbChatEntry()
+    CHATS[chat.id] = chat
+    return CreateChatResponse(id=chat.id)
+
+
+@app.get("/chat/{id}")
+async def get_chat(id: str) -> protocol.Conversation:
+    """Allows a client to get the current state of a chat."""
+    return CHATS[id].conversation
+
+
+@app.post("/chat/{id}/message")
+async def create_message(id: str, message_request: MessageRequest, fastapi_request: fastapi.Request):
+    """Allows the client to stream the results of a request."""
+
+    chat = CHATS[id]
+    if not chat.conversation.is_prompter_turn:
+        raise fastapi.HTTPException(status_code=400, detail="Not your turn")
+    if chat.pending_message_request is not None:
+        raise fastapi.HTTPException(status_code=400, detail="Already pending")
+
+    chat.conversation.messages.append(
+        protocol.ConversationMessage(
+            text=message_request.message,
+            is_assistant=False,
+        )
+    )
+
+    chat.pending_message_request = message_request
+    chat.message_request_state = MessageRequestState.pending
+
+    async def event_generator():
+        result_data = []
+
+        try:
+            while True:
+                if await fastapi_request.is_disconnected():
+                    logger.warning("Client disconnected")
+                    break
+
+                item = await redisClient.blpop(chat.id, 1)
+                if item is None:
+                    continue
+
+                _, response_packet_str = item
+                response_packet = inference.WorkResponsePacket.parse_raw(response_packet_str)
+                result_data.append(response_packet)
+
+                if response_packet.is_end:
+                    break
+
+                yield {
+                    "retry": settings.sse_retry_timeout,
+                    "data": TokenResponseEvent(token=response_packet.token).json(),
+                }
+            logger.info(f"Finished streaming {chat.id} {len(result_data)=}")
+        except Exception:
+            logger.exception(f"Error streaming {chat.id}")
+
+        chat.conversation.messages.append(
+            protocol.ConversationMessage(
+                text="".join([d.token for d in result_data[:-1]]),
+                is_assistant=True,
+            )
+        )
+        chat.pending_message_request = None
+
+    return EventSourceResponse(event_generator())
+
+
+@app.websocket("/work")
+async def work(websocket: fastapi.WebSocket):
+    await websocket.accept()
+    worker_config = inference.WorkerConfig.parse_raw(await websocket.receive_text())
+    while True:
+        # find a pending task that matches the worker's config
+        # could also be implemented using task queues
+        # but general compatibility matching is tricky
+        for chat in CHATS.values():
+            if (request := chat.pending_message_request) is not None:
+                if chat.message_request_state == MessageRequestState.pending:
+                    if request.compatible_with(worker_config):
+                        break
+        else:
+            logger.debug("No pending tasks")
+            await asyncio.sleep(1)
+            continue
+
+        chat.message_request_state = MessageRequestState.in_progress
+
+        work_request = inference.WorkRequest(
+            conversation=chat.conversation,
+            model_name=request.model_name,
+            max_new_tokens=request.max_new_tokens,
+        )
+
+        logger.info(f"Created {work_request}")
+        try:
+            await websocket.send_text(work_request.json())
+            while True:
+                # maybe unnecessary to parse and re-serialize
+                # could just pass the raw string and mark end via empty string
+                response_packet = inference.WorkResponsePacket.parse_raw(await websocket.receive_text())
+                await redisClient.rpush(chat.id, response_packet.json())
+                if response_packet.is_end:
+                    break
+        except fastapi.WebSocketException:
+            # TODO: handle this better
+            logger.exception(f"Websocket closed during handling of {chat.id}")
+
+        chat.message_request_state = MessageRequestState.complete
diff --git a/inference/server/requirements.txt b/inference/server/requirements.txt
new file mode 100644
index 00000000..e0a00339
--- /dev/null
+++ b/inference/server/requirements.txt
@@ -0,0 +1,6 @@
+fastapi[all]
+loguru
+pydantic
+redis
+sse-starlette
+websockets
diff --git a/inference/text-client/__main__.py b/inference/text-client/__main__.py
new file mode 100644
index 00000000..bf1f8b02
--- /dev/null
+++ b/inference/text-client/__main__.py
@@ -0,0 +1,40 @@
+"""Simple REPL frontend."""
+
+import json
+
+import requests
+import sseclient
+import typer
+
+app = typer.Typer()
+
+
+@app.command()
+def main(backend_url: str = "http://127.0.0.1:8000"):
+    """Simple REPL client."""
+    chat_id = requests.post(f"{backend_url}/chat", json={}).json()["id"]
+    while True:
+        message = typer.prompt("User").strip()
+
+        # wait for stream to be ready
+        # could implement a queue position indicator
+        # could be implemented with long polling
+        # but server load needs to be considered
+        response = requests.post(
+            f"{backend_url}/chat/{chat_id}/message",
+            json={"message": message},
+            stream=True,
+            headers={"Accept": "text/event-stream"},
+        )
+        response.raise_for_status()
+
+        client = sseclient.SSEClient(response)
+        print("Assistant: ", end="", flush=True)
+        for event in client.events():
+            data = json.loads(event.data)
+            print(data["token"], end="", flush=True)
+        print()
+
+
+if __name__ == "__main__":
+    app()
diff --git a/inference/text-client/requirements.txt b/inference/text-client/requirements.txt
new file mode 100644
index 00000000..8d7bff7d
--- /dev/null
+++ b/inference/text-client/requirements.txt
@@ -0,0 +1,3 @@
+requests
+sseclient-py
+typer
diff --git a/inference/worker/__main__.py b/inference/worker/__main__.py
new file mode 100644
index 00000000..ad5e5cef
--- /dev/null
+++ b/inference/worker/__main__.py
@@ -0,0 +1,79 @@
+import re
+import time
+
+import rel
+import torch
+import typer
+import websocket
+from loguru import logger
+from oasst_shared.schemas import inference, protocol
+from transformers import pipeline
+
+app = typer.Typer()
+
+
+@app.command()
+def main(
+    backend_url: str = "ws://localhost:8000",
+    model_name: str = "distilgpt2",
+):
+    pipe = pipeline("text-generation", model=model_name)
+
+    def on_open(ws: websocket.WebSocket):
+        worker_config = inference.WorkerConfig(model_name=model_name)
+        ws.send(worker_config.json())
+
+    def on_message(ws: websocket.WebSocket, message: str):
+        # TODO: what if this comes in, but one is already in progress?
+        # also need to think of enabling batching
+        work_request = inference.WorkRequest.parse_raw(message)
+
+        def _prepare_message(message: protocol.ConversationMessage) -> str:
+            prefix = "Assistant: " if message.is_assistant else "User: "
+            return prefix + message.text
+
+        # construct prompt
+        messages = [_prepare_message(message) for message in work_request.conversation.messages]
+
+        prompt = "\n".join(messages) + "\nAssistant:"
+
+        # TODO: replace this with incremental generation
+        torch.manual_seed(work_request.seed)
+        model_output = pipe(prompt, max_new_tokens=work_request.max_new_tokens, do_sample=True, return_full_text=False)[
+            0
+        ]["generated_text"]
+        model_output = model_output.strip()
+
+        # fake streaming
+        split_idcs = [m.start() for m in re.finditer(r"([\w:]+)", model_output)]
+        pieces = [model_output[a:b] for a, b in zip([0] + split_idcs, split_idcs + [None])]
+        for piece in pieces:
+            if not piece:
+                continue
+            if piece.strip() in ("User:", "Assistant:"):
+                break
+            ws.send(inference.WorkResponsePacket(token=piece).json())
+            time.sleep(0.1)
+        ws.send(inference.WorkResponsePacket(is_end=True).json())
+
+    def on_error(ws: websocket.WebSocket, error: Exception):
+        logger.error(f"Connection error: {error}")
+
+    def on_close(ws: websocket.WebSocket, close_status_code: int, close_msg: str):
+        logger.warning(f"Connection closed: {close_status_code=} {close_msg=}")
+
+    ws = websocket.WebSocketApp(
+        f"{backend_url}/work",
+        on_message=on_message,
+        on_error=on_error,
+        on_close=on_close,
+        on_open=on_open,
+    )
+
+    ws.run_forever(dispatcher=rel, reconnect=5)
+    rel.signal(2, rel.abort)
+    rel.dispatch()
+
+
+if __name__ == "__main__":
+    app()
diff --git a/inference/worker/requirements.txt b/inference/worker/requirements.txt
new file mode 100644
index 00000000..c248c652
--- /dev/null
+++ b/inference/worker/requirements.txt
@@ -0,0 +1,6 @@
+loguru
+rel
+torch
+transformers
+typer
+websocket-client
diff --git a/oasst-shared/oasst_shared/schemas/inference.py b/oasst-shared/oasst_shared/schemas/inference.py
new file mode 100644
index 00000000..0acb5014
--- /dev/null
+++ b/oasst-shared/oasst_shared/schemas/inference.py
@@ -0,0 +1,21 @@
+import random
+
+import pydantic
+
+from . import protocol
+
+
+class WorkerConfig(pydantic.BaseModel):
+    model_name: str = "distilgpt2"
+
+
+class WorkRequest(pydantic.BaseModel):
+    conversation: protocol.Conversation = pydantic.Field(..., repr=False)
+    model_name: str = "distilgpt2"
+    max_new_tokens: int = 100
+    seed: int = pydantic.Field(default_factory=lambda: random.randint(0, 2**32 - 1))
+
+
+class WorkResponsePacket(pydantic.BaseModel):
+    token: str | None = None
+    is_end: bool = False
diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py
index f2164e8f..20bbdf9b 100644
--- a/oasst-shared/oasst_shared/schemas/protocol.py
+++ b/oasst-shared/oasst_shared/schemas/protocol.py
@@ -64,6 +64,18 @@ class Conversation(BaseModel):
 
     messages: list[ConversationMessage] = []
 
+    def __len__(self):
+        return len(self.messages)
+
+    @property
+    def is_prompter_turn(self) -> bool:
+        if len(self) == 0:
+            return True
+        last_message = self.messages[-1]
+        if last_message.is_assistant:
+            return True
+        return False
+
 
 class Message(ConversationMessage):
     parent_id: Optional[UUID] = None

From 76b5bdc5d795aedd0f65beff8862f51b594e821e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20K=C3=B6pf?= 
Date: Sun, 22 Jan 2023 00:58:03 +0100
Subject: [PATCH 44/59] fix cursor queries
 query_users_ordered_by_username/query_users_ordered_by_display_name

---
 backend/oasst_backend/api/v1/users.py    |  2 +-
 backend/oasst_backend/user_repository.py | 24 ++++++++++++++++++------
 2 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/backend/oasst_backend/api/v1/users.py b/backend/oasst_backend/api/v1/users.py
index 04b3cc99..ed07eb21 100644
--- a/backend/oasst_backend/api/v1/users.py
+++ b/backend/oasst_backend/api/v1/users.py
@@ -111,7 +111,7 @@ def get_users_cursor(
                 n = lt
         return p, n
 
-    def remove_extra_item(items: list[protocol.FrontEndUser], lt: str | None, gt: str):
+    def remove_extra_item(items: list[protocol.FrontEndUser], lt: str | None, gt: str | None):
         num_rows = len(items)
         if qry_max_count > max_count and num_rows == qry_max_count:
             assert not (lt and gt)
diff --git a/backend/oasst_backend/user_repository.py b/backend/oasst_backend/user_repository.py
index 1e9ac78f..44a7e685 100644
--- a/backend/oasst_backend/user_repository.py
+++ b/backend/oasst_backend/user_repository.py
@@ -153,7 +153,7 @@ class UserRepository:
             if api_client_id != self.api_client.id:
                 raise OasstError("Forbidden", OasstErrorCode.API_CLIENT_NOT_AUTHORIZED, HTTP_403_FORBIDDEN)
 
-        qry = self.db.query(User).order_by(User.username, User.id)
+        qry = self.db.query(User)
 
         if gte_username is not None:
             if gt_id:
@@ -184,8 +184,14 @@ class UserRepository:
             pattern = "%{}%".format(search_text.replace("\\", "\\\\").replace("_", "\\_").replace("%", "\\%"))
             qry = qry.filter(User.username.like(pattern))
 
-        if limit is not None:
-            qry = qry.limit(limit)
+        if limit is not None and lte_username and not gte_username:
+            # select top rows but return results in ascernding order
+            sub_qry = qry.order_by(User.username.desc(), User.id.desc()).limit(limit).subquery("u")
+            qry = self.db.query(User).select_entity_from(sub_qry).order_by(User.username, User.id)
+        else:
+            qry = qry.order_by(User.username, User.id)
+            if limit is not None:
+                qry = qry.limit(limit)
 
         return qry.all()
 
@@ -209,7 +215,7 @@ class UserRepository:
                 # Unprivileged api client asks for foreign users
                 raise OasstError("Forbidden", OasstErrorCode.API_CLIENT_NOT_AUTHORIZED, HTTP_403_FORBIDDEN)
 
-        qry = self.db.query(User).order_by(User.display_name, User.id)
+        qry = self.db.query(User)
 
         if gte_display_name is not None:
             if gt_id:
@@ -249,7 +255,13 @@ class UserRepository:
         if auth_method:
             qry = qry.filter(User.auth_method == auth_method)
 
-        if limit is not None:
-            qry = qry.limit(limit)
+        if limit is not None and lte_display_name and not gte_display_name:
+            # select top rows but return results in ascernding order
+            sub_qry = qry.order_by(User.display_name.desc(), User.id.desc()).limit(limit).subquery("u")
+            qry = self.db.query(User).select_entity_from(sub_qry).order_by(User.display_name, User.id)
+        else:
+            qry = qry.order_by(User.display_name, User.id)
+            if limit is not None:
+                qry = qry.limit(limit)
 
         return qry.all()

From 19652d5cec066195ba62b158e2cff1019d3d08a4 Mon Sep 17 00:00:00 2001
From: ml729 <85370083+ml729@users.noreply.github.com>
Date: Sat, 21 Jan 2023 20:42:34 -0500
Subject: [PATCH 45/59] remove unnecessary imports

---
 website/src/pages/admin/status/index.tsx | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx
index ae0f2f1d..12eb0785 100644
--- a/website/src/pages/admin/status/index.tsx
+++ b/website/src/pages/admin/status/index.tsx
@@ -14,16 +14,13 @@ import {
   Thead,
   Tr,
   useColorMode,
-  useToast,
 } from "@chakra-ui/react";
 import Head from "next/head";
 import { useRouter } from "next/router";
 import { useSession } from "next-auth/react";
-import { useEffect, useState } from "react";
+import { useEffect } from "react";
 import useSWRImmutable from "swr/immutable";
 import { getAdminLayout } from "src/components/Layout";
-import poster from "src/lib/poster";
-import prisma from "src/lib/prismadb";
 import { get } from "src/lib/api";
 
 /**
@@ -31,13 +28,11 @@ import { get } from "src/lib/api";
  * namely /api/v1/tasks/availability, /api/v1/stats/, /api/v1/stats/tree_manager
  */
 
-const StatusIndex = ({ user }) => {
-  const toast = useToast();
+const StatusIndex = () => {
   const router = useRouter();
   const { data: session, status } = useSession();
 
   const { colorMode } = useColorMode();
-  const backgroundColor = colorMode === "light" ? "white" : "gray.700";
   const dataBackgroundColor = colorMode === "light" ? "gray.100" : "gray.800";
   // Check when the user session is loaded and re-route if the user is not an
   // admin.  This follows the suggestion by NextJS for handling private pages:

From a31da35fb0eb272461b82c4ca3ce39b800d0068a Mon Sep 17 00:00:00 2001
From: ml729 <85370083+ml729@users.noreply.github.com>
Date: Sat, 21 Jan 2023 21:08:09 -0500
Subject: [PATCH 46/59] use dummy user for tasks availability

---
 website/src/pages/api/admin/status.ts | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/website/src/pages/api/admin/status.ts b/website/src/pages/api/admin/status.ts
index 1f0ba915..1da03da8 100644
--- a/website/src/pages/api/admin/status.ts
+++ b/website/src/pages/api/admin/status.ts
@@ -7,10 +7,13 @@ import { getBackendUserCore } from "src/lib/users";
  * Returns tasks availability, stats, and tree manager stats.
  */
 const handler = withRole("admin", async (req, res) => {
-  const token = await getToken({ req });
-  const currentUser = await getBackendUserCore(token.sub);
+  const dummyUser = {
+    id: "__dummy_user__",
+    display_name: "Dummy User",
+    auth_method: "local",
+  };
   const [tasksAvailabilityOutcome, statsOutcome, treeManagerOutcome] = await Promise.allSettled([
-    oasstApiClient.fetch_tasks_availability(currentUser),
+    oasstApiClient.fetch_tasks_availability(dummyUser),
     oasstApiClient.fetch_stats(),
     oasstApiClient.fetch_tree_manager(),
   ]);

From ab09a3f50fe17242c32c7e82f2b94e409561e73e Mon Sep 17 00:00:00 2001
From: Keith Stevens 
Date: Sun, 22 Jan 2023 16:13:35 +0900
Subject: [PATCH 47/59] Adding a valid language check

---
 website/src/lib/users.ts | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/website/src/lib/users.ts b/website/src/lib/users.ts
index 3dbe5a08..3f1cb65d 100644
--- a/website/src/lib/users.ts
+++ b/website/src/lib/users.ts
@@ -3,13 +3,26 @@ import type { NextApiRequest } from "next";
 import prisma from "src/lib/prismadb";
 import type { BackendUserCore } from "src/types/Users";
 
+import { i18n } from "src/../next-i18next.config";
+
+const LOCALE_SET = new Set(i18n.locales);
+
+/**
+ * Returns the most appropriate user language using the following priority:
+ *
+ * 1. The `NEXT_LOCALE` cookie which is set by the client side and will be in
+ *    the set of supported locales.
+ * 2. The `accept-language` header if it contains a supported locale as set by
+ *    the i18n module.
+ * 3. "en" as a final fallback.
+ */
 const getUserLanguage = (req: NextApiRequest) => {
   const cookieLanguage = req.cookies["NEXT_LOCALE"];
   if (cookieLanguage) {
     return cookieLanguage;
   }
   const headerLanguages = parser.parse(req.headers["accept-language"]);
-  if (headerLanguages.length > 0) {
+  if (headerLanguages.length > 0 && LOCALE_SET.has(headerLanguages[0].code)) {
     return headerLanguages[0].code;
   }
   return "en";

From 6945cc5fe7c7028901c0834ea8d24a9ef628c268 Mon Sep 17 00:00:00 2001
From: notmd 
Date: Sun, 22 Jan 2023 14:14:41 +0700
Subject: [PATCH 48/59] remove `reverse` method

---
 backend/oasst_backend/user_repository.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/backend/oasst_backend/user_repository.py b/backend/oasst_backend/user_repository.py
index 670bbc3e..118f4e82 100644
--- a/backend/oasst_backend/user_repository.py
+++ b/backend/oasst_backend/user_repository.py
@@ -269,9 +269,4 @@ class UserRepository:
             if limit is not None:
                 qry = qry.limit(limit)
 
-        users = qry.all()
-
-        if lte_display_name and lt_id:
-            users.reverse()
-
-        return users
+        return qry.all()

From 79331df366f3a95bb5209b210552189457ab9723 Mon Sep 17 00:00:00 2001
From: Keith Stevens 
Date: Sun, 22 Jan 2023 16:15:54 +0900
Subject: [PATCH 49/59] Fixing an import order

---
 website/src/components/LanguageSelector/LanguageSelector.tsx | 2 +-
 website/src/lib/users.ts                                     | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/website/src/components/LanguageSelector/LanguageSelector.tsx b/website/src/components/LanguageSelector/LanguageSelector.tsx
index e611bf0f..37659265 100644
--- a/website/src/components/LanguageSelector/LanguageSelector.tsx
+++ b/website/src/components/LanguageSelector/LanguageSelector.tsx
@@ -1,7 +1,7 @@
 import { Select } from "@chakra-ui/react";
 import { useRouter } from "next/router";
 import { useTranslation } from "next-i18next";
-import { useCallback, useMemo, useState } from "react";
+import { useCallback, useMemo } from "react";
 import cookie from "react-cookies";
 
 const LanguageSelector = () => {
diff --git a/website/src/lib/users.ts b/website/src/lib/users.ts
index 3f1cb65d..637e93a9 100644
--- a/website/src/lib/users.ts
+++ b/website/src/lib/users.ts
@@ -1,10 +1,9 @@
 import parser from "accept-language-parser";
 import type { NextApiRequest } from "next";
+import { i18n } from "src/../next-i18next.config";
 import prisma from "src/lib/prismadb";
 import type { BackendUserCore } from "src/types/Users";
 
-import { i18n } from "src/../next-i18next.config";
-
 const LOCALE_SET = new Set(i18n.locales);
 
 /**

From 101f2c536a3ebbfe744637fc9bace22768755eb3 Mon Sep 17 00:00:00 2001
From: notmd 
Date: Sun, 22 Jan 2023 14:20:39 +0700
Subject: [PATCH 50/59] revert change in user_repository

---
 backend/oasst_backend/user_repository.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/backend/oasst_backend/user_repository.py b/backend/oasst_backend/user_repository.py
index 118f4e82..79df99ab 100644
--- a/backend/oasst_backend/user_repository.py
+++ b/backend/oasst_backend/user_repository.py
@@ -233,15 +233,11 @@ class UserRepository:
 
         if lte_display_name is not None:
             if lt_id:
-                qry = (
-                    qry.filter(
-                        or_(
-                            User.display_name < lte_display_name,
-                            and_(User.display_name == lte_display_name, User.id < lt_id),
-                        )
+                qry = qry.filter(
+                    or_(
+                        User.display_name < lte_display_name,
+                        and_(User.display_name == lte_display_name, User.id < lt_id),
                     )
-                    .order_by(None)
-                    .order_by(User.display_name.desc(), User.id.desc())
                 )
             else:
                 qry = qry.filter(User.display_name <= lte_display_name)

From 3b5b6669a503114019f08ffe09c605851c90b304 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20K=C3=B6pf?= 
Date: Sun, 22 Jan 2023 08:42:38 +0100
Subject: [PATCH 51/59] move lt-desc order to users-cursor function

---
 backend/oasst_backend/api/v1/users.py    | 14 ++++++++++----
 backend/oasst_backend/user_repository.py | 24 ++++++++++++------------
 2 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/backend/oasst_backend/api/v1/users.py b/backend/oasst_backend/api/v1/users.py
index ed07eb21..c7ff9f9c 100644
--- a/backend/oasst_backend/api/v1/users.py
+++ b/backend/oasst_backend/api/v1/users.py
@@ -28,6 +28,7 @@ def get_users_ordered_by_username(
     search_text: Optional[str] = None,
     auth_method: Optional[str] = None,
     max_count: Optional[int] = Query(100, gt=0, le=10000),
+    desc: Optional[bool] = False,
     api_client: ApiClient = Depends(deps.get_api_client),
     db: Session = Depends(deps.get_db),
 ):
@@ -41,6 +42,7 @@ def get_users_ordered_by_username(
         auth_method=auth_method,
         search_text=search_text,
         limit=max_count,
+        desc=desc,
     )
     return [u.to_protocol_frontend_user() for u in users]
 
@@ -55,6 +57,7 @@ def get_users_ordered_by_display_name(
     auth_method: Optional[str] = None,
     search_text: Optional[str] = None,
     max_count: Optional[int] = Query(100, gt=0, le=10000),
+    desc: Optional[bool] = False,
     api_client: ApiClient = Depends(deps.get_api_client),
     db: Session = Depends(deps.get_db),
 ):
@@ -68,6 +71,7 @@ def get_users_ordered_by_display_name(
         auth_method=auth_method,
         search_text=search_text,
         limit=max_count,
+        desc=desc,
     )
     return [u.to_protocol_frontend_user() for u in users]
 
@@ -96,6 +100,7 @@ def get_users_cursor(
 
     items: list[protocol.FrontEndUser]
     qry_max_count = max_count + 1 if lt is None or gt is None else max_count
+    desc = lt and not gt
 
     def get_next_prev(num_rows: int, lt: str | None, gt: str | None, key_fn: Callable[[protocol.FrontEndUser], str]):
         p, n = None, None
@@ -115,10 +120,9 @@ def get_users_cursor(
         num_rows = len(items)
         if qry_max_count > max_count and num_rows == qry_max_count:
             assert not (lt and gt)
-            if lt:
-                items = items[1:]
-            else:
-                items = items[:-1]
+            items = items[:-1]
+        if desc:
+            items.reverse()
         return items, num_rows
 
     n, p = None, None
@@ -134,6 +138,7 @@ def get_users_cursor(
             auth_method=auth_method,
             search_text=search_text,
             max_count=qry_max_count,
+            desc=desc,
             api_client=api_client,
             db=db,
         )
@@ -152,6 +157,7 @@ def get_users_cursor(
             auth_method=auth_method,
             search_text=search_text,
             max_count=qry_max_count,
+            desc=desc,
             api_client=api_client,
             db=db,
         )
diff --git a/backend/oasst_backend/user_repository.py b/backend/oasst_backend/user_repository.py
index 44a7e685..b467bcaf 100644
--- a/backend/oasst_backend/user_repository.py
+++ b/backend/oasst_backend/user_repository.py
@@ -145,6 +145,7 @@ class UserRepository:
         auth_method: Optional[str] = None,
         search_text: Optional[str] = None,
         limit: Optional[int] = 100,
+        desc: bool = False,
     ) -> list[User]:
         if not self.api_client.trusted:
             if not api_client_id:
@@ -184,14 +185,13 @@ class UserRepository:
             pattern = "%{}%".format(search_text.replace("\\", "\\\\").replace("_", "\\_").replace("%", "\\%"))
             qry = qry.filter(User.username.like(pattern))
 
-        if limit is not None and lte_username and not gte_username:
-            # select top rows but return results in ascernding order
-            sub_qry = qry.order_by(User.username.desc(), User.id.desc()).limit(limit).subquery("u")
-            qry = self.db.query(User).select_entity_from(sub_qry).order_by(User.username, User.id)
+        if desc:
+            qry = qry.order_by(User.username.desc(), User.id.desc())
         else:
             qry = qry.order_by(User.username, User.id)
-            if limit is not None:
-                qry = qry.limit(limit)
+
+        if limit is not None:
+            qry = qry.limit(limit)
 
         return qry.all()
 
@@ -205,6 +205,7 @@ class UserRepository:
         auth_method: Optional[str] = None,
         search_text: Optional[str] = None,
         limit: Optional[int] = 100,
+        desc: bool = False,
     ) -> list[User]:
         if not self.api_client.trusted:
             if not api_client_id:
@@ -255,13 +256,12 @@ class UserRepository:
         if auth_method:
             qry = qry.filter(User.auth_method == auth_method)
 
-        if limit is not None and lte_display_name and not gte_display_name:
-            # select top rows but return results in ascernding order
-            sub_qry = qry.order_by(User.display_name.desc(), User.id.desc()).limit(limit).subquery("u")
-            qry = self.db.query(User).select_entity_from(sub_qry).order_by(User.display_name, User.id)
+        if desc:
+            qry = qry.order_by(User.display_name.desc(), User.id.desc())
         else:
             qry = qry.order_by(User.display_name, User.id)
-            if limit is not None:
-                qry = qry.limit(limit)
+
+        if limit is not None:
+            qry = qry.limit(limit)
 
         return qry.all()

From 28089d9ecf6766086298c4a1034f06046062b9cb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20K=C3=B6pf?= 
Date: Sun, 22 Jan 2023 09:29:21 +0100
Subject: [PATCH 52/59] fix username+auth combo check

---
 backend/oasst_backend/prompt_repository.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py
index abf5b721..8c259dda 100644
--- a/backend/oasst_backend/prompt_repository.py
+++ b/backend/oasst_backend/prompt_repository.py
@@ -693,7 +693,7 @@ class PromptRepository:
         if user_id:
             qry = qry.filter(Message.user_id == user_id)
         if username or auth_method:
-            if not username and auth_method:
+            if not (username and auth_method):
                 raise OasstError("Auth method or username missing.", OasstErrorCode.AUTH_AND_USERNAME_REQUIRED)
             qry = qry.join(User)
             qry = qry.filter(User.username == username, User.auth_method == auth_method)

From 952c61f4613a74e387c1819b56ea7a0fd12e9c44 Mon Sep 17 00:00:00 2001
From: notmd 
Date: Sun, 22 Jan 2023 15:50:23 +0700
Subject: [PATCH 53/59] Fix recent messages

---
 website/src/pages/api/messages/user.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/website/src/pages/api/messages/user.ts b/website/src/pages/api/messages/user.ts
index e5f361b8..bd651acc 100644
--- a/website/src/pages/api/messages/user.ts
+++ b/website/src/pages/api/messages/user.ts
@@ -4,6 +4,7 @@ const handler = withoutRole("banned", async (req, res, token) => {
   //TODO: add params if needed
   const params = new URLSearchParams({
     username: token.sub,
+    auth_method: "local",
   });
 
   const messagesRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/messages?${params}`, {

From 554e730d348f766e49af2e71760c303757919815 Mon Sep 17 00:00:00 2001
From: notmd 
Date: Sun, 22 Jan 2023 16:01:01 +0700
Subject: [PATCH 54/59] user `getBackendUserCore`

---
 website/src/pages/api/messages/user.ts | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/website/src/pages/api/messages/user.ts b/website/src/pages/api/messages/user.ts
index bd651acc..cbf658f1 100644
--- a/website/src/pages/api/messages/user.ts
+++ b/website/src/pages/api/messages/user.ts
@@ -1,10 +1,12 @@
 import { withoutRole } from "src/lib/auth";
+import { getBackendUserCore } from "src/lib/users";
 
 const handler = withoutRole("banned", async (req, res, token) => {
   //TODO: add params if needed
+  const user = await getBackendUserCore(token.sub);
   const params = new URLSearchParams({
     username: token.sub,
-    auth_method: "local",
+    auth_method: user.auth_method,
   });
 
   const messagesRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/messages?${params}`, {

From 0f0d0e00b5494775355035aa9c5cad7cd170d7d9 Mon Sep 17 00:00:00 2001
From: notmd 
Date: Sun, 22 Jan 2023 17:05:22 +0700
Subject: [PATCH 55/59] use `user.id`

---
 website/src/pages/api/messages/user.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/website/src/pages/api/messages/user.ts b/website/src/pages/api/messages/user.ts
index cbf658f1..6f39aad1 100644
--- a/website/src/pages/api/messages/user.ts
+++ b/website/src/pages/api/messages/user.ts
@@ -5,7 +5,7 @@ const handler = withoutRole("banned", async (req, res, token) => {
   //TODO: add params if needed
   const user = await getBackendUserCore(token.sub);
   const params = new URLSearchParams({
-    username: token.sub,
+    username: user.id,
     auth_method: user.auth_method,
   });
 

From c0391a6df9aff9bf83e2baa1b59d0d5478ad434b Mon Sep 17 00:00:00 2001
From: James Melvin Ebenezer 
Date: Sun, 22 Jan 2023 15:38:02 +0530
Subject: [PATCH 56/59] fix: redundant row updates with no Task id in
 text_labels table (#876)

* fix: redundant row updates with no Task id in text_labels table

* fix: review comments incorporated

* fix: better error handling and function name

* fix: review comments

Co-authored-by: James Melvin 
---
 backend/oasst_backend/api/v1/text_labels.py | 16 +++++++++++-----
 backend/oasst_backend/prompt_repository.py  | 15 +++++++++++++++
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/backend/oasst_backend/api/v1/text_labels.py b/backend/oasst_backend/api/v1/text_labels.py
index c9afd88c..dc6cc889 100644
--- a/backend/oasst_backend/api/v1/text_labels.py
+++ b/backend/oasst_backend/api/v1/text_labels.py
@@ -4,8 +4,9 @@ from loguru import logger
 from oasst_backend.api import deps
 from oasst_backend.prompt_repository import PromptRepository
 from oasst_backend.schemas.text_labels import LabelOption, ValidLabelsResponse
+from oasst_backend.utils.database_utils import CommitMode, managed_tx_function
+from oasst_shared.exceptions import OasstError
 from oasst_shared.schemas import protocol as protocol_schema
-from sqlmodel import Session
 from starlette.status import HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST
 
 router = APIRouter()
@@ -14,20 +15,25 @@ router = APIRouter()
 @router.post("/", status_code=HTTP_204_NO_CONTENT)
 def label_text(
     *,
-    db: Session = Depends(deps.get_db),
     api_key: APIKey = Depends(deps.get_api_key),
     text_labels: protocol_schema.TextLabels,
 ) -> None:
     """
     Label a piece of text.
     """
-    api_client = deps.api_auth(api_key, db)
+
+    @managed_tx_function(CommitMode.COMMIT)
+    def store_text_labels(session: deps.Session):
+        api_client = deps.api_auth(api_key, session)
+        pr = PromptRepository(session, api_client, client_user=text_labels.user)
+        pr.store_text_labels(text_labels)
 
     try:
         logger.info(f"Labeling text {text_labels=}.")
-        pr = PromptRepository(db, api_client, client_user=text_labels.user)
-        pr.store_text_labels(text_labels)
+        store_text_labels()
 
+    except OasstError:
+        raise
     except Exception:
         logger.exception("Failed to store label.")
         raise HTTPException(
diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py
index 8c259dda..0a0fa61d 100644
--- a/backend/oasst_backend/prompt_repository.py
+++ b/backend/oasst_backend/prompt_repository.py
@@ -448,6 +448,11 @@ class PromptRepository:
             if task:
                 message.review_count += 1
                 self.db.add(message)
+            # for the same User id with no task id associated with the message, then update existing record for repeated updates
+            existing_text_label = self.fetch_non_task_text_labels(message_id, self.user_id)
+            if existing_text_label is not None:
+                existing_text_label.labels = text_labels.labels
+                model = existing_text_label
 
         self.db.add(model)
         return model, task, message
@@ -561,6 +566,16 @@ class PromptRepository:
             raise OasstError("Message not found", OasstErrorCode.MESSAGE_NOT_FOUND, HTTP_404_NOT_FOUND)
         return message
 
+    def fetch_non_task_text_labels(self, message_id: UUID, user_id: UUID) -> Optional[TextLabels]:
+
+        query = (
+            self.db.query(TextLabels)
+            .outerjoin(Task, Task.id == TextLabels.id)
+            .filter(Task.id.is_(None), TextLabels.message_id == message_id, TextLabels.user_id == user_id)
+        )
+        text_label = query.one_or_none()
+        return text_label
+
     @staticmethod
     def trace_conversation(messages: list[Message] | dict[UUID, Message], last_message: Message) -> list[Message]:
         """

From 6167f63467e0bc98a644b52f9dd57090e1b55287 Mon Sep 17 00:00:00 2001
From: AbdBarho 
Date: Sun, 22 Jan 2023 11:53:34 +0100
Subject: [PATCH 57/59] Present available task types on dashboard

---
 .../src/components/Dashboard/TaskOption.tsx   | 77 +++++++++++--------
 website/src/components/Tasks/Task/Task.tsx    |  4 +-
 website/src/components/Tasks/TaskTypes.tsx    |  8 +-
 website/src/pages/dashboard.tsx               | 10 ++-
 4 files changed, 57 insertions(+), 42 deletions(-)

diff --git a/website/src/components/Dashboard/TaskOption.tsx b/website/src/components/Dashboard/TaskOption.tsx
index e2bafac3..5a759d40 100644
--- a/website/src/components/Dashboard/TaskOption.tsx
+++ b/website/src/components/Dashboard/TaskOption.tsx
@@ -1,48 +1,61 @@
 import { Box, Flex, GridItem, Heading, SimpleGrid, Text, useColorModeValue } from "@chakra-ui/react";
 import Link from "next/link";
+import { useMemo } from "react";
+import { TaskType } from "src/types/Task";
 
-import { TaskCategory, TaskCategoryLabels, TaskTypes } from "../Tasks/TaskTypes";
+import { TaskCategory, TaskCategoryLabels, TaskInfo, TaskInfos } from "../Tasks/TaskTypes";
 
-export const TaskOption = ({ displayTaskCategories }: { displayTaskCategories: TaskCategory[] }) => {
+export interface TasksOptionProps {
+  content: Partial>;
+}
+
+export const TaskOption = ({ content }: TasksOptionProps) => {
   const backgroundColor = useColorModeValue("white", "gray.700");
 
+  const taskInfoMap = useMemo(
+    () =>
+      Object.values(content)
+        .flat()
+        .reduce((obj, taskType) => {
+          obj[taskType] = TaskInfos.filter((t) => t.type === taskType).pop();
+          return obj;
+        }, {} as Record),
+    [content]
+  );
+
   return (
     
-      {displayTaskCategories.map((category) => (
+      {Object.entries(content).map(([category, taskTypes]) => (
         

-          {TaskCategoryLabels[category]}
+          
+            {TaskCategoryLabels[category]}
+          
           
-            {TaskTypes.filter((task) => task.category === category).map((item) => (
-              
-                
-                  
-                    
-                      
-                        {item.label}
-                      
-                      
-                        {item.desc}
-                      
-                    
-                  
-                   taskInfoMap[taskType])
+              .map((item) => (
+                
+                  
-                    
+                    
+                      {item.label}
+                      {item.desc}
+                    
+                    
                       Go ->
                     
-                  
-                
-              
-            ))}
+                  
+                
+              ))}
           
         

       ))}
diff --git a/website/src/components/Tasks/Task/Task.tsx b/website/src/components/Tasks/Task/Task.tsx
index 3d393575..45fb83d0 100644
--- a/website/src/components/Tasks/Task/Task.tsx
+++ b/website/src/components/Tasks/Task/Task.tsx
@@ -3,7 +3,7 @@ import { TaskControls } from "src/components/Survey/TaskControls";
 import { CreateTask } from "src/components/Tasks/CreateTask";
 import { EvaluateTask } from "src/components/Tasks/EvaluateTask";
 import { LabelTask } from "src/components/Tasks/LabelTask";
-import { TaskCategory, TaskInfo, TaskTypes } from "src/components/Tasks/TaskTypes";
+import { TaskCategory, TaskInfo, TaskInfos } from "src/components/Tasks/TaskTypes";
 import { UnchangedWarning } from "src/components/Tasks/UnchangedWarning";
 import { post } from "src/lib/api";
 import { TaskContent } from "src/types/Task";
@@ -29,7 +29,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
 
   const rootEl = useRef(null);
 
-  const taskType = TaskTypes.find((taskType) => taskType.type === task.type && taskType.mode === task.mode);
+  const taskType = TaskInfos.find((taskType) => taskType.type === task.type && taskType.mode === task.mode);
 
   const { trigger: sendRejection } = useSWRMutation("/api/reject_task", post, {
     onSuccess: async () => {
diff --git a/website/src/components/Tasks/TaskTypes.tsx b/website/src/components/Tasks/TaskTypes.tsx
index 4c6da92c..d10159d9 100644
--- a/website/src/components/Tasks/TaskTypes.tsx
+++ b/website/src/components/Tasks/TaskTypes.tsx
@@ -21,16 +21,16 @@ export interface TaskInfo {
 }
 
 export const TaskCategoryLabels: { [key in TaskCategory]: string } = {
-  [TaskCategory.Random]: "I'm feeling lucky",
+  [TaskCategory.Random]: "Grab a task!",
   [TaskCategory.Create]: "Create",
   [TaskCategory.Evaluate]: "Evaluate",
   [TaskCategory.Label]: "Label",
 };
 
-export const TaskTypes: TaskInfo[] = [
+export const TaskInfos: TaskInfo[] = [
   // general/random
   {
-    label: "Start a Task",
+    label: "I'm feeling lucky",
     desc: "Help us improve Open Assistant by starting a random task.",
     category: TaskCategory.Random,
     pathname: "/tasks/random",
@@ -104,7 +104,7 @@ export const TaskTypes: TaskInfo[] = [
     category: TaskCategory.Evaluate,
     pathname: "/evaluate/rank_initial_prompts",
     help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
-    overview: "Given the following inital prompts, sort them from best to worst, best being first, worst being last.",
+    overview: "Given the following initial prompts, sort them from best to worst, best being first, worst being last.",
     type: "rank_initial_prompts",
     update_type: "message_ranking",
     unchanged_title: "Order Unchanged",
diff --git a/website/src/pages/dashboard.tsx b/website/src/pages/dashboard.tsx
index e0b8bba4..4def6196 100644
--- a/website/src/pages/dashboard.tsx
+++ b/website/src/pages/dashboard.tsx
@@ -5,15 +5,17 @@ import { LeaderboardTable, TaskOption, WelcomeCard } from "src/components/Dashbo
 import { getDashboardLayout } from "src/components/Layout";
 import { TaskCategory } from "src/components/Tasks/TaskTypes";
 import { get } from "src/lib/api";
-import type { AvailableTasks, TaskType } from "src/types/Task";
+import { AvailableTasks, TaskType } from "src/types/Task";
 export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props";
 import useSWRImmutable from "swr/immutable";
 
 const Dashboard = () => {
   const { data } = useSWRImmutable("/api/available_tasks", get);
 
-  // TODO: show only these tasks:
-  const availableTasks = useMemo(() => filterAvailableTasks(data ?? {}), [data]);
+  const availableTaskTypes = useMemo(() => {
+    const taskTypes = filterAvailableTasks(data ?? {});
+    return { [TaskCategory.Random]: taskTypes };
+  }, [data]);
 
   return (
     <>
@@ -23,7 +25,7 @@ const Dashboard = () => {
       
       
         
-        
+        
         
       
     

From fd703663bd1bea984a09869da6124d4dd49641a2 Mon Sep 17 00:00:00 2001
From: AbdBarho 
Date: Sun, 22 Jan 2023 12:00:55 +0100
Subject: [PATCH 58/59] Fix all tasks page

---
 website/src/components/Dashboard/TaskOption.tsx | 11 +++++++++++
 website/src/pages/tasks/all.tsx                 |  4 ++--
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/website/src/components/Dashboard/TaskOption.tsx b/website/src/components/Dashboard/TaskOption.tsx
index 5a759d40..e73a06c8 100644
--- a/website/src/components/Dashboard/TaskOption.tsx
+++ b/website/src/components/Dashboard/TaskOption.tsx
@@ -62,3 +62,14 @@ export const TaskOption = ({ content }: TasksOptionProps) => {
     
   );
 };
+
+export const allTaskOptions: TasksOptionProps["content"] = {
+  [TaskCategory.Random]: [TaskType.random],
+  [TaskCategory.Create]: [TaskType.initial_prompt, TaskType.prompter_reply, TaskType.assistant_reply],
+  [TaskCategory.Evaluate]: [
+    TaskType.rank_initial_prompts,
+    TaskType.rank_prompter_replies,
+    TaskType.rank_assistant_replies,
+  ],
+  [TaskCategory.Label]: [TaskType.label_initial_prompt, TaskType.label_prompter_reply, TaskType.label_assistant_reply],
+};
diff --git a/website/src/pages/tasks/all.tsx b/website/src/pages/tasks/all.tsx
index 3ccfd4e8..01954c2f 100644
--- a/website/src/pages/tasks/all.tsx
+++ b/website/src/pages/tasks/all.tsx
@@ -1,7 +1,7 @@
 import Head from "next/head";
 import { TaskOption } from "src/components/Dashboard";
+import { allTaskOptions } from "src/components/Dashboard/TaskOption";
 import { getDashboardLayout } from "src/components/Layout";
-import { TaskCategory } from "src/components/Tasks/TaskTypes";
 export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props";
 
 const AllTasks = () => {
@@ -11,7 +11,7 @@ const AllTasks = () => {
         All Tasks - Open Assistant
         
       
-      
+      
     
   );
 };

From fb4e94487cd407c941096400e10c4704c88a6913 Mon Sep 17 00:00:00 2001
From: AbdBarho 
Date: Sun, 22 Jan 2023 12:15:21 +0100
Subject: [PATCH 59/59] Redirect users to dashboard if there are no tasks

---
 website/src/components/EmptyState.tsx | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/website/src/components/EmptyState.tsx b/website/src/components/EmptyState.tsx
index 14715518..51e51a00 100644
--- a/website/src/components/EmptyState.tsx
+++ b/website/src/components/EmptyState.tsx
@@ -1,5 +1,5 @@
-import { Box, Link, Text, useColorModeValue } from "@chakra-ui/react";
-import { useRouter } from "next/router";
+import { Box, Text, useColorModeValue } from "@chakra-ui/react";
+import NextLink from "next/link";
 import { FiAlertTriangle } from "react-icons/fi";
 import { IconType } from "react-icons/lib";
 
@@ -10,16 +10,15 @@ type EmptyStateProps = {
 
 export const EmptyState = (props: EmptyStateProps) => {
   const backgroundColor = useColorModeValue("white", "gray.800");
-  const router = useRouter();
 
   return (
     
       
         
         {props.text}
-         router.back()} color="blue.500" textUnderlineOffset="3px">
-          Click here to go back
-        
+        
+          Go back to the dashboard
+        
       
     
   );