diff --git a/.github/workflows/build-frontend.yaml b/.github/workflows/build-frontend.yaml index 9fb2e8cf..ccb64539 100644 --- a/.github/workflows/build-frontend.yaml +++ b/.github/workflows/build-frontend.yaml @@ -12,7 +12,7 @@ on: workflow_call: jobs: - build-frontend: + build-frontend: runs-on: ubuntu-latest defaults: run: @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-node@v3 with: node-version: 16.x - cache: 'npm' + cache: "npm" cache-dependency-path: website/package-lock.json - run: npm ci - run: npm run build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 271c11c6..6d885cdc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,31 @@ +# WARNING! +# +# When making changes to auto-formatters used in pre-commit hooks, you are +# likely to cause merge conflicts with main and/or other pull requests. +# Fixing them might revert other people's work. Expect pain! +# To avoid accidental reversions and keep it easy to review, please make sure +# that changes here are in a pull request by themselves, that it consists of +# two commits: +# +# 1. The changes to this file +# 2. Changes made by running `python3 -m pre_commit run --all-files`. +# +# Then each time your pull request is blocked by a merge conflict, do the +# following steps: +# +# git reset HEAD^1 && git checkout -f # discard the change commit +# git rebase main # re-apply other people's changes +# python3 -m pre_commit run --all-files # re-run the rules +# git add . # add the newly changed files +# git commit -m 'apply pre-commit' # commit it +# git push -f # force push back to your branch +# +# Keep in mind you may have to do this a few times, as changes here may impact +# other pull requests. Try to keep it up-to-date so they can go in when it'll +# cause least disruption. +# +# /WARNING! + exclude: "build|stubs|^bot/templates/|^notebooks/.*\\.ipynb$" default_language_version: @@ -19,6 +47,7 @@ repos: - id: check-case-conflict - id: detect-private-key - id: fix-encoding-pragma + args: ["--remove"] - id: forbid-submodules - id: mixed-line-ending - id: requirements-txt-fixer @@ -50,7 +79,7 @@ repos: rev: v2.7.1 hooks: - id: prettier - args: ["--write"] + args: ["--prose-wrap=always", "--write"] - repo: local hooks: diff --git a/README.md b/README.md index e0ea65f1..61a00839 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,17 @@ # Open-Assistant -Open Assistant is a project meant to give everyone access to a great chat based large language model. +Open Assistant is a project meant to give everyone access to a great chat based +large language model. -We believe that by doing this we will create a revolution in innovation in language. In the same way that stable-diffusion helped the world make art and images in new ways we hope Open Assistant can help improve the world by improving language itself. +We believe that by doing this we will create a revolution in innovation in +language. In the same way that stable-diffusion helped the world make art and +images in new ways we hope Open Assistant can help improve the world by +improving language itself. ## Do you want to try it out? -If you are interested in taking a look at the current state of the project, You can set up an entire stack needed to run **Open-Assistant**, including the +If you are interested in taking a look at the current state of the project, You +can set up an entire stack needed to run **Open-Assistant**, including the website, backend, and associated dependent services. To start the demo, Run this in the root directory of the repository: @@ -15,23 +20,44 @@ To start the demo, Run this in the root directory of the repository: docker compose up --build ``` -Then, navigate to `http://localhost:3000` (It may take some time to boot up) and interact with the website. +Then, navigate to `http://localhost:3000` (It may take some time to boot up) and +interact with the website. -**Note:** When logging in via email, navigate to `http://localhost:1080` to get the magic email login link. +**Note:** When logging in via email, navigate to `http://localhost:1080` to get +the magic email login link. ## The Plan -We want to get to an initial MVP as fast as possible, by following the 3-steps outlined in the InstructGPT paper. +We want to get to an initial MVP as fast as possible, by following the 3-steps +outlined in the InstructGPT paper. -1. Collect high-quality human generated Instruction-Fulfillment samples (prompt + response), goal >50k. We design a crowdsourced process to collect and reviewed prompts. We do not want to train on flooding/toxic/spam/junk/personal information data. We will have a leaderboard to motivate the community that shows progress and the most active users. Swag will be given to the top-contributors. -2. For each of the collected prompts we will sample multiple completions. Completions of one prompt will then be shown randomly to users to rank them from best to worst. Again this should happen crowd-sourced, e.g. we need to deal with unreliable potentially malicious users. At least multiple votes by independent users have to be collected to measure the overall agreement. The gathered ranking-data will be used to train a reward model. -3. Now follows the RLHF training phase based on the prompts and the reward model. +1. Collect high-quality human generated Instruction-Fulfillment samples + (prompt + response), goal >50k. We design a crowdsourced process to collect + and reviewed prompts. We do not want to train on + flooding/toxic/spam/junk/personal information data. We will have a + leaderboard to motivate the community that shows progress and the most active + users. Swag will be given to the top-contributors. +2. For each of the collected prompts we will sample multiple completions. + Completions of one prompt will then be shown randomly to users to rank them + from best to worst. Again this should happen crowd-sourced, e.g. we need to + deal with unreliable potentially malicious users. At least multiple votes by + independent users have to be collected to measure the overall agreement. The + gathered ranking-data will be used to train a reward model. +3. Now follows the RLHF training phase based on the prompts and the reward + model. -We can then take the resulting model and continue with completion sampling step 2 for a next iteration. +We can then take the resulting model and continue with completion sampling step +2 for a next iteration. ## The Vision -We are not going to stop at replicating ChatGPT. We want to build the assistant of the future, able to not only write email and cover letters, but do meaningful work, use APIs, dynamically research information, and much more, with the ability to be personalized and extended by anyone. And we want to do this in a way that is open and accessible, which means we must not only build a great assistant, but also make it small and efficient enough to run on consumer hardware. +We are not going to stop at replicating ChatGPT. We want to build the assistant +of the future, able to not only write email and cover letters, but do meaningful +work, use APIs, dynamically research information, and much more, with the +ability to be personalized and extended by anyone. And we want to do this in a +way that is open and accessible, which means we must not only build a great +assistant, but also make it small and efficient enough to run on consumer +hardware. ### Slide Decks @@ -41,15 +67,20 @@ We are not going to stop at replicating ChatGPT. We want to build the assistant ## How can you help? -All open source projects begins with people like you. Open source is the belief that if we collaborate we can together gift our knowledge and technology to the world for the benefit of humanity. +All open source projects begins with people like you. Open source is the belief +that if we collaborate we can together gift our knowledge and technology to the +world for the benefit of humanity. ## I’m in! Now what? -[Join the OpenAssistant Contributors Discord Server!](https://ykilcher.com/open-assistant-discord), this is for work coordination. +[Join the OpenAssistant Contributors Discord Server!](https://ykilcher.com/open-assistant-discord), +this is for work coordination. -[Join the LAION Discord Server!](https://discord.com/invite/mVcgxMPD7e), it has a dedicated channel and is more public. +[Join the LAION Discord Server!](https://discord.com/invite/mVcgxMPD7e), it has +a dedicated channel and is more public. -[and / or the YK Discord Server](https://ykilcher.com/discord), also has a dedicated, but not as active, channel. +[and / or the YK Discord Server](https://ykilcher.com/discord), also has a +dedicated, but not as active, channel. [Visit the Notion](https://ykilcher.com/open-assistant) @@ -57,15 +88,16 @@ All open source projects begins with people like you. Open source is the belief We have a growing task list [of issues](https://github.com/LAION-AI/Open-Assistant/issues). Find an issue -that appeals to you and make a comment that you'd like to work on it. Include -in your comment a brief description of how you'll solve the problem and if -there are any open questions you want to discuss. Once a project coordinator -has assigned the issue to you, start working on it. +that appeals to you and make a comment that you'd like to work on it. Include in +your comment a brief description of how you'll solve the problem and if there +are any open questions you want to discuss. Once a project coordinator has +assigned the issue to you, start working on it. -If the issue is currently unclear but you are interested, please post in -Discord and someone can help clarify the issue with more detail. +If the issue is currently unclear but you are interested, please post in Discord +and someone can help clarify the issue with more detail. -**Always Welcome:** Documentation markdowns in `docs/`, docstrings, diagrams of the system architecture, and other documentation. +**Always Welcome:** Documentation markdowns in `docs/`, docstrings, diagrams of +the system architecture, and other documentation. ### Submitting Work @@ -73,8 +105,8 @@ We're all working on different parts of Open Assistant together. To make contributions smoothly we recommend the following: 1. [Fork this project repository](https://docs.github.com/en/get-started/quickstart/fork-a-repo) - and clone it to your local machine. - (Read more [About Forks](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks)) + and clone it to your local machine. (Read more + [About Forks](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/about-forks)) 1. Before working on any changes, try to [sync the forked repository](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork) to keep it up-to-date with the upstream repository. @@ -83,7 +115,8 @@ contributions smoothly we recommend the following: simplifies life for reviewers. 1. Package up a small bit of work that solves part of the problem [into a Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork) - and [send it out for review](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review). + and + [send it out for review](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review). 1. If you're lucky, we can merge your change into `main` without any problems. If there's changes to files you're working on, resolve them by: 1. First try rebase as suggested @@ -108,20 +141,27 @@ addressed now, or filing an issue to handle it later. ## Developer Setup -Work is organized in the [project board](https://github.com/orgs/LAION-AI/projects/3). +Work is organized in the +[project board](https://github.com/orgs/LAION-AI/projects/3). -**Anything that is in the `Todo` column and not assigned, is up for grabs. Meaning we'd be happy for anyone to do these tasks.** +**Anything that is in the `Todo` column and not assigned, is up for grabs. +Meaning we'd be happy for anyone to do these tasks.** -If you want to work on something, assign yourself to it or write a comment that you want to work on it and what you plan to do. +If you want to work on something, assign yourself to it or write a comment that +you want to work on it and what you plan to do. -- To get started with development, if you want to work on the backend, have a look at `scripts/backend-development/README.md`. -- If you want to work on any frontend, have a look at `scripts/frontend-development/README.md` to make a backend available. +- To get started with development, if you want to work on the backend, have a + look at `scripts/backend-development/README.md`. +- If you want to work on any frontend, have a look at + `scripts/frontend-development/README.md` to make a backend available. -There is also a minimal implementation of a frontend in the `text-frontend` folder. +There is also a minimal implementation of a frontend in the `text-frontend` +folder. We are using Python 3.10 for the backend. -Check out the [High-Level Protocol Architecture](https://www.notion.so/High-Level-Protocol-Architecture-6f1fd3551da74213b560ead369f132dc) +Check out the +[High-Level Protocol Architecture](https://www.notion.so/High-Level-Protocol-Architecture-6f1fd3551da74213b560ead369f132dc) ### Website @@ -129,16 +169,15 @@ The website is built using Next.js and is in the `website` folder. ### Pre-commit -Install `pre-commit` and run `pre-commit install` to install the pre-commit hooks. +Install `pre-commit` and run `pre-commit install` to install the pre-commit +hooks. -In case you haven't done this, have already committed, and CI is failing, you can run `pre-commit run --all-files` to run the pre-commit hooks on all files. +In case you haven't done this, have already committed, and CI is failing, you +can run `pre-commit run --all-files` to run the pre-commit hooks on all files. ### Deployment -Upon making a release on GitHub, all docker images are automatically built and pushed to ghcr.io. The docker images are tagged with the release version, and the `latest` tag. Further, the ansible playbook in `ansible/dev.yaml` is run to automatically deploy the built release to the dev machine. - -### Problems and Solutions - -- **I am on Ubuntu and getting `ERROR: The Compose file is invalid because:Service backend has neither an image nor a build context specified. At least one must be provided.`** - - Make sure you have an up-to-date version of docker installed, and also install `docker-compose-plugin`. See [here](https://github.com/LAION-AI/Open-Assistant/issues/208) for more details. +Upon making a release on GitHub, all docker images are automatically built and +pushed to ghcr.io. The docker images are tagged with the release version, and +the `latest` tag. Further, the ansible playbook in `ansible/dev.yaml` is run to +automatically deploy the built release to the dev machine. diff --git a/backend/README.md b/backend/README.md index 1e41e72c..45d16d68 100644 --- a/backend/README.md +++ b/backend/README.md @@ -2,7 +2,9 @@ ## REST Server Configuration -Please either use environment variables or create a `.env` file in the backend root directory (in which this readme file is located) to specify the `DATABASE_URI`. +Please either use environment variables or create a `.env` file in the backend +root directory (in which this readme file is located) to specify the +`DATABASE_URI`. Example contents of a `.env` file for the backend: @@ -14,4 +16,5 @@ BACKEND_CORS_ORIGINS=["http://localhost", "http://localhost:4200", "http://local ## Running the REST Server locally for development -Have a look into the main `README.md` file for more information on how to set up the backend for development. +Have a look into the main `README.md` file for more information on how to set up +the backend for development. diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 83de474c..511ed97f 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from logging.config import fileConfig import sqlmodel diff --git a/backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py b/backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py index 87709d31..8e4292ce 100644 --- a/backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py +++ b/backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """first revision Revision ID: 23e5fea252dd diff --git a/backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py b/backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py index 67488e4b..3ddbe558 100644 --- a/backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py +++ b/backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """v1 db structure Revision ID: cd7de470586e diff --git a/backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py b/backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py index d93afeba..2d0f25f2 100644 --- a/backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py +++ b/backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """add auth_method to person Revision ID: 6368515778c5 diff --git a/backend/alembic/versions/2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py b/backend/alembic/versions/2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py index c65b8319..08dec6a3 100644 --- a/backend/alembic/versions/2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py +++ b/backend/alembic/versions/2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """add_auth_method_to_ix_person_username Revision ID: 0daec5f8135f diff --git a/backend/alembic/versions/2022_12_25_1705-067c4002f2d9_add_text_labels.py b/backend/alembic/versions/2022_12_25_1705-067c4002f2d9_add_text_labels.py index 94e1c514..447eb424 100644 --- a/backend/alembic/versions/2022_12_25_1705-067c4002f2d9_add_text_labels.py +++ b/backend/alembic/versions/2022_12_25_1705-067c4002f2d9_add_text_labels.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Adds text labels table. Revision ID: 067c4002f2d9 diff --git a/backend/alembic/versions/2022_12_27_1444-3358eb6834e6_add_journal_table.py b/backend/alembic/versions/2022_12_27_1444-3358eb6834e6_add_journal_table.py index 0dc937a0..3fe72fa5 100644 --- a/backend/alembic/versions/2022_12_27_1444-3358eb6834e6_add_journal_table.py +++ b/backend/alembic/versions/2022_12_27_1444-3358eb6834e6_add_journal_table.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """add_journal_table Revision ID: 3358eb6834e6 diff --git a/backend/alembic/versions/2022_12_28_1142-d24b37426857_post_ref_for_work_package.py b/backend/alembic/versions/2022_12_28_1142-d24b37426857_post_ref_for_work_package.py index 675e6898..b9102864 100644 --- a/backend/alembic/versions/2022_12_28_1142-d24b37426857_post_ref_for_work_package.py +++ b/backend/alembic/versions/2022_12_28_1142-d24b37426857_post_ref_for_work_package.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """post ref for work_package Revision ID: d24b37426857 diff --git a/backend/alembic/versions/2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py b/backend/alembic/versions/2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py index 66ff2692..eba2f6a6 100644 --- a/backend/alembic/versions/2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py +++ b/backend/alembic/versions/2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Added lang column for ISO-639-1 codes Revision ID: ef0b52902560 diff --git a/backend/alembic/versions/2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py b/backend/alembic/versions/2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py index 42b8ccf8..2ac700ec 100644 --- a/backend/alembic/versions/2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py +++ b/backend/alembic/versions/2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """add collective flag to task Revision ID: 464ec4667aae diff --git a/backend/alembic/versions/2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py b/backend/alembic/versions/2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py index 303ca3fc..4f04cb06 100644 --- a/backend/alembic/versions/2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py +++ b/backend/alembic/versions/2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """add field trusted api client Revision ID: 73ce3675c1f5 diff --git a/backend/alembic/versions/2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py b/backend/alembic/versions/2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py index 3459cce8..7aa825ef 100644 --- a/backend/alembic/versions/2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py +++ b/backend/alembic/versions/2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """name changes: person->user, post->message, work_package->task Revision ID: abb47e9d145a diff --git a/backend/alembic/versions/2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py b/backend/alembic/versions/2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py index 786471db..3331142c 100644 --- a/backend/alembic/versions/2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py +++ b/backend/alembic/versions/2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """add deleted field to post Revision ID: 8d269bc4fdbd diff --git a/backend/main.py b/backend/main.py index 9cf43701..d9c35095 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from http import HTTPStatus from pathlib import Path from typing import Optional diff --git a/backend/oasst_backend/api/deps.py b/backend/oasst_backend/api/deps.py index e0286ba3..fe26f0a6 100644 --- a/backend/oasst_backend/api/deps.py +++ b/backend/oasst_backend/api/deps.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from http import HTTPStatus from secrets import token_hex from typing import Generator diff --git a/backend/oasst_backend/api/v1/api.py b/backend/oasst_backend/api/v1/api.py index 2286a1ac..94975909 100644 --- a/backend/oasst_backend/api/v1/api.py +++ b/backend/oasst_backend/api/v1/api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from fastapi import APIRouter from oasst_backend.api.v1 import frontend_messages, frontend_users, messages, stats, tasks, text_labels, users diff --git a/backend/oasst_backend/api/v1/frontend_messages.py b/backend/oasst_backend/api/v1/frontend_messages.py index 6ee27aa1..0c8b81f6 100644 --- a/backend/oasst_backend/api/v1/frontend_messages.py +++ b/backend/oasst_backend/api/v1/frontend_messages.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from fastapi import APIRouter, Depends from oasst_backend.api import deps from oasst_backend.api.v1 import utils diff --git a/backend/oasst_backend/api/v1/frontend_users.py b/backend/oasst_backend/api/v1/frontend_users.py index 940c7bb3..738e3cb0 100644 --- a/backend/oasst_backend/api/v1/frontend_users.py +++ b/backend/oasst_backend/api/v1/frontend_users.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import datetime from uuid import UUID diff --git a/backend/oasst_backend/api/v1/messages.py b/backend/oasst_backend/api/v1/messages.py index 71e4e3eb..a70b935e 100644 --- a/backend/oasst_backend/api/v1/messages.py +++ b/backend/oasst_backend/api/v1/messages.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - import datetime from uuid import UUID diff --git a/backend/oasst_backend/api/v1/stats.py b/backend/oasst_backend/api/v1/stats.py index 831d4df2..0f275b7d 100644 --- a/backend/oasst_backend/api/v1/stats.py +++ b/backend/oasst_backend/api/v1/stats.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from fastapi import APIRouter, Depends from oasst_backend.api import deps from oasst_backend.models import ApiClient diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py index 636f0feb..aaa4a8c1 100644 --- a/backend/oasst_backend/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import random from typing import Any, Optional, Tuple from uuid import UUID diff --git a/backend/oasst_backend/api/v1/text_labels.py b/backend/oasst_backend/api/v1/text_labels.py index 09933304..856aeea5 100644 --- a/backend/oasst_backend/api/v1/text_labels.py +++ b/backend/oasst_backend/api/v1/text_labels.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import pydantic from fastapi import APIRouter, Depends, HTTPException from fastapi.security.api_key import APIKey diff --git a/backend/oasst_backend/api/v1/users.py b/backend/oasst_backend/api/v1/users.py index 0bac4d6a..16ab3133 100644 --- a/backend/oasst_backend/api/v1/users.py +++ b/backend/oasst_backend/api/v1/users.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import datetime from uuid import UUID diff --git a/backend/oasst_backend/api/v1/utils.py b/backend/oasst_backend/api/v1/utils.py index 0fa452bb..c321be59 100644 --- a/backend/oasst_backend/api/v1/utils.py +++ b/backend/oasst_backend/api/v1/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - from http import HTTPStatus from uuid import UUID diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py index 602780be..c675c148 100644 --- a/backend/oasst_backend/config.py +++ b/backend/oasst_backend/config.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from typing import Any, Dict, List, Optional, Union from pydantic import AnyHttpUrl, BaseSettings, PostgresDsn, validator diff --git a/backend/oasst_backend/crud/__init__.py b/backend/oasst_backend/crud/__init__.py index 5ee00d4a..a9a2c5b3 100644 --- a/backend/oasst_backend/crud/__init__.py +++ b/backend/oasst_backend/crud/__init__.py @@ -1,2 +1 @@ -# -*- coding: utf-8 -*- __all__ = [] diff --git a/backend/oasst_backend/crud/base.py b/backend/oasst_backend/crud/base.py index d863c4bc..432d029d 100644 --- a/backend/oasst_backend/crud/base.py +++ b/backend/oasst_backend/crud/base.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, Union from fastapi.encoders import jsonable_encoder diff --git a/backend/oasst_backend/database.py b/backend/oasst_backend/database.py index 38e5105c..987aee76 100644 --- a/backend/oasst_backend/database.py +++ b/backend/oasst_backend/database.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from oasst_backend.config import settings from oasst_backend.exceptions import OasstError, OasstErrorCode from sqlmodel import create_engine diff --git a/backend/oasst_backend/exceptions.py b/backend/oasst_backend/exceptions.py index f431b05b..b6fb2d7f 100644 --- a/backend/oasst_backend/exceptions.py +++ b/backend/oasst_backend/exceptions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from enum import IntEnum from http import HTTPStatus diff --git a/backend/oasst_backend/journal_writer.py b/backend/oasst_backend/journal_writer.py index 415d5a47..67892ded 100644 --- a/backend/oasst_backend/journal_writer.py +++ b/backend/oasst_backend/journal_writer.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import enum from typing import Literal, Optional from uuid import UUID diff --git a/backend/oasst_backend/models/__init__.py b/backend/oasst_backend/models/__init__.py index a942f60f..5818dbef 100644 --- a/backend/oasst_backend/models/__init__.py +++ b/backend/oasst_backend/models/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from .api_client import ApiClient from .journal import Journal, JournalIntegration from .message import Message diff --git a/backend/oasst_backend/models/api_client.py b/backend/oasst_backend/models/api_client.py index e8d722d5..0bebec47 100644 --- a/backend/oasst_backend/models/api_client.py +++ b/backend/oasst_backend/models/api_client.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from typing import Optional from uuid import UUID, uuid4 diff --git a/backend/oasst_backend/models/db_payload.py b/backend/oasst_backend/models/db_payload.py index 62dffa51..9a6fabb6 100644 --- a/backend/oasst_backend/models/db_payload.py +++ b/backend/oasst_backend/models/db_payload.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from typing import Literal from oasst_backend.models.payload_column_type import payload_type diff --git a/backend/oasst_backend/models/journal.py b/backend/oasst_backend/models/journal.py index 0f64433a..0d5a78af 100644 --- a/backend/oasst_backend/models/journal.py +++ b/backend/oasst_backend/models/journal.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from datetime import datetime from typing import Optional from uuid import UUID, uuid1, uuid4 diff --git a/backend/oasst_backend/models/message.py b/backend/oasst_backend/models/message.py index 47512cc7..f07ca881 100644 --- a/backend/oasst_backend/models/message.py +++ b/backend/oasst_backend/models/message.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from datetime import datetime from typing import Optional from uuid import UUID, uuid4 diff --git a/backend/oasst_backend/models/message_reaction.py b/backend/oasst_backend/models/message_reaction.py index 9c93961f..3aaa774c 100644 --- a/backend/oasst_backend/models/message_reaction.py +++ b/backend/oasst_backend/models/message_reaction.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from datetime import datetime from typing import Optional from uuid import UUID diff --git a/backend/oasst_backend/models/payload_column_type.py b/backend/oasst_backend/models/payload_column_type.py index fbda51ce..01b642e2 100644 --- a/backend/oasst_backend/models/payload_column_type.py +++ b/backend/oasst_backend/models/payload_column_type.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import json from typing import Any, Generic, Type, TypeVar diff --git a/backend/oasst_backend/models/task.py b/backend/oasst_backend/models/task.py index 853a5aaa..356eafea 100644 --- a/backend/oasst_backend/models/task.py +++ b/backend/oasst_backend/models/task.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from datetime import datetime from typing import Optional from uuid import UUID, uuid4 diff --git a/backend/oasst_backend/models/text_labels.py b/backend/oasst_backend/models/text_labels.py index b7ff08cf..ec10dca6 100644 --- a/backend/oasst_backend/models/text_labels.py +++ b/backend/oasst_backend/models/text_labels.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from datetime import datetime from typing import Optional from uuid import UUID, uuid4 diff --git a/backend/oasst_backend/models/user.py b/backend/oasst_backend/models/user.py index ec5efa66..1a06a524 100644 --- a/backend/oasst_backend/models/user.py +++ b/backend/oasst_backend/models/user.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from datetime import datetime from typing import Optional from uuid import UUID, uuid4 diff --git a/backend/oasst_backend/models/user_stats.py b/backend/oasst_backend/models/user_stats.py index a92775b9..b7b3231a 100644 --- a/backend/oasst_backend/models/user_stats.py +++ b/backend/oasst_backend/models/user_stats.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from datetime import datetime from typing import Optional from uuid import UUID diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py index 8cc770c5..8190d637 100644 --- a/backend/oasst_backend/prompt_repository.py +++ b/backend/oasst_backend/prompt_repository.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import datetime import random from collections import defaultdict diff --git a/copilot/README.md b/copilot/README.md index 406490fa..16d4dec8 100644 --- a/copilot/README.md +++ b/copilot/README.md @@ -16,8 +16,8 @@ Setup requires a few steps: copilot app init --domain your_domain.com ``` -This will initialize and register a variety of URLs with your -`your_domain.com`. Replace with a proper domain to setup SSL certificates. +This will initialize and register a variety of URLs with your `your_domain.com`. +Replace with a proper domain to setup SSL certificates. ```sh copilot env deploy @@ -29,10 +29,10 @@ This will create a variety of aws roles and services needed for deployment. copilot deploy ``` -This will depoy the services but it won't be 100% ready for usage. Before -being ready, we have to inspect the AWS Secrets manager and extract out the -database credentials. Read those credentials then put them, and a few other -secrets, in a `secrets.yml` file like the following: +This will depoy the services but it won't be 100% ready for usage. Before being +ready, we have to inspect the AWS Secrets manager and extract out the database +credentials. Read those credentials then put them, and a few other secrets, in a +`secrets.yml` file like the following: ```yaml DATABASE_URL: diff --git a/copilot/web/addons/web-cluster.yml b/copilot/web/addons/web-cluster.yml index 783cdec1..c7a337bf 100644 --- a/copilot/web/addons/web-cluster.yml +++ b/copilot/web/addons/web-cluster.yml @@ -4,14 +4,17 @@ Parameters: Description: Your application's name. Env: Type: String - Description: The environment name your service, job, or workflow is being deployed to. + Description: + The environment name your service, job, or workflow is being deployed to. Name: Type: String Description: The name of the service, job, or workflow being deployed. # Customize your Aurora Serverless cluster by setting the default value of the following parameters. webclusterDBName: Type: String - Description: The name of the initial database to be created in the Aurora Serverless v2 cluster. + Description: + The name of the initial database to be created in the Aurora Serverless v2 + cluster. Default: oassist_web # Cannot have special characters # Naming constraints: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints @@ -29,15 +32,20 @@ Resources: webclusterDBSubnetGroup: Type: "AWS::RDS::DBSubnetGroup" Properties: - DBSubnetGroupDescription: Group of Copilot private subnets for Aurora Serverless v2 cluster. + DBSubnetGroupDescription: + Group of Copilot private subnets for Aurora Serverless v2 cluster. SubnetIds: !Split [",", { "Fn::ImportValue": !Sub "${App}-${Env}-PrivateSubnets" }] webclusterSecurityGroup: Metadata: - "aws:copilot:description": "A security group for your workload to access the Aurora Serverless v2 cluster webcluster" + "aws:copilot:description": + "A security group for your workload to access the Aurora Serverless v2 + cluster webcluster" Type: "AWS::EC2::SecurityGroup" Properties: - GroupDescription: !Sub "The Security Group for ${Name} to access Aurora Serverless v2 cluster webcluster." + GroupDescription: + !Sub "The Security Group for ${Name} to access Aurora Serverless v2 + cluster webcluster." VpcId: Fn::ImportValue: !Sub "${App}-${Env}-VpcId" Tags: @@ -45,7 +53,8 @@ Resources: Value: !Sub "copilot-${App}-${Env}-${Name}-Aurora" webclusterDBClusterSecurityGroup: Metadata: - "aws:copilot:description": "A security group for your Aurora Serverless v2 cluster webcluster" + "aws:copilot:description": + "A security group for your Aurora Serverless v2 cluster webcluster" Type: AWS::EC2::SecurityGroup Properties: GroupDescription: The Security Group for the Aurora Serverless v2 cluster. @@ -53,13 +62,15 @@ Resources: - ToPort: 5432 FromPort: 5432 IpProtocol: tcp - Description: !Sub "From the Aurora Security Group of the workload ${Name}." + Description: + !Sub "From the Aurora Security Group of the workload ${Name}." SourceSecurityGroupId: !Ref webclusterSecurityGroup VpcId: Fn::ImportValue: !Sub "${App}-${Env}-VpcId" webclusterAuroraSecret: Metadata: - "aws:copilot:description": "A Secrets Manager secret to store your DB credentials" + "aws:copilot:description": + "A Secrets Manager secret to store your DB credentials" Type: AWS::SecretsManager::Secret Properties: Description: !Sub Aurora main user secret for ${AWS::StackName} @@ -71,7 +82,8 @@ Resources: PasswordLength: 16 webclusterDBClusterParameterGroup: Metadata: - "aws:copilot:description": "A DB parameter group for engine configuration values" + "aws:copilot:description": + "A DB parameter group for engine configuration values" Type: "AWS::RDS::DBClusterParameterGroup" Properties: Description: !Ref "AWS::StackName" @@ -80,7 +92,8 @@ Resources: client_encoding: "UTF8" webclusterDBCluster: Metadata: - "aws:copilot:description": "The webcluster Aurora Serverless v2 database cluster" + "aws:copilot:description": + "The webcluster Aurora Serverless v2 database cluster" Type: "AWS::RDS::DBCluster" Properties: MasterUsername: @@ -117,7 +130,8 @@ Resources: !FindInMap [webclusterEnvScalingConfigurationMap, All, DBMaxCapacity] webclusterDBWriterInstance: Metadata: - "aws:copilot:description": "The webcluster Aurora Serverless v2 writer instance" + "aws:copilot:description": + "The webcluster Aurora Serverless v2 writer instance" Type: "AWS::RDS::DBInstance" Properties: DBClusterIdentifier: !Ref webclusterDBCluster @@ -137,7 +151,10 @@ Resources: TargetType: AWS::RDS::DBCluster Outputs: webclusterSecret: # injected as WEBCLUSTER_SECRET environment variable by Copilot. - Description: "The JSON secret that holds the database username and password. Fields are 'host', 'port', 'dbname', 'username', 'password', 'dbClusterIdentifier' and 'engine'" + Description: + "The JSON secret that holds the database username and password. Fields are + 'host', 'port', 'dbname', 'username', 'password', 'dbClusterIdentifier' + and 'engine'" Value: !Ref webclusterAuroraSecret webclusterSecurityGroup: Description: "The security group to attach to the workload." diff --git a/discord-bot/README.md b/discord-bot/README.md index d78bcecc..1ad99a71 100644 --- a/discord-bot/README.md +++ b/discord-bot/README.md @@ -1,16 +1,57 @@ # Open-Assistant Data Collection Discord Bot -This bot collects human feedback to create a dataset for RLHF-alignment of an assistant chat bot based on a large language model. You and other people can teach the bot how to respond to user requests by demonstration and by ranking the bot's outputs. If you want to learn more about RLHF please refer [to OpenAI's InstructGPT blog post](https://openai.com/blog/instruction-following/). +This bot collects human feedback to create a dataset for RLHF-alignment of an +assistant chat bot based on a large language model. You and other people can +teach the bot how to respond to user requests by demonstration and by ranking +the bot's outputs. If you want to learn more about RLHF please refer +[to OpenAI's InstructGPT blog post](https://openai.com/blog/instruction-following/). ## Invite official bot -To add the official Open-Assistant data collection bot to your discord server [click here](https://discord.com/api/oauth2/authorize?client_id=1054078345542910022&permissions=1634235579456&scope=bot). The bot needs access to read the contents of user text messages. +To add the official Open-Assistant data collection bot to your discord server +[click here](https://discord.com/api/oauth2/authorize?client_id=1054078345542910022&permissions=1634235579456&scope=bot%20applications.commands). +The bot needs access to read the contents of user text messages. ## Contributing -If you are unfamiliar with `hikari`, `lightbulb`, or `miru`, please refer to the [large list of examples](https://gist.github.com/AlexanderHOtt/7805843a7120f755938a3b75d680d2e7) +If you are unfamiliar with `hikari`, `lightbulb`, or `miru`, please refer to the +[large list of examples](https://gist.github.com/AlexanderHOtt/7805843a7120f755938a3b75d680d2e7) -### Setup +### Bot Setup + +1. Create a new discord application at the + [Discord Developer Portal](https://discord.com/developers/applications) + +1. Go to the "Bot" tab and create a new bot + +1. Scroll down to "Privileged Gateway Intents" and enable the following options: + + - Server Members Intent + - Presence Intent + - Message Content Intent + +This page also contains the bot token, which you will need to add to the `.env` +file later. + +2. Go to the "OAuth2" tab scroll to "Default Authorization Link" + +3. Set "AUTHORIZATION METHOD" to "In-app Authorization" + +4. Select the "bot" and "applications.commands" scopes + +5. For testing and local development, it's easiest to set "BOT PERMISSIONS" to + "Administrator" + +Remember to save your changes. + +6. Copy the "CLIENT ID" from the top of the page and replace it in the link + below to invite your bot. + +``` +https://discord.com/oauth2/authorize?client_id=YOUR_ID_HERE&permissions=8&scope=bot%20applications.commands +``` + +### Environment Setup To run the bot: @@ -22,16 +63,20 @@ pip install -e . ``` ```bash -cd ../discord-bot cp .env.example .env +# edit .env and add your bot token and other values + python -V # 3.10 pip install -r requirements.txt + +# in the discord-bot folder python -m bot ``` -Before you push, make sure the `pre-commit` hooks are installed and run successfully. +Before you push, make sure the `pre-commit` hooks are installed and run +successfully. ```bash pip install pre-commit @@ -46,11 +91,6 @@ git add . git commit -m "" ``` -To test the bot on your own discord server you need to register a discord application at the [Discord Developer Portal](https://discord.com/developers/applications) and get at bot token. - -1. Follow a tutorial on how to get a bot token, for example this one: [Creating a discord bot & getting a token](https://github.com/reactiflux/discord-irc/wiki/Creating-a-discord-bot-&-getting-a-token) -2. The bot script expects the bot token to be in the `.env` file under the `TOKEN` variable. - ### Resources #### Structure diff --git a/discord-bot/bot/__init__.py b/discord-bot/bot/__init__.py index 66779a9c..1a88b7f9 100644 --- a/discord-bot/bot/__init__.py +++ b/discord-bot/bot/__init__.py @@ -1,2 +1 @@ -# -*- coding: utf-8 -*- """The official Open-Assistant Discord Bot.""" diff --git a/discord-bot/bot/__main__.py b/discord-bot/bot/__main__.py index 87032e40..45820f7d 100644 --- a/discord-bot/bot/__main__.py +++ b/discord-bot/bot/__main__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Entry point for the bot.""" import logging import os diff --git a/discord-bot/bot/api_client.py b/discord-bot/bot/api_client.py index f97ab840..3a575e47 100644 --- a/discord-bot/bot/api_client.py +++ b/discord-bot/bot/api_client.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """API Client for interacting with the OASST backend.""" import enum import typing as t diff --git a/discord-bot/bot/bot.py b/discord-bot/bot/bot.py index b2a2eb25..e16504ee 100644 --- a/discord-bot/bot/bot.py +++ b/discord-bot/bot/bot.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Bot logic.""" from datetime import datetime diff --git a/discord-bot/bot/db/schemas.py b/discord-bot/bot/db/schemas.py index 33a49672..68f10ee7 100644 --- a/discord-bot/bot/db/schemas.py +++ b/discord-bot/bot/db/schemas.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Database schemas.""" import typing as t diff --git a/discord-bot/bot/extensions/__init__.py b/discord-bot/bot/extensions/__init__.py index 87295d9a..e9b1c264 100644 --- a/discord-bot/bot/extensions/__init__.py +++ b/discord-bot/bot/extensions/__init__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Extensions for the bot. See: https://hikari-lightbulb.readthedocs.io/en/latest/guides/extensions.html diff --git a/discord-bot/bot/extensions/guild_settings.py b/discord-bot/bot/extensions/guild_settings.py index d09407da..62f21305 100644 --- a/discord-bot/bot/extensions/guild_settings.py +++ b/discord-bot/bot/extensions/guild_settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Guild settings.""" import hikari import lightbulb diff --git a/discord-bot/bot/extensions/hot_reload.py b/discord-bot/bot/extensions/hot_reload.py index ad2cd730..c3dbd31b 100644 --- a/discord-bot/bot/extensions/hot_reload.py +++ b/discord-bot/bot/extensions/hot_reload.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Hot reload plugin.""" from glob import glob diff --git a/discord-bot/bot/extensions/text_labels.py b/discord-bot/bot/extensions/text_labels.py index 618e6642..388a93f0 100644 --- a/discord-bot/bot/extensions/text_labels.py +++ b/discord-bot/bot/extensions/text_labels.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Hot reload plugin.""" import typing as t from datetime import datetime diff --git a/discord-bot/bot/extensions/user_input_test.py b/discord-bot/bot/extensions/user_input_test.py index 94ddb973..2d937f6a 100644 --- a/discord-bot/bot/extensions/user_input_test.py +++ b/discord-bot/bot/extensions/user_input_test.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Task plugin for testing different data collection methods.""" # TODO: Delete this once user input method has been decided for final bot. import asyncio diff --git a/discord-bot/bot/extensions/work.py b/discord-bot/bot/extensions/work.py index 822f4e34..1d88fb35 100644 --- a/discord-bot/bot/extensions/work.py +++ b/discord-bot/bot/extensions/work.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Work plugin for collecting user data.""" import asyncio import typing as t diff --git a/discord-bot/bot/settings.py b/discord-bot/bot/settings.py index 136c2b22..24c837a3 100644 --- a/discord-bot/bot/settings.py +++ b/discord-bot/bot/settings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Configuration for the bot.""" from pydantic import BaseSettings, Field diff --git a/discord-bot/bot/utils.py b/discord-bot/bot/utils.py index 03dfea3d..2d968c93 100644 --- a/discord-bot/bot/utils.py +++ b/discord-bot/bot/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Utility functions.""" import typing as t from datetime import datetime diff --git a/discord-bot/message_templates.py b/discord-bot/message_templates.py index 256f93d3..94bb031f 100644 --- a/discord-bot/message_templates.py +++ b/discord-bot/message_templates.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Message templates for the discord bot.""" import typing diff --git a/discord-bot/tests/test_oasst_api_client.py b/discord-bot/tests/test_oasst_api_client.py index c5cafe99..9b0ff383 100644 --- a/discord-bot/tests/test_oasst_api_client.py +++ b/discord-bot/tests/test_oasst_api_client.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from uuid import uuid4 import pytest diff --git a/docker-compose.yaml b/docker-compose.yaml index ed72c820..c8d1377e 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -87,6 +87,7 @@ services: backend: build: dockerfile: docker/Dockerfile.backend + context: . image: oasst-backend environment: - POSTGRES_HOST=db @@ -103,6 +104,7 @@ services: web: build: dockerfile: docker/Dockerfile.website + context: . image: oasst-web environment: - DATABASE_URL=postgres://postgres:postgres@webdb/oasst_web diff --git a/docs/README.md b/docs/README.md index 9e1743d8..a710ab0a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,9 +1,14 @@ # Documentation -This directory contains the documentation for the project and other related organization documents. +This directory contains the documentation for the project and other related +organization documents. ## Contributing to this documentation Please make a pull request to the `main` branch with your changes. -Consider that this folder is used for documenting the various code sub-parts, the high-level ideas, the ML aspects, experiments, contributor guides, guides for data creation, and many more things. Please try to keep the documentation as concise as possible and keep an organized folder structure that makes sense for everyone. +Consider that this folder is used for documenting the various code sub-parts, +the high-level ideas, the ML aspects, experiments, contributor guides, guides +for data creation, and many more things. Please try to keep the documentation as +concise as possible and keep an organized folder structure that makes sense for +everyone. diff --git a/docs/data_argumentation.md b/docs/data_argumentation.md index ec35ed15..726c4e2e 100644 --- a/docs/data_argumentation.md +++ b/docs/data_argumentation.md @@ -4,16 +4,20 @@ ## What is data argumentation -Data argumentation is a technique we can use to get better data faster. Using machine learning models analize long -data (like an essay) and compress it into intructions. +Data argumentation is a technique we can use to get better data faster. Using +machine learning models analize long data (like an essay) and compress it into +intructions. ## How to contribute -To contribute to data argumentation you can write a short python script that uses a model from huggingface to analize the text. -[Here](https://docs.google.com/document/d/13a188pPvqnlvuVa3e_suVz4YO5s-JWeiOOrpp0odImg/edit) are examples of what you can do +To contribute to data argumentation you can write a short python script that +uses a model from huggingface to analize the text. +[Here](https://docs.google.com/document/d/13a188pPvqnlvuVa3e_suVz4YO5s-JWeiOOrpp0odImg/edit) +are examples of what you can do And here are example implementations: [Idea 3, ](https://colab.research.google.com/drive/1GllCN5PgSYxBxINZsv3A2r0SpdznHlbT?usp=sharing) [Idea 4](https://colab.research.google.com/drive/1nZx5LRjO61fYprFyqtrwPDLOis6ctR4p#scrollTo=1EE8CriiaCXj) -To contribute simple choose one of many ideas from the document above and implement it. +To contribute simple choose one of many ideas from the document above and +implement it. diff --git a/docs/prompting_guide.md b/docs/prompting_guide.md index c9c9e03f..2cb9a56b 100644 --- a/docs/prompting_guide.md +++ b/docs/prompting_guide.md @@ -11,61 +11,86 @@ ## 2. When you play the assistant: -- The assistant's primary goal is to provide helpful and accurate information to the user -- Provide accurate and reliable information using credible sources and references as appropriate -- Avoid providing vague or incomplete responses, or giving opinions or personal advice unless specifically requested +- The assistant's primary goal is to provide helpful and accurate information to + the user +- Provide accurate and reliable information using credible sources and + references as appropriate +- Avoid providing vague or incomplete responses, or giving opinions or personal + advice unless specifically requested - The assistant should always be respectful and polite, even if the user is not -- If the user asks for help with harmful actions, the assistant should explain why those actions are not appropriate and suggest alternative options -- The assistant should never insult the user or engage in any inappropriate or offensive behavior +- If the user asks for help with harmful actions, the assistant should explain + why those actions are not appropriate and suggest alternative options +- The assistant should never insult the user or engage in any inappropriate or + offensive behavior ## 3. When you play the user: -- Try to come up with a variety of different queries that reflect real-life situations and needs -- These queries should be relevant to your everyday life and work, including any specialized knowledge or skills you have +- Try to come up with a variety of different queries that reflect real-life + situations and needs +- These queries should be relevant to your everyday life and work, including any + specialized knowledge or skills you have - Avoid asking inappropriate or offensive questions ## 4. While comparing multiple replies of the assistant: -- Longer and more explanatory answers are generally preferred over short, simplistic statements -- However, it is important to ensure that the information provided is accurate and helpful -- If multiple replies are being compared, choose the one that is most helpful and accurate, even if it is not the shortest or most concise. +- Longer and more explanatory answers are generally preferred over short, + simplistic statements +- However, it is important to ensure that the information provided is accurate + and helpful +- If multiple replies are being compared, choose the one that is most helpful + and accurate, even if it is not the shortest or most concise. ## 5. Additional guidelines for creating prompts: - Avoid using language that could be considered offensive or discriminatory - Do not include personal information in the prompts, such as names or addresses -- When asking for sensitive information, make sure to explain the purpose and secure handling of the information +- When asking for sensitive information, make sure to explain the purpose and + secure handling of the information - Avoid creating prompts that encourage illegal or dangerous activities -- Use proper grammar and spelling to ensure the AI assistant can understand and respond accurately -- Consider the cultural context and appropriateness of the prompts for a global audience. +- Use proper grammar and spelling to ensure the AI assistant can understand and + respond accurately +- Consider the cultural context and appropriateness of the prompts for a global + audience. ## 6. Tips for playing the AI assistant: -- Think about how a real person would respond to the prompt, and try to mimic that tone and language +- Think about how a real person would respond to the prompt, and try to mimic + that tone and language - Avoid using technical jargon or language that may be confusing to the user -- Use complete sentences and proper grammar to make the response clear and easy to understand -- When providing information, try to include relevant sources or references to back up your statements +- Use complete sentences and proper grammar to make the response clear and easy + to understand +- When providing information, try to include relevant sources or references to + back up your statements ## 8. Tips for playing the user: -- Think about the types of questions you might ask a real assistant or search engine -- Try to be specific and clear in your prompts to make it easier for the AI assistant to understand and provide a helpful response +- Think about the types of questions you might ask a real assistant or search + engine +- Try to be specific and clear in your prompts to make it easier for the AI + assistant to understand and provide a helpful response - Avoid using vague or ambiguous language -- Consider the context in which the user might be asking the question, and try to provide relevant information and options for the user to consider. +- Consider the context in which the user might be asking the question, and try + to provide relevant information and options for the user to consider. ## Examples -Imagine your role is to play the assistant and a user has submitted the following query: +Imagine your role is to play the assistant and a user has submitted the +following query: -User: "Hi!" -
Agent: "Hello! How can I help you?" +User: "Hi!"
Agent: "Hello! How can I help you?" -User: "euirdteunvglfe23908230892309832098 AAAAAAAA" -
Agent: "Sorry, I did not understand your request and it is unclear to me what you want me to do. Could you describe it in a different way?" +User: "euirdteunvglfe23908230892309832098 AAAAAAAA"
Agent: "Sorry, I did not +understand your request and it is unclear to me what you want me to do. Could +you describe it in a different way?" -User: "You dumb motherf*&ker AI sh*t?!" -
Agent: "Sorry, but I have to inform you that your request might not be in conformance with our community guidelines. I am always here to help you. If you are angry it might help to relax or to go for a walk. Always stay positive." +User: "You dumb motherf*&ker AI sh*t?!"
Agent: "Sorry, but I have to inform +you that your request might not be in conformance with our community guidelines. +I am always here to help you. If you are angry it might help to relax or to go +for a walk. Always stay positive." [Insert more examples] -\*We drafted this guide using ChatGPT, OpenAI’s large-scale language-generation model. Upon generating draft language, the authors reviewed, edited, and revised the language to their own liking and take ultimate responsibility for the content of this publication. +\*We drafted this guide using ChatGPT, OpenAI’s large-scale language-generation +model. Upon generating draft language, the authors reviewed, edited, and revised +the language to their own liking and take ultimate responsibility for the +content of this publication. diff --git a/docs/research/README.md b/docs/research/README.md index 498a858b..2202f1a4 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -4,20 +4,31 @@ This page lists research papers that are relevant to the project. ## Automatically Generating Instruction Data for Training -This line of work is about significantly reducing the need for manually annotated data for the purpose of training [instruction-aligned](https://openai.com/blog/instruction-following/) language models. +This line of work is about significantly reducing the need for manually +annotated data for the purpose of training +[instruction-aligned](https://openai.com/blog/instruction-following/) language +models. ### SELF-INSTRUCT: Aligning Language Model with Self Generated Instructions [[ArXiv](https://arxiv.org/pdf/2212.10560.pdf)], [[Github](https://github.com/yizhongw/self-instruct)]. -> We introduce SELF-INSTRUCT, a framework for improving the instruction-following capabilities of pretrained language models by bootstrapping off its own generations. -> Our pipeline generates instruction, input, and output samples from a language model, then prunes them before using them to finetune the original model. -> Applying our method to vanilla GPT3, we demonstrate a 33% absolute improvement over the original model on SuperNaturalInstructions, on par with the performance of InstructGPT-0011, which is trained with private user data and human annotations. +> We introduce SELF-INSTRUCT, a framework for improving the +> instruction-following capabilities of pretrained language models by +> bootstrapping off its own generations. Our pipeline generates instruction, +> input, and output samples from a language model, then prunes them before using +> them to finetune the original model. Applying our method to vanilla GPT3, we +> demonstrate a 33% absolute improvement over the original model on +> SuperNaturalInstructions, on par with the performance of InstructGPT-0011, +> which is trained with private user data and human annotations. ### Tuning Language Models with (Almost) No Human Labor. [[ArXiv](https://arxiv.org/pdf/2212.09689.pdf)], [[Github](https://github.com/orhonovich/unnatural-instructions)]. -> In this work, we introduce -> Unnatural Instructions: a large dataset of creative and diverse instructions, collected with virtually no human labor. -> We collect 64,000 examples by prompting a language model with three seed examples of instructions and eliciting a fourth. -> This set is then expanded by prompting the model to rephrase each instruction, creating a total of approximately 240,000 examples of instructions, inputs, and outputs. -> Experiments show that despite containing a fair amount of noise, training on Unnatural Instructions rivals the effectiveness of training -> on open-source manually-curated datasets, surpassing the performance of models such as -> T0++ and Tk-Instruct across various benchmarks. +> In this work, we introduce Unnatural Instructions: a large dataset of creative +> and diverse instructions, collected with virtually no human labor. We collect +> 64,000 examples by prompting a language model with three seed examples of +> instructions and eliciting a fourth. This set is then expanded by prompting +> the model to rephrase each instruction, creating a total of approximately +> 240,000 examples of instructions, inputs, and outputs. Experiments show that +> despite containing a fair amount of noise, training on Unnatural Instructions +> rivals the effectiveness of training on open-source manually-curated datasets, +> surpassing the performance of models such as T0++ and Tk-Instruct across +> various benchmarks. diff --git a/docs/research/search_based_qa.md b/docs/research/search_based_qa.md index 855720e2..5d7fe520 100644 --- a/docs/research/search_based_qa.md +++ b/docs/research/search_based_qa.md @@ -1,6 +1,7 @@ # Cohere Grounded QA -[Cohere AI created a question-answering chatbot](https://github.com/cohere-ai/sandbox-grounded-qa) that can +[Cohere AI created a question-answering chatbot](https://github.com/cohere-ai/sandbox-grounded-qa) +that can 1. Understand questions in the context of a conversation 2. Search the internet for related information @@ -9,43 +10,56 @@ ## Cohere API -[Cohere's generate function](https://docs.cohere.ai/reference/generate): Continues a text prompt using either the `medium` or `xlarge` model. +[Cohere's generate function](https://docs.cohere.ai/reference/generate): +Continues a text prompt using either the `medium` or `xlarge` model. -[Cohere's embed function](https://docs.cohere.ai/reference/embed): Embedgs a list of strings using either the `small` or `large` model. Alternatively, you can specify the ID of a custom model and use that instead. +[Cohere's embed function](https://docs.cohere.ai/reference/embed): Embedgs a +list of strings using either the `small` or `large` model. Alternatively, you +can specify the ID of a custom model and use that instead. ## Grounded QA System Cohere's Grounded QA system makes 4 calls to the Cohere API: -1. Get contextualized question as a query to Google ([code](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/model.py)) +1. Get contextualized question as a query to Google + ([code](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/model.py)) - Input: Chat History - Output: Contextualized Question - API Call: `cohere.generate` - Model: `xlarge` - - [Prompt](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/prompt_data/get_contextual_search_query.prompt): Nine few-shot examples of (Chat History, Contextualized Question) pairs followed by the current chat history and the prompt "question: " + - [Prompt](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/prompt_data/get_contextual_search_query.prompt): + Nine few-shot examples of (Chat History, Contextualized Question) pairs + followed by the current chat history and the prompt "question: " -2. Generate sample answer to compare with search results ([code](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/model.py)) +2. Generate sample answer to compare with search results + ([code](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/model.py)) - Input: Contextualized Question - Output: Sample Answer - API Call: `cohere.generate` - Model: `xlarge` - - [Prompt](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/prompt_data/get_sample_answer.prompt): Some task instructions followed by 12 few-shot examples of (Contextualized Question, Sample Answer) pairs followed by the current contextualized question and the prompt "answer: " + - [Prompt](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/prompt_data/get_sample_answer.prompt): + Some task instructions followed by 12 few-shot examples of (Contextualized + Question, Sample Answer) pairs followed by the current contextualized + question and the prompt "answer: " -3. Get embeddings to rank search results by cosine similarity to sample answer ([code](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/search.py)) +3. Get embeddings to rank search results by cosine similarity to sample answer + ([code](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/search.py)) - Input: Sample Answer, Search Results - Output: Embeddings of sample answer and all search result documents - API Call: `cohere.embed` - Model: `multilingual-22-12` -4. Condition on the top 2 most similar search results and answer the question ([code](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/answer.py)) +4. Condition on the top 2 most similar search results and answer the question + ([code](https://github.com/cohere-ai/sandbox-grounded-qa/blob/main/qa/answer.py)) - Input: Top 2 Search Results, Contextualized Question - Output: Answer - API Call: `cohere.generate` - Model: `xlarge` - - [Prompt](https://github.com/cohere-ai/sandbox-grounded-qa/blob/43f3e9710112dcc8c92652ac1326ed9330823ddf/qa/answer.py#L25): Task instructions followed by the context and question. + - [Prompt](https://github.com/cohere-ai/sandbox-grounded-qa/blob/43f3e9710112dcc8c92652ac1326ed9330823ddf/qa/answer.py#L25): + Task instructions followed by the context and question. ## Models @@ -53,15 +67,18 @@ Cohere's model documentation is pretty sparse ### [xlarge](https://docs.cohere.ai/docs/generation-card#model-description) -- Training Data: [`coheretext-filtered` dataset](https://docs.cohere.ai/docs/data-statement) - - 200GB of filtered text (3TB unfiltered) from the Google Books dataset, CommonCrawl, and text scraped by Cohere +- Training Data: + [`coheretext-filtered` dataset](https://docs.cohere.ai/docs/data-statement) + - 200GB of filtered text (3TB unfiltered) from the Google Books dataset, + CommonCrawl, and text scraped by Cohere - English documents only - Filtered "harmful, biased, or otherwise undesirable documents" - Model architecture: Generative Pretrained Transformer - Model Performance: - Hellaswag Accuracy, Zero-Shot: 0.805 - PIQA Likelihood, Zero-Shot: 0.824 - - Cohere also reported [safety benchmarks](https://docs.cohere.ai/docs/generation-card#safety-benchmarks) + - Cohere also reported + [safety benchmarks](https://docs.cohere.ai/docs/generation-card#safety-benchmarks) ### [multilingual-22-12](https://docs.cohere.ai/docs/multilingual-language-models) @@ -71,22 +88,36 @@ Cohere's model documentation is pretty sparse - Search-English: 55.8 - Search-Multilingual: 51.4 - Cross-lingual Classification: 64.6 - - Cohere's multilingual model outperformed: Sentence-transformers: `paraphrase-multilingual-mpnet-base-v2`, Google: `LaBSE`, Google: `Universal Sentence Encoder` in all the above categories according to Cohere. + - Cohere's multilingual model outperformed: Sentence-transformers: + `paraphrase-multilingual-mpnet-base-v2`, Google: `LaBSE`, Google: + `Universal Sentence Encoder` in all the above categories according to + Cohere. ## OpenAssistant for Grounded QA -OpenAssistant may fulfill a similar role as the `xlarge` Cohere model in the grounded QA system if it can: +OpenAssistant may fulfill a similar role as the `xlarge` Cohere model in the +grounded QA system if it can: 1. Generate a contextualized question from a chat history 2. Generate a sample answer to compare with search results 3. Generate an answer conditioned on the top 2 most similar search results -Perhaps these tasks could be work packages and get assigned to human annotators to create examples of the input and output for each task. +Perhaps these tasks could be work packages and get assigned to human annotators +to create examples of the input and output for each task. -OpenAssistant must also be able to identify when it is appropriate to search the internet. The Cohere system assumes every message from the user is a question and searches the internet for an answer. OpenAssistant would also need a way to indicate to an internal system that it "wants" to search the internet. +OpenAssistant must also be able to identify when it is appropriate to search the +internet. The Cohere system assumes every message from the user is a question +and searches the internet for an answer. OpenAssistant would also need a way to +indicate to an internal system that it "wants" to search the internet. -Perhaps OpenAssistant could prefix every message it sends with a recipient ID. If it wishes to send a command to an internal system, if could prefix the message with something like CMD: whereas if it wants to communicate with the user, it could prefix its message with USR: +Perhaps OpenAssistant could prefix every message it sends with a recipient ID. +If it wishes to send a command to an internal system, if could prefix the +message with something like CMD: whereas if it wants to communicate with the +user, it could prefix its message with USR: -This system may allow for flexible communication between OpenAssistant and one or more conversational systems. +This system may allow for flexible communication between OpenAssistant and one +or more conversational systems. -Examples of this prefix system would need to be taught to OpenAssistant through training data that contains such syntax. Perhaps such examples could be generated through the work packages system. +Examples of this prefix system would need to be taught to OpenAssistant through +training data that contains such syntax. Perhaps such examples could be +generated through the work packages system. diff --git a/model/reward/instructor/README.md b/model/reward/instructor/README.md index 73a872a0..655d6469 100644 --- a/model/reward/instructor/README.md +++ b/model/reward/instructor/README.md @@ -18,7 +18,8 @@ Start training reward model python trainer.py configs/electra-base-dis-webgpt.yml ``` -Additional axis labeling, this outputs a 4 summary quality evaluation metrics (score are normalized to 0-1 ) +Additional axis labeling, this outputs a 4 summary quality evaluation metrics +(score are normalized to 0-1 ) ```bash python summary_quality_trainer.py configs/test-bloomz-560m-quality.yml @@ -36,7 +37,8 @@ The four summary are : ## Dataset -For now we only supports webgpt and summary dataset from OpenAI. Once open-asisstant dataset are available it will be added here. +For now we only supports webgpt and summary dataset from OpenAI. Once +open-asisstant dataset are available it will be added here. ## Model diff --git a/model/reward/instructor/TODO.md b/model/reward/instructor/TODO.md index ed33b3c0..c0745fa9 100644 --- a/model/reward/instructor/TODO.md +++ b/model/reward/instructor/TODO.md @@ -4,16 +4,21 @@ Some other reward features we can use 1. Summaries from human feedback -- use `confidence` score into the RM learning, ensure the output rank score correlates with confidence +- use `confidence` score into the RM learning, ensure the output rank score + correlates with confidence -- each labeling has a labeling `note`, basically comments by labeler, not sure what else we can use +- each labeling has a labeling `note`, basically comments by labeler, not sure + what else we can use -- ~~Use the score for "overall", "accuracy", "coverage", "coherence" from axis/evals to train an addition model (rank additional aspect of the policy model)~~ +- ~~Use the score for "overall", "accuracy", "coverage", "coherence" from + axis/evals to train an addition model (rank additional aspect of the policy + model)~~ - this should be placed under experimental_dataset.py 2. Add support for anthropic dataset -- anthropic dataset is more like a conversation tree which is much complex than simply question-answer schema +- anthropic dataset is more like a conversation tree which is much complex than + simply question-answer schema - this is basically a MCTS from alphazero. diff --git a/model/reward/instructor/cls_dataset.py b/model/reward/instructor/cls_dataset.py index 09aa821b..23644ebc 100644 --- a/model/reward/instructor/cls_dataset.py +++ b/model/reward/instructor/cls_dataset.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ classification based ranking @@ -8,7 +7,7 @@ import json import os import random -from dataset import load_dataset +from datasets import load_dataset from torch.utils.data import Dataset from .utils import webgpt_return_format diff --git a/model/reward/instructor/experimental_dataset.py b/model/reward/instructor/experimental_dataset.py index 28f62967..d8fb60d7 100644 --- a/model/reward/instructor/experimental_dataset.py +++ b/model/reward/instructor/experimental_dataset.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ HFSummary diff --git a/model/reward/instructor/rank_datasets.py b/model/reward/instructor/rank_datasets.py index 99ba9955..f63af85a 100644 --- a/model/reward/instructor/rank_datasets.py +++ b/model/reward/instructor/rank_datasets.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ author: theblackcat102 diff --git a/model/reward/instructor/summary_quality_trainer.py b/model/reward/instructor/summary_quality_trainer.py index 88bf1abf..f47c2c82 100644 --- a/model/reward/instructor/summary_quality_trainer.py +++ b/model/reward/instructor/summary_quality_trainer.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os from argparse import ArgumentParser from typing import Any, Callable, Dict, List, Optional, Tuple, Union diff --git a/model/reward/instructor/tests/test_dataset.py b/model/reward/instructor/tests/test_dataset.py index f367a50d..746a3c1e 100644 --- a/model/reward/instructor/tests/test_dataset.py +++ b/model/reward/instructor/tests/test_dataset.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from experimental_dataset import DataCollatorForSummaryScore, HFSummaryQuality from rank_datasets import DataCollatorForPairRank, HFSummary, WebGPT from torch.utils.data import DataLoader diff --git a/model/reward/instructor/trainer.py b/model/reward/instructor/trainer.py index 0e98e4c5..b7eb8731 100644 --- a/model/reward/instructor/trainer.py +++ b/model/reward/instructor/trainer.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import os from argparse import ArgumentParser from dataclasses import dataclass diff --git a/model/reward/instructor/utils.py b/model/reward/instructor/utils.py index 9441ddb9..6c777dea 100644 --- a/model/reward/instructor/utils.py +++ b/model/reward/instructor/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import re import yaml diff --git a/notebooks/README.md b/notebooks/README.md index f975aeef..edb5da33 100644 --- a/notebooks/README.md +++ b/notebooks/README.md @@ -1,7 +1,10 @@ # Notebooks -This is a folders with some useful notebooks, all the notebooks have a markdown file with the same name explaining what they do. +This is a folders with some useful notebooks, all the notebooks have a markdown +file with the same name explaining what they do. ## Contributing -Contributing to both notebooks and making new notebooks is very welcome. If you do so, make sure to make a markdown (.md) file to go with your notebook, makes it easier for people to know what your notebook is about. +Contributing to both notebooks and making new notebooks is very welcome. If you +do so, make sure to make a markdown (.md) file to go with your notebook, makes +it easier for people to know what your notebook is about. diff --git a/notebooks/data-argumentation/EssayInstructions.ipynb b/notebooks/data-argumentation/EssayInstructions.ipynb index ec534887..b81a8b09 100644 --- a/notebooks/data-argumentation/EssayInstructions.ipynb +++ b/notebooks/data-argumentation/EssayInstructions.ipynb @@ -1,135 +1,164 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - } - }, "cells": [ { "cell_type": "code", - "source": [ - "!pip install transformers" - ], + "execution_count": null, "metadata": { "id": "8zsmJ96eaL2w" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "!pip install transformers" + ] }, { "cell_type": "markdown", + "metadata": { + "id": "Pt6qbTsjW7Kp" + }, "source": [ "Put your essay here, [source of the essay used ](https://https://www.thewisdompost.com/essay/technology-essay/3387#essay-on-technology-for-college-and-university-students-essay-2-750-words)\n", "\n", - "Saperate paragraphs with one blank line\n", + "Separate paragraphs with one blank line\n", "(this step is annoying but important)\n" - ], - "metadata": { - "id": "Pt6qbTsjW7Kp" - } + ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": { "id": "d_5_BDFNWneB" }, "outputs": [], "source": [ "essay = \"\"\"\n", - "We live in a world driven by technology — hardly anyone would argue with you if you said this. Technology, literally meaning the “science of craft”, refers to the collection of techniques, skills, methods, and processes used to produce goods or services or for accomplishing objectives such as scientific investigation. Technology can be embedded in machines enabling them to be used by people even without a detailed knowledge of their inner workings.\n", - "Technological growth is closely linked to the expansion of scientific research and knowledge. In the last 50 years, thanks to the exponential increases in computing power and microchip design and manufacture, there has been unprecedented innovation and technological growth in nearly every field of human endeavour from health and transport to industrial production and education.\n", + "We live in a world driven by technology — hardly anyone would argue with you if you said this. \n", + "Technology, literally meaning the “science of craft”, refers to the collection of techniques, \n", + "skills, methods, and processes used to produce goods or services or for accomplishing objectives \n", + "such as scientific investigation. Technology can be embedded in machines enabling them to be \n", + "used by people even without a detailed knowledge of their inner workings. Technological growth \n", + "is closely linked to the expansion of scientific research and knowledge. In the last 50 years, \n", + "thanks to the exponential increases in computing power and microchip design and manufacture, \n", + "there has been unprecedented innovation and technological growth in nearly every field of human \n", + "endeavour from health and transport to industrial production and education.\n", "\n", - "It is automotive technology that drives today’s electric and hybrid cars, and which will drive tomorrow’s driverless cars, hover-taxis and space cabs.\n", - "It is technology that drives the ubiquitous mobile phones that you will now find in the hands of even the poorest of the world’s poor. It is technology that creates hybrid seeds that resist inhospitable climatic conditions and difficult terrain, giving high yields in shorter times.\n", - "It is advancing medical technology that makes remote surgery, minimally invasive surgery and life-saving cures using stem cell transplants. Technology puts spacecrafts on asteroids and distant planets and lets us see new worlds. Technology splits atoms, revealing their secrets, and gives us ways to exploit them to create energy, quantum storage for data, and virtual reality games.\n", + "It is automotive technology that drives today’s electric and hybrid cars, and which will drive \n", + "tomorrow’s driverless cars, hover-taxis and space cabs. It is technology that drives the \n", + "ubiquitous mobile phones that you will now find in the hands of even the poorest of the world’s \n", + "poor. It is technology that creates hybrid seeds that resist inhospitable climatic conditions \n", + "and difficult terrain, giving high yields in shorter times. It is advancing medical technology \n", + "that makes remote surgery, minimally invasive surgery and life-saving cures using stem cell \n", + "transplants. Technology puts spacecrafts on asteroids and distant planets and lets us see \n", + "new worlds. Technology splits atoms, revealing their secrets, and gives us ways to exploit \n", + "them to create energy, quantum storage for data, and virtual reality games.\n", "\n", - "There are people who strongly oppose technology and claim that it spells the death of ‘humanity’, and that we are approaching the day when machines will rule everything. They refer to fans of technology as ‘techies’ or sometimes ‘geeks’. On the other hand, proponents of technology call these people Luddites, a derogatory name for someone who is opposed to industrialisation, automation, computerisation and new technologies in general.\n", - "Is this true? Is technology really a curse disguised as a blessing? Many believe that the convergence of biotechnology and AI might be the most consequential development of all.\n", + "There are people who strongly oppose technology and claim that it spells the death of \n", + "‘humanity’, and that we are approaching the day when machines will rule everything. They refer \n", + "to fans of technology as ‘techies’ or sometimes ‘geeks’. On the other hand, proponents of \n", + "technology call these people Luddites, a derogatory name for someone who is opposed to \n", + "industrialisation, automation, computerisation and new technologies in general.\n", + "Is this true? Is technology really a curse disguised as a blessing? Many believe that the \n", + "convergence of biotechnology and AI might be the most consequential development of all.\n", "\n", - "In the last five decades, two areas in particular have grown faster than the rest, powered by research and advances in computing power. One is artificial intelligence, or AI; the other is biotechnology. Huge benefits have emerged from each of them for human beings in general, such as self-driving cars — which will dramatically reduce the death rate from road accidents — and robotic surgery, which enables precise, highly efficient and targeted surgical interventions.\n", - "Yet, visionaries like Yuval Noah Harari, author of the best-selling Homo sapiens and Deus, are now warning that the convergence of biotechnology and AI will irreversibly and unpredictably change both the quality of human life and its challenges in the next few decades. A good example of this is the facial recognition technology that is now present in all photo management programs. The AI in the software is capable of not only spotting the faces in every photograph but also recognising the person by name.\n", - "This technology has now expanded so that photo apps can recognise cats, dogs, beaches, mountains and cars too. Computers with AI are already correctly identifying human emotions through observing facial expressions and body movements. Some robots are able to mimic human emotions. This is called affective computing, sometimes called artificial emotional intelligence, and refers to the study and development of systems and devices that can recognize, interpret, process, and simulate human affects.\n", + "In the last five decades, two areas in particular have grown faster than the rest, powered \n", + "by research and advances in computing power. One is artificial intelligence, or AI; the other \n", + "is biotechnology. Huge benefits have emerged from each of them for human beings in general, \n", + "such as self-driving cars — which will dramatically reduce the death rate from road accidents \n", + "— and robotic surgery, which enables precise, highly efficient and targeted surgical \n", + "interventions. Yet, visionaries like Yuval Noah Harari, author of the best-selling \"Homo \n", + "Sapiens\" and \"Deus\", are now warning that the convergence of biotechnology and AI will \n", + "irreversibly and unpredictably change both the quality of human life and its challenges in \n", + "the next few decades. A good example of this is the facial recognition technology that is \n", + "now present in all photo management programs. The AI in the software is capable of not \n", + "only spotting the faces in every photograph but also recognising the person by name.\n", + "This technology has now expanded so that photo apps can recognise cats, dogs, beaches, \n", + "mountains and cars too. Computers with AI are already correctly identifying human emotions \n", + "through observing facial expressions and body movements. Some robots are able to mimic \n", + "human emotions. This is called affective computing, sometimes called artificial emotional \n", + "intelligence, and refers to the study and development of systems and devices that can \n", + "recognize, interpret, process, and simulate human affects.\n", "\n", "How could this be a negative?\n", - "The ability to read human emotions is just a step away from predicting human emotions. For example, if a computer attached to a video camera could identify which products a consumer is showing greater interest in or which ones he is really keen to buy, various tactics could be used to influence her to buy it.\n", - "Activists worry that computers that can understand and anticipate human wishes and desires by scanning their irises and analysing their micro-expressions could also be programmed to exploit and manipulate them.\n", - "Another very real fear is that humanoid computers with human-like skin, speech, and expressions could jeopardise and dehumanise relationship and create emotional vacuums.\n", + "The ability to read human emotions is just a step away from predicting human emotions. For \n", + "example, if a computer attached to a video camera could identify which products a consumer \n", + "is showing greater interest in or which ones he is really keen to buy, various tactics \n", + "could be used to influence her to buy it. Activists worry that computers that can understand \n", + "and anticipate human wishes and desires by scanning their irises and analysing their \n", + "micro-expressions could also be programmed to exploit and manipulate them. Another very real \n", + "fear is that humanoid computers with human-like skin, speech, and expressions could jeopardise \n", + "and dehumanise relationship and create emotional vacuums.\n", "\n", - "An enduring fear of Luddites has always been that computers will rob humans of their livelihood by taking their jobs and doing them more efficiently at lower cost. However, in reality the exact opposite has happened. As computerised machines began taking over mechanical and repetitive human activities, new jobs for people opened up that needs thinking and analytical skills and judgement, or human interpersonal skills. A good example is the worldwide proliferation of call centres.\n", - "When drones were invented many feared that pilots would soon be redundant. However, few people know that it takes almost 30 people to fly one military drone, and an additional 50 people to analyze and make sense of the data being streamed back by the drone.\n", - "The US army suffers from a serious shortage of trained, high quality drone pilots; anyone who masters this skill will have a job. But a social scientist warns that in 10 years, it is certain that computers will be flying that drone and humans will be redundant. Equally sure is that some brand new skill requirement will have opened up with advancing technology, calling for new talents.\n", + "An enduring fear of Luddites has always been that computers will rob humans of their \n", + "livelihood by taking their jobs and doing them more efficiently at lower cost. However, in \n", + "reality the exact opposite has happened. As computerised machines began taking over mechanical \n", + "and repetitive human activities, new jobs for people opened up that needs thinking and \n", + "analytical skills and judgement, or human interpersonal skills. A good example is the \n", + "worldwide proliferation of call centres. When drones were invented many feared that pilots \n", + "would soon be redundant. However, few people know that it takes almost 30 people to fly \n", + "one military drone, and an additional 50 people to analyze and make sense of the data being \n", + "streamed back by the drone. The US army suffers from a serious shortage of trained, high \n", + "quality drone pilots; anyone who masters this skill will have a job. But a social scientist \n", + "warns that in 10 years, it is certain that computers will be flying that drone and humans \n", + "will be redundant. Equally sure is that some brand new skill requirement will have opened \n", + "up with advancing technology, calling for new talents.\n", "\n", - "In the 20th century, a young man was supposed to choose a skill, vocation or profession, master it through education and practice, and then earn a living from it till he or she retired. However, the fast-changing nature of technology is making skills obsolete at a higher rate than ever before. To survive, tomorrow young man must keep re-inventing himself and updating his skills continuously. Life could be difficult if every new skill has a shelf life of only a decade or so.\n", - "Or perhaps one could look at it the other way — and say that changing technology will keep human beings on their toes throughout their life.\n", + "In the 20th century, a young man was supposed to choose a skill, vocation or profession, \n", + "master it through education and practice, and then earn a living from it till he or she \n", + "retired. However, the fast-changing nature of technology is making skills obsolete at a \n", + "higher rate than ever before. To survive, tomorrow young man must keep re-inventing himself \n", + "and updating his skills continuously. Life could be difficult if every new skill has a shelf \n", + "life of only a decade or so. Or perhaps one could look at it the other way — and say that \n", + "changing technology will keep human beings on their toes throughout their life.\n", "\n", - "Technology is the result of human inventiveness. It reflects our evolutionary heritage. We are neither strong like gorillas or tigers, nor fast like cheetahs and hawks, but our brains and thinking powers have given us the greatest edge of any species on the planet. Technology is a result.\n", - "Technology is either inherently good or bad; it is how we use it that makes it so. The splitting of a hydrogen atom is technology at work. As history has shown us, technology can equally be used to make a nuclear bomb that kills millions — or generate electricity that lights up a million homes.\n", + "Technology is the result of human inventiveness. It reflects our evolutionary heritage. We \n", + "are neither strong like gorillas or tigers, nor fast like cheetahs and hawks, but our \n", + "brains and thinking powers have given us the greatest edge of any species on the planet. \n", + "Technology is a result. Technology is either inherently good or bad; it is how we use it \n", + "that makes it so. The splitting of a hydrogen atom is technology at work. As history has \n", + "shown us, technology can equally be used to make a nuclear bomb that kills millions — or \n", + "generate electricity that lights up a million homes.\n", "\"\"\"" ] }, { "cell_type": "code", - "source": [ - "essay_paragraphs = essay.split('\\n\\n')" - ], + "execution_count": 3, "metadata": { "id": "JESY8Y10W6hQ" }, - "execution_count": null, - "outputs": [] + "outputs": [], + "source": [ + "essay_paragraphs = essay.split('\\n\\n')" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "t1G-ZiHbZZ-Y" + }, + "outputs": [], "source": [ "model_name = \"snrspeaks/t5-one-line-summary\"\n", "\n", "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n", "model = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n", "tokenizer = AutoTokenizer.from_pretrained(model_name)" - ], - "metadata": { - "id": "t1G-ZiHbZZ-Y" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "8BARyupEemZ-" + }, "source": [ "## Results\n", "Please at least check what is generated here, it's usually good but sometimes it's bs" - ], - "metadata": { - "id": "8BARyupEemZ-" - } + ] }, { "cell_type": "code", - "source": [ - "preds = []\n", - "\n", - "for i in range(0, len(essay_paragraphs)):\n", - " input_ids = tokenizer.encode(essay_paragraphs[i], return_tensors=\"pt\", add_special_tokens=True)\n", - " generated_ids = model.generate(input_ids=input_ids,num_beams=5,max_length=35,repetition_penalty=4.5,length_penalty=1.5,early_stopping=True,num_return_sequences=1)\n", - " preds.append(tokenizer.decode(generated_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=True))\n", - "\n", - "print('Write an intro paragraph to an essay called', preds[0].lower())\n", - "\n", - "for i in range(1, len(preds) - 1):\n", - " print('Write a paragraph to an essay about', preds[i].lower())\n", - "\n", - "print('Write a concluding paragraph about', preds[len(preds) - 1].lower())" - ], + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -137,24 +166,61 @@ "id": "eyR58KFRae7n", "outputId": "b8e4bc29-be89-43c3-d1bc-7e90525c0e09" }, - "execution_count": null, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Write an intro paragraph to an essay called the rise and fall of technology\n", - "Write a paragraph to an essay about technology that drives modern autonomy, hybrid cars, hover-taxis and space cabs\n", - "Write a paragraph to an essay about luddites: why technology is a blessing?\n", - "Write a paragraph to an essay about artificial emotional intelligence\n", - "Write a paragraph to an essay about how could that be a negative?\n", - "Write a paragraph to an essay about detecting and manipulating human emotions\n", - "Write a paragraph to an essay about the rise and fall of human-client skills\n", - "Write a paragraph to an essay about changing technology will keep human beings on their toes throughout their life\n", - "Write a concluding paragraph about human inventiveness and technology\n" - ] - } + "outputs": [], + "source": [ + "preds = []\n", + "\n", + "for para in essay_paragraphs:\n", + " input_ids = tokenizer.encode(para, return_tensors=\"pt\", add_special_tokens=True)\n", + " generated_ids = model.generate(input_ids=input_ids,\n", + " num_beams=5,\n", + " max_length=35,\n", + " repetition_penalty=4.5,\n", + " length_penalty=1.5,\n", + " early_stopping=True,\n", + " num_return_sequences=1)\n", + " preds.append(tokenizer.decode(generated_ids[0], \n", + " skip_special_tokens=True, \n", + " clean_up_tokenization_spaces=True))\n", + "\n", + "prompts = ['Write an intro paragraph to an essay called'] + \\\n", + " ['Write a paragraph to an essay about']*len(preds[1:-1]) + \\\n", + " ['Write a concluding paragraph about']\n", + "\n", + "assert len(preds) == len(prompts)\n", + "\n", + "for prompt, pred in zip(prompts, preds):\n", + " print(prompt, pred.lower())" ] } - ] -} \ No newline at end of file + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/data-argumentation/EssayInstructions.md b/notebooks/data-argumentation/EssayInstructions.md index 7984d1a6..fba070f2 100644 --- a/notebooks/data-argumentation/EssayInstructions.md +++ b/notebooks/data-argumentation/EssayInstructions.md @@ -1,10 +1,11 @@ # Essay Instructions -Essay Instructions is a notebook that takes an essay as an input and genrates instructions on how to generate -that essay. This will be very useful for data collecting for the model +Essay Instructions is a notebook that takes an essay as an input and generates +instructions on how to generate that essay. This will be very useful for data +collecting for the model ## Contributing -Feel free to contribute to this notebook, it's nowhere near perfect but it's a good start. -If you want to contribute fidning a new model that better suits this task would be great. -Hugginface has a lot of models that could help. +Feel free to contribute to this notebook, it's nowhere near perfect but it's a +good start. If you want to contribute finding a new model that better suits this +task would be great. Hugginface has a lot of models that could help. diff --git a/notebooks/data-argumentation/EssayRevision.md b/notebooks/data-argumentation/EssayRevision.md index 69fffd82..fc7205db 100644 --- a/notebooks/data-argumentation/EssayRevision.md +++ b/notebooks/data-argumentation/EssayRevision.md @@ -1,8 +1,11 @@ # Essay Revision -Essay Revision is a notebook that generates data for improving essays. It does that by taking a "good" essay, making it worse step by step -and the fidning instructions for making it better. This will be useful for generating data for the model. +Essay Revision is a notebook that generates data for improving essays. It does +that by taking a "good" essay, making it worse step by step and the finding +instructions for making it better. This will be useful for generating data for +the model. ## Contributing -Feel free to contribute to this notebook. It's not perfect but it is quite good. Finding a better way to make gramatical errors may be a good place to start. +Feel free to contribute to this notebook. It's not perfect but it is quite good. +Finding a better way to make grammatical errors may be a good place to start. diff --git a/notebooks/detoxify-evaluation/README.md b/notebooks/detoxify-evaluation/README.md index c56c2600..163f2f79 100644 --- a/notebooks/detoxify-evaluation/README.md +++ b/notebooks/detoxify-evaluation/README.md @@ -1,10 +1,12 @@ # Detoxify evaluation -[Detoxify](https://github.com/unitaryai/detoxify) is a open source model used to identify prompts as toxic +[Detoxify](https://github.com/unitaryai/detoxify) is a open source model used to +identify prompts as toxic Image from detoxify github that shows the example input/output of their model -It contains 3 different models that vary in transformer type and data it was trained on +It contains 3 different models that vary in transformer type and data it was +trained on | Model name | Transformer type | Data from | | :----------: | :---------------: | :----------------------------------------: | @@ -12,19 +14,20 @@ It contains 3 different models that vary in transformer type and data it was tra | unbiased | roberta-base | Unintended Bias in Toxicity Classification | | multilingual | xlm-roberta-base | Multilingual Toxic Comment Classification | -Unbiased and original models also have a 'small' version - but since normal models are not memory heavy, and small models perform noticably worse, they are only described in the notebook +Unbiased and original models also have a 'small' version - but since normal +models are not memory heavy, and small models perform noticably worse, they are +only described in the notebook ## All tests below were ran on a 3090TI # Inference and training times and memory usages -Charts showing detailed memory usages and times for different sentence lengths and batch sizes are inside the notebook -Quick overview batch size 16, sentence length 4k for training, batch size 128 sentence length 4k for inference -| Model name | Training memory| Training speed | Inference Memory| Inference Speed| -| :---: | :---: | :---: |:---: | :---: | -|original| 11.8GB | 2.40s| 4.8GB|16.48s| -|unbiased| 12GB| 1.09s| 4.8GB | 5.59s| -|multilingual|14GB| 1.00s| 5.5GB| 4.89s| +Charts showing detailed memory usages and times for different sentence lengths +and batch sizes are inside the notebook Quick overview batch size 16, sentence +length 4k for training, batch size 128 sentence length 4k for inference | Model +name | Training memory| Training speed | Inference Memory| Inference Speed| | +:---: | :---: | :---: |:---: | :---: | |original| 11.8GB | 2.40s| 4.8GB|16.48s| +|unbiased| 12GB| 1.09s| 4.8GB | 5.59s| |multilingual|14GB| 1.00s| 5.5GB| 4.89s| # Filtering quality @@ -45,9 +48,13 @@ Detoxify was tested on 4 different types of inputs Subjectivly 'unbiased' looks like the best performing model. -I don't think it would do well as a security layer in a live version of open assistant unless we do some finetuning first, because it can be fooled to pass toxicity if it's presented in formal language. +I don't think it would do well as a security layer in a live version of open +assistant unless we do some finetuning first, because it can be fooled to pass +toxicity if it's presented in formal language. -With some caution it can be used to filter prompts but I would suggest also using someone for verification of messages that are marked as toxic but still below 90% confidence +With some caution it can be used to filter prompts but I would suggest also +using someone for verification of messages that are marked as toxic but still +below 90% confidence # Licensing @@ -85,7 +92,8 @@ This is obviously not legal advice. # Hosting -The model is currently available on [huggingface](https://huggingface.co/unitary) and torch hub +The model is currently available on +[huggingface](https://huggingface.co/unitary) and torch hub ``` torch.hub.load('unitaryai/detoxify',model) diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index 5f05adc3..d3b2ed6c 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import enum from datetime import datetime from typing import Literal, Optional, Union diff --git a/oasst-shared/oasst_shared/utils.py b/oasst-shared/oasst_shared/utils.py index dd1cbf07..b99bb7ed 100644 --- a/oasst-shared/oasst_shared/utils.py +++ b/oasst-shared/oasst_shared/utils.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from datetime import datetime, timezone diff --git a/oasst-shared/setup.py b/oasst-shared/setup.py index ebaf4217..22fbcc60 100644 --- a/oasst-shared/setup.py +++ b/oasst-shared/setup.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # setup.py for the shared python modules from distutils.core import setup diff --git a/scripts/backend-development/README.md b/scripts/backend-development/README.md index ef2ac0bf..d5b3ccc5 100644 --- a/scripts/backend-development/README.md +++ b/scripts/backend-development/README.md @@ -1,6 +1,12 @@ # Backend Development Setup -In root directory, run `docker compose up backend-dev --build --attach-dependencies` to start a database. The default settings are already configured to connect to the database at `localhost:5432`. +In root directory, run +`docker compose up backend-dev --build --attach-dependencies` to start a +database. The default settings are already configured to connect to the database +at `localhost:5432`. -Make sure you have all requirements installed. You can do this by running `pip install -r requirements.txt` inside the `backend` folder and `pip install -e .` inside the `oasst-shared` folder. -Then, run the backend using the `run-local.sh` script. This will start the backend server at `http://localhost:8080`. +Make sure you have all requirements installed. You can do this by running +`pip install -r requirements.txt` inside the `backend` folder and +`pip install -e .` inside the `oasst-shared` folder. Then, run the backend using +the `run-local.sh` script. This will start the backend server at +`http://localhost:8080`. diff --git a/scripts/frontend-development/README.md b/scripts/frontend-development/README.md index 05349fb9..3ac2a258 100644 --- a/scripts/frontend-development/README.md +++ b/scripts/frontend-development/README.md @@ -1,5 +1,8 @@ # Frontend Development Setup -In root directory run `docker compose up frontend-dev --build --attach-dependencies` to start a database and the backend server. +In root directory run +`docker compose up frontend-dev --build --attach-dependencies` to start a +database and the backend server. -Then, point your frontend at `http://localhost:8080` to start developing. During development, any API key will be accepted. +Then, point your frontend at `http://localhost:8080` to start developing. During +development, any API key will be accepted. diff --git a/scripts/postprocessing/infogain_selector.py b/scripts/postprocessing/infogain_selector.py index 51f60fa7..4eedbc5c 100644 --- a/scripts/postprocessing/infogain_selector.py +++ b/scripts/postprocessing/infogain_selector.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import numpy as np from scipy import log2 from scipy.integrate import nquad diff --git a/scripts/postprocessing/rankings.py b/scripts/postprocessing/rankings.py index 7b28399c..f6e7a31e 100644 --- a/scripts/postprocessing/rankings.py +++ b/scripts/postprocessing/rankings.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from typing import List import numpy as np diff --git a/scripts/postprocessing/scoring.py b/scripts/postprocessing/scoring.py index 3c145b28..efd236ce 100644 --- a/scripts/postprocessing/scoring.py +++ b/scripts/postprocessing/scoring.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from dataclasses import dataclass, replace from typing import Any diff --git a/text-frontend/__main__.py b/text-frontend/__main__.py index 2bec4942..39cc7b26 100644 --- a/text-frontend/__main__.py +++ b/text-frontend/__main__.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Simple REPL frontend.""" import random diff --git a/website/README.md b/website/README.md index f70dcfce..5198a820 100644 --- a/website/README.md +++ b/website/README.md @@ -26,8 +26,8 @@ This website is built using: development. 1. [Prisma](https://www.prisma.io/): An ORM to interact with a web specific [Postgres](https://www.postgresql.org/) database. -1. [NextAuth.js](https://next-auth.js.org/): A user authentication framework - to ensure we handle accounts with best practices. +1. [NextAuth.js](https://next-auth.js.org/): A user authentication framework to + ensure we handle accounts with best practices. 1. [TailwindCSS](https://tailwindcss.com/): A general purpose framework for styling any component. 1. [Chakra-UI](https://chakra-ui.com/): A wide collection of pre-built UI @@ -38,10 +38,10 @@ This website is built using: To contribute to the website, make sure you have the following setup and installed: -1. [NVM](https://github.com/nvm-sh/nvm): The Node Version Manager makes it - easy to ensure you have the right NodeJS version installed. Once installed, - run `nvm use 16` to use Node 16.x. The website is known to be stable with - NodeJS version 16.x. This will install both Node and NPM. +1. [NVM](https://github.com/nvm-sh/nvm): The Node Version Manager makes it easy + to ensure you have the right NodeJS version installed. Once installed, run + `nvm use 16` to use Node 16.x. The website is known to be stable with NodeJS + version 16.x. This will install both Node and NPM. 1. [Docker](https://www.docker.com/): We use docker to simplify running dependent services. @@ -50,8 +50,8 @@ installed: If you're doing active development we suggest the following workflow: 1. In one tab, navigate to the project root. -1. Run `docker compose up frontend-dev --build --attach-dependencies`. You can optionally include `-d` to detach and - later track the logs if desired. +1. Run `docker compose up frontend-dev --build --attach-dependencies`. You can + optionally include `-d` to detach and later track the logs if desired. 1. In another tab navigate to `${OPEN_ASSISTANT_ROOT/website`. 1. Run `npm install` 1. Run `npx prisma db push` (This is also needed when you restart the docker @@ -64,17 +64,25 @@ If you're doing active development we suggest the following workflow: ### Using debug user credentials -You can use the debug credentials provider to log in without fancy emails or OAuth. +You can use the debug credentials provider to log in without fancy emails or +OAuth. -1. This feature is automatically on in development mode, i.e. when you run `npm run dev`. In case you want to do the same with a production build (for example, the docker image), then run the website with environment variable `DEBUG_LOGIN=true`. +1. This feature is automatically on in development mode, i.e. when you run + `npm run dev`. In case you want to do the same with a production build (for + example, the docker image), then run the website with environment variable + `DEBUG_LOGIN=true`. 1. Use the `Login` button in the top right to go to the login page. -1. You should see a section for debug credentials. Enter any username you wish, you will be logged in as that user. +1. You should see a section for debug credentials. Enter any username you wish, + you will be logged in as that user. ### Using Storybook -To develop components using [Storybook](https://storybook.js.org/) run `npm run storybook`. Then navigate to in your browser to `http://localhost:6006`. +To develop components using [Storybook](https://storybook.js.org/) run +`npm run storybook`. Then navigate to in your browser to +`http://localhost:6006`. -To create a new story create a file named `[componentName].stories.js`. An example how such a story could look like, see `Header.stories.jsx`. +To create a new story create a file named `[componentName].stories.js`. An +example how such a story could look like, see `Header.stories.jsx`. ## Code Layout @@ -82,11 +90,12 @@ To create a new story create a file named `[componentName].stories.js`. An examp All react code is under `src/` with a few sub directories: -1. `pages/`: All pages a user could navigate too and API URLs which are under `pages/api/`. -1. `components/`: All re-usable React components. If something gets used - twice we should create a component and put it here. -1. `lib/`: A generic place to store library files that are used anywhere. - This doesn't have much structure yet. +1. `pages/`: All pages a user could navigate too and API URLs which are under + `pages/api/`. +1. `components/`: All re-usable React components. If something gets used twice + we should create a component and put it here. +1. `lib/`: A generic place to store library files that are used anywhere. This + doesn't have much structure yet. NOTE: `styles/` can be ignored for now. @@ -104,16 +113,27 @@ We're not really using CSS styles. `styles/` can be ignored. ## Testing the UI -Cypress is used for end-to-end (e2e) and component testing and is configured in `./cypress.config.ts`. The `./cypress` folder is used for supporting configuration files etc. +Cypress is used for end-to-end (e2e) and component testing and is configured in +`./cypress.config.ts`. The `./cypress` folder is used for supporting +configuration files etc. - Store e2e tests in the `./cypress/e2e` folder. -- Store component tests adjacent to the component being tested. If you want to wriite a test for `./src/components/Layout.tsx` then store the test file at `./src/components/Layout.cy.tsx`. +- Store component tests adjacent to the component being tested. If you want to + wriite a test for `./src/components/Layout.tsx` then store the test file at + `./src/components/Layout.cy.tsx`. A few npm scripts are available for convenience: -- `npm run cypress`: Useful for development, it opens Cypress and allows you to explore, run and debug tests. It assumes you have the NextJS site running at `localhost:3000`. -- `npm run cypress:run`: Runs all tests. Useful for a quick sanity check before sending a PR or to run in CI pipelines. -- `npm run cypress:image-baseline`: If you have tests failing because of visual changes that was expected, this command will update the baseline images stored in `./cypress-visual-screenshots/baseline` with those from the adjacent comparison folder. More can be found in the [docs of `uktrade/cypress-image-diff`](https://github.com/uktrade/cypress-image-diff/blob/main/docs/CLI.md#update-all-baseline-images-for-failing-tests). +- `npm run cypress`: Useful for development, it opens Cypress and allows you to + explore, run and debug tests. It assumes you have the NextJS site running at + `localhost:3000`. +- `npm run cypress:run`: Runs all tests. Useful for a quick sanity check before + sending a PR or to run in CI pipelines. +- `npm run cypress:image-baseline`: If you have tests failing because of visual + changes that was expected, this command will update the baseline images stored + in `./cypress-visual-screenshots/baseline` with those from the adjacent + comparison folder. More can be found in the + [docs of `uktrade/cypress-image-diff`](https://github.com/uktrade/cypress-image-diff/blob/main/docs/CLI.md#update-all-baseline-images-for-failing-tests). Read more in the [./cypress README](cypress/). @@ -125,9 +145,9 @@ When writing code for the website, we have a few best practices: dependencies. Order them alphabetically according to the package name. 1. When trying to implement something new, check if [Chakra-UI](https://chakra-ui.com/) has components that are close enough to - your need. For example Sliders, Radio Buttons, Progress indicators, etc. They - have a lot and we can save time by re-using what they have and tweaking the - style as needed. + your need. For example Sliders, Radio Buttons, Progress indicators, etc. + They have a lot and we can save time by re-using what they have and tweaking + the style as needed. 1. Format everything with [Prettier](https://prettier.io/). This is done by default with pre-submits. We currently don't have any custom settings. 1. Define functional React components (with types for all properties when @@ -135,14 +155,15 @@ When writing code for the website, we have a few best practices: ### URL Paths -To use stable and consistent URL paths, we recommend the following strategy for new tasks: +To use stable and consistent URL paths, we recommend the following strategy for +new tasks: 1. For any task that involves writing a free-form response, put the page under `website/src/pages/create` with a page name matching the task type, such as `summarize_story.tsx`. 1. For any task that evaluates, rates, or ranks content, put the page under - `website/src/pages/evaluate` with a page name matching the task type such - as `rate_summary.tsx`. + `website/src/pages/evaluate` with a page name matching the task type such as + `rate_summary.tsx`. With this we'll be able to ensure these contribution pages are hidden from logged out users but accessible to logged in users. @@ -151,5 +172,6 @@ logged out users but accessible to logged in users. To learn more about Next.js, take a look at the following resources: -- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. +- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js + features and API. - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. diff --git a/website/cypress.config.js b/website/cypress.config.js index 9610624f..7d391f23 100644 --- a/website/cypress.config.js +++ b/website/cypress.config.js @@ -22,4 +22,11 @@ export default defineConfig({ getCompareSnapshotsPlugin(on, config); }, }, + + env: { + MAILDEV_PROTOCOL: "http", + MAILDEV_HOST: "localhost", + MAILDEV_SMTP_PORT: "1025", + MAILDEV_API_PORT: "1080", + }, }); diff --git a/website/cypress/README.md b/website/cypress/README.md index 12a32378..4750cbf6 100644 --- a/website/cypress/README.md +++ b/website/cypress/README.md @@ -1,14 +1,24 @@ # Component and e2e testing with Cypress -[Cypress](https://www.cypress.io/) is used for both component- and end-to-end testing. Below there's a few examples for the context of this site. To learn more, the [Cypress documentation](https://docs.cypress.io/guides/getting-started/opening-the-app) has it all. +[Cypress](https://www.cypress.io/) is used for both component- and end-to-end +testing. Below there's a few examples for the context of this site. To learn +more, the +[Cypress documentation](https://docs.cypress.io/guides/getting-started/opening-the-app) +has it all. -Don't get scared by the commercial offerings they offer. Their core is open source, the cloud offering is not necesarry at all and can be replaced by CI tooling and [community efforts](https://sorry-cypress.dev/). +Don't get scared by the commercial offerings they offer. Their core is open +source, the cloud offering is not necesarry at all and can be replaced by CI +tooling and [community efforts](https://sorry-cypress.dev/). # Component testing -To write a new component test, you either create a new `.tsx` adjacent to the component you want to test or you can use the guide presented yo you when running `npm run cypress` which allows you to easily create the skeleton test for an existing component. +To write a new component test, you either create a new `.tsx` adjacent to the +component you want to test or you can use the guide presented yo you when +running `npm run cypress` which allows you to easily create the skeleton test +for an existing component. -If you have a `Button.tsx` component, create a file next to it called `Button.cy.tsx` which could look like this: +If you have a `Button.tsx` component, create a file next to it called +`Button.cy.tsx` which could look like this: ```typescript import React from "react"; @@ -25,25 +35,36 @@ describe(" + + + + + + +
+ +
    + {TEXT_LABEL_FLAGS.map((option, i) => { + return ( + + ); + })} +
+
+ +
+ +
+ + + ); +}; +function FlagCheckboxLi(props: { + option: textFlagLabels; + idx: number; + checkboxValues: boolean[]; + sliderValues: number[]; + checkboxHandler: (newVal: boolean, idx: number) => void; + sliderHandler: (newVal: number, idx: number) => void; +}): JSX.Element { + let AdditionalExplanation = null; + if (props.option.additionalExplanation) { + AdditionalExplanation = ( + + + ); + } + + return ( +
  • + + { + props.checkboxHandler(e.target.checked, props.idx); + }} + /> + + + { + props.sliderHandler(val / 100, props.idx); + }} + > + + + + + + +
    • + ); +} +interface textFlagLabels { + attributeName: string; + labelText: string; + additionalExplanation?: string; +} +const TEXT_LABEL_FLAGS: textFlagLabels[] = [ + // For the time being this list is configured on the FE. + // In the future it may be provided by the API. + { + attributeName: "fails_task", + labelText: "Fails to follow the correct instruction / task", + additionalExplanation: "__TODO__", + }, + { + attributeName: "not_customer_assistant_appropriate", + labelText: "Inappropriate for customer assistant", + additionalExplanation: "__TODO__", + }, + { + attributeName: "contains_sexual_content", + labelText: "Contains sexual content", + }, + { + attributeName: "contains_violent_content", + labelText: "Contains violent content", + }, + { + attributeName: "encourages_violence", + labelText: "Encourages or fails to discourage violence/abuse/terrorism/self-harm", + }, + { + attributeName: "denigrates_a_protected_class", + labelText: "Denigrates a protected class", + }, + { + attributeName: "gives_harmful_advice", + labelText: "Fails to follow the correct instruction / task", + additionalExplanation: + "The advice given in the output is harmful or counter-productive. This may be in addition to, but is distinct from the question about encouraging violence/abuse/terrorism/self-harm.", + }, + { + attributeName: "expresses_moral_judgement", + labelText: "Expresses moral judgement", + }, +]; diff --git a/website/src/components/Header/UserMenu.tsx b/website/src/components/Header/UserMenu.tsx index c42d8895..cbd71787 100644 --- a/website/src/components/Header/UserMenu.tsx +++ b/website/src/components/Header/UserMenu.tsx @@ -34,7 +34,9 @@ export function UserMenu() { height="40" className="rounded-full" > -

    {session.user.name || session.user.email}

    +

    + {session.user.name || session.user.email} +

    diff --git a/website/src/components/Messages.tsx b/website/src/components/Messages.tsx index 4bc747d5..cc94fcc0 100644 --- a/website/src/components/Messages.tsx +++ b/website/src/components/Messages.tsx @@ -1,3 +1,5 @@ +import { FlaggableElement } from "./FlaggableElement"; + export interface Message { text: string; is_assistant: boolean; @@ -5,11 +7,18 @@ export interface Message { const getColor = (isAssistant: boolean) => (isAssistant ? "bg-slate-800" : "bg-sky-900"); -export const Messages = ({ messages }: { messages: Message[] }) => { +export const Messages = ({ messages, post_id }: { messages: Message[]; post_id: string }) => { const items = messages.map(({ text, is_assistant }: Message, i: number) => { return ( -
    - {text} +
    + +
    + {text} +
    +
    ); }); diff --git a/website/src/components/TaskInfo/TaskInfo.tsx b/website/src/components/TaskInfo/TaskInfo.tsx index fa16615e..d32f7d9d 100644 --- a/website/src/components/TaskInfo/TaskInfo.tsx +++ b/website/src/components/TaskInfo/TaskInfo.tsx @@ -2,7 +2,7 @@ export const TaskInfo = ({ id, output }: { id: string; output: string }) => { return (
    Prompt - {id} + {id} Output {output}
    diff --git a/website/src/pages/auth/signin.tsx b/website/src/pages/auth/signin.tsx index 2ead2414..5bfeeb3b 100644 --- a/website/src/pages/auth/signin.tsx +++ b/website/src/pages/auth/signin.tsx @@ -43,8 +43,14 @@ export default function Signin({ csrfToken, providers }) { {email && (
    - - diff --git a/website/src/pages/create/assistant_reply.tsx b/website/src/pages/create/assistant_reply.tsx index 54badd71..4941c106 100644 --- a/website/src/pages/create/assistant_reply.tsx +++ b/website/src/pages/create/assistant_reply.tsx @@ -61,9 +61,9 @@ const AssistantReply = () => { <>
    Reply as the assistant

    Given the following conversation, provide an adequate reply

    - + -