Merge branch 'main' into dark-mode-implementation

.devcontainer/devcontainer.json

@@ -0,0 +1,10 @@
+{
+  "service": "frontend-dev",
+  "dockerComposeFile": "../docker-compose.yaml",
+  "forwardPorts": [3000],
+  "customizations": {
+    "vscode": {
+      "extensions": ["GitHub.copilot"]
+    }
+  }
+}

.dockerignore

@@ -0,0 +1 @@
+**/node_modules

.github/workflows/test-api-contract.yaml

@@ -18,18 +18,13 @@ jobs:
 
       - run: cd oasst-shared && pip install -e .
 
+      - run: cd oasst-shared && pip install -r requirements.dev.txt
+
       - run: cd backend && pip install -r requirements.txt
 
-      - run: cd discord-bot && pip install -r requirements.txt
-
-      - run: cd discord-bot && pip install -r requirements.dev.txt
-
       - run: ./scripts/backend-development/start-mock-server.sh
 
-      # runs the contract tests. currently the api client is
-      # found in the discord bot code, but this should be updated
-      # once the client moves into oasst-shared.
       - name: Run contract tests
-        run: ./scripts/discord-bot-development/test.sh
+        run: ./scripts/oasst-shared-development/test.sh
 
       - run: ./scripts/backend-development/stop-mock-server.sh

.pre-commit-config.yaml

@@ -1,3 +1,31 @@
+# WARNING!
+#
+# When making changes to auto-formatters used in pre-commit hooks, you are
+# likely to cause merge conflicts with main and/or other pull requests.
+# Fixing them might revert other people's work. Expect pain!
+# To avoid accidental reversions and keep it easy to review, please make sure
+# that changes here are in a pull request by themselves, that it consists of
+# two commits:
+#
+# 1. The changes to this file
+# 2. Changes made by running `python3 -m pre_commit run --all-files`.
+#
+# Then each time your pull request is blocked by a merge conflict, do the
+# following steps:
+#
+#    git reset HEAD^1 && git checkout -f    # discard the change commit
+#    git rebase main                        # re-apply other people's changes
+#    python3 -m pre_commit run --all-files  # re-run the rules
+#    git add .                              # add the newly changed files
+#    git commit -m 'apply pre-commit'       # commit it
+#    git push -f                            # force push back to your branch
+#
+# Keep in mind you may have to do this a few times, as changes here may impact
+# other pull requests. Try to keep it up-to-date so they can go in when it'll
+# cause least disruption.
+#
+# /WARNING!
+
 exclude: "build|stubs|^bot/templates/|^notebooks/.*\\.ipynb$"
 
 default_language_version:
@@ -19,6 +47,7 @@ repos:
       - id: check-case-conflict
       - id: detect-private-key
       - id: fix-encoding-pragma
+        args: ["--remove"]
       - id: forbid-submodules
       - id: mixed-line-ending
       - id: requirements-txt-fixer

README.md

@@ -10,11 +10,11 @@ improving language itself.
 
 ## Do you want to try it out?
 
-If you are interested in taking a look at the current state of the project, You
+If you are interested in taking a look at the current state of the project, you
 can set up an entire stack needed to run **Open-Assistant**, including the
 website, backend, and associated dependent services.
 
-To start the demo, Run this in the root directory of the repository:
+To start the demo, run this in the root directory of the repository:
 
 ```sh
 docker compose up --build
@@ -67,7 +67,7 @@ hardware.
 
 ## How can you help?
 
-All open source projects begins with people like you. Open source is the belief
+All open source projects begin with people like you. Open source is the belief
 that if we collaborate we can together gift our knowledge and technology to the
 world for the benefit of humanity.
 
@@ -181,13 +181,3 @@ Upon making a release on GitHub, all docker images are automatically built and
 pushed to ghcr.io. The docker images are tagged with the release version, and
 the `latest` tag. Further, the ansible playbook in `ansible/dev.yaml` is run to
 automatically deploy the built release to the dev machine.
-
-### Problems and Solutions
-
-- **I am on Ubuntu and getting
-  `ERROR: The Compose file is invalid because:Service backend has neither an image nor a build context specified. At least one must be provided.`**
-
-  Make sure you have an up-to-date version of docker installed, and also install
-  `docker-compose-plugin`. See
-  [here](https://github.com/LAION-AI/Open-Assistant/issues/208) for more
-  details.

backend/alembic/env.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from logging.config import fileConfig
 
 import sqlmodel

backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """first revision
 
 Revision ID: 23e5fea252dd

backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """v1 db structure
 
 Revision ID: cd7de470586e

backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """add auth_method to person
 
 Revision ID: 6368515778c5

backend/alembic/versions/2022_12_22_1835-0daec5f8135f_add_auth_method_to_ix_person_username.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """add_auth_method_to_ix_person_username
 
 Revision ID: 0daec5f8135f

backend/alembic/versions/2022_12_25_1705-067c4002f2d9_add_text_labels.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Adds text labels table.
 
 Revision ID: 067c4002f2d9

backend/alembic/versions/2022_12_27_1444-3358eb6834e6_add_journal_table.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """add_journal_table
 
 Revision ID: 3358eb6834e6

backend/alembic/versions/2022_12_28_1142-d24b37426857_post_ref_for_work_package.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """post ref for work_package
 
 Revision ID: d24b37426857

backend/alembic/versions/2022_12_28_1824-ef0b52902560_added_lang_column_for_iso_639_1_codes.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Added lang column for ISO-639-1 codes
 
 Revision ID: ef0b52902560

backend/alembic/versions/2022_12_29_2103-464ec4667aae_add_collective_flag_to_task.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """add collective flag to task
 
 Revision ID: 464ec4667aae

backend/alembic/versions/2022_12_30_0109-73ce3675c1f5_add_field_trusted_api_client.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """add field trusted api client
 
 Revision ID: 73ce3675c1f5

backend/alembic/versions/2022_12_30_2054-abb47e9d145a_name_changes_person_user_post_message_.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """name changes: person->user, post->message, work_package->task
 
 Revision ID: abb47e9d145a

backend/alembic/versions/2022_12_31_0438-8d269bc4fdbd_add_deleted_field_to_post.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """add deleted field to post
 
 Revision ID: 8d269bc4fdbd

backend/main.py

@@ -1,5 +1,5 @@
-# -*- coding: utf-8 -*-
 from http import HTTPStatus
+from math import ceil
 from pathlib import Path
 from typing import Optional
 
@@ -7,13 +7,15 @@ import alembic.command
 import alembic.config
 import fastapi
 import pydantic
+import redis.asyncio as redis
+from fastapi_limiter import FastAPILimiter
 from loguru import logger
 from oasst_backend.api.deps import get_dummy_api_client
 from oasst_backend.api.v1.api import api_router
 from oasst_backend.config import settings
 from oasst_backend.database import engine
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
 from sqlmodel import Session
 from starlette.middleware.cors import CORSMiddleware
@@ -63,6 +65,29 @@ if settings.UPDATE_ALEMBIC:
             logger.exception("Alembic upgrade failed on startup")
 
 
+if settings.RATE_LIMIT:
+
+    @app.on_event("startup")
+    async def connect_redis():
+        async def http_callback(request: fastapi.Request, response: fastapi.Response, pexpire: int):
+            """Error callback function when too many requests"""
+            expire = ceil(pexpire / 1000)
+            raise OasstError(
+                f"Too Many Requests. Retry After {expire} seconds.",
+                OasstErrorCode.TOO_MANY_REQUESTS,
+                HTTPStatus.TOO_MANY_REQUESTS,
+            )
+
+        try:
+            redis_client = redis.from_url(
+                f"redis://{settings.REDIS_HOST}:{settings.REDIS_PORT}/0", encoding="utf-8", decode_responses=True
+            )
+            logger.info(f"Connected to {redis_client=}")
+            await FastAPILimiter.init(redis_client, http_callback=http_callback)
+        except Exception:
+            logger.exception("Failed to establish Redis connection")
+
+
 if settings.DEBUG_USE_SEED_DATA:
 
     @app.on_event("startup")

backend/oasst_backend/api/deps.py

@@ -1,16 +1,16 @@
-# -*- coding: utf-8 -*-
 from http import HTTPStatus
 from secrets import token_hex
 from typing import Generator
 from uuid import UUID
 
-from fastapi import Depends, Security
+from fastapi import Depends, Request, Response, Security
 from fastapi.security.api_key import APIKey, APIKeyHeader, APIKeyQuery
+from fastapi_limiter.depends import RateLimiter
 from loguru import logger
 from oasst_backend.config import settings
 from oasst_backend.database import engine
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import ApiClient
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import Session
 
 
@@ -85,3 +85,58 @@ def get_trusted_api_client(
             http_status_code=HTTPStatus.FORBIDDEN,
         )
     return client
+
+
+class UserRateLimiter(RateLimiter):
+    def __init__(
+        self, times: int = 100, milliseconds: int = 0, seconds: int = 0, minutes: int = 1, hours: int = 0
+    ) -> None:
+        async def identifier(request: Request) -> str:
+            """Identify a request based on api_key and user.id"""
+            api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
+            user = (await request.json()).get("user")
+            return f"{api_key}:{user.get('id')}"
+
+        super().__init__(times, milliseconds, seconds, minutes, hours, identifier)
+
+    async def __call__(self, request: Request, response: Response, api_key: str = Depends(get_api_key)) -> None:
+        # Skip if rate limiting is disabled
+        if not settings.RATE_LIMIT:
+            return
+
+        # Attempt to retrieve api_key and user information
+        user = (await request.json()).get("user")
+
+        # Skip when api_key and user information are not available
+        # (such that it will be handled by `APIClientRateLimiter`)
+        if not api_key or not user or not user.get("id"):
+            return
+
+        return await super().__call__(request, response)
+
+
+class APIClientRateLimiter(RateLimiter):
+    def __init__(
+        self, times: int = 10_000, milliseconds: int = 0, seconds: int = 0, minutes: int = 1, hours: int = 0
+    ) -> None:
+        async def identifier(request: Request) -> str:
+            """Identify a request based on api_key and user.id"""
+            api_key = request.headers.get("X-API-Key") or request.query_params.get("api_key")
+            return f"{api_key}"
+
+        super().__init__(times, milliseconds, seconds, minutes, hours, identifier)
+
+    async def __call__(self, request: Request, response: Response, api_key: str = Depends(get_api_key)) -> None:
+        # Skip if rate limiting is disabled
+        if not settings.RATE_LIMIT:
+            return
+
+        # Attempt to retrieve api_key and user information
+        user = (await request.json()).get("user")
+
+        # Skip if user information is available
+        # (such that it will be handled by `UserRateLimiter`)
+        if not api_key or user:
+            return
+
+        return await super().__call__(request, response)

backend/oasst_backend/api/v1/api.py

@@ -1,6 +1,14 @@
-# -*- coding: utf-8 -*-
 from fastapi import APIRouter
-from oasst_backend.api.v1 import frontend_messages, frontend_users, messages, stats, tasks, text_labels, users
+from oasst_backend.api.v1 import (
+    frontend_messages,
+    frontend_users,
+    leaderboards,
+    messages,
+    stats,
+    tasks,
+    text_labels,
+    users,
+)
 
 api_router = APIRouter()
 api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"])
@@ -10,3 +18,4 @@ api_router.include_router(frontend_messages.router, prefix="/frontend_messages",
 api_router.include_router(users.router, prefix="/users", tags=["users"])
 api_router.include_router(frontend_users.router, prefix="/frontend_users", tags=["frontend_users"])
 api_router.include_router(stats.router, prefix="/stats", tags=["stats"])
+api_router.include_router(leaderboards.router, prefix="/experimental/leaderboards", tags=["leaderboards"])

backend/oasst_backend/api/v1/frontend_messages.py

@@ -1,18 +1,17 @@
-# -*- coding: utf-8 -*-
-
 from fastapi import APIRouter, Depends
 from oasst_backend.api import deps
 from oasst_backend.api.v1 import utils
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import ApiClient
 from oasst_backend.models.db_payload import MessagePayload
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
+from oasst_shared.schemas import protocol
 from sqlmodel import Session
 
 router = APIRouter()
 
 
-@router.get("/{message_id}")
+@router.get("/{message_id}", response_model=protocol.Message)
 def get_message_by_frontend_id(
     message_id: str, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -29,7 +28,7 @@ def get_message_by_frontend_id(
     return utils.prepare_message(message)
 
 
-@router.get("/{message_id}/conversation")
+@router.get("/{message_id}/conversation", response_model=protocol.Conversation)
 def get_conv_by_frontend_id(
     message_id: str, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -43,7 +42,7 @@ def get_conv_by_frontend_id(
     return utils.prepare_conversation(messages)
 
 
-@router.get("/{message_id}/tree")
+@router.get("/{message_id}/tree", response_model=protocol.MessageTree)
 def get_tree_by_frontend_id(
     message_id: str, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -57,7 +56,7 @@ def get_tree_by_frontend_id(
     return utils.prepare_tree(tree, message.message_tree_id)
 
 
-@router.get("/{message_id}/children")
+@router.get("/{message_id}/children", response_model=list[protocol.Message])
 def get_children_by_frontend_id(
     message_id: str, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -70,7 +69,7 @@ def get_children_by_frontend_id(
     return utils.prepare_message_list(messages)
 
 
-@router.get("/{message_id}/descendants")
+@router.get("/{message_id}/descendants", response_model=protocol.MessageTree)
 def get_descendants_by_frontend_id(
     message_id: str, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -84,7 +83,7 @@ def get_descendants_by_frontend_id(
     return utils.prepare_tree(descendants, message.id)
 
 
-@router.get("/{message_id}/longest_conversation_in_tree")
+@router.get("/{message_id}/longest_conversation_in_tree", response_model=protocol.Conversation)
 def get_longest_conv_by_frontend_id(
     message_id: str, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -98,7 +97,7 @@ def get_longest_conv_by_frontend_id(
     return utils.prepare_conversation(conv)
 
 
-@router.get("/{message_id}/max_children_in_tree")
+@router.get("/{message_id}/max_children_in_tree", response_model=protocol.MessageTree)
 def get_max_children_by_frontend_id(
     message_id: str, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):

backend/oasst_backend/api/v1/frontend_users.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 from uuid import UUID
 
@@ -7,14 +6,14 @@ from oasst_backend.api import deps
 from oasst_backend.api.v1 import utils
 from oasst_backend.models import ApiClient
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.schemas import protocol
 from sqlmodel import Session
-from starlette.responses import Response
-from starlette.status import HTTP_200_OK
+from starlette.status import HTTP_204_NO_CONTENT
 
 router = APIRouter()
 
 
-@router.get("/{username}/messages")
+@router.get("/{username}/messages", response_model=list[protocol.Message])
 def query_frontend_user_messages(
     username: str,
     api_client_id: UUID = None,
@@ -44,11 +43,10 @@ def query_frontend_user_messages(
     return utils.prepare_message_list(messages)
 
 
-@router.delete("/{username}/messages")
+@router.delete("/{username}/messages", status_code=HTTP_204_NO_CONTENT)
 def mark_frontend_user_messages_deleted(
     username: str, api_client: ApiClient = Depends(deps.get_trusted_api_client), db: Session = Depends(deps.get_db)
 ):
     pr = PromptRepository(db, api_client, None)
     messages = pr.query_messages(username=username, api_client_id=api_client.id)
     pr.mark_messages_deleted(messages)
-    return Response(status_code=HTTP_200_OK)

backend/oasst_backend/api/v1/leaderboards.py

@@ -0,0 +1,25 @@
+from fastapi import APIRouter, Depends
+from oasst_backend.api import deps
+from oasst_backend.models import ApiClient
+from oasst_backend.prompt_repository import PromptRepository
+from sqlmodel import Session
+
+router = APIRouter()
+
+
+@router.get("/create/assistant")
+def get_assistant_leaderboard(
+    db: Session = Depends(deps.get_db),
+    api_client: ApiClient = Depends(deps.get_trusted_api_client),
+):
+    pr = PromptRepository(db, api_client, None)
+    return pr.get_user_leaderboard(role="assistant")
+
+
+@router.get("/create/prompter")
+def get_prompter_leaderboard(
+    db: Session = Depends(deps.get_db),
+    api_client: ApiClient = Depends(deps.get_trusted_api_client),
+):
+    pr = PromptRepository(db, api_client, None)
+    return pr.get_user_leaderboard(role="prompter")

backend/oasst_backend/api/v1/messages.py

@@ -1,22 +1,21 @@
-# -*- coding: utf-8 -*-
-
 import datetime
 from uuid import UUID
 
-from fastapi import APIRouter, Depends, Query, Response
+from fastapi import APIRouter, Depends, Query
 from oasst_backend.api import deps
 from oasst_backend.api.v1 import utils
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import ApiClient
 from oasst_backend.models.db_payload import MessagePayload
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
+from oasst_shared.schemas import protocol
 from sqlmodel import Session
-from starlette.status import HTTP_200_OK
+from starlette.status import HTTP_204_NO_CONTENT
 
 router = APIRouter()
 
 
-@router.get("/")
+@router.get("/", response_model=list[protocol.Message])
 def query_messages(
     username: str = None,
     api_client_id: str = None,
@@ -47,7 +46,7 @@ def query_messages(
     return utils.prepare_message_list(messages)
 
 
-@router.get("/{message_id}")
+@router.get("/{message_id}", response_model=protocol.Message)
 def get_message(
     message_id: UUID, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -63,7 +62,7 @@ def get_message(
     return utils.prepare_message(message)
 
 
-@router.get("/{message_id}/conversation")
+@router.get("/{message_id}/conversation", response_model=protocol.Conversation)
 def get_conv(
     message_id: UUID, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -76,7 +75,7 @@ def get_conv(
     return utils.prepare_conversation(messages)
 
 
-@router.get("/{message_id}/tree")
+@router.get("/{message_id}/tree", response_model=protocol.MessageTree)
 def get_tree(
     message_id: UUID, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -89,7 +88,7 @@ def get_tree(
     return utils.prepare_tree(tree, message.message_tree_id)
 
 
-@router.get("/{message_id}/children")
+@router.get("/{message_id}/children", response_model=list[protocol.Message])
 def get_children(
     message_id: UUID, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -101,7 +100,7 @@ def get_children(
     return utils.prepare_message_list(messages)
 
 
-@router.get("/{message_id}/descendants")
+@router.get("/{message_id}/descendants", response_model=protocol.MessageTree)
 def get_descendants(
     message_id: UUID, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -114,7 +113,7 @@ def get_descendants(
     return utils.prepare_tree(descendants, message.id)
 
 
-@router.get("/{message_id}/longest_conversation_in_tree")
+@router.get("/{message_id}/longest_conversation_in_tree", response_model=protocol.Conversation)
 def get_longest_conv(
     message_id: UUID, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -127,7 +126,7 @@ def get_longest_conv(
     return utils.prepare_conversation(conv)
 
 
-@router.get("/{message_id}/max_children_in_tree")
+@router.get("/{message_id}/max_children_in_tree", response_model=protocol.MessageTree)
 def get_max_children(
     message_id: UUID, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db)
 ):
@@ -140,10 +139,9 @@ def get_max_children(
     return utils.prepare_tree([message, *children], message.id)
 
 
-@router.delete("/{message_id}")
+@router.delete("/{message_id}", status_code=HTTP_204_NO_CONTENT)
 def mark_message_deleted(
     message_id: UUID, api_client: ApiClient = Depends(deps.get_trusted_api_client), db: Session = Depends(deps.get_db)
 ):
     pr = PromptRepository(db, api_client, None)
     pr.mark_messages_deleted(message_id)
-    return Response(status_code=HTTP_200_OK)

backend/oasst_backend/api/v1/stats.py

@@ -1,14 +1,14 @@
-# -*- coding: utf-8 -*-
 from fastapi import APIRouter, Depends
 from oasst_backend.api import deps
 from oasst_backend.models import ApiClient
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.schemas import protocol
 from sqlmodel import Session
 
 router = APIRouter()
 
 
-@router.get("/")
+@router.get("/", response_model=protocol.SystemStats)
 def get_message_stats(
     db: Session = Depends(deps.get_db),
     api_client: ApiClient = Depends(deps.get_trusted_api_client),

backend/oasst_backend/api/v1/tasks.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import random
 from typing import Any, Optional, Tuple
 from uuid import UUID
@@ -7,10 +6,11 @@ from fastapi import APIRouter, Depends
 from fastapi.security.api_key import APIKey
 from loguru import logger
 from oasst_backend.api import deps
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
 from sqlmodel import Session
+from starlette.status import HTTP_204_NO_CONTENT
 
 router = APIRouter()
 
@@ -128,7 +128,14 @@ def generate_task(
     return task, message_tree_id, parent_message_id
 
 
-@router.post("/", response_model=protocol_schema.AnyTask)  # work with Union once more types are added
+@router.post(
+    "/",
+    response_model=protocol_schema.AnyTask,
+    dependencies=[
+        Depends(deps.UserRateLimiter(times=100, minutes=5)),
+        Depends(deps.APIClientRateLimiter(times=10_000, minutes=1)),
+    ],
+)  # work with Union once more types are added
 def request_task(
     *,
     db: Session = Depends(deps.get_db),
@@ -153,14 +160,14 @@ def request_task(
     return task
 
 
-@router.post("/{task_id}/ack", response_model=None)
+@router.post("/{task_id}/ack", response_model=None, status_code=HTTP_204_NO_CONTENT)
 def tasks_acknowledge(
     *,
     db: Session = Depends(deps.get_db),
     api_key: APIKey = Depends(deps.get_api_key),
     task_id: UUID,
     ack_request: protocol_schema.TaskAck,
-) -> Any:
+) -> None:
     """
     The frontend acknowledges a task.
     """
@@ -181,14 +188,14 @@ def tasks_acknowledge(
         raise OasstError("Failed to acknowledge task.", OasstErrorCode.TASK_ACK_FAILED)
 
 
-@router.post("/{task_id}/nack", response_model=None)
+@router.post("/{task_id}/nack", response_model=None, status_code=HTTP_204_NO_CONTENT)
 def tasks_acknowledge_failure(
     *,
     db: Session = Depends(deps.get_db),
     api_key: APIKey = Depends(deps.get_api_key),
     task_id: UUID,
     nack_request: protocol_schema.TaskNAck,
-) -> Any:
+) -> None:
     """
     The frontend reports failure to implement a task.
     """
@@ -259,7 +266,7 @@ def tasks_interaction(
         raise OasstError("Interaction request failed.", OasstErrorCode.TASK_INTERACTION_REQUEST_FAILED)
 
 
-@router.post("/close")
+@router.post("/close", response_model=protocol_schema.TaskDone)
 def close_collective_task(
     close_task_request: protocol_schema.TaskClose,
     db: Session = Depends(deps.get_db),

backend/oasst_backend/api/v1/text_labels.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import pydantic
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.security.api_key import APIKey
@@ -7,7 +6,7 @@ from oasst_backend.api import deps
 from oasst_backend.prompt_repository import PromptRepository
 from oasst_shared.schemas import protocol as protocol_schema
 from sqlmodel import Session
-from starlette.status import HTTP_400_BAD_REQUEST
+from starlette.status import HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST
 
 router = APIRouter()
 
@@ -17,7 +16,7 @@ class LabelTextRequest(pydantic.BaseModel):
     user: protocol_schema.User
 
 
-@router.post("/")
+@router.post("/", status_code=HTTP_204_NO_CONTENT)
 def label_text(
     *,
     db: Session = Depends(deps.get_db),

backend/oasst_backend/api/v1/users.py

@@ -1,20 +1,19 @@
-# -*- coding: utf-8 -*-
 import datetime
 from uuid import UUID
 
 from fastapi import APIRouter, Depends, Query
 from oasst_backend.api import deps
+from oasst_backend.api.v1 import utils
 from oasst_backend.models import ApiClient
 from oasst_backend.prompt_repository import PromptRepository
 from oasst_shared.schemas import protocol
 from sqlmodel import Session
-from starlette.responses import Response
-from starlette.status import HTTP_200_OK
+from starlette.status import HTTP_204_NO_CONTENT
 
 router = APIRouter()
 
 
-@router.get("/{user_id}/messages")
+@router.get("/{user_id}/messages", response_model=list[protocol.Message])
 def query_user_messages(
     user_id: UUID,
     api_client_id: UUID = None,
@@ -42,19 +41,13 @@ def query_user_messages(
         deleted=None if include_deleted else False,
     )
 
-    return [
-        protocol.Message(
-            id=m.id, parent_id=m.parent_id, text=m.payload.payload.text, is_assistant=(m.role == "assistant")
-        )
-        for m in messages
-    ]
+    return utils.prepare_message_list(messages)
 
 
-@router.delete("/{user_id}/messages")
+@router.delete("/{user_id}/messages", status_code=HTTP_204_NO_CONTENT)
 def mark_user_messages_deleted(
     user_id: UUID, api_client: ApiClient = Depends(deps.get_trusted_api_client), db: Session = Depends(deps.get_db)
 ):
     pr = PromptRepository(db, api_client, None)
     messages = pr.query_messages(user_id=user_id)
     pr.mark_messages_deleted(messages)
-    return Response(status_code=HTTP_200_OK)

backend/oasst_backend/api/v1/utils.py

@@ -1,11 +1,9 @@
-# -*- coding: utf-8 -*-
-
 from http import HTTPStatus
 from uuid import UUID
 
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import Message
 from oasst_backend.models.db_payload import MessagePayload
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol
 
 

backend/oasst_backend/config.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from typing import Any, Dict, List, Optional, Union
 
 from pydantic import AnyHttpUrl, BaseSettings, PostgresDsn, validator
@@ -15,6 +14,10 @@ class Settings(BaseSettings):
     POSTGRES_DB: str = "postgres"
     DATABASE_URI: Optional[PostgresDsn] = None
 
+    RATE_LIMIT: bool = True
+    REDIS_HOST: str = "localhost"
+    REDIS_PORT: str = "6379"
+
     DEBUG_ALLOW_ANY_API_KEY: bool = False
     DEBUG_SKIP_API_KEY_CHECK: bool = False
     DEBUG_USE_SEED_DATA: bool = False

backend/oasst_backend/crud/__init__.py

@@ -1,2 +1 @@
-# -*- coding: utf-8 -*-
 __all__ = []

backend/oasst_backend/crud/base.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from typing import Any, Dict, Generic, List, Optional, Type, TypeVar, Union
 
 from fastapi.encoders import jsonable_encoder

backend/oasst_backend/database.py

@@ -1,6 +1,5 @@
-# -*- coding: utf-8 -*-
 from oasst_backend.config import settings
-from oasst_backend.exceptions import OasstError, OasstErrorCode
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import create_engine
 
 if settings.DATABASE_URI is None:

backend/oasst_backend/journal_writer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import enum
 from typing import Literal, Optional
 from uuid import UUID

backend/oasst_backend/models/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from .api_client import ApiClient
 from .journal import Journal, JournalIntegration
 from .message import Message

backend/oasst_backend/models/api_client.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from typing import Optional
 from uuid import UUID, uuid4
 

backend/oasst_backend/models/db_payload.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from typing import Literal
 
 from oasst_backend.models.payload_column_type import payload_type

backend/oasst_backend/models/journal.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from datetime import datetime
 from typing import Optional
 from uuid import UUID, uuid1, uuid4

backend/oasst_backend/models/message.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from datetime import datetime
 from typing import Optional
 from uuid import UUID, uuid4

backend/oasst_backend/models/message_reaction.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from datetime import datetime
 from typing import Optional
 from uuid import UUID

backend/oasst_backend/models/payload_column_type.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import json
 from typing import Any, Generic, Type, TypeVar
 

backend/oasst_backend/models/task.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from datetime import datetime
 from typing import Optional
 from uuid import UUID, uuid4

backend/oasst_backend/models/text_labels.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from datetime import datetime
 from typing import Optional
 from uuid import UUID, uuid4

backend/oasst_backend/models/user.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from datetime import datetime
 from typing import Optional
 from uuid import UUID, uuid4

backend/oasst_backend/models/user_stats.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from datetime import datetime
 from typing import Optional
 from uuid import UUID

backend/oasst_backend/prompt_repository.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import datetime
 import random
 from collections import defaultdict
@@ -8,12 +7,12 @@ from uuid import UUID, uuid4
 
 import oasst_backend.models.db_payload as db_payload
 from loguru import logger
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.journal_writer import JournalWriter
 from oasst_backend.models import ApiClient, Message, MessageReaction, Task, TextLabels, User
 from oasst_backend.models.payload_column_type import PayloadContainer
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
-from oasst_shared.schemas.protocol import SystemStats
+from oasst_shared.schemas.protocol import LeaderboardStats, SystemStats
 from sqlalchemy import update
 from sqlmodel import Session, func
 from starlette.status import HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND
@@ -706,3 +705,24 @@ class PromptRepository:
             deleted=result.get(True, 0),
             message_trees=result.get(None, 0),
         )
+
+    def get_user_leaderboard(self, role: str) -> LeaderboardStats:
+        """
+        Get leaderboard stats for Messages created,
+        separate leaderboard for prompts & assistants
+
+        """
+        query = (
+            self.db.query(Message.user_id, User.username, User.display_name, func.count(Message.user_id))
+            .join(User, User.id == Message.user_id, isouter=True)
+            .filter(Message.deleted is not True, Message.role == role)
+            .group_by(Message.user_id, User.username, User.display_name)
+            .order_by(func.count(Message.user_id).desc())
+        )
+
+        result = [
+            {"ranking": i, "user_id": j[0], "username": j[1], "display_name": j[2], "score": j[3]}
+            for i, j in enumerate(query.all(), start=1)
+        ]
+
+        return LeaderboardStats(leaderboard=result)

backend/requirements.txt

@@ -1,5 +1,6 @@
 alembic==1.8.1
 fastapi==0.88.0
+fastapi-limiter==0.1.5
 loguru==0.6.0
 numpy==1.22.4
 psycopg2-binary==2.9.5

discord-bot/README.md

@@ -9,7 +9,7 @@ the bot's outputs. If you want to learn more about RLHF please refer
 ## Invite official bot
 
 To add the official Open-Assistant data collection bot to your discord server
-[click here](https://discord.com/api/oauth2/authorize?client_id=1054078345542910022&permissions=1634235579456&scope=bot).
+[click here](https://discord.com/api/oauth2/authorize?client_id=1054078345542910022&permissions=1634235579456&scope=bot%20applications.commands).
 The bot needs access to read the contents of user text messages.
 
 ## Contributing
@@ -17,7 +17,41 @@ The bot needs access to read the contents of user text messages.
 If you are unfamiliar with `hikari`, `lightbulb`, or `miru`, please refer to the
 [large list of examples](https://gist.github.com/AlexanderHOtt/7805843a7120f755938a3b75d680d2e7)
 
-### Setup
+### Bot Setup
+
+1. Create a new discord application at the
+   [Discord Developer Portal](https://discord.com/developers/applications)
+
+1. Go to the "Bot" tab and create a new bot
+
+1. Scroll down to "Privileged Gateway Intents" and enable the following options:
+
+   - Server Members Intent
+   - Presence Intent
+   - Message Content Intent
+
+This page also contains the bot token, which you will need to add to the `.env`
+file later.
+
+2. Go to the "OAuth2" tab scroll to "Default Authorization Link"
+
+3. Set "AUTHORIZATION METHOD" to "In-app Authorization"
+
+4. Select the "bot" and "applications.commands" scopes
+
+5. For testing and local development, it's easiest to set "BOT PERMISSIONS" to
+   "Administrator"
+
+Remember to save your changes.
+
+6. Copy the "CLIENT ID" from the top of the page and replace it in the link
+   below to invite your bot.
+
+```
+https://discord.com/oauth2/authorize?client_id=YOUR_ID_HERE&permissions=8&scope=bot%20applications.commands
+```
+
+### Environment Setup
 
 To run the bot:
 
@@ -29,12 +63,15 @@ pip install -e .
 ```
 
 ```bash
-cd ../discord-bot
 cp .env.example .env
 
+# edit .env and add your bot token and other values
+
 python -V  # 3.10
 
 pip install -r requirements.txt
+
+# in the discord-bot folder
 python -m bot
 ```
 
@@ -54,16 +91,6 @@ git add .
 git commit -m "<good commit message>"
 ```
 
-To test the bot on your own discord server you need to register a discord
-application at the
-[Discord Developer Portal](https://discord.com/developers/applications) and get
-at bot token.
-
-1. Follow a tutorial on how to get a bot token, for example this one:
-   [Creating a discord bot & getting a token](https://github.com/reactiflux/discord-irc/wiki/Creating-a-discord-bot-&-getting-a-token)
-2. The bot script expects the bot token to be in the `.env` file under the
-   `TOKEN` variable.
-
 ### Resources
 
 #### Structure

discord-bot/bot/__init__.py

@@ -1,2 +1 @@
-# -*- coding: utf-8 -*-
 """The official Open-Assistant Discord Bot."""

discord-bot/bot/__main__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Entry point for the bot."""
 import logging
 import os

discord-bot/bot/bot.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Bot logic."""
 from datetime import datetime
 
@@ -6,9 +5,9 @@ import aiosqlite
 import hikari
 import lightbulb
 import miru
-from bot.api_client import OasstApiClient
 from bot.settings import Settings
 from bot.utils import EMPTY, mention
+from oasst_shared.api_client import OasstApiClient
 
 settings = Settings()
 
@@ -35,6 +34,9 @@ async def on_starting(event: hikari.StartingEvent):
 
     bot.d.oasst_api = OasstApiClient(settings.oasst_api_url, settings.oasst_api_key)
 
+    # A set of user id's that are currently doing work.
+    bot.d.currently_working = set()
+
 
 @bot.listen()
 async def on_stopping(event: hikari.StoppingEvent):

discord-bot/bot/db/schemas.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Database schemas."""
 import typing as t
 

discord-bot/bot/extensions/__init__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Extensions for the bot.
 
 See: https://hikari-lightbulb.readthedocs.io/en/latest/guides/extensions.html

discord-bot/bot/extensions/guild_settings.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Guild settings."""
 import hikari
 import lightbulb

discord-bot/bot/extensions/hot_reload.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Hot reload plugin."""
 from glob import glob
 

discord-bot/bot/extensions/text_labels.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Hot reload plugin."""
 import typing as t
 from datetime import datetime

discord-bot/bot/extensions/user_input_test.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Task plugin for testing different data collection methods."""
 # TODO: Delete this once user input method has been decided for final bot.
 import asyncio

discord-bot/bot/extensions/work.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Work plugin for collecting user data."""
 import asyncio
 import typing as t
@@ -9,10 +8,10 @@ import lightbulb
 import lightbulb.decorators
 import miru
 from aiosqlite import Connection
-from bot.api_client import OasstApiClient, TaskType
 from bot.db.schemas import GuildSettings
 from bot.utils import EMPTY
 from loguru import logger
+from oasst_shared.api_client import OasstApiClient, TaskType
 from oasst_shared.schemas import protocol as protocol_schema
 from oasst_shared.schemas.protocol import TaskRequestType
 
@@ -35,12 +34,25 @@ MAX_TASK_ACCEPT_TIME = 60  # 1 minute
 @lightbulb.implements(lightbulb.SlashCommand)
 async def work(ctx: lightbulb.SlashContext):
     """Create and handle a task."""
+    # make sure the user isn't currently doing a task
+    currently_working: set[hikari.Snowflakeish] = ctx.bot.d.currently_working
+    if ctx.author.id in currently_working:
+        await ctx.respond(
+            "You are already performing a task. Please complete that one first.", flags=hikari.MessageFlag.EPHEMERAL
+        )
+        return
+
+    currently_working.add(ctx.author.id)
+
     task_type: TaskRequestType = TaskRequestType(ctx.options.type.split(".")[-1])
 
     await ctx.respond("Sending you a task, check your DMs", flags=hikari.MessageFlag.EPHEMERAL)
     logger.debug(f"Starting task_type: {task_type!r}")
 
-    await _handle_task(ctx, task_type)
+    try:
+        await _handle_task(ctx, task_type)
+    finally:
+        currently_working.remove(ctx.author.id)
 
 
 async def _handle_task(ctx: lightbulb.SlashContext, task_type: TaskRequestType) -> None:

discord-bot/bot/settings.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Configuration for the bot."""
 from pydantic import BaseSettings, Field
 

discord-bot/bot/utils.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Utility functions."""
 import typing as t
 from datetime import datetime

discord-bot/message_templates.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Message templates for the discord bot."""
 import typing
 

discord-bot/requirements.txt

@@ -6,6 +6,6 @@ hikari-lightbulb    # command handler
 hikari-miru         # modals and buttons
 hikari[speedups]
 loguru
-pydantic
+pydantic[dotenv]
 
 uvloop; os_name != 'nt'  # Faster drop-in replacement for asyncio event loop

docker-compose.yaml

@@ -4,7 +4,7 @@ services:
   # Use `docker compose up backend-dev --build --attach-dependencies` to start a database and work and the backend.
   backend-dev:
     image: sverrirab/sleep
-    depends_on: [db, adminer]
+    depends_on: [db, adminer, redis, redis-insights]
 
   # Use `docker compose up frontend-dev --build --attach-dependencies` to start all services needed to work on the frontend.
   frontend-dev:
@@ -87,9 +87,11 @@ services:
   backend:
     build:
       dockerfile: docker/Dockerfile.backend
+      context: .
     image: oasst-backend
     environment:
       - POSTGRES_HOST=db
+      - REDIS_HOST=redis
       - DEBUG_SKIP_API_KEY_CHECK=True
       - DEBUG_USE_SEED_DATA=True
       - MAX_WORKERS=1
@@ -103,6 +105,7 @@ services:
   web:
     build:
       dockerfile: docker/Dockerfile.website
+      context: .
     image: oasst-web
     environment:
       - DATABASE_URL=postgres://postgres:postgres@webdb/oasst_web

docs/data_argumentation.md

@@ -1,23 +0,0 @@
-# Data Argumentation
-
-(pull request welcome)
-
-## What is data argumentation
-
-Data argumentation is a technique we can use to get better data faster. Using
-machine learning models analize long data (like an essay) and compress it into
-intructions.
-
-## How to contribute
-
-To contribute to data argumentation you can write a short python script that
-uses a model from huggingface to analize the text.
-[Here](https://docs.google.com/document/d/13a188pPvqnlvuVa3e_suVz4YO5s-JWeiOOrpp0odImg/edit)
-are examples of what you can do
-
-And here are example implementations:
-[Idea 3, ](https://colab.research.google.com/drive/1GllCN5PgSYxBxINZsv3A2r0SpdznHlbT?usp=sharing)
-[Idea 4](https://colab.research.google.com/drive/1nZx5LRjO61fYprFyqtrwPDLOis6ctR4p#scrollTo=1EE8CriiaCXj)
-
-To contribute simple choose one of many ideas from the document above and
-implement it.

docs/data_augmentation.md

@@ -0,0 +1,23 @@
+# Data Augmentation
+
+(pull request welcome)
+
+## What is data augmentation
+
+Data augmentation is a technique we can use to get better data faster. Using
+machine learning models to analyze long data (like an essay) and compress it
+into instructions.
+
+## How to contribute
+
+To contribute to data augmentation you can write a short Python script that uses
+a model from HuggingFace to analyze the text.
+[Here](https://docs.google.com/document/d/13a188pPvqnlvuVa3e_suVz4YO5s-JWeiOOrpp0odImg/edit)
+are examples of what you can do.
+
+And here are example implementations:
+[Idea 3](https://colab.research.google.com/drive/1GllCN5PgSYxBxINZsv3A2r0SpdznHlbT?usp=sharing),
+[Idea 4](https://colab.research.google.com/drive/1nZx5LRjO61fYprFyqtrwPDLOis6ctR4p#scrollTo=1EE8CriiaCXj)
+
+To contribute simply choose one of many ideas from the document above and
+implement it.

docs/data_schemas.md

@@ -0,0 +1,202 @@
+# OpenAssistant Data Schemas
+
+## Introduction
+
+This document describes the data schemas used by OpenAssistant. The schemas are
+defined as Python classes, but can be implemented in any format, be that Python,
+JSON, XML, SQL, Parquet files, etc.
+
+Also, the schemas are leaning heavily on the
+[OpenAssistant Data Structures](https://docs.google.com/presentation/d/1iaX_nxasVWlvPiSNs0cllR9L_1neZq0RJxd6MFEalUY/edit?usp=sharing)
+presentation.
+
+## Data Schemas
+
+### Main structure: conversation trees
+
+Conversation trees are the fundamental data structure. Many of the datasets we
+want to collect can be represented as conversation trees, such as QA datasets,
+chat logs, reddit dumps, etc. The main idea is that a conversation tree starts
+with a prompt and branches out from there. Every node can also have metadata,
+such as collected rankings, labels, or other information.
+
+Datasets that just represent linear data, such as a list of questions and
+answers, can be represented as a conversation tree with just a single branch.
+
+```python
+class ConversationTreeNode:
+    text: str # The text of the node
+    role: Literal['prompter', 'assistant'] # Whether the node is a user prompt/follow-up or an assistant response
+    children: list[ConversationTreeNode] # The children of the node (if you have a linear conversation, this will be of length 0 or 1)
+    metadata: dict[str, Any] # Node metadata (see below)
+
+class ConversationTree:
+    root: ConversationTreeNode # The node containing the initial prompt
+    metadata: dict[str, Any] # Tree metadata, different from root node metadata.
+
+```
+
+### Metadata
+
+Metadata encapsulates all the information that is not part of the conversation
+itself. This includes data about how the node was created (i.e. where it is
+from: crowd-sourced, templated, scraped, etc.), when it was created, its labels,
+tags, collected rankings, and other information.
+
+## Example: Reddit AMA dataset
+
+- Represent each question-follow-up set as a conversation tree.
+- Store things like usernames, timestamps, upvotes, etc. as metadata of the
+  nodes.
+- Store things like the AMA title, the AMA author, the AMA subreddit, etc. as
+  metadata of the tree.
+
+## Example: QA dataset
+
+- Represent each question-answer pair as a conversation tree.
+  - The question is the prompt, the answer is the assistant response.
+- If the dataset contains multiple answers to each question, each answer can be
+  a child of the question node.
+- If the dataset contains context text, it can be added as metadata to the
+  question node.
+
+## Example: Templated math problem dataset
+
+- Represent each problem as a conversation tree with the problem text as the
+  prompt and the solution as the assistant response.
+- Store the problem type (e.g. algebra, geometry, etc.) as metadata of the tree.
+- Store the template used also as metadata of the tree, as well as the source of
+  the data used to fill the template.
+
+## File Formats
+
+The above data should be representable in most file formats, but some care has
+to be taken with respect to the recursive nature of the data.
+
+Most row-major formats (JSON, Avro, Protobuf, etc.), as well as many databases,
+have no trouble with recursive (or arbitrary) schemas, but column-major formats,
+such as Parquet, do. For datasets with linear conversations, like many of the
+datasets we are collecting, this is not a problem. Instead of a tree of nodes,
+simply represent the conversation as a list of nodes. For true tree-like
+conversations, we should use a row-major format.
+
+## Other considerations
+
+- For text data of moderate size, it really doesn't matter much. It's more
+  important to use consistent data structures and naming, than to worry about
+  the exact file format.
+- For crowd-sourced data, we are collecting it into a SQL database already.
+- Parquet files are a good choice for large datasets, modulo the issues with
+  recursive schemas.
+- If parquet can't be used, gzipped JSON-line files are a good choice. So are
+  Avro files and protobufs. Keep in mind that column-major files are better for
+  reading, filtering, and aggregating, but row-major files are better for
+  writing.
+
+# Task-Specific Data Schemas
+
+The main tasks are a) generation of response text and b) ranking of responses.
+The following sections describe the data schemas for each of these tasks. Both
+should be implementable in parquet files.
+
+## Common Data Structures
+
+```python
+
+class Message:
+  text: str # The text of the message
+  role: Literal['prompter', 'assistant'] # Whether the message is a user prompt/follow-up or an assistant response
+
+class Thread:
+  messages: list[Message] # The messages in the conversation
+
+```
+
+The corresponding parquet schemas are:
+
+```parquet
+message Message {
+  required binary text (UTF8);
+  required binary role (UTF8);
+}
+
+message Thread {
+  required group messages (LIST) {
+    repeated group list {
+      required group element {
+        required binary text (UTF8);
+        required binary role (UTF8);
+      }
+    }
+  }
+}
+
+```
+
+## Generation
+
+```python
+
+class GenerationExample:
+  thread: Thread # The conversation thread before the message to be generated
+  message: Message # The message to be generated
+
+```
+
+The corresponding parquet schema is:
+
+```parquet
+message GenerationExample {
+  required group thread (LIST) {
+    repeated group list {
+      required group element {
+        required binary text (UTF8);
+        required binary role (UTF8);
+      }
+    }
+  }
+  required group message (LIST) {
+    repeated group list {
+      required group element {
+        required binary text (UTF8);
+        required binary role (UTF8);
+      }
+    }
+  }
+}
+
+```
+
+## Ranking
+
+```python
+
+class RankingExample:
+  thread: Thread # The conversation thread before the message to be ranked
+  messages: list[Message] # The messages to be ranked, in oder of decreasing preference
+
+```
+
+The corresponding parquet schema is:
+
+```parquet
+message RankingExample {
+  required group thread (LIST) {
+    repeated group list {
+      required group element {
+        required binary text (UTF8);
+        required binary role (UTF8);
+      }
+    }
+  }
+  required group messages (LIST) {
+    repeated group list {
+      required group element {
+        required binary text (UTF8);
+        required binary role (UTF8);
+      }
+    }
+  }
+}
+
+```

model/reward/instructor/cls_dataset.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
 
     classification based ranking

model/reward/instructor/experimental_dataset.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
     HFSummary
 

model/reward/instructor/rank_datasets.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """
     author: theblackcat102
 

model/reward/instructor/summary_quality_trainer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 from argparse import ArgumentParser
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union

model/reward/instructor/tests/test_dataset.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from experimental_dataset import DataCollatorForSummaryScore, HFSummaryQuality
 from rank_datasets import DataCollatorForPairRank, HFSummary, WebGPT
 from torch.utils.data import DataLoader

model/reward/instructor/trainer.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import os
 from argparse import ArgumentParser
 from dataclasses import dataclass

model/reward/instructor/utils.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import re
 
 import yaml

notebooks/closed-book-qa/T5_closed_book_QA_generator.md

@@ -0,0 +1,21 @@
+# Generate Topics, Questions, and Answers from a text
+
+This python code can be used to generate topics, questions, and answers from a
+paragraph of text. This is a good way to generate ground truth knowledge about a
+topic from a trusted source.
+
+The output of this is a dictionary with:
+
+1. submitted paragraph
+1. generated topics
+1. generated questions
+1. generated topic prefixes that can be prepended to the questions
+1. open book answer based only on the provided paragraph
+1. closed book answers generated by FLAN-T5-11B
+
+## Contributing
+
+This code is verified to work on a 24GB vram graphics card (like an RTX3090). We
+are working on getting it to run on google colab TPUs and also it may be
+possible to use smaller T5 models like the 3 billion parameter model and still
+get acceptable results.

notebooks/closed-book-qa/T5_closed_book_QA_generator.py

@@ -0,0 +1,406 @@
+# This notebook will run on a system with a single RTX3090 (24 GB vram).
+# You need to install accelerate, bitsandbytes, and transformers
+
+import math
+import pickle
+import time
+
+import torch
+
+# load all needed libraries
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+
+# This device map will work a GPU with > 24GB vram.
+# It uses nearly all the memory.
+device_map_T5_13B = {
+    "shared": 0,
+    "decoder.embed_tokens": 0,
+    "encoder.embed_tokens": 0,
+    "encoder.block.0": 0,
+    "encoder.block.1": 0,
+    "encoder.block.2": 0,
+    "encoder.block.3": 0,
+    "encoder.block.4": 0,
+    "encoder.block.5": 0,
+    "encoder.block.6": 0,
+    "encoder.block.7": 0,
+    "encoder.block.8": 0,
+    "encoder.block.9": 0,
+    "encoder.block.10": 0,
+    "encoder.block.11": 0,
+    "encoder.block.12": 0,
+    "encoder.block.13": 0,
+    "encoder.block.14": 0,
+    "encoder.block.15": 0,
+    "encoder.block.16": 0,
+    "encoder.block.17": 0,
+    "encoder.block.18": 0,
+    "encoder.block.19": 0,
+    "encoder.block.20": 0,
+    "encoder.block.21": 0,
+    "encoder.block.22": 0,
+    "encoder.block.23": 0,
+    "encoder.final_layer_norm": 0,
+    "encoder.dropout": 0,
+    "decoder.block.0": 0,
+    "decoder.block.1": 0,
+    "decoder.block.2": 0,
+    "decoder.block.3": 0,
+    "decoder.block.4": 0,
+    "decoder.block.5": 0,
+    "decoder.block.6": 0,
+    "decoder.block.7": 0,
+    "decoder.block.8": 0,
+    "decoder.block.9": 0,
+    "decoder.block.10": 0,
+    "decoder.block.11": 0,
+    "decoder.block.12": 0,
+    "decoder.block.13": 0,
+    "decoder.block.14": 0,
+    "decoder.block.15": 0,
+    "decoder.block.16": 0,
+    "decoder.block.17": 0,
+    "decoder.block.18": 0,
+    "decoder.block.19": 0,
+    "decoder.block.20": 0,
+    "decoder.block.21": 0,
+    "decoder.block.22": 0,
+    "decoder.block.23": 0,
+    "decoder.final_layer_norm": 0,
+    "decoder.dropout": 0,
+    "lm_head": 0,
+}
+
+
+# Load the model in bfloat16. Make sure to use bfloat16
+# if you are doing inference with 16bit precision.
+tokenizer = AutoTokenizer.from_pretrained("flan-t5-xxl")
+model = AutoModelForSeq2SeqLM.from_pretrained(
+    "flan-t5-xxl",
+    device_map=device_map_T5_13B,
+    torch_dtype=torch.bfloat16,
+    load_in_8bit=False,
+)
+
+
+# Load strings as knowledge sources for QA generation.
+# You can do this with a pickle.
+objects = []
+with (open("paragraphs.pkl", "rb")) as openfile:
+    while True:
+        try:
+            objects.append(pickle.load(openfile))
+        except EOFError:
+            break
+paragraphs = objects[0]
+
+# Make sure no paragraphs are too long for T5.
+# It handles up to 512 tokens context length.
+fixed_paragraphs = []
+for k in paragraphs:
+    if len(k) > 1100:
+        pass
+    else:
+        fixed_paragraphs.append(k)
+print("Original number of paragraphs:", len(paragraphs))
+print("Length filtered number of paragraphs:", len(fixed_paragraphs))
+paragraphs = fixed_paragraphs
+
+
+# Sort_Tuple sorts a list of tuples
+# by the second element.
+def Sort_Tuple(tup):
+    tup.sort(key=lambda x: x[1], reverse=True)
+    return tup
+
+
+# ask_flan_T5 takes a text input and returns the
+# response of FLAN_T5 and a normalized logits
+# score for the generation.
+def ask_flan_T5(input_text):
+    inputs = tokenizer.encode(input_text, return_tensors="pt").cuda(0)
+    outputs = model.generate(
+        inputs,
+        do_sample=True,
+        top_p=0.95,
+        eos_token_id=1,
+        max_new_tokens=50,
+        bos_token_id=0,
+        temperature=0.9,
+        return_dict_in_generate=True,
+        output_scores=True,
+    )
+    out_text = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)
+    probs = torch.stack(outputs.scores, dim=1).softmax(-1)
+    for i in outputs.sequences:
+        logprobs = 0
+        counter = 0
+        for k in i[1:]:
+            word_prob = (round(probs[0][counter][k.item()].item(), 2)) + 0.001
+            logprobs = logprobs + math.log(word_prob)
+            counter += 1
+        out_tuple = (out_text, round(logprobs, 2))
+    return out_tuple
+
+
+# ask_flan_T5D is a function that takes an input text and
+# returns the deterministic(do_sample=False) output of
+# FLAN_T5 and logits.
+def ask_flan_T5D(input_text):
+    inputs = tokenizer.encode(input_text, return_tensors="pt").cuda(0)
+    outputs = model.generate(
+        inputs,
+        do_sample=False,
+        eos_token_id=1,
+        max_new_tokens=50,
+        bos_token_id=0,
+        return_dict_in_generate=True,
+        output_scores=True,
+    )
+    out_text = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)
+    probs = torch.stack(outputs.scores, dim=1).softmax(-1)
+    for i in outputs.sequences:
+        logprobs = 0
+        counter = 0
+        for k in i[1:]:
+            word_prob = (round(probs[0][counter][k.item()].item(), 2)) + 0.001
+            logprobs = logprobs + math.log(word_prob)
+            counter += 1
+        out_tuple = (out_text, round(logprobs, 2))
+    return out_tuple
+
+
+# Generate a topic classifier for a paragraph of text
+def generate_topic(paragraph):
+    results = set()
+    input_text = (
+        "Task: Create a topic classifier for the provided \
+        paragraph.\nParagraph:\n"
+        + paragraph
+        + "\nTopic: "
+    )
+    for k in range(0, 20):
+        result = ask_flan_T5(input_text)
+        if result[1] > -4:
+            results.add(result)
+        if len(results) < 3:
+            results.add(("I was wondering", -3.3))
+            results.add(("I have a question", -3.3))
+    sorted_results = Sort_Tuple(list(results))
+    return sorted_results[0:5]
+
+
+# Generate a topic classifier for a paragraph of text
+def generate_topic_prefix(topic_set):
+    results = set()
+    for entry in topic_set:
+        topic = entry[0]
+        input_text = (
+            "Task: Create a prepositional phrase about the topic.\n\
+            Example 1\n Topic: climbing mount everest\nPrepositional \
+            Phrase: With regards to climbing mount everest,\nExample \
+            2\nTopic: United States Air Force\nPrepositional Phrase: \
+            On the topic of the United States Air Force,\n Example 3\nTopic: "
+            + topic
+            + "\nPrepositional Phrase: "
+        )
+        for k in range(0, 5):
+            results.add(ask_flan_T5(input_text))
+        sorted_results = Sort_Tuple(list(results))
+        return sorted_results[0:5]
+
+
+# Generate who/what/where/when/why questions from a paragraph.
+# Number of questions variable is an integer which indicates how
+# many of each question type to try to generate.
+def generate_questions(paragraph, number_of_questions):
+    if len(tokenizer.encode(paragraph)) > 480:
+        print("Warning, the context length is too long.")
+    question_set = set()
+    question_types = [
+        "What",
+        "Where",
+        "Why",
+        "How",
+        "Who",
+    ]
+    for qtype in question_types:
+        question = (
+            "Please generate a question that starts with '"
+            + qtype
+            + "' based on the following paragraph.\nText:\n"
+            + paragraph
+            + "\nQuestion:\n"
+        )
+        for k in range(0, number_of_questions):
+            new_question = ask_flan_T5(question)
+            if qtype in new_question[0]:
+                question_set.add((qtype, new_question))
+    return question_set
+
+
+# Generate answers for a set of questions.
+# Input is the paragraph of text and a set of questions where each question
+# is a tuple generated from the generate_questions() function.
+def generate_answers(paragraph, question_set):
+    possible_answers = set()
+    for question in question_set:
+        input_text = (
+            "Please read the following paragraph and \
+            then answer the question using only data \
+            found in the text. If no answer is possible, respond \
+            'NA'.\nText:\n"
+            + paragraph
+            + "\nQuestion:\n"
+            + question[1][0]
+            + "\nAnswer:\n"
+        )
+        answer = ask_flan_T5D(input_text)
+        if "NA" in answer[0]:
+            pass
+        else:
+            possible_answers.add((question[0], question[1], answer))
+    return possible_answers
+
+
+# Generate questions from a paragraph and set of answers.
+# Input is the paragraph of text and a set of answers where each question
+# is a tuple generated from the generate_answers() function.
+def generate_question2(paragraph, qa_set):
+    qaq_results = set()
+    for qa_item in qa_set:
+        answer = qa_item[2][0]
+        input_text = (
+            "Please read the following paragraph and \
+            then generate a question whose answer is: "
+            + answer
+            + "\nParagraph:\n"
+            + paragraph
+            + "\nQuestion:\n"
+        )
+        result = ask_flan_T5D(input_text)
+        qaq_results.add((qa_item[0], qa_item[1], qa_item[2], result))
+    return qaq_results
+
+
+# Generate answers from a paragraph and set of questions.
+# Input is the paragraph of text and a set of questions where each answer
+#  is a tuple generated from the generate_questions2() function.
+def generate_answers2(paragraph, question_set):
+    possible_answers = set()
+    for question in question_set:
+        input_text = (
+            "Please read the following paragraph and \
+            then answer the question using only data \
+            found in the text. If no answer is possible, respond \
+            'NA'.\nText:\n"
+            + paragraph
+            + "\nQuestion:\n"
+            + question
+            + "\nAnswer:\n"
+        )
+        answer = ask_flan_T5D(input_text)
+        possible_answers.add((question, answer))
+    return possible_answers
+
+
+# Generate declarative statement from question and answer pair.
+def generate_declarative(qaq_set):
+    qaqd_results = set()
+    for qa_item in qaq_set:
+        question = qa_item[0]
+        answer = qa_item[1][0]
+        if "NA" in answer:
+            pass
+        else:
+            input_text = (
+                "Generate a declarative statement based on the \
+                given question and answer pair.\nQ: What is \
+                sitting on the couch?\nA: poodle\nA poodle is \
+                sitting on the couch.\nQ: "
+                + question
+                + "\nA: "
+                + answer
+                + "\n"
+            )
+            result = ask_flan_T5D(input_text)
+            qaqd_results.add((question, answer, result))
+    return qaqd_results
+
+
+# Generate closed book answer to question.
+def generate_closed_answer(qaqd_set):
+    qaqd_results = set()
+    for qa_item in qaqd_set:
+        question = qa_item[0]
+        answer = qa_item[2][0]
+        if "NA" in answer:
+            # print(answer)
+            pass
+        else:
+            input_text = (
+                "Task: Answer the question in a detailed fashion. \
+                If the question cannot be answered without more \
+                information, please answer NA.\nExample 1:\nQuestion: \
+                Why does Shala like cookies?\nAnswer: It is not possible \
+                to know why Shala likes cookies without more information, \
+                but many people that like cookies enjoy their taste or \
+                some of their ingredients (e.g. chocolate chips or \
+                peanut butter).\nExample 2:\nQuestion: Why would someone \
+                vote in an election?\nAnswer: There are many reasons \
+                someone might vote in an election, for instance to have \
+                their voice heard or to help a candidate they like win the \
+                race.\nExample 3\nQuestion: What decoration goes on top of \
+                a Christmas tree?\nAnswer: Usually a star is placed at the \
+                top of a Christmas tree.\nExample 4:\nQuestion: "
+                + question
+                + "\nAnswer: "
+            )
+            result = ask_flan_T5D(input_text)
+            qaqd_results.add((qa_item[0], qa_item[1], qa_item[2], result))
+    return qaqd_results
+
+
+# Create a dictionary of questions and answers from a list of paragraphs.
+# Takes about 20 seconds per paragraph to process.
+start_time = time.perf_counter()
+questions_dict = {}
+uniq_id = 100000
+for paragraph in paragraphs[0:1500]:
+    topic_list = generate_topic(paragraph)
+    topic_prefix = generate_topic_prefix(topic_list)
+    question_set = generate_questions(paragraph, 2)
+    qa_set = generate_answers(paragraph, question_set)
+    qaq_set = generate_question2(paragraph, qa_set)
+    q2_set = set()
+    for q in qaq_set:
+        q2_set.add(q[3][0])
+    q2a2_set = generate_answers2(paragraph, q2_set)
+    a2d_set = generate_declarative(q2a2_set)
+    a3cb_set = generate_closed_answer(a2d_set)
+    questions_dict[uniq_id] = {}
+    questions_dict[uniq_id]["topics"] = topic_list
+    questions_dict[uniq_id]["topic prepositions"] = topic_prefix
+    questions_dict[uniq_id]["paragraph"] = paragraph
+    entry_count = 0
+    entry_dict = {}
+    for entry in a3cb_set:
+        entry_dict[entry_count] = {}
+        entry_dict[entry_count]["question"] = entry[0]
+        entry_dict[entry_count]["answer_T5_ob"] = entry[2][0]
+        entry_dict[entry_count]["answer_T5_cb"] = entry[3][0]
+        entry_count += 1
+    questions_dict[uniq_id]["QA_set"] = entry_dict
+    uniq_id += 1
+    print(uniq_id, "topics:", topic_prefix)
+
+stop_time = time.perf_counter()
+generation_time = stop_time - start_time
+print(questions_dict[uniq_id - 1])
+print(generation_time)
+
+
+# create a binary pickle file to save your dictionary
+f = open("questions_dict.pkl", "wb")
+pickle.dump(questions_dict, f)
+f.close()

notebooks/data-argumentation/EssayInstructions.md

@@ -1,11 +1,11 @@
 # Essay Instructions
 
-Essay Instructions is a notebook that takes an essay as an input and genrates
+Essay Instructions is a notebook that takes an essay as an input and generates
 instructions on how to generate that essay. This will be very useful for data
 collecting for the model
 
 ## Contributing
 
 Feel free to contribute to this notebook, it's nowhere near perfect but it's a
-good start. If you want to contribute fidning a new model that better suits this
+good start. If you want to contribute finding a new model that better suits this
 task would be great. Hugginface has a lot of models that could help.

notebooks/data-argumentation/EssayRevision.md

@@ -1,11 +1,11 @@
 # Essay Revision
 
 Essay Revision is a notebook that generates data for improving essays. It does
-that by taking a "good" essay, making it worse step by step and the fidning
+that by taking a "good" essay, making it worse step by step and the finding
 instructions for making it better. This will be useful for generating data for
 the model.
 
 ## Contributing
 
 Feel free to contribute to this notebook. It's not perfect but it is quite good.
-Finding a better way to make gramatical errors may be a good place to start.
+Finding a better way to make grammatical errors may be a good place to start.

oasst-shared/oasst_shared/api_client.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """API Client for interacting with the OASST backend."""
 import enum
 import typing as t

oasst-shared/oasst_shared/exceptions/__init__.py

@@ -0,0 +1,3 @@
+# Ignore unused imports; these are re-exported
+from .oasst_api_error import OasstError as OasstError  # noqa: F401
+from .oasst_api_error import OasstErrorCode as OasstErrorCode  # noqa: F401

oasst-shared/oasst_shared/exceptions/oasst_api_error.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from enum import IntEnum
 from http import HTTPStatus
 
@@ -18,6 +17,7 @@ class OasstErrorCode(IntEnum):
     DATABASE_URI_NOT_SET = 1
     API_CLIENT_NOT_AUTHORIZED = 2
     SERVER_ERROR = 3
+    TOO_MANY_REQUESTS = 429
 
     # 1000-2000: tasks endpoint
     TASK_INVALID_REQUEST_TYPE = 1000

oasst-shared/oasst_shared/schemas/protocol.py

@@ -1,7 +1,6 @@
-# -*- coding: utf-8 -*-
 import enum
 from datetime import datetime
-from typing import Literal, Optional, Union
+from typing import List, Literal, Optional, Union
 from uuid import UUID, uuid4
 
 import pydantic
@@ -282,3 +281,15 @@ class SystemStats(BaseModel):
     active: int = 0
     deleted: int = 0
     message_trees: int = 0
+
+
+class UserScore(BaseModel):
+    ranking: int
+    user_id: UUID
+    username: str
+    display_name: str
+    score: int
+
+
+class LeaderboardStats(BaseModel):
+    leaderboard: List[UserScore]

oasst-shared/oasst_shared/utils.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from datetime import datetime, timezone
 
 

oasst-shared/setup.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 # setup.py for the shared python modules
 
 from distutils.core import setup

oasst-shared/tests/test_oasst_api_client.py

@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
 from uuid import uuid4
 
 import pytest
-from bot.api_client import OasstApiClient
+from oasst_shared.api_client import OasstApiClient
 from oasst_shared.schemas import protocol as protocol_schema
 
 

scripts/discord/verify-lobby.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+"""This file is for moderators to verify new users in the lobby.
+
+First, moderators read the brief introduction people write in the lobby.
+If all people's introductions are acceptable, moderators run this script.
+
+Needs BOT_TOKEN environment variable to be set to the bot token.
+
+"""
+
+
+import discord
+import pydantic
+import tqdm.asyncio as tqdm
+
+
+class Settings(pydantic.BaseSettings):
+    bot_token: str
+
+
+settings = Settings()
+
+intents = discord.Intents.default()
+intents.message_content = True
+intents.members = True
+client = discord.Client(intents=intents)
+
+
+@client.event
+async def on_ready():
+    lobby_channel = discord.utils.get(client.get_all_channels(), name="lobby")
+    # obtain the role object for the verified role
+    verified_role = discord.utils.get(lobby_channel.guild.roles, name="verified")
+    async for message in tqdm.tqdm(lobby_channel.history(limit=None)):
+        if not isinstance(message.author, discord.Member):
+            print(f"{message.author} is not a member")
+            continue
+        for role in message.author.roles:
+            if role.name == "unverified":
+                print(f"{message.author} has the unverified role.")
+                break
+        else:
+            continue
+        # un-assign the unverified role
+        await message.author.remove_roles(role)
+        # assign the verified role
+        await message.author.add_roles(verified_role)
+        print(f"Assigned verified role to {message.author}")
+    await client.close()
+
+
+client.run(settings.bot_token)

scripts/oasst-shared-development/test.sh

@@ -2,7 +2,7 @@
 parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
 
 # switch to backend directory
-pushd "$parent_path/../../discord-bot"
+pushd "$parent_path/../../oasst-shared"
 
 pytest .
 

scripts/postprocessing/infogain_selector.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 import numpy as np
 from scipy import log2
 from scipy.integrate import nquad

scripts/postprocessing/rankings.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from typing import List
 
 import numpy as np

scripts/postprocessing/scoring.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 from dataclasses import dataclass, replace
 from typing import Any
 

text-frontend/__main__.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """Simple REPL frontend."""
 
 import random

website/cypress.config.js

@@ -22,4 +22,11 @@ export default defineConfig({
       getCompareSnapshotsPlugin(on, config);
     },
   },
+
+  env: {
+    MAILDEV_PROTOCOL: "http",
+    MAILDEV_HOST: "localhost",
+    MAILDEV_SMTP_PORT: "1025",
+    MAILDEV_API_PORT: "1080",
+  },
 });

website/cypress/README.md

@@ -58,13 +58,13 @@ under test.
 When running `npm run cypress` and selecting e2e testing, we assume you have the
 NextJS site running at `localhost:3000`.
 
-An example test from this time of writing, could look as follows:
+An example test could look as follows:
 
 ```typescript
 describe("signin flow", () => {
   it("redirects to a confirmation page on submit of valid email address", () => {
     cy.visit("/auth/signin");
-    cy.get(".chakra-input").type(`test@example.com{enter}`);
+    cy.get('[data-cy="email-address"]').type(`test@example.com{enter}`);
     cy.url().should("contain", "/auth/verify");
   });
 });
@@ -82,10 +82,49 @@ Cypress will
 [automatically await](https://docs.cypress.io/guides/core-concepts/introduction-to-cypress#Timeouts)
 almost anything you do, but fail if the default timeout is reached.
 
-Then we get the email input field and type our email address. Notice the
-`{enter}` keyword, this will cause Cypress to hit the return key which we expect
-to submit the form.
+Then we get the email input field and type our email address. We find the input
+field using the data-cy attribute that we added in the source code of the
+element on the page.
+
+```jsx
+<Input data-cy="email-address" placeholder="Email Address" />
+```
+
+Using `data-cy` is how we ensure that selecting the element is robust to changes
+in page design or function and is one of the
+[best practices recommended by Cypress](https://docs.cypress.io/guides/references/best-practices#Selecting-Elements).
+
+Next we call `type()` to use the keyboard, cypress will automatically focus the
+element and send the keypress events. Notice the `{enter}` keyword, this will
+cause Cypress to hit the return key which we expect to submit the form.
 
 We then assert that the URL should contain `/auth/verify`. Again the timeout
 will make sure we are not waiting forever, and the test will fail if we do not
 manage to get there in a reasonable time.
+
+## Authenticating in e2e tests
+
+For end-to-end tests almost every test will need to first sign in to the
+website. To make this easier we have a custom command for Cypress that makes
+logging in with an email address a single command, `cy.signInWithEmail()`.
+
+```typescript
+describe("replying as the assistant", () => {
+  it("completes the current task on submit", () => {
+    cy.signInWithEmail("cypress@example.com");
+
+    cy.visit("/create/assistant_reply");
+
+    cy.get('[data-cy="reply"').type(
+      "You need to run pre-commit to make the reviewer happy."
+    );
+    cy.get('[data-cy="submit"]').click();
+  });
+});
+```
+
+In this example we sign in as `cypress@example.com` before visiting the
+`/create/assistant_reply` page that is only available when authenticated. We can
+then continue on with our test as normal. Note: using `cy.signInWithEmail()`
+requires that the maildev is running, which should have been started as part of
+the `docker compose up` command that is required to do any end-to-end testing.

website/cypress/e2e/auth/signin.cy.ts

@@ -1,10 +1,39 @@
+import { faker } from "@faker-js/faker";
+
 describe("signin flow", () => {
   it("redirects to a confirmation page on submit of valid email address", () => {
     cy.visit("/auth/signin");
-    cy.get(".chakra-input").type(`test@example.com`);
-    cy.get(".chakra-stack > .chakra-button").click();
+    cy.get('[data-cy="email-address"]').type(`test@example.com`);
+    cy.get('[data-cy="signin-email-button"]').click();
     cy.url().should("contain", "/auth/verify");
   });
+  it("emails a login link to the user when signing in with email", () => {
+    // Use random email to avoid possibility of tests passing just due to other tests or previous runs also causing emails to be sent
+    const emailAddress = faker.internet.email();
+    cy.log("emailAddress", emailAddress);
+    cy.request("GET", "/api/auth/csrf")
+      .then((response) => {
+        const csrfToken = response.body.csrfToken;
+        cy.request("POST", "/api/auth/signin/email", {
+          callbackUrl: "/",
+          email: emailAddress,
+          csrfToken,
+          json: "true",
+        });
+      })
+      .then((response) => {
+        cy.signInUsingEmailedLink(emailAddress).then(() => {
+          cy.get('[data-cy="username"]').should("exist");
+        });
+      });
+  });
+  it("shows the logged in users email address if logged in with email", () => {
+    const emailAddress = "user@example.com";
+    cy.signInWithEmail(emailAddress);
+    // The user will only see the email address if the window is wide enough, not technically required as even when hidden this will find it in the page.
+    cy.viewport(1920, 1000);
+    cy.contains('[data-cy="username"]', emailAddress);
+  });
 });
 
 export {};

website/cypress/e2e/create/assistant_reply.cy.ts

@@ -0,0 +1,26 @@
+import { faker } from "@faker-js/faker";
+
+describe("replying as the assistant", () => {
+  it("completes the current task on submit and on request shows a new task", () => {
+    cy.signInWithEmail("cypress@example.com");
+    cy.visit("/create/assistant_reply");
+
+    cy.get('[data-cy="task-id"').then((taskIdElement) => {
+      const taskId = taskIdElement.text();
+
+      const reply = faker.lorem.sentence();
+      cy.log("reply", reply);
+      cy.get('[data-cy="reply"').type(reply);
+
+      cy.get('[data-cy="submit"]').click();
+
+      cy.get('[data-cy="next-task"]').click();
+
+      cy.get('[data-cy="task-id"').should((taskIdElement) => {
+        expect(taskIdElement.text()).not.to.eq(taskId);
+      });
+    });
+  });
+});
+
+export {};

website/cypress/e2e/create/user_reply.cy.ts

@@ -0,0 +1,26 @@
+import { faker } from "@faker-js/faker";
+
+describe("replying as the prompter", () => {
+  it("completes the current task on submit and on request shows a new task", () => {
+    cy.signInWithEmail("cypress@example.com");
+    cy.visit("/create/user_reply");
+
+    cy.get('[data-cy="task-id"').then((taskIdElement) => {
+      const taskId = taskIdElement.text();
+
+      const reply = faker.lorem.sentence();
+      cy.log("reply", reply);
+      cy.get('[data-cy="reply"').type(reply);
+
+      cy.get('[data-cy="submit"]').click();
+
+      cy.get('[data-cy="next-task"]').click();
+
+      cy.get('[data-cy="task-id"').should((taskIdElement) => {
+        expect(taskIdElement.text()).not.to.eq(taskId);
+      });
+    });
+  });
+});
+
+export {};

website/cypress/e2e/evaluate/rank_assistant_replies.cy.ts

@@ -0,0 +1,30 @@
+describe("ranking prompter replies", () => {
+  it("completes the current task on submit and on request shows a new task", () => {
+    cy.signInWithEmail("cypress@example.com");
+    cy.visit("/evaluate/rank_user_replies");
+
+    cy.get('[data-cy="task-id"').then((taskIdElement) => {
+      const taskId = taskIdElement.text();
+
+      // Rank an item using the keyboard so that the submit button is enabled
+      cy.get('button[aria-roledescription="sortable"]')
+        .first()
+        .click()
+        .type("{enter}")
+        .wait(100)
+        .type("{downArrow}")
+        .wait(100)
+        .type("{enter}");
+
+      cy.get('[data-cy="submit"]').click();
+
+      cy.get('[data-cy="next-task"]').click();
+
+      cy.get('[data-cy="task-id"').should((taskIdElement) => {
+        expect(taskIdElement.text()).not.to.eq(taskId);
+      });
+    });
+  });
+});
+
+export {};

website/cypress/e2e/evaluate/rank_initial_prompts.cy.ts

@@ -0,0 +1,30 @@
+describe("ranking initial prompts", () => {
+  it("completes the current task on submit and on request shows a new task", () => {
+    cy.signInWithEmail("cypress@example.com");
+    cy.visit("/evaluate/rank_initial_prompts");
+
+    cy.get('[data-cy="task-id"').then((taskIdElement) => {
+      const taskId = taskIdElement.text();
+
+      // Rank an item using the keyboard so that the submit button is enabled
+      cy.get('button[aria-roledescription="sortable"]')
+        .first()
+        .click()
+        .type("{enter}")
+        .wait(100)
+        .type("{downArrow}")
+        .wait(100)
+        .type("{enter}");
+
+      cy.get('[data-cy="submit"]').click();
+
+      cy.get('[data-cy="next-task"]').click();
+
+      cy.get('[data-cy="task-id"').should((taskIdElement) => {
+        expect(taskIdElement.text()).not.to.eq(taskId);
+      });
+    });
+  });
+});
+
+export {};

website/cypress/e2e/evaluate/rank_user_replies.cy.ts

@@ -0,0 +1,30 @@
+describe("ranking assistant replies", () => {
+  it("completes the current task on submit and on request shows a new task", () => {
+    cy.signInWithEmail("cypress@example.com");
+    cy.visit("/evaluate/rank_assistant_replies");
+
+    cy.get('[data-cy="task-id"').then((taskIdElement) => {
+      const taskId = taskIdElement.text();
+
+      // Rank an item using the keyboard so that the submit button is enabled
+      cy.get('button[aria-roledescription="sortable"]')
+        .first()
+        .click()
+        .type("{enter}")
+        .wait(100)
+        .type("{downArrow}")
+        .wait(100)
+        .type("{enter}");
+
+      cy.get('[data-cy="submit"]').click();
+
+      cy.get('[data-cy="next-task"]').click();
+
+      cy.get('[data-cy="task-id"').should((taskIdElement) => {
+        expect(taskIdElement.text()).not.to.eq(taskId);
+      });
+    });
+  });
+});
+
+export {};