Merge branch 'main' of github.com:LAION-AI/Open-Chat-GPT

.github/workflows/test-api-contract.yaml

@@ -18,18 +18,13 @@ jobs:
 
       - run: cd oasst-shared && pip install -e .
 
+      - run: cd oasst-shared && pip install -r requirements.dev.txt
+
       - run: cd backend && pip install -r requirements.txt
 
-      - run: cd discord-bot && pip install -r requirements.txt
-
-      - run: cd discord-bot && pip install -r requirements.dev.txt
-
       - run: ./scripts/backend-development/start-mock-server.sh
 
-      # runs the contract tests. currently the api client is
-      # found in the discord bot code, but this should be updated
-      # once the client moves into oasst-shared.
       - name: Run contract tests
-        run: ./scripts/discord-bot-development/test.sh
+        run: ./scripts/oasst-shared-development/test.sh
 
       - run: ./scripts/backend-development/stop-mock-server.sh

backend/main.py

@@ -14,8 +14,8 @@ from oasst_backend.api.deps import get_dummy_api_client
 from oasst_backend.api.v1.api import api_router
 from oasst_backend.config import settings
 from oasst_backend.database import engine
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
 from sqlmodel import Session
 from starlette.middleware.cors import CORSMiddleware

backend/oasst_backend/api/deps.py

@@ -9,8 +9,8 @@ from fastapi_limiter.depends import RateLimiter
 from loguru import logger
 from oasst_backend.config import settings
 from oasst_backend.database import engine
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import ApiClient
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import Session
 
 

backend/oasst_backend/api/v1/frontend_messages.py

@@ -1,10 +1,10 @@
 from fastapi import APIRouter, Depends
 from oasst_backend.api import deps
 from oasst_backend.api.v1 import utils
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import ApiClient
 from oasst_backend.models.db_payload import MessagePayload
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import Session
 
 router = APIRouter()

backend/oasst_backend/api/v1/messages.py

@@ -4,10 +4,10 @@ from uuid import UUID
 from fastapi import APIRouter, Depends, Query, Response
 from oasst_backend.api import deps
 from oasst_backend.api.v1 import utils
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import ApiClient
 from oasst_backend.models.db_payload import MessagePayload
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import Session
 from starlette.status import HTTP_200_OK
 

backend/oasst_backend/api/v1/tasks.py

@@ -6,8 +6,8 @@ from fastapi import APIRouter, Depends
 from fastapi.security.api_key import APIKey
 from loguru import logger
 from oasst_backend.api import deps
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
 from sqlmodel import Session
 

backend/oasst_backend/api/v1/utils.py

@@ -1,9 +1,9 @@
 from http import HTTPStatus
 from uuid import UUID
 
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import Message
 from oasst_backend.models.db_payload import MessagePayload
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol
 
 

backend/oasst_backend/database.py

@@ -1,5 +1,5 @@
 from oasst_backend.config import settings
-from oasst_backend.exceptions import OasstError, OasstErrorCode
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import create_engine
 
 if settings.DATABASE_URI is None:

backend/oasst_backend/prompt_repository.py

@@ -7,10 +7,10 @@ from uuid import UUID, uuid4
 
 import oasst_backend.models.db_payload as db_payload
 from loguru import logger
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.journal_writer import JournalWriter
 from oasst_backend.models import ApiClient, Message, MessageReaction, Task, TextLabels, User
 from oasst_backend.models.payload_column_type import PayloadContainer
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
 from oasst_shared.schemas.protocol import SystemStats
 from sqlalchemy import update

discord-bot/bot/bot.py

@@ -5,9 +5,9 @@ import aiosqlite
 import hikari
 import lightbulb
 import miru
-from bot.api_client import OasstApiClient
 from bot.settings import Settings
 from bot.utils import EMPTY, mention
+from oasst_shared.api_client import OasstApiClient
 
 settings = Settings()
 

discord-bot/bot/extensions/work.py

@@ -8,10 +8,10 @@ import lightbulb
 import lightbulb.decorators
 import miru
 from aiosqlite import Connection
-from bot.api_client import OasstApiClient, TaskType
 from bot.db.schemas import GuildSettings
 from bot.utils import EMPTY
 from loguru import logger
+from oasst_shared.api_client import OasstApiClient, TaskType
 from oasst_shared.schemas import protocol as protocol_schema
 from oasst_shared.schemas.protocol import TaskRequestType
 

docs/data_schemas.md

@@ -0,0 +1,94 @@
+# OpenAssistant Data Schemas
+
+## Introduction
+
+This document describes the data schemas used by OpenAssistant. The schemas are
+defined as Python classes, but can be implemented in any format, be that Python,
+JSON, XML, SQL, Parquet files, etc.
+
+Also, the schemas are leaning heavily on the
+[OpenAssistant Data Structures](https://docs.google.com/presentation/d/1iaX_nxasVWlvPiSNs0cllR9L_1neZq0RJxd6MFEalUY/edit?usp=sharing)
+presentation.
+
+## Data Schemas
+
+### Main structure: conversation trees
+
+Conversation trees are the fundamental data structure. Many of the datasets we
+want to collect can be represented as conversation trees, such as QA datasets,
+chat logs, reddit dumps, etc. The main idea is that a conversation tree starts
+with a prompt and branches out from there. Every node can also have metadata,
+such as collected rankings, labels, or other information.
+
+Datasets that just represent linear data, such as a list of questions and
+answers, can be represented as a conversation tree with just a single branch.
+
+```python
+class ConversationTreeNode:
+    text: str # The text of the node
+    role: Literal['prompter', 'assistant'] # Whether the node is a user prompt/follow-up or an assistant response
+    children: list[ConversationTreeNode] # The children of the node (if you have a linear conversation, this will be of length 0 or 1)
+    metadata: dict[str, Any] # Node metadata (see below)
+
+class ConversationTree:
+    root: ConversationTreeNode # The node containing the initial prompt
+    metadata: dict[str, Any] # Tree metadata, different from root node metadata.
+
+```
+
+### Metadata
+
+Metadata encapsulates all the information that is not part of the conversation
+itself. This includes data about how the node was created (i.e. where it is
+from: crowd-sourced, templated, scraped, etc.), when it was created, its labels,
+tags, collected rankings, and other information.
+
+## Example: Reddit AMA dataset
+
+- Represent each question-follow-up set as a conversation tree.
+- Store things like usernames, timestamps, upvotes, etc. as metadata of the
+  nodes.
+- Store things like the AMA title, the AMA author, the AMA subreddit, etc. as
+  metadata of the tree.
+
+## Example: QA dataset
+
+- Represent each question-answer pair as a conversation tree.
+  - The question is the prompt, the answer is the assistant response.
+- If the dataset contains multiple answers to each question, each answer can be
+  a child of the question node.
+- If the dataset contains context text, it can be added as metadata to the
+  question node.
+
+## Example: Templated math problem dataset
+
+- Represent each problem as a conversation tree with the problem text as the
+  prompt and the solution as the assistant response.
+- Store the problem type (e.g. algebra, geometry, etc.) as metadata of the tree.
+- Store the template used also as metadata of the tree, as well as the source of
+  the data used to fill the template.
+
+## File Formats
+
+The above data should be representable in most file formats, but some care has
+to be taken with respect to the recursive nature of the data.
+
+Most row-major formats (JSON, Avro, Protobuf, etc.), as well as many databases,
+have no trouble with recursive (or arbitrary) schemas, but column-major formats,
+such as Parquet, do. For datasets with linear conversations, like many of the
+datasets we are collecting, this is not a problem. Instead of a tree of nodes,
+simply represent the conversation as a list of nodes. For true tree-like
+conversations, we should use a row-major format.
+
+## Other considerations
+
+- For text data of moderate size, it really doesn't matter much. It's more
+  important to use consistent data structures and naming, than to worry about
+  the exact file format.
+- For crowd-sourced data, we are collecting it into a SQL database already.
+- Parquet files are a good choice for large datasets, modulo the issues with
+  recursive schemas.
+- If parquet can't be used, gzipped JSON-line files are a good choice. So are
+  Avro files and protobufs. Keep in mind that column-major files are better for
+  reading, filtering, and aggregating, but row-major files are better for
+  writing.

oasst-shared/oasst_shared/exceptions/__init__.py

@@ -0,0 +1,3 @@
+# Ignore unused imports; these are re-exported
+from .oasst_api_error import OasstError as OasstError  # noqa: F401
+from .oasst_api_error import OasstErrorCode as OasstErrorCode  # noqa: F401

oasst-shared/tests/test_oasst_api_client.py

@@ -1,7 +1,7 @@
 from uuid import uuid4
 
 import pytest
-from bot.api_client import OasstApiClient
+from oasst_shared.api_client import OasstApiClient
 from oasst_shared.schemas import protocol as protocol_schema
 
 

scripts/oasst-shared-development/test.sh

@@ -2,7 +2,7 @@
 parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
 
 # switch to backend directory
-pushd "$parent_path/../../discord-bot"
+pushd "$parent_path/../../oasst-shared"
 
 pytest .