From c7ac01718c682f9a42c0d6695294b7483e4ded12 Mon Sep 17 00:00:00 2001
From: Jack Michaud <jack-michaud@users.noreply.github.com>
Date: Mon, 2 Jan 2023 21:03:53 +0000
Subject: [PATCH 1/3] refactor: move OasstError into oasst_shared.exceptions
 (#289)

* refactor: move OasstError into oasst_shared.exceptions

* fix: update OasstError import

* fix: linter errors
---
 backend/main.py                                                | 2 +-
 backend/oasst_backend/api/deps.py                              | 2 +-
 backend/oasst_backend/api/v1/frontend_messages.py              | 2 +-
 backend/oasst_backend/api/v1/messages.py                       | 2 +-
 backend/oasst_backend/api/v1/tasks.py                          | 2 +-
 backend/oasst_backend/api/v1/utils.py                          | 2 +-
 backend/oasst_backend/database.py                              | 2 +-
 backend/oasst_backend/prompt_repository.py                     | 2 +-
 oasst-shared/oasst_shared/exceptions/__init__.py               | 3 +++
 .../oasst_shared/exceptions/oasst_api_error.py                 | 0
 10 files changed, 11 insertions(+), 8 deletions(-)
 create mode 100644 oasst-shared/oasst_shared/exceptions/__init__.py
 rename backend/oasst_backend/exceptions.py => oasst-shared/oasst_shared/exceptions/oasst_api_error.py (100%)

diff --git a/backend/main.py b/backend/main.py
index 4eb4a436..e6b3bdce 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -14,8 +14,8 @@ from oasst_backend.api.deps import get_dummy_api_client
 from oasst_backend.api.v1.api import api_router
 from oasst_backend.config import settings
 from oasst_backend.database import engine
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
 from sqlmodel import Session
 from starlette.middleware.cors import CORSMiddleware
diff --git a/backend/oasst_backend/api/deps.py b/backend/oasst_backend/api/deps.py
index 103de4fd..f61947cd 100644
--- a/backend/oasst_backend/api/deps.py
+++ b/backend/oasst_backend/api/deps.py
@@ -9,8 +9,8 @@ from fastapi_limiter.depends import RateLimiter
 from loguru import logger
 from oasst_backend.config import settings
 from oasst_backend.database import engine
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import ApiClient
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import Session
 
 
diff --git a/backend/oasst_backend/api/v1/frontend_messages.py b/backend/oasst_backend/api/v1/frontend_messages.py
index 0c8b81f6..261b24ea 100644
--- a/backend/oasst_backend/api/v1/frontend_messages.py
+++ b/backend/oasst_backend/api/v1/frontend_messages.py
@@ -1,10 +1,10 @@
 from fastapi import APIRouter, Depends
 from oasst_backend.api import deps
 from oasst_backend.api.v1 import utils
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import ApiClient
 from oasst_backend.models.db_payload import MessagePayload
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import Session
 
 router = APIRouter()
diff --git a/backend/oasst_backend/api/v1/messages.py b/backend/oasst_backend/api/v1/messages.py
index a70b935e..20420690 100644
--- a/backend/oasst_backend/api/v1/messages.py
+++ b/backend/oasst_backend/api/v1/messages.py
@@ -4,10 +4,10 @@ from uuid import UUID
 from fastapi import APIRouter, Depends, Query, Response
 from oasst_backend.api import deps
 from oasst_backend.api.v1 import utils
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import ApiClient
 from oasst_backend.models.db_payload import MessagePayload
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import Session
 from starlette.status import HTTP_200_OK
 
diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py
index 6571a5a9..49e82880 100644
--- a/backend/oasst_backend/api/v1/tasks.py
+++ b/backend/oasst_backend/api/v1/tasks.py
@@ -6,8 +6,8 @@ from fastapi import APIRouter, Depends
 from fastapi.security.api_key import APIKey
 from loguru import logger
 from oasst_backend.api import deps
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.prompt_repository import PromptRepository
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
 from sqlmodel import Session
 
diff --git a/backend/oasst_backend/api/v1/utils.py b/backend/oasst_backend/api/v1/utils.py
index c321be59..55a7c572 100644
--- a/backend/oasst_backend/api/v1/utils.py
+++ b/backend/oasst_backend/api/v1/utils.py
@@ -1,9 +1,9 @@
 from http import HTTPStatus
 from uuid import UUID
 
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.models import Message
 from oasst_backend.models.db_payload import MessagePayload
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol
 
 
diff --git a/backend/oasst_backend/database.py b/backend/oasst_backend/database.py
index 987aee76..b160da61 100644
--- a/backend/oasst_backend/database.py
+++ b/backend/oasst_backend/database.py
@@ -1,5 +1,5 @@
 from oasst_backend.config import settings
-from oasst_backend.exceptions import OasstError, OasstErrorCode
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from sqlmodel import create_engine
 
 if settings.DATABASE_URI is None:
diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py
index 8190d637..570d48b8 100644
--- a/backend/oasst_backend/prompt_repository.py
+++ b/backend/oasst_backend/prompt_repository.py
@@ -7,10 +7,10 @@ from uuid import UUID, uuid4
 
 import oasst_backend.models.db_payload as db_payload
 from loguru import logger
-from oasst_backend.exceptions import OasstError, OasstErrorCode
 from oasst_backend.journal_writer import JournalWriter
 from oasst_backend.models import ApiClient, Message, MessageReaction, Task, TextLabels, User
 from oasst_backend.models.payload_column_type import PayloadContainer
+from oasst_shared.exceptions import OasstError, OasstErrorCode
 from oasst_shared.schemas import protocol as protocol_schema
 from oasst_shared.schemas.protocol import SystemStats
 from sqlalchemy import update
diff --git a/oasst-shared/oasst_shared/exceptions/__init__.py b/oasst-shared/oasst_shared/exceptions/__init__.py
new file mode 100644
index 00000000..9cf37f87
--- /dev/null
+++ b/oasst-shared/oasst_shared/exceptions/__init__.py
@@ -0,0 +1,3 @@
+# Ignore unused imports; these are re-exported
+from .oasst_api_error import OasstError as OasstError  # noqa: F401
+from .oasst_api_error import OasstErrorCode as OasstErrorCode  # noqa: F401
diff --git a/backend/oasst_backend/exceptions.py b/oasst-shared/oasst_shared/exceptions/oasst_api_error.py
similarity index 100%
rename from backend/oasst_backend/exceptions.py
rename to oasst-shared/oasst_shared/exceptions/oasst_api_error.py

From 0a59aeb8b0c973792866aa29b43e1a02be7cf1ac Mon Sep 17 00:00:00 2001
From: Yannic Kilcher <yk@users.noreply.github.com>
Date: Mon, 2 Jan 2023 22:10:48 +0100
Subject: [PATCH 2/3] added data schemas md (#288)

---
 docs/data_schemas.md | 94 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 94 insertions(+)
 create mode 100644 docs/data_schemas.md

diff --git a/docs/data_schemas.md b/docs/data_schemas.md
new file mode 100644
index 00000000..b1fe16c1
--- /dev/null
+++ b/docs/data_schemas.md
@@ -0,0 +1,94 @@
+# OpenAssistant Data Schemas
+
+## Introduction
+
+This document describes the data schemas used by OpenAssistant. The schemas are
+defined as Python classes, but can be implemented in any format, be that Python,
+JSON, XML, SQL, Parquet files, etc.
+
+Also, the schemas are leaning heavily on the
+[OpenAssistant Data Structures](https://docs.google.com/presentation/d/1iaX_nxasVWlvPiSNs0cllR9L_1neZq0RJxd6MFEalUY/edit?usp=sharing)
+presentation.
+
+## Data Schemas
+
+### Main structure: conversation trees
+
+Conversation trees are the fundamental data structure. Many of the datasets we
+want to collect can be represented as conversation trees, such as QA datasets,
+chat logs, reddit dumps, etc. The main idea is that a conversation tree starts
+with a prompt and branches out from there. Every node can also have metadata,
+such as collected rankings, labels, or other information.
+
+Datasets that just represent linear data, such as a list of questions and
+answers, can be represented as a conversation tree with just a single branch.
+
+```python
+class ConversationTreeNode:
+    text: str # The text of the node
+    role: Literal['prompter', 'assistant'] # Whether the node is a user prompt/follow-up or an assistant response
+    children: list[ConversationTreeNode] # The children of the node (if you have a linear conversation, this will be of length 0 or 1)
+    metadata: dict[str, Any] # Node metadata (see below)
+
+class ConversationTree:
+    root: ConversationTreeNode # The node containing the initial prompt
+    metadata: dict[str, Any] # Tree metadata, different from root node metadata.
+
+```
+
+### Metadata
+
+Metadata encapsulates all the information that is not part of the conversation
+itself. This includes data about how the node was created (i.e. where it is
+from: crowd-sourced, templated, scraped, etc.), when it was created, its labels,
+tags, collected rankings, and other information.
+
+## Example: Reddit AMA dataset
+
+- Represent each question-follow-up set as a conversation tree.
+- Store things like usernames, timestamps, upvotes, etc. as metadata of the
+  nodes.
+- Store things like the AMA title, the AMA author, the AMA subreddit, etc. as
+  metadata of the tree.
+
+## Example: QA dataset
+
+- Represent each question-answer pair as a conversation tree.
+  - The question is the prompt, the answer is the assistant response.
+- If the dataset contains multiple answers to each question, each answer can be
+  a child of the question node.
+- If the dataset contains context text, it can be added as metadata to the
+  question node.
+
+## Example: Templated math problem dataset
+
+- Represent each problem as a conversation tree with the problem text as the
+  prompt and the solution as the assistant response.
+- Store the problem type (e.g. algebra, geometry, etc.) as metadata of the tree.
+- Store the template used also as metadata of the tree, as well as the source of
+  the data used to fill the template.
+
+## File Formats
+
+The above data should be representable in most file formats, but some care has
+to be taken with respect to the recursive nature of the data.
+
+Most row-major formats (JSON, Avro, Protobuf, etc.), as well as many databases,
+have no trouble with recursive (or arbitrary) schemas, but column-major formats,
+such as Parquet, do. For datasets with linear conversations, like many of the
+datasets we are collecting, this is not a problem. Instead of a tree of nodes,
+simply represent the conversation as a list of nodes. For true tree-like
+conversations, we should use a row-major format.
+
+## Other considerations
+
+- For text data of moderate size, it really doesn't matter much. It's more
+  important to use consistent data structures and naming, than to worry about
+  the exact file format.
+- For crowd-sourced data, we are collecting it into a SQL database already.
+- Parquet files are a good choice for large datasets, modulo the issues with
+  recursive schemas.
+- If parquet can't be used, gzipped JSON-line files are a good choice. So are
+  Avro files and protobufs. Keep in mind that column-major files are better for
+  reading, filtering, and aggregating, but row-major files are better for
+  writing.

From ed115ea17c8d70c72312d9f7f0bb29d6a2702770 Mon Sep 17 00:00:00 2001
From: Jack Michaud <jack-michaud@users.noreply.github.com>
Date: Mon, 2 Jan 2023 21:13:20 +0000
Subject: [PATCH 3/3] refactor: move OasstApiClient into oasst-shared (#287)

* refactor: move api_client into oasst-shared

* refactor: move contract tests into oasst-shared

* fix: use new OasstApiClient imports in discord bot
---
 .github/workflows/test-api-contract.yaml              | 11 +++--------
 discord-bot/bot/bot.py                                |  2 +-
 discord-bot/bot/extensions/work.py                    |  2 +-
 .../bot => oasst-shared/oasst_shared}/api_client.py   |  0
 {discord-bot => oasst-shared}/requirements.dev.txt    |  0
 .../tests/test_oasst_api_client.py                    |  2 +-
 .../test.sh                                           |  2 +-
 7 files changed, 7 insertions(+), 12 deletions(-)
 rename {discord-bot/bot => oasst-shared/oasst_shared}/api_client.py (100%)
 rename {discord-bot => oasst-shared}/requirements.dev.txt (100%)
 rename {discord-bot => oasst-shared}/tests/test_oasst_api_client.py (97%)
 rename scripts/{discord-bot-development => oasst-shared-development}/test.sh (76%)

diff --git a/.github/workflows/test-api-contract.yaml b/.github/workflows/test-api-contract.yaml
index e75e5375..3707f4de 100644
--- a/.github/workflows/test-api-contract.yaml
+++ b/.github/workflows/test-api-contract.yaml
@@ -18,18 +18,13 @@ jobs:
 
       - run: cd oasst-shared && pip install -e .
 
+      - run: cd oasst-shared && pip install -r requirements.dev.txt
+
       - run: cd backend && pip install -r requirements.txt
 
-      - run: cd discord-bot && pip install -r requirements.txt
-
-      - run: cd discord-bot && pip install -r requirements.dev.txt
-
       - run: ./scripts/backend-development/start-mock-server.sh
 
-      # runs the contract tests. currently the api client is
-      # found in the discord bot code, but this should be updated
-      # once the client moves into oasst-shared.
       - name: Run contract tests
-        run: ./scripts/discord-bot-development/test.sh
+        run: ./scripts/oasst-shared-development/test.sh
 
       - run: ./scripts/backend-development/stop-mock-server.sh
diff --git a/discord-bot/bot/bot.py b/discord-bot/bot/bot.py
index 35a3fb23..df3c5f2f 100644
--- a/discord-bot/bot/bot.py
+++ b/discord-bot/bot/bot.py
@@ -5,9 +5,9 @@ import aiosqlite
 import hikari
 import lightbulb
 import miru
-from bot.api_client import OasstApiClient
 from bot.settings import Settings
 from bot.utils import EMPTY, mention
+from oasst_shared.api_client import OasstApiClient
 
 settings = Settings()
 
diff --git a/discord-bot/bot/extensions/work.py b/discord-bot/bot/extensions/work.py
index 767e05ac..19802c64 100644
--- a/discord-bot/bot/extensions/work.py
+++ b/discord-bot/bot/extensions/work.py
@@ -8,10 +8,10 @@ import lightbulb
 import lightbulb.decorators
 import miru
 from aiosqlite import Connection
-from bot.api_client import OasstApiClient, TaskType
 from bot.db.schemas import GuildSettings
 from bot.utils import EMPTY
 from loguru import logger
+from oasst_shared.api_client import OasstApiClient, TaskType
 from oasst_shared.schemas import protocol as protocol_schema
 from oasst_shared.schemas.protocol import TaskRequestType
 
diff --git a/discord-bot/bot/api_client.py b/oasst-shared/oasst_shared/api_client.py
similarity index 100%
rename from discord-bot/bot/api_client.py
rename to oasst-shared/oasst_shared/api_client.py
diff --git a/discord-bot/requirements.dev.txt b/oasst-shared/requirements.dev.txt
similarity index 100%
rename from discord-bot/requirements.dev.txt
rename to oasst-shared/requirements.dev.txt
diff --git a/discord-bot/tests/test_oasst_api_client.py b/oasst-shared/tests/test_oasst_api_client.py
similarity index 97%
rename from discord-bot/tests/test_oasst_api_client.py
rename to oasst-shared/tests/test_oasst_api_client.py
index 9b0ff383..be757e8f 100644
--- a/discord-bot/tests/test_oasst_api_client.py
+++ b/oasst-shared/tests/test_oasst_api_client.py
@@ -1,7 +1,7 @@
 from uuid import uuid4
 
 import pytest
-from bot.api_client import OasstApiClient
+from oasst_shared.api_client import OasstApiClient
 from oasst_shared.schemas import protocol as protocol_schema
 
 
diff --git a/scripts/discord-bot-development/test.sh b/scripts/oasst-shared-development/test.sh
similarity index 76%
rename from scripts/discord-bot-development/test.sh
rename to scripts/oasst-shared-development/test.sh
index a45adf00..e9324196 100755
--- a/scripts/discord-bot-development/test.sh
+++ b/scripts/oasst-shared-development/test.sh
@@ -2,7 +2,7 @@
 parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
 
 # switch to backend directory
-pushd "$parent_path/../../discord-bot"
+pushd "$parent_path/../../oasst-shared"
 
 pytest .