From b34ef754233b1035d95121e7da4eb84af743202e Mon Sep 17 00:00:00 2001 From: Yannic Kilcher Date: Mon, 19 Dec 2022 18:00:09 +0100 Subject: [PATCH] started re-working python code --- .gitignore | 2 ++ backend/Dockerfile | 9 ------ backend/{app => }/alembic.ini | 0 backend/{app => }/alembic/README | 0 backend/{app => }/alembic/env.py | 2 +- backend/{app => }/alembic/script.py.mako | 0 ..._12_15_0000-23e5fea252dd_first_revision.py | 0 ...12_16_0000-cd7de470586e_v1_db_structure.py | 0 ...-6368515778c5_add_auth_method_to_person.py | 0 backend/app/oasst/schemas/__init__.py | 2 -- backend/{app => }/main.py | 4 +-- .../{app/oasst => oasst_backend}/__init__.py | 0 .../oasst => oasst_backend}/api/__init__.py | 0 .../{app/oasst => oasst_backend}/api/deps.py | 6 ++-- .../api/v1/__init__.py | 0 .../oasst => oasst_backend}/api/v1/api.py | 2 +- .../oasst => oasst_backend}/api/v1/tasks.py | 8 ++--- .../{app/oasst => oasst_backend}/config.py | 0 .../oasst => oasst_backend}/crud/__init__.py | 0 .../{app/oasst => oasst_backend}/crud/base.py | 0 .../{app/oasst => oasst_backend}/database.py | 2 +- .../models/__init__.py | 0 .../models/api_client.py | 0 .../models/db_payload.py | 4 +-- .../models/payload_column_type.py | 0 .../oasst => oasst_backend}/models/person.py | 0 .../models/person_stats.py | 0 .../oasst => oasst_backend}/models/post.py | 0 .../models/post_reaction.py | 0 .../models/work_package.py | 0 .../prompt_repository.py | 8 ++--- bot/setup.py | 29 ------------------- docker/Dockerfile.backend | 15 ++++++++++ .../Dockerfile.discord-bot | 6 ++-- oasst-shared/README.md | 3 ++ .../oasst_shared}/schemas/protocol.py | 0 oasst-shared/setup.py | 16 ++++++++++ scripts/backend-development/run-local.sh | 2 +- .../frontend-development/docker-compose.yaml | 4 ++- .../postprocessing/rankings.py | 0 40 files changed, 61 insertions(+), 63 deletions(-) delete mode 100644 backend/Dockerfile rename backend/{app => }/alembic.ini (100%) rename backend/{app => }/alembic/README (100%) rename backend/{app => }/alembic/env.py (97%) rename backend/{app => }/alembic/script.py.mako (100%) rename backend/{app => }/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py (100%) rename backend/{app => }/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py (100%) rename backend/{app => }/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py (100%) delete mode 100644 backend/app/oasst/schemas/__init__.py rename backend/{app => }/main.py (93%) rename backend/{app/oasst => oasst_backend}/__init__.py (100%) rename backend/{app/oasst => oasst_backend}/api/__init__.py (100%) rename backend/{app/oasst => oasst_backend}/api/deps.py (93%) rename backend/{app/oasst => oasst_backend}/api/v1/__init__.py (100%) rename backend/{app/oasst => oasst_backend}/api/v1/api.py (79%) rename backend/{app/oasst => oasst_backend}/api/v1/tasks.py (97%) rename backend/{app/oasst => oasst_backend}/config.py (100%) rename backend/{app/oasst => oasst_backend}/crud/__init__.py (100%) rename backend/{app/oasst => oasst_backend}/crud/base.py (100%) rename backend/{app/oasst => oasst_backend}/database.py (81%) rename backend/{app/oasst => oasst_backend}/models/__init__.py (100%) rename backend/{app/oasst => oasst_backend}/models/api_client.py (100%) rename backend/{app/oasst => oasst_backend}/models/db_payload.py (94%) rename backend/{app/oasst => oasst_backend}/models/payload_column_type.py (100%) rename backend/{app/oasst => oasst_backend}/models/person.py (100%) rename backend/{app/oasst => oasst_backend}/models/person_stats.py (100%) rename backend/{app/oasst => oasst_backend}/models/post.py (100%) rename backend/{app/oasst => oasst_backend}/models/post_reaction.py (100%) rename backend/{app/oasst => oasst_backend}/models/work_package.py (100%) rename backend/{app/oasst => oasst_backend}/prompt_repository.py (97%) delete mode 100644 bot/setup.py create mode 100644 docker/Dockerfile.backend rename bot/Dockerfile => docker/Dockerfile.discord-bot (60%) create mode 100644 oasst-shared/README.md rename {backend/app/oasst => oasst-shared/oasst_shared}/schemas/protocol.py (100%) create mode 100644 oasst-shared/setup.py rename {backend => scripts}/postprocessing/rankings.py (100%) diff --git a/.gitignore b/.gitignore index ce7a9b8a..2c698b44 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .venv *.pyc *.swp +*.egg-info +__pycache__ diff --git a/backend/Dockerfile b/backend/Dockerfile deleted file mode 100644 index 8074ef3a..00000000 --- a/backend/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM tiangolo/uvicorn-gunicorn-fastapi:python3.10 - -COPY ./requirements.txt /app/requirements.txt - -RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt - -ENV PORT 8080 - -COPY ./app /app diff --git a/backend/app/alembic.ini b/backend/alembic.ini similarity index 100% rename from backend/app/alembic.ini rename to backend/alembic.ini diff --git a/backend/app/alembic/README b/backend/alembic/README similarity index 100% rename from backend/app/alembic/README rename to backend/alembic/README diff --git a/backend/app/alembic/env.py b/backend/alembic/env.py similarity index 97% rename from backend/app/alembic/env.py rename to backend/alembic/env.py index 6d2ec0c3..83de474c 100644 --- a/backend/app/alembic/env.py +++ b/backend/alembic/env.py @@ -3,7 +3,7 @@ from logging.config import fileConfig import sqlmodel from alembic import context -from oasst import models # noqa: F401 +from oasst_backend import models # noqa: F401 from sqlalchemy import engine_from_config, pool # this is the Alembic Config object, which provides diff --git a/backend/app/alembic/script.py.mako b/backend/alembic/script.py.mako similarity index 100% rename from backend/app/alembic/script.py.mako rename to backend/alembic/script.py.mako diff --git a/backend/app/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py b/backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py similarity index 100% rename from backend/app/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py rename to backend/alembic/versions/2022_12_15_0000-23e5fea252dd_first_revision.py diff --git a/backend/app/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py b/backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py similarity index 100% rename from backend/app/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py rename to backend/alembic/versions/2022_12_16_0000-cd7de470586e_v1_db_structure.py diff --git a/backend/app/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py b/backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py similarity index 100% rename from backend/app/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py rename to backend/alembic/versions/2022_12_17_2230-6368515778c5_add_auth_method_to_person.py diff --git a/backend/app/oasst/schemas/__init__.py b/backend/app/oasst/schemas/__init__.py deleted file mode 100644 index 5ee00d4a..00000000 --- a/backend/app/oasst/schemas/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# -*- coding: utf-8 -*- -__all__ = [] diff --git a/backend/app/main.py b/backend/main.py similarity index 93% rename from backend/app/main.py rename to backend/main.py index f78f608c..abcd2391 100644 --- a/backend/app/main.py +++ b/backend/main.py @@ -5,8 +5,8 @@ import alembic.command import alembic.config import fastapi from loguru import logger -from oasst.api.v1.api import api_router -from oasst.config import settings +from oasst_backend.api.v1.api import api_router +from oasst_backend.config import settings from starlette.middleware.cors import CORSMiddleware app = fastapi.FastAPI(title=settings.PROJECT_NAME, openapi_url=f"{settings.API_V1_STR}/openapi.json") diff --git a/backend/app/oasst/__init__.py b/backend/oasst_backend/__init__.py similarity index 100% rename from backend/app/oasst/__init__.py rename to backend/oasst_backend/__init__.py diff --git a/backend/app/oasst/api/__init__.py b/backend/oasst_backend/api/__init__.py similarity index 100% rename from backend/app/oasst/api/__init__.py rename to backend/oasst_backend/api/__init__.py diff --git a/backend/app/oasst/api/deps.py b/backend/oasst_backend/api/deps.py similarity index 93% rename from backend/app/oasst/api/deps.py rename to backend/oasst_backend/api/deps.py index bfa54931..96af5c5e 100644 --- a/backend/app/oasst/api/deps.py +++ b/backend/oasst_backend/api/deps.py @@ -6,9 +6,9 @@ from uuid import UUID from fastapi import HTTPException, Security from fastapi.security.api_key import APIKey, APIKeyHeader, APIKeyQuery from loguru import logger -from oasst.config import settings -from oasst.database import engine -from oasst.models import ApiClient +from oasst_backend.config import settings +from oasst_backend.database import engine +from oasst_backend.models import ApiClient from sqlmodel import Session from starlette.status import HTTP_403_FORBIDDEN diff --git a/backend/app/oasst/api/v1/__init__.py b/backend/oasst_backend/api/v1/__init__.py similarity index 100% rename from backend/app/oasst/api/v1/__init__.py rename to backend/oasst_backend/api/v1/__init__.py diff --git a/backend/app/oasst/api/v1/api.py b/backend/oasst_backend/api/v1/api.py similarity index 79% rename from backend/app/oasst/api/v1/api.py rename to backend/oasst_backend/api/v1/api.py index 3d568cb9..cd1119d6 100644 --- a/backend/app/oasst/api/v1/api.py +++ b/backend/oasst_backend/api/v1/api.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from fastapi import APIRouter -from oasst.api.v1 import tasks +from oasst_backend.api.v1 import tasks api_router = APIRouter() api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"]) diff --git a/backend/app/oasst/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py similarity index 97% rename from backend/app/oasst/api/v1/tasks.py rename to backend/oasst_backend/api/v1/tasks.py index 41f01f3c..40965022 100644 --- a/backend/app/oasst/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -6,10 +6,10 @@ from uuid import UUID from fastapi import APIRouter, Depends, HTTPException from fastapi.security.api_key import APIKey from loguru import logger -from oasst.api import deps -from oasst.models.db_payload import TaskPayload -from oasst.prompt_repository import PromptRepository -from oasst.schemas import protocol as protocol_schema +from oasst_backend.api import deps +from oasst_backend.models.db_payload import TaskPayload +from oasst_backend.prompt_repository import PromptRepository +from oasst_shared.schemas import protocol as protocol_schema from sqlmodel import Session from starlette.status import HTTP_400_BAD_REQUEST diff --git a/backend/app/oasst/config.py b/backend/oasst_backend/config.py similarity index 100% rename from backend/app/oasst/config.py rename to backend/oasst_backend/config.py diff --git a/backend/app/oasst/crud/__init__.py b/backend/oasst_backend/crud/__init__.py similarity index 100% rename from backend/app/oasst/crud/__init__.py rename to backend/oasst_backend/crud/__init__.py diff --git a/backend/app/oasst/crud/base.py b/backend/oasst_backend/crud/base.py similarity index 100% rename from backend/app/oasst/crud/base.py rename to backend/oasst_backend/crud/base.py diff --git a/backend/app/oasst/database.py b/backend/oasst_backend/database.py similarity index 81% rename from backend/app/oasst/database.py rename to backend/oasst_backend/database.py index ca729f4e..66d7a857 100644 --- a/backend/app/oasst/database.py +++ b/backend/oasst_backend/database.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from oasst.config import settings +from oasst_backend.config import settings from sqlmodel import create_engine if settings.DATABASE_URI is None: diff --git a/backend/app/oasst/models/__init__.py b/backend/oasst_backend/models/__init__.py similarity index 100% rename from backend/app/oasst/models/__init__.py rename to backend/oasst_backend/models/__init__.py diff --git a/backend/app/oasst/models/api_client.py b/backend/oasst_backend/models/api_client.py similarity index 100% rename from backend/app/oasst/models/api_client.py rename to backend/oasst_backend/models/api_client.py diff --git a/backend/app/oasst/models/db_payload.py b/backend/oasst_backend/models/db_payload.py similarity index 94% rename from backend/app/oasst/models/db_payload.py rename to backend/oasst_backend/models/db_payload.py index b01cecce..2a4438e2 100644 --- a/backend/app/oasst/models/db_payload.py +++ b/backend/oasst_backend/models/db_payload.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- from typing import Literal -from oasst.models.payload_column_type import payload_type -from oasst.schemas import protocol as protocol_schema +from oasst_backend.models.payload_column_type import payload_type +from oasst_shared.schemas import protocol as protocol_schema from pydantic import BaseModel diff --git a/backend/app/oasst/models/payload_column_type.py b/backend/oasst_backend/models/payload_column_type.py similarity index 100% rename from backend/app/oasst/models/payload_column_type.py rename to backend/oasst_backend/models/payload_column_type.py diff --git a/backend/app/oasst/models/person.py b/backend/oasst_backend/models/person.py similarity index 100% rename from backend/app/oasst/models/person.py rename to backend/oasst_backend/models/person.py diff --git a/backend/app/oasst/models/person_stats.py b/backend/oasst_backend/models/person_stats.py similarity index 100% rename from backend/app/oasst/models/person_stats.py rename to backend/oasst_backend/models/person_stats.py diff --git a/backend/app/oasst/models/post.py b/backend/oasst_backend/models/post.py similarity index 100% rename from backend/app/oasst/models/post.py rename to backend/oasst_backend/models/post.py diff --git a/backend/app/oasst/models/post_reaction.py b/backend/oasst_backend/models/post_reaction.py similarity index 100% rename from backend/app/oasst/models/post_reaction.py rename to backend/oasst_backend/models/post_reaction.py diff --git a/backend/app/oasst/models/work_package.py b/backend/oasst_backend/models/work_package.py similarity index 100% rename from backend/app/oasst/models/work_package.py rename to backend/oasst_backend/models/work_package.py diff --git a/backend/app/oasst/prompt_repository.py b/backend/oasst_backend/prompt_repository.py similarity index 97% rename from backend/app/oasst/prompt_repository.py rename to backend/oasst_backend/prompt_repository.py index 35f1c9b3..9f7bb1dd 100644 --- a/backend/app/oasst/prompt_repository.py +++ b/backend/oasst_backend/prompt_repository.py @@ -3,11 +3,11 @@ from datetime import datetime from typing import Optional from uuid import UUID, uuid4 -import oasst.models.db_payload as db_payload +import oasst_backend.models.db_payload as db_payload from loguru import logger -from oasst.models import ApiClient, Person, Post, PostReaction, WorkPackage -from oasst.models.payload_column_type import PayloadContainer -from oasst.schemas import protocol as protocol_schema +from oasst_backend.models import ApiClient, Person, Post, PostReaction, WorkPackage +from oasst_backend.models.payload_column_type import PayloadContainer +from oasst_shared.schemas import protocol as protocol_schema from sqlmodel import Session diff --git a/bot/setup.py b/bot/setup.py deleted file mode 100644 index 06f7326e..00000000 --- a/bot/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -from setuptools import find_packages, setup - -if __name__ == "__main__": - import os - - def _read_reqs(relpath): - fullpath = os.path.join(os.path.dirname(__file__), relpath) - with open(fullpath) as f: - return [s.strip() for s in f.readlines() if (s.strip() and not s.startswith("#"))] - - REQUIREMENTS = _read_reqs("requirements.txt") - - setup( - name="open-assistant", - packages=find_packages(), - version="0.0.1", - license="Apache 2.0", - description="A Discord Bot for collecting and ranking prompts to train an Open Assistant", - keywords=["machine learning", "natural language processing", "discord"], - install_requires=REQUIREMENTS, - classifiers=[ - "Development Status :: Alpha", - "Intended Audience :: Developers", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "License :: OSI Approved :: Apache License", - "Programming Language :: Python :: 3.6", - ], - ) diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend new file mode 100644 index 00000000..d9458ae0 --- /dev/null +++ b/docker/Dockerfile.backend @@ -0,0 +1,15 @@ +FROM tiangolo/uvicorn-gunicorn-fastapi:python3.10 + +COPY ./backend/requirements.txt /app/requirements.txt + +RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt + +ENV PORT 8080 + +COPY ./oasst-shared /oasst-shared +RUN pip install -e /oasst-shared + +COPY ./backend/alembic /app/alembic +COPY ./backend/alembic.ini /app/alembic.ini +COPY ./backend/main.py /app/main.py +COPY ./backend/oasst_backend /app/oasst_backend diff --git a/bot/Dockerfile b/docker/Dockerfile.discord-bot similarity index 60% rename from bot/Dockerfile rename to docker/Dockerfile.discord-bot index ab215b5b..13ae308a 100644 --- a/bot/Dockerfile +++ b/docker/Dockerfile.discord-bot @@ -1,7 +1,7 @@ FROM python:3.10-slim-bullseye RUN mkdir /app -ADD requirements.txt /app/requirements.txt -WORKDIR /app +COPY ./discord-bot/requirements.txt /requirements.txt RUN pip install -r requirements.txt -ADD . /app +WORKDIR /app +COPY ./discord-bot /app CMD ["python", "bot.py"] diff --git a/oasst-shared/README.md b/oasst-shared/README.md new file mode 100644 index 00000000..28761ced --- /dev/null +++ b/oasst-shared/README.md @@ -0,0 +1,3 @@ +# Shared Python code for Open Assisstant + +Run `pip install -e .` to install the package in editable mode. diff --git a/backend/app/oasst/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py similarity index 100% rename from backend/app/oasst/schemas/protocol.py rename to oasst-shared/oasst_shared/schemas/protocol.py diff --git a/oasst-shared/setup.py b/oasst-shared/setup.py new file mode 100644 index 00000000..ebaf4217 --- /dev/null +++ b/oasst-shared/setup.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# setup.py for the shared python modules + +from distutils.core import setup + +from setuptools import find_namespace_packages + +setup( + name="oasst-shared", + version="1.0", + packages=find_namespace_packages(), + author="OASST Team", + install_requires=[ + "pydantic==1.9.1", + ], +) diff --git a/scripts/backend-development/run-local.sh b/scripts/backend-development/run-local.sh index 064612c3..e9df6ca2 100755 --- a/scripts/backend-development/run-local.sh +++ b/scripts/backend-development/run-local.sh @@ -2,7 +2,7 @@ parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) # switch to backend directory -pushd "$parent_path/../../backend/app" +pushd "$parent_path/../../backend" export ALLOW_ANY_API_KEY=True diff --git a/scripts/frontend-development/docker-compose.yaml b/scripts/frontend-development/docker-compose.yaml index 1127d35d..7ff5dd28 100644 --- a/scripts/frontend-development/docker-compose.yaml +++ b/scripts/frontend-development/docker-compose.yaml @@ -15,7 +15,9 @@ services: file: ../backend-development/docker-compose.yaml service: adminer backend: - build: ../../backend/. + build: + dockerfile: docker/Dockerfile.backend + context: ../.. image: oasst-backend environment: - POSTGRES_HOST=db diff --git a/backend/postprocessing/rankings.py b/scripts/postprocessing/rankings.py similarity index 100% rename from backend/postprocessing/rankings.py rename to scripts/postprocessing/rankings.py