diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-to-node.yaml similarity index 73% rename from .github/workflows/deploy-dev.yaml rename to .github/workflows/deploy-to-node.yaml index f9938ebf..63b6f78f 100644 --- a/.github/workflows/deploy-dev.yaml +++ b/.github/workflows/deploy-to-node.yaml @@ -1,4 +1,4 @@ -name: Deploy to dev machine +name: Deploy to node on: workflow_call: @@ -19,6 +19,14 @@ on: required: false type: string default: 3000 + postgres-password: + required: false + type: string + default: postgres + web-api-key: + required: false + type: string + default: "1234" jobs: deploy: @@ -39,7 +47,7 @@ jobs: uses: dawidd6/action-ansible-playbook@v2 with: # Required, playbook filepath - playbook: deploy-dev.yaml + playbook: deploy-to-node.yaml # Optional, directory where playbooks live directory: ansible # Optional, SSH private key @@ -49,4 +57,9 @@ jobs: [dev] dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team options: | - --extra-vars "stack_name=${{inputs.stack-name}} image_tag=${{inputs.image-tag}} backend_port=${{inputs.backend-port}} website_port=${{inputs.website-port}}" + --extra-vars "stack_name=${{inputs.stack-name}} \ + image_tag=${{inputs.image-tag}} \ + backend_port=${{inputs.backend-port}} \ + website_port=${{inputs.website-port}} \ + postgres_password=${{inputs.postgres-password}} \ + web_api_key=${{inputs.web-api-key}}" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index ea239492..51086a78 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -35,9 +35,9 @@ jobs: context: . dockerfile: docker/Dockerfile.discord-bot build-args: "" - deploy-dev: + deploy-to-node: needs: [build-backend, build-web, build-bot] - uses: ./.github/workflows/deploy-dev.yaml + uses: ./.github/workflows/deploy-to-node.yaml secrets: inherit with: stack-name: ${{ github.event_name == 'release' && 'staging' || 'dev' }} @@ -46,3 +46,9 @@ jobs: 'latest' }} backend-port: ${{ github.event_name == 'release' && '8180' || '8080' }} website-port: ${{ github.event_name == 'release' && '3100' || '3000' }} + postgres-password: + ${{ github.event_name == 'release' && secrets.STAGING_POSTGRES_PASSWORD + || 'postgres' }} + web-api-key: + ${{ github.event_name == 'release' && secrets.STAGING_WEB_API_KEY || + '1234' }} diff --git a/ansible/deploy-dev.yaml b/ansible/deploy-to-node.yaml similarity index 81% rename from ansible/deploy-dev.yaml rename to ansible/deploy-to-node.yaml index 8d701fb2..4fc6fe4e 100644 --- a/ansible/deploy-dev.yaml +++ b/ansible/deploy-to-node.yaml @@ -1,6 +1,6 @@ # ansible playbook to set up some docker containers -- name: Deploy to dev node +- name: Deploy to node hosts: dev gather_facts: true vars: @@ -8,6 +8,8 @@ image_tag: latest backend_port: 8080 website_port: 3000 + postgres_password: postgres + web_api_key: "1234" tasks: - name: Create network community.docker.docker_network: @@ -44,6 +46,14 @@ volumes: - "./{{ stack_name }}/redis.conf:/usr/local/etc/redis/redis.conf" + - name: Create volumes for postgres + community.docker.docker_volume: + name: "oasst-{{ stack_name }}-postgres-{{ item.name }}" + state: present + loop: + - name: backend + - name: web + - name: Create postgres containers community.docker.docker_container: name: "oasst-{{ stack_name }}-postgres-{{ item.name }}" @@ -54,8 +64,12 @@ network_mode: "oasst-{{ stack_name }}" env: POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_PASSWORD: "{{ postgres_password }}" POSTGRES_DB: postgres + OFFICIAL_WEB_API_KEY: "{{ web_api_key }}" + volumes: + - "oasst-{{ stack_name }}-postgres-{{ item.name + }}:/var/lib/postgresql/data" healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] interval: 2s @@ -76,15 +90,17 @@ network_mode: "oasst-{{ stack_name }}" env: POSTGRES_HOST: "oasst-{{ stack_name }}-postgres-backend" + POSTGRES_PASSWORD: "{{ postgres_password }}" REDIS_HOST: "oasst-{{ stack_name }}-redis" - DEBUG_ALLOW_DEBUG_API_KEY: "true" - DEBUG_USE_SEED_DATA: "true" + DEBUG_USE_SEED_DATA: + "{{ 'true' if stack_name == 'dev' else 'false' }}" DEBUG_ALLOW_SELF_LABELING: "{{ 'true' if stack_name == 'dev' else 'false' }}" MAX_WORKERS: "1" RATE_LIMIT: "{{ 'false' if stack_name == 'dev' else 'true' }}" DEBUG_SKIP_EMBEDDING_COMPUTATION: "true" - DEBUG_SKIP_TOXICITY_CALCULATION: "true" + DEBUG_SKIP_TOXICITY_CALCULATION: + "{{ 'true' if stack_name == 'dev' else 'false' }}" ports: - "{{ backend_port }}:8080" @@ -100,9 +116,9 @@ env: ADMIN_USERS: "{{ lookup('ansible.builtin.env', 'WEB_ADMIN_USERS') }}" DATABASE_URL: - "postgres://postgres:postgres@oasst-{{ stack_name + "postgres://postgres:{{ postgres_password }}@oasst-{{ stack_name }}-postgres-web/postgres" - DEBUG_LOGIN: "true" + DEBUG_LOGIN: "{{ 'true' if stack_name == 'dev' else 'false' }}" DISCORD_CLIENT_ID: "{{ lookup('ansible.builtin.env', 'WEB_DISCORD_CLIENT_ID') }}" DISCORD_CLIENT_SECRET: @@ -117,7 +133,7 @@ EMAIL_SERVER_USER: "{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_USER') }}" FASTAPI_URL: "http://oasst-{{ stack_name }}-backend:8080" - FASTAPI_KEY: "1234" + FASTAPI_KEY: "{{ web_api_key }}" NEXTAUTH_SECRET: "{{ lookup('ansible.builtin.env', 'WEB_NEXTAUTH_SECRET') }}" NEXTAUTH_URL: http://web.{{ stack_name }}.open-assistant.io/ diff --git a/backend/main.py b/backend/main.py index 3787100e..8037032c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -11,7 +11,7 @@ import redis.asyncio as redis from fastapi_limiter import FastAPILimiter from fastapi_utils.tasks import repeat_every from loguru import logger -from oasst_backend.api.deps import get_dummy_api_client +from oasst_backend.api.deps import api_auth, create_api_client from oasst_backend.api.v1.api import api_router from oasst_backend.api.v1.utils import prepare_conversation from oasst_backend.config import settings @@ -76,6 +76,20 @@ if settings.UPDATE_ALEMBIC: logger.exception("Alembic upgrade failed on startup") +if settings.OFFICIAL_WEB_API_KEY: + + @app.on_event("startup") + def create_official_web_api_client(): + with Session(engine) as session: + create_api_client( + session=session, + api_key=settings.OFFICIAL_WEB_API_KEY, + description="The official web client for the OASST backend.", + frontend_type="web", + trusted=True, + ) + + if settings.RATE_LIMIT: @app.on_event("startup") @@ -111,10 +125,13 @@ if settings.DEBUG_USE_SEED_DATA: role: str tree_state: Optional[message_tree_state.State] + if not settings.OFFICIAL_WEB_API_KEY: + raise ValueError("Cannot use seed data without OFFICIAL_WEB_API_KEY") + try: logger.info("Seed data check began") with Session(engine) as db: - api_client = get_dummy_api_client(db) + api_client = api_auth(settings.OFFICIAL_WEB_API_KEY, db=db) dummy_user = protocol_schema.User(id="__dummy_user__", display_name="Dummy User", auth_method="local") ur = UserRepository(db=db, api_client=api_client) diff --git a/backend/oasst_backend/api/deps.py b/backend/oasst_backend/api/deps.py index b4d27870..985b72a0 100644 --- a/backend/oasst_backend/api/deps.py +++ b/backend/oasst_backend/api/deps.py @@ -61,33 +61,11 @@ def create_api_client( return api_client -def get_dummy_api_client(session: Session) -> ApiClient: - # make sure that a dummy api key exits in db (foreign key references) - DUMMY_API_KEY = "1234" - api_client: ApiClient = session.query(ApiClient).filter(ApiClient.api_key == DUMMY_API_KEY).first() - if api_client is None: - logger.info(f"ANY_API_KEY missing, inserting api_key: {DUMMY_API_KEY}") - api_client = create_api_client( - session=session, - api_key=DUMMY_API_KEY, - description="Dummy api key for debugging", - trusted=True, - frontend_type="Test frontend", - ) - session.add(api_client) - session.commit() - return api_client - - def api_auth( api_key: APIKey, db: Session, ) -> ApiClient: - if api_key or settings.DEBUG_SKIP_API_KEY_CHECK: - - if settings.DEBUG_SKIP_API_KEY_CHECK or settings.DEBUG_ALLOW_DEBUG_API_KEY: - return get_dummy_api_client(db) - + if api_key: api_client = db.query(ApiClient).filter(ApiClient.api_key == api_key).first() if api_client is not None and api_client.enabled: return api_client diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py index 005f43dd..b0c6b7f5 100644 --- a/backend/oasst_backend/config.py +++ b/backend/oasst_backend/config.py @@ -59,6 +59,7 @@ class TreeManagerConfiguration(BaseModel): class Settings(BaseSettings): PROJECT_NAME: str = "open-assistant backend" API_V1_STR: str = "/api/v1" + OFFICIAL_WEB_API_KEY: str = "1234" POSTGRES_HOST: str = "localhost" POSTGRES_PORT: str = "5432" @@ -71,8 +72,6 @@ class Settings(BaseSettings): REDIS_HOST: str = "localhost" REDIS_PORT: str = "6379" - DEBUG_ALLOW_DEBUG_API_KEY: bool = False - DEBUG_SKIP_API_KEY_CHECK: bool = False DEBUG_USE_SEED_DATA: bool = False DEBUG_USE_SEED_DATA_PATH: Optional[FilePath] = ( Path(__file__).parent.parent / "test_data/realistic/realistic_seed_data.json" diff --git a/copilot/api/manifest.yml b/copilot/api/manifest.yml index 59848a25..893689ea 100644 --- a/copilot/api/manifest.yml +++ b/copilot/api/manifest.yml @@ -29,8 +29,6 @@ environments: variables: # Note: this has to be a valid JSON list for Pydantic to parse it. BACKEND_CORS_ORIGINS: '["https://web.staging.open-assistant.surfacedata.org"]' - DEBUG_ALLOW_DEBUG_API_KEY: True - DEBUG_SKIP_API_KEY_CHECK: True MAX_WORKERS: 1 secrets: diff --git a/docker-compose.yaml b/docker-compose.yaml index cde65166..78192eb3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -97,7 +97,6 @@ services: environment: - POSTGRES_HOST=db - REDIS_HOST=redis - - DEBUG_SKIP_API_KEY_CHECK=True - DEBUG_USE_SEED_DATA=True - DEBUG_ALLOW_SELF_LABELING=True - MAX_WORKERS=1 diff --git a/scripts/backend-development/run-local.sh b/scripts/backend-development/run-local.sh index 7d3f715c..7366cde6 100755 --- a/scripts/backend-development/run-local.sh +++ b/scripts/backend-development/run-local.sh @@ -4,7 +4,6 @@ parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) # switch to backend directory pushd "$parent_path/../../backend" -export DEBUG_SKIP_API_KEY_CHECK=False export DEBUG_USE_SEED_DATA=True export DEBUG_SKIP_TOXICITY_CALCULATION=True export DEBUG_ALLOW_SELF_LABELING=True