wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-06-27 16:10:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

prepared playbook for prod

Browse Source
This commit is contained in:
Yannic Kilcher
2023-01-15 23:30:16 +01:00
parent cc03376d86
commit 5d441b1570
9 changed files with 69 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/deploy-dev.yaml → .github/workflows/deploy-to-node.yaml
+16 -3
View File
@@ -1,4 +1,4 @@
name: Deploy to dev machine
name: Deploy to node
on:
workflow_call:
@@ -19,6 +19,14 @@ on:
required: false
type: string
default: 3000
postgres-password:
required: false
type: string
default: postgres
web-api-key:
required: false
type: string
default: "1234"
jobs:
deploy:
@@ -39,7 +47,7 @@ jobs:
uses: dawidd6/action-ansible-playbook@v2
with:
# Required, playbook filepath
playbook: deploy-dev.yaml
playbook: deploy-to-node.yaml
# Optional, directory where playbooks live
directory: ansible
# Optional, SSH private key
@@ -49,4 +57,9 @@ jobs:
[dev]
dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team
options: |
--extra-vars "stack_name=${{inputs.stack-name}} image_tag=${{inputs.image-tag}} backend_port=${{inputs.backend-port}} website_port=${{inputs.website-port}}"
--extra-vars "stack_name=${{inputs.stack-name}} \
image_tag=${{inputs.image-tag}} \
backend_port=${{inputs.backend-port}} \
website_port=${{inputs.website-port}} \
postgres_password=${{inputs.postgres-password}} \
web_api_key=${{inputs.web-api-key}}"
.github/workflows/release.yaml
+8 -2
View File
@@ -35,9 +35,9 @@ jobs:
context: .
dockerfile: docker/Dockerfile.discord-bot
build-args: ""
deploy-dev:
deploy-to-node:
needs: [build-backend, build-web, build-bot]
uses: ./.github/workflows/deploy-dev.yaml
uses: ./.github/workflows/deploy-to-node.yaml
secrets: inherit
with:
stack-name: ${{ github.event_name == 'release' && 'staging' || 'dev' }}
@@ -46,3 +46,9 @@ jobs:
'latest' }}
backend-port: ${{ github.event_name == 'release' && '8180' || '8080' }}
website-port: ${{ github.event_name == 'release' && '3100' || '3000' }}
postgres-password:
${{ github.event_name == 'release' && secrets.STAGING_POSTGRES_PASSWORD
|| 'postgres' }}
web-api-key:
${{ github.event_name == 'release' && secrets.STAGING_WEB_API_KEY ||
'1234' }}
ansible/deploy-dev.yaml → ansible/deploy-to-node.yaml
+24 -8
View File
@@ -1,6 +1,6 @@
# ansible playbook to set up some docker containers
- name: Deploy to dev node
- name: Deploy to node
hosts: dev
gather_facts: true
vars:
@@ -8,6 +8,8 @@
image_tag: latest
backend_port: 8080
website_port: 3000
postgres_password: postgres
web_api_key: "1234"
tasks:
- name: Create network
community.docker.docker_network:
@@ -44,6 +46,14 @@
volumes:
- "./{{ stack_name }}/redis.conf:/usr/local/etc/redis/redis.conf"
- name: Create volumes for postgres
community.docker.docker_volume:
name: "oasst-{{ stack_name }}-postgres-{{ item.name }}"
state: present
loop:
- name: backend
- name: web
- name: Create postgres containers
community.docker.docker_container:
name: "oasst-{{ stack_name }}-postgres-{{ item.name }}"
@@ -54,8 +64,12 @@
network_mode: "oasst-{{ stack_name }}"
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PASSWORD: "{{ postgres_password }}"
POSTGRES_DB: postgres
OFFICIAL_WEB_API_KEY: "{{ web_api_key }}"
volumes:
- "oasst-{{ stack_name }}-postgres-{{ item.name
}}:/var/lib/postgresql/data"
healthcheck:
test: ["CMD", "pg_isready", "-U", "postgres"]
interval: 2s
@@ -76,15 +90,17 @@
network_mode: "oasst-{{ stack_name }}"
env:
POSTGRES_HOST: "oasst-{{ stack_name }}-postgres-backend"
POSTGRES_PASSWORD: "{{ postgres_password }}"
REDIS_HOST: "oasst-{{ stack_name }}-redis"
DEBUG_ALLOW_DEBUG_API_KEY: "true"
DEBUG_USE_SEED_DATA: "true"
DEBUG_USE_SEED_DATA:
"{{ 'true' if stack_name == 'dev' else 'false' }}"
DEBUG_ALLOW_SELF_LABELING:
"{{ 'true' if stack_name == 'dev' else 'false' }}"
MAX_WORKERS: "1"
RATE_LIMIT: "{{ 'false' if stack_name == 'dev' else 'true' }}"
DEBUG_SKIP_EMBEDDING_COMPUTATION: "true"
DEBUG_SKIP_TOXICITY_CALCULATION: "true"
DEBUG_SKIP_TOXICITY_CALCULATION:
"{{ 'true' if stack_name == 'dev' else 'false' }}"
ports:
- "{{ backend_port }}:8080"
@@ -100,9 +116,9 @@
env:
ADMIN_USERS: "{{ lookup('ansible.builtin.env', 'WEB_ADMIN_USERS') }}"
DATABASE_URL:
"postgres://postgres:postgres@oasst-{{ stack_name
"postgres://postgres:{{ postgres_password }}@oasst-{{ stack_name
}}-postgres-web/postgres"
DEBUG_LOGIN: "true"
DEBUG_LOGIN: "{{ 'true' if stack_name == 'dev' else 'false' }}"
DISCORD_CLIENT_ID:
"{{ lookup('ansible.builtin.env', 'WEB_DISCORD_CLIENT_ID') }}"
DISCORD_CLIENT_SECRET:
@@ -117,7 +133,7 @@
EMAIL_SERVER_USER:
"{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_USER') }}"
FASTAPI_URL: "http://oasst-{{ stack_name }}-backend:8080"
FASTAPI_KEY: "1234"
FASTAPI_KEY: "{{ web_api_key }}"
NEXTAUTH_SECRET:
"{{ lookup('ansible.builtin.env', 'WEB_NEXTAUTH_SECRET') }}"
NEXTAUTH_URL: http://web.{{ stack_name }}.open-assistant.io/
backend/main.py
+19 -2
View File
@@ -11,7 +11,7 @@ import redis.asyncio as redis
from fastapi_limiter import FastAPILimiter
from fastapi_utils.tasks import repeat_every
from loguru import logger
from oasst_backend.api.deps import get_dummy_api_client
from oasst_backend.api.deps import api_auth, create_api_client
from oasst_backend.api.v1.api import api_router
from oasst_backend.api.v1.utils import prepare_conversation
from oasst_backend.config import settings
@@ -76,6 +76,20 @@ if settings.UPDATE_ALEMBIC:
logger.exception("Alembic upgrade failed on startup")
if settings.OFFICIAL_WEB_API_KEY:
@app.on_event("startup")
def create_official_web_api_client():
with Session(engine) as session:
create_api_client(
session=session,
api_key=settings.OFFICIAL_WEB_API_KEY,
description="The official web client for the OASST backend.",
frontend_type="web",
trusted=True,
)
if settings.RATE_LIMIT:
@app.on_event("startup")
@@ -111,10 +125,13 @@ if settings.DEBUG_USE_SEED_DATA:
role: str
tree_state: Optional[message_tree_state.State]
if not settings.OFFICIAL_WEB_API_KEY:
raise ValueError("Cannot use seed data without OFFICIAL_WEB_API_KEY")
try:
logger.info("Seed data check began")
with Session(engine) as db:
api_client = get_dummy_api_client(db)
api_client = api_auth(settings.OFFICIAL_WEB_API_KEY, db=db)
dummy_user = protocol_schema.User(id="__dummy_user__", display_name="Dummy User", auth_method="local")
ur = UserRepository(db=db, api_client=api_client)
backend/oasst_backend/api/deps.py
+1 -23
View File
@@ -61,33 +61,11 @@ def create_api_client(
return api_client
def get_dummy_api_client(session: Session) -> ApiClient:
# make sure that a dummy api key exits in db (foreign key references)
DUMMY_API_KEY = "1234"
api_client: ApiClient = session.query(ApiClient).filter(ApiClient.api_key == DUMMY_API_KEY).first()
if api_client is None:
logger.info(f"ANY_API_KEY missing, inserting api_key: {DUMMY_API_KEY}")
api_client = create_api_client(
session=session,
api_key=DUMMY_API_KEY,
description="Dummy api key for debugging",
trusted=True,
frontend_type="Test frontend",
)
session.add(api_client)
session.commit()
return api_client
def api_auth(
api_key: APIKey,
db: Session,
) -> ApiClient:
if api_key or settings.DEBUG_SKIP_API_KEY_CHECK:
if settings.DEBUG_SKIP_API_KEY_CHECK or settings.DEBUG_ALLOW_DEBUG_API_KEY:
return get_dummy_api_client(db)
if api_key:
api_client = db.query(ApiClient).filter(ApiClient.api_key == api_key).first()
if api_client is not None and api_client.enabled:
return api_client
backend/oasst_backend/config.py
+1 -2
View File
@@ -59,6 +59,7 @@ class TreeManagerConfiguration(BaseModel):
class Settings(BaseSettings):
PROJECT_NAME: str = "open-assistant backend"
API_V1_STR: str = "/api/v1"
OFFICIAL_WEB_API_KEY: str = "1234"
POSTGRES_HOST: str = "localhost"
POSTGRES_PORT: str = "5432"
@@ -71,8 +72,6 @@ class Settings(BaseSettings):
REDIS_HOST: str = "localhost"
REDIS_PORT: str = "6379"
DEBUG_ALLOW_DEBUG_API_KEY: bool = False
DEBUG_SKIP_API_KEY_CHECK: bool = False
DEBUG_USE_SEED_DATA: bool = False
DEBUG_USE_SEED_DATA_PATH: Optional[FilePath] = (
Path(__file__).parent.parent / "test_data/realistic/realistic_seed_data.json"
copilot/api/manifest.yml
-2
View File
@@ -29,8 +29,6 @@ environments:
variables:
# Note: this has to be a valid JSON list for Pydantic to parse it.
BACKEND_CORS_ORIGINS: '["https://web.staging.open-assistant.surfacedata.org"]'
DEBUG_ALLOW_DEBUG_API_KEY: True
DEBUG_SKIP_API_KEY_CHECK: True
MAX_WORKERS: 1
secrets:
docker-compose.yaml
-1
View File
@@ -97,7 +97,6 @@ services:
environment:
- POSTGRES_HOST=db
- REDIS_HOST=redis
- DEBUG_SKIP_API_KEY_CHECK=True
- DEBUG_USE_SEED_DATA=True
- DEBUG_ALLOW_SELF_LABELING=True
- MAX_WORKERS=1
scripts/backend-development/run-local.sh
-1
View File
@@ -4,7 +4,6 @@ parent_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
# switch to backend directory
pushd "$parent_path/../../backend"
export DEBUG_SKIP_API_KEY_CHECK=False
export DEBUG_USE_SEED_DATA=True
export DEBUG_SKIP_TOXICITY_CALCULATION=True
export DEBUG_ALLOW_SELF_LABELING=True