From 39cf2cf6db674620317123b066c435a7efe16414 Mon Sep 17 00:00:00 2001 From: Yannic Kilcher Date: Sat, 14 Jan 2023 20:06:27 +0100 Subject: [PATCH] added deploy workflow --- .github/workflows/deploy-dev.yaml | 50 +++++++++++++++++++++ .github/workflows/docker-build.yaml | 5 ++- .github/workflows/pre-commit.yaml | 6 +-- .github/workflows/release.yaml | 42 +++++++----------- ansible/{dev.yaml => deploy-dev.yaml} | 63 ++++++++++++--------------- 5 files changed, 99 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/deploy-dev.yaml rename ansible/{dev.yaml => deploy-dev.yaml} (69%) diff --git a/.github/workflows/deploy-dev.yaml b/.github/workflows/deploy-dev.yaml new file mode 100644 index 00000000..1fe88dbe --- /dev/null +++ b/.github/workflows/deploy-dev.yaml @@ -0,0 +1,50 @@ +name: Deploy to dev machine + +on: + workflow_call: + inputs: + stack-name: + required: false + type: string + default: dev + image-tag: + required: false + type: string + default: latest + backend-port: + required: false + type: string + default: 8080 + website-port: + required: false + type: string + default: 3000 + +jobs: + deploy: + runs-on: ubuntu-latest + env: + WEB_ADMIN_USERS: ${{ secrets.DEV_WEB_ADMIN_USERS }} + WEB_DISCORD_CLIENT_ID: ${{ secrets.DEV_WEB_DISCORD_CLIENT_ID }} + WEB_DISCORD_CLIENT_SECRET: ${{ secrets.DEV_WEB_DISCORD_CLIENT_SECRET }} + WEB_EMAIL_SERVER_HOST: ${{ secrets.DEV_WEB_EMAIL_SERVER_HOST }} + WEB_EMAIL_SERVER_PASSWORD: ${{ secrets.DEV_WEB_EMAIL_SERVER_PASSWORD }} + WEB_EMAIL_SERVER_PORT: ${{ secrets.DEV_WEB_EMAIL_SERVER_PORT }} + WEB_EMAIL_SERVER_USER: ${{ secrets.DEV_WEB_EMAIL_SERVER_USER }} + WEB_NEXTAUTH_SECRET: ${{ secrets.DEV_WEB_NEXTAUTH_SECRET }} + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Run playbook + uses: dawidd6/action-ansible-playbook@v2 + with: + # Required, playbook filepath + playbook: deploy-dev.yaml + # Optional, directory where playbooks live + directory: ansible + # Optional, SSH private key + key: ${{secrets.DEV_NODE_PRIVATE_KEY}} + # Optional, literal inventory file contents + inventory: | + [dev] + dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team stack_name=${{inputs.stack-name}} image_tag=${{inputs.image-tag}} backend_port=${{inputs.backend-port}} website_port=${{inputs.website-port}} diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 2a8c2ff9..9fdce19c 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -46,8 +46,9 @@ jobs: with: images: ${{ env.REGISTRY }}/${{ inputs.image-name }} tags: | - type=sha,prefix=${{ env.TAG_PREFIX }},format=short - type=ref,event=tag + type=raw,value=latest,enable={{is_default_branch}} + type=sha,prefix=${{ env.TAG_PREFIX }},format=short,enable={{!is_default_branch}} + type=ref,event=tag,enable={{!is_default_branch}} - name: Build and push Docker image uses: docker/build-push-action@v3.2.0 with: diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 47f21feb..3f587f28 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -1,9 +1,7 @@ name: pre-commit on: - push: - branches: - - main + workflow_call: pull_request_target: jobs: @@ -18,7 +16,7 @@ jobs: # in case of push, check out the main branch - uses: actions/checkout@v3 - if: github.event_name == 'push' + if: github.event_name != 'pull_request_target' - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1bf0ac6a..6eafe30c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,10 +1,16 @@ name: Release on: + push: + branches: + - main release: - types: [released] + types: + - released jobs: + pre-commit: + uses: ./.github/workflows/pre-commit.yaml build-backend: uses: ./.github/workflows/docker-build.yaml with: @@ -28,29 +34,11 @@ jobs: build-args: "" deploy-dev: needs: [build-backend, build-web, build-bot] - runs-on: ubuntu-latest - env: - WEB_ADMIN_USERS: ${{ secrets.DEV_WEB_ADMIN_USERS }} - WEB_DISCORD_CLIENT_ID: ${{ secrets.DEV_WEB_DISCORD_CLIENT_ID }} - WEB_DISCORD_CLIENT_SECRET: ${{ secrets.DEV_WEB_DISCORD_CLIENT_SECRET }} - WEB_EMAIL_SERVER_HOST: ${{ secrets.DEV_WEB_EMAIL_SERVER_HOST }} - WEB_EMAIL_SERVER_PASSWORD: ${{ secrets.DEV_WEB_EMAIL_SERVER_PASSWORD }} - WEB_EMAIL_SERVER_PORT: ${{ secrets.DEV_WEB_EMAIL_SERVER_PORT }} - WEB_EMAIL_SERVER_USER: ${{ secrets.DEV_WEB_EMAIL_SERVER_USER }} - WEB_NEXTAUTH_SECRET: ${{ secrets.DEV_WEB_NEXTAUTH_SECRET }} - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Run playbook - uses: dawidd6/action-ansible-playbook@v2 - with: - # Required, playbook filepath - playbook: dev.yaml - # Optional, directory where playbooks live - directory: ansible - # Optional, SSH private key - key: ${{secrets.DEV_NODE_PRIVATE_KEY}} - # Optional, literal inventory file contents - inventory: | - [dev] - dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team + uses: ./.github/workflows/deploy-dev.yaml + with: + stack-name: ${{ github.event_name == 'release' && 'staging' || 'dev' }} + image-tag: + ${{ github.event_name == 'release' && github.event.release.tag_name || + 'latest' }} + backend-port: ${{ github.event_name == 'release' && '8080' || '8180' }} + website-port: ${{ github.event_name == 'release' && '3000' || '3100' }} diff --git a/ansible/dev.yaml b/ansible/deploy-dev.yaml similarity index 69% rename from ansible/dev.yaml rename to ansible/deploy-dev.yaml index 2bf67b01..7d11e3c4 100644 --- a/ansible/dev.yaml +++ b/ansible/deploy-dev.yaml @@ -1,29 +1,32 @@ # ansible playbook to set up some docker containers -- name: Set up a dev node +- name: Deploy to dev node hosts: dev gather_facts: true + vars: + stack_name: "dev" + image_tag: latest + backend_port: 8080 + website_port: 3000 tasks: - name: Create network community.docker.docker_network: - name: oasst + name: "oasst-{{ stack_name }}" state: present driver: bridge - name: Copy redis.conf to managed node ansible.builtin.copy: src: ./redis.conf - dest: ./redis.conf + dest: "./{{ stack_name }}/redis.conf" - name: Set up Redis community.docker.docker_container: - name: oasst-redis + name: "oasst-redis-{{ stack_name }}" image: redis state: started restart_policy: always - network_mode: oasst - ports: - - 6379:6379 + network_mode: "oasst-{{ stack_name }}" healthcheck: test: ["CMD-SHELL", "redis-cli ping | grep PONG"] interval: 2s @@ -31,31 +34,19 @@ retries: 10 command: redis-server /usr/local/etc/redis/redis.conf volumes: - - "./redis.conf:/usr/local/etc/redis/redis.conf" - - - name: Set up Redis Insights - community.docker.docker_container: - name: oasst-redis-insights - image: redislabs/redisinsight:latest - state: started - restart_policy: always - network_mode: oasst - ports: - - 8001:8001 + - "./{{ stack_name }}/redis.conf:/usr/local/etc/redis/redis.conf" - name: Create postgres containers community.docker.docker_container: - name: "{{ item.name }}" + name: "{{ item.name }}-{{ stack_name }}" image: postgres:15 state: started restart_policy: always - network_mode: oasst + network_mode: "oasst-{{ stack_name }}" env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres POSTGRES_DB: postgres - volumes: - - "{{ item.name }}:/var/lib/postgresql/data" healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] interval: 2s @@ -67,16 +58,17 @@ - name: Run the oasst oasst-backend community.docker.docker_container: - name: oasst-backend - image: ghcr.io/laion-ai/open-assistant/oasst-backend + name: "oasst-backend-{{ stack_name }}" + image: "ghcr.io/laion-ai/open-assistant/oasst-backend:{{ image_tag }}" state: started recreate: true pull: true restart_policy: always - network_mode: oasst + network_mode: "oasst-{{ stack_name }}" env: - POSTGRES_HOST: oasst-postgres - REDIS_HOST: oasst-redis + PORT: "{{ backend_port }}" + POSTGRES_HOST: "oasst-postgres-{{ stack_name }}" + REDIS_HOST: "oasst-redis-{{ stack_name }}" DEBUG_ALLOW_ANY_API_KEY: "true" DEBUG_USE_SEED_DATA: "true" DEBUG_ALLOW_SELF_LABELING: "true" @@ -85,20 +77,23 @@ DEBUG_SKIP_EMBEDDING_COMPUTATION: "true" DEBUG_SKIP_TOXICITY_CALCULATION: "true" ports: - - 8080:8080 + - "{{ backend_port }}:8080" - name: Run the oasst oasst-web frontend community.docker.docker_container: - name: oasst-web - image: ghcr.io/laion-ai/open-assistant/oasst-web + name: "oasst-web-{{ stack_name }}" + image: "ghcr.io/laion-ai/open-assistant/oasst-web:{{ image_tag }}" state: started recreate: true pull: true restart_policy: always - network_mode: oasst + network_mode: "oasst-{{ stack_name }}" env: + PORT: "{{ website_port }}" ADMIN_USERS: "{{ lookup('ansible.builtin.env', 'WEB_ADMIN_USERS') }}" - DATABASE_URL: postgres://postgres:postgres@oasst-postgres-web/postgres + DATABASE_URL: + "postgres://postgres:postgres@oasst-postgres-web-{{ stack_name + }}/postgres" DEBUG_LOGIN: "true" DISCORD_CLIENT_ID: "{{ lookup('ansible.builtin.env', 'WEB_DISCORD_CLIENT_ID') }}" @@ -113,11 +108,11 @@ "{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_PORT') }}" EMAIL_SERVER_USER: "{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_USER') }}" - FASTAPI_URL: http://oasst-backend:8080 + FASTAPI_URL: "http://oasst-backend-{{ stack_name }}:8080" FASTAPI_KEY: "1234" NEXTAUTH_SECRET: "{{ lookup('ansible.builtin.env', 'WEB_NEXTAUTH_SECRET') }}" NEXTAUTH_URL: http://web.dev.open-assistant.io/ ports: - - 3000:3000 + - "{{ website_port }}:3000" command: bash wait-for-postgres.sh node server.js