added deploy workflow

.github/workflows/deploy-dev.yaml

@@ -0,0 +1,50 @@
+name: Deploy to dev machine
+
+on:
+  workflow_call:
+    inputs:
+      stack-name:
+        required: false
+        type: string
+        default: dev
+      image-tag:
+        required: false
+        type: string
+        default: latest
+      backend-port:
+        required: false
+        type: string
+        default: 8080
+      website-port:
+        required: false
+        type: string
+        default: 3000
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    env:
+      WEB_ADMIN_USERS: ${{ secrets.DEV_WEB_ADMIN_USERS }}
+      WEB_DISCORD_CLIENT_ID: ${{ secrets.DEV_WEB_DISCORD_CLIENT_ID }}
+      WEB_DISCORD_CLIENT_SECRET: ${{ secrets.DEV_WEB_DISCORD_CLIENT_SECRET }}
+      WEB_EMAIL_SERVER_HOST: ${{ secrets.DEV_WEB_EMAIL_SERVER_HOST }}
+      WEB_EMAIL_SERVER_PASSWORD: ${{ secrets.DEV_WEB_EMAIL_SERVER_PASSWORD }}
+      WEB_EMAIL_SERVER_PORT: ${{ secrets.DEV_WEB_EMAIL_SERVER_PORT }}
+      WEB_EMAIL_SERVER_USER: ${{ secrets.DEV_WEB_EMAIL_SERVER_USER }}
+      WEB_NEXTAUTH_SECRET: ${{ secrets.DEV_WEB_NEXTAUTH_SECRET }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Run playbook
+        uses: dawidd6/action-ansible-playbook@v2
+        with:
+          # Required, playbook filepath
+          playbook: deploy-dev.yaml
+          # Optional, directory where playbooks live
+          directory: ansible
+          # Optional, SSH private key
+          key: ${{secrets.DEV_NODE_PRIVATE_KEY}}
+          # Optional, literal inventory file contents
+          inventory: |
+            [dev]
+            dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team stack_name=${{inputs.stack-name}} image_tag=${{inputs.image-tag}} backend_port=${{inputs.backend-port}} website_port=${{inputs.website-port}}

.github/workflows/docker-build.yaml

@@ -46,8 +46,9 @@ jobs:
         with:
           images: ${{ env.REGISTRY }}/${{ inputs.image-name }}
           tags: |
-            type=sha,prefix=${{ env.TAG_PREFIX }},format=short
-            type=ref,event=tag
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=sha,prefix=${{ env.TAG_PREFIX }},format=short,enable={{!is_default_branch}}
+            type=ref,event=tag,enable={{!is_default_branch}}
       - name: Build and push Docker image
         uses: docker/build-push-action@v3.2.0
         with:

.github/workflows/pre-commit.yaml

@@ -1,9 +1,7 @@
 name: pre-commit
 
 on:
-  push:
-    branches:
-      - main
+  workflow_call:
   pull_request_target:
 
 jobs:
@@ -18,7 +16,7 @@ jobs:
 
       # in case of push, check out the main branch
       - uses: actions/checkout@v3
-        if: github.event_name == 'push'
+        if: github.event_name != 'pull_request_target'
 
       - uses: actions/setup-python@v4
         with:

.github/workflows/release.yaml

@@ -1,10 +1,16 @@
 name: Release
 
 on:
+  push:
+    branches:
+      - main
   release:
-    types: [released]
+    types:
+      - released
 
 jobs:
+  pre-commit:
+    uses: ./.github/workflows/pre-commit.yaml
   build-backend:
     uses: ./.github/workflows/docker-build.yaml
     with:
@@ -28,29 +34,11 @@ jobs:
       build-args: ""
   deploy-dev:
     needs: [build-backend, build-web, build-bot]
-    runs-on: ubuntu-latest
-    env:
-      WEB_ADMIN_USERS: ${{ secrets.DEV_WEB_ADMIN_USERS }}
-      WEB_DISCORD_CLIENT_ID: ${{ secrets.DEV_WEB_DISCORD_CLIENT_ID }}
-      WEB_DISCORD_CLIENT_SECRET: ${{ secrets.DEV_WEB_DISCORD_CLIENT_SECRET }}
-      WEB_EMAIL_SERVER_HOST: ${{ secrets.DEV_WEB_EMAIL_SERVER_HOST }}
-      WEB_EMAIL_SERVER_PASSWORD: ${{ secrets.DEV_WEB_EMAIL_SERVER_PASSWORD }}
-      WEB_EMAIL_SERVER_PORT: ${{ secrets.DEV_WEB_EMAIL_SERVER_PORT }}
-      WEB_EMAIL_SERVER_USER: ${{ secrets.DEV_WEB_EMAIL_SERVER_USER }}
-      WEB_NEXTAUTH_SECRET: ${{ secrets.DEV_WEB_NEXTAUTH_SECRET }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Run playbook
-        uses: dawidd6/action-ansible-playbook@v2
-        with:
-          # Required, playbook filepath
-          playbook: dev.yaml
-          # Optional, directory where playbooks live
-          directory: ansible
-          # Optional, SSH private key
-          key: ${{secrets.DEV_NODE_PRIVATE_KEY}}
-          # Optional, literal inventory file contents
-          inventory: |
-            [dev]
-            dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team
+    uses: ./.github/workflows/deploy-dev.yaml
+    with:
+      stack-name: ${{ github.event_name == 'release' && 'staging' || 'dev' }}
+      image-tag:
+        ${{ github.event_name == 'release' && github.event.release.tag_name ||
+        'latest' }}
+      backend-port: ${{ github.event_name == 'release' && '8080' || '8180' }}
+      website-port: ${{ github.event_name == 'release' && '3000' || '3100' }}

ansible/deploy-dev.yaml

@@ -1,29 +1,32 @@
 # ansible playbook to set up some docker containers
 
-- name: Set up a dev node
+- name: Deploy to dev node
   hosts: dev
   gather_facts: true
+  vars:
+    stack_name: "dev"
+    image_tag: latest
+    backend_port: 8080
+    website_port: 3000
   tasks:
     - name: Create network
       community.docker.docker_network:
-        name: oasst
+        name: "oasst-{{ stack_name }}"
         state: present
         driver: bridge
 
     - name: Copy redis.conf to managed node
       ansible.builtin.copy:
         src: ./redis.conf
-        dest: ./redis.conf
+        dest: "./{{ stack_name }}/redis.conf"
 
     - name: Set up Redis
       community.docker.docker_container:
-        name: oasst-redis
+        name: "oasst-redis-{{ stack_name }}"
         image: redis
         state: started
         restart_policy: always
-        network_mode: oasst
-        ports:
-          - 6379:6379
+        network_mode: "oasst-{{ stack_name }}"
         healthcheck:
           test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
           interval: 2s
@@ -31,31 +34,19 @@
           retries: 10
         command: redis-server /usr/local/etc/redis/redis.conf
         volumes:
-          - "./redis.conf:/usr/local/etc/redis/redis.conf"
-
-    - name: Set up Redis Insights
-      community.docker.docker_container:
-        name: oasst-redis-insights
-        image: redislabs/redisinsight:latest
-        state: started
-        restart_policy: always
-        network_mode: oasst
-        ports:
-          - 8001:8001
+          - "./{{ stack_name }}/redis.conf:/usr/local/etc/redis/redis.conf"
 
     - name: Create postgres containers
       community.docker.docker_container:
-        name: "{{ item.name }}"
+        name: "{{ item.name }}-{{ stack_name }}"
         image: postgres:15
         state: started
         restart_policy: always
-        network_mode: oasst
+        network_mode: "oasst-{{ stack_name }}"
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: postgres
-        volumes:
-          - "{{ item.name }}:/var/lib/postgresql/data"
         healthcheck:
           test: ["CMD", "pg_isready", "-U", "postgres"]
           interval: 2s
@@ -67,16 +58,17 @@
 
     - name: Run the oasst oasst-backend
       community.docker.docker_container:
-        name: oasst-backend
-        image: ghcr.io/laion-ai/open-assistant/oasst-backend
+        name: "oasst-backend-{{ stack_name }}"
+        image: "ghcr.io/laion-ai/open-assistant/oasst-backend:{{ image_tag }}"
         state: started
         recreate: true
         pull: true
         restart_policy: always
-        network_mode: oasst
+        network_mode: "oasst-{{ stack_name }}"
         env:
-          POSTGRES_HOST: oasst-postgres
-          REDIS_HOST: oasst-redis
+          PORT: "{{ backend_port }}"
+          POSTGRES_HOST: "oasst-postgres-{{ stack_name }}"
+          REDIS_HOST: "oasst-redis-{{ stack_name }}"
           DEBUG_ALLOW_ANY_API_KEY: "true"
           DEBUG_USE_SEED_DATA: "true"
           DEBUG_ALLOW_SELF_LABELING: "true"
@@ -85,20 +77,23 @@
           DEBUG_SKIP_EMBEDDING_COMPUTATION: "true"
           DEBUG_SKIP_TOXICITY_CALCULATION: "true"
         ports:
-          - 8080:8080
+          - "{{ backend_port }}:8080"
 
     - name: Run the oasst oasst-web frontend
       community.docker.docker_container:
-        name: oasst-web
-        image: ghcr.io/laion-ai/open-assistant/oasst-web
+        name: "oasst-web-{{ stack_name }}"
+        image: "ghcr.io/laion-ai/open-assistant/oasst-web:{{ image_tag }}"
         state: started
         recreate: true
         pull: true
         restart_policy: always
-        network_mode: oasst
+        network_mode: "oasst-{{ stack_name }}"
         env:
+          PORT: "{{ website_port }}"
           ADMIN_USERS: "{{ lookup('ansible.builtin.env', 'WEB_ADMIN_USERS') }}"
-          DATABASE_URL: postgres://postgres:postgres@oasst-postgres-web/postgres
+          DATABASE_URL:
+            "postgres://postgres:postgres@oasst-postgres-web-{{ stack_name
+            }}/postgres"
           DEBUG_LOGIN: "true"
           DISCORD_CLIENT_ID:
             "{{ lookup('ansible.builtin.env', 'WEB_DISCORD_CLIENT_ID') }}"
@@ -113,11 +108,11 @@
             "{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_PORT') }}"
           EMAIL_SERVER_USER:
             "{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_USER') }}"
-          FASTAPI_URL: http://oasst-backend:8080
+          FASTAPI_URL: "http://oasst-backend-{{ stack_name }}:8080"
           FASTAPI_KEY: "1234"
           NEXTAUTH_SECRET:
             "{{ lookup('ansible.builtin.env', 'WEB_NEXTAUTH_SECRET') }}"
           NEXTAUTH_URL: http://web.dev.open-assistant.io/
         ports:
-          - 3000:3000
+          - "{{ website_port }}:3000"
         command: bash wait-for-postgres.sh node server.js