From ab90d7810acbf9a52b27d8707377a506c8409506 Mon Sep 17 00:00:00 2001 From: Yannic Kilcher Date: Tue, 17 Jan 2023 11:13:43 +0100 Subject: [PATCH 1/3] added postgres dockerfile --- .github/workflows/build-postgres.yaml | 17 +++++++++++++++++ docker/Dockerfile.postgres | 9 +++++++++ 2 files changed, 26 insertions(+) create mode 100644 .github/workflows/build-postgres.yaml create mode 100644 docker/Dockerfile.postgres diff --git a/.github/workflows/build-postgres.yaml b/.github/workflows/build-postgres.yaml new file mode 100644 index 00000000..6f6595f7 --- /dev/null +++ b/.github/workflows/build-postgres.yaml @@ -0,0 +1,17 @@ +name: Build OASST Postgres image + +on: + push: + branches: + - main + paths: + - docker/Dockerfile.postgres + +jobs: + build-postgres: + uses: ./.github/workflows/docker-build.yaml + with: + image-name: oasst-postgres + context: ./docker/ + dockerfile: docker/Dockerfile.postgres + build-args: "" \ No newline at end of file diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres new file mode 100644 index 00000000..247dc46f --- /dev/null +++ b/docker/Dockerfile.postgres @@ -0,0 +1,9 @@ +FROM postgres + +# install unzip +RUN apt-get update && apt-get install -y unzip curl && rm -rf /var/lib/apt/lists/* + +# download aws cli +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +RUN unzip awscliv2.zip +RUN ./aws/install From 4285438fe719f82664be131288f6a9ee20606e89 Mon Sep 17 00:00:00 2001 From: Yannic Kilcher Date: Tue, 17 Jan 2023 11:15:35 +0100 Subject: [PATCH 2/3] added postgres dockerfile --- docker/Dockerfile.postgres | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 247dc46f..0eb4fec0 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -5,5 +5,5 @@ RUN apt-get update && apt-get install -y unzip curl && rm -rf /var/lib/apt/lists # download aws cli RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -RUN unzip awscliv2.zip +RUN unzip -q awscliv2.zip RUN ./aws/install From 137f733fef9539d77d0f5a87cd41239697b4793a Mon Sep 17 00:00:00 2001 From: Yannic Kilcher Date: Tue, 17 Jan 2023 11:30:52 +0100 Subject: [PATCH 3/3] added oasst postgres --- ansible/deploy-to-node.yaml | 33 ++++++++------------------------- docker-compose.yaml | 3 ++- docker/Dockerfile.postgres | 2 +- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/ansible/deploy-to-node.yaml b/ansible/deploy-to-node.yaml index 4bb13bb6..bbadca6a 100644 --- a/ansible/deploy-to-node.yaml +++ b/ansible/deploy-to-node.yaml @@ -57,8 +57,9 @@ - name: Create postgres containers community.docker.docker_container: name: "oasst-{{ stack_name }}-postgres-{{ item.name }}" - image: postgres:15 + image: ghcr.io/laion-ai/open-assistant/oasst-postgres state: started + pull: true recreate: "{{ (stack_name == 'dev') | bool }}" restart_policy: always network_mode: "oasst-{{ stack_name }}" @@ -66,6 +67,12 @@ POSTGRES_USER: postgres POSTGRES_PASSWORD: "{{ postgres_password }}" POSTGRES_DB: postgres + S3_BUCKET_NAME: + "{{ lookup('ansible.builtin.env', 'S3_BUCKET_NAME') }}" + AWS_ACCESS_KEY_ID: + "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY') }}" + AWS_SECRET_ACCESS_KEY: + "{{ lookup('ansible.builtin.env', 'AWS_SECRET_KEY') }}" volumes: - "oasst-{{ stack_name }}-postgres-{{ item.name }}:/var/lib/postgresql/data" @@ -78,30 +85,6 @@ - name: backend - name: web - - name: Copy pgbackrest.conf to managed node - ansible.builtin.copy: - src: ./pgbackrest.conf - dest: "./{{ stack_name }}/pgbackrest.conf" - mode: 0644 - - - name: Create pgbackrest container - community.docker.docker_container: - name: "oasst-{{ stack_name }}-pgbackrest" - image: woblerr/pgbackrest:2.43 - state: "{{ 'started' if stack_name == 'production' else 'absent' }}" - network_mode: "oasst-{{ stack_name }}" - entrypoint: ["/bin/bash", "-c", "sleep infinity"] - volumes: - - "./{{ stack_name }}/pgbackrest.conf:/etc/pgbackrest/pgbackrest.conf" - - "oasst-{{ stack_name }}-postgres-backend:/var/lib/postgresql/data" - env: - PGBACKREST_REPO1_S3_BUCKET: - "{{ lookup('ansible.builtin.env', 'S3_BUCKET_NAME') }}" - PGBACKREST_REPO1_S3_KEY: - "{{ lookup('ansible.builtin.env', 'AWS_ACCESS_KEY') }}" - PGBACKREST_REPO1_S3_KEY_SECRET: - "{{ lookup('ansible.builtin.env', 'AWS_SECRET_KEY') }}" - - name: Run the oasst oasst-backend community.docker.docker_container: name: "oasst-{{ stack_name }}-backend" diff --git a/docker-compose.yaml b/docker-compose.yaml index 78192eb3..908457cd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -18,7 +18,8 @@ services: # This DB is for the FastAPI Backend. db: - image: postgres + image: ghcr.io/laion-ai/open-assistant/oasst-postgres + pull_policy: always restart: always ports: - 5432:5432 diff --git a/docker/Dockerfile.postgres b/docker/Dockerfile.postgres index 0eb4fec0..b5b98c07 100644 --- a/docker/Dockerfile.postgres +++ b/docker/Dockerfile.postgres @@ -1,4 +1,4 @@ -FROM postgres +FROM postgres:15 # install unzip RUN apt-get update && apt-get install -y unzip curl && rm -rf /var/lib/apt/lists/*