From 4a7a1513a70dc85fb494779611a745ee22df5825 Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Mon, 30 Jan 2023 22:30:29 +0400
Subject: [PATCH 01/10] =?UTF-8?q?feat(inference):=20add=20Dockerfile=20for?=
 =?UTF-8?q?=20the=20inference=20stack's=20orchestrator=20image=20?=
 =?UTF-8?q?=F0=9F=90=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Initial definition of the inference stack's coordination/orchestration
server container image.

Notes:
- The base layer uses python:3-slim to avoid having to install cargo on
  top of alpine. Cargo is required to install orjson, which is in turns a
  dependency of fastapi[all].

- The oasst-shared package is installed as root as a workaround: it seems
  that python 3.10+ doesn't pick up on a .egg-link file installed on the
  PYTHONPATH anywhere other than at a global site-package directory.
  This quirk is similar to the behaviour described here
  https://groups.google.com/g/python-virtualenv/c/sKVq_6gG5z4.

- pydantic 1.9.1 (specified by oasst-shared) is incompatible with python 3.11,
  and needs to be upgraded. For now, this image uses python 3.10, as
  specified by the .python-version file
---
 docker/inference/Dockerfile.server | 68 ++++++++++++++++++++++++++++++
 1 file changed, 68 insertions(+)
 create mode 100644 docker/inference/Dockerfile.server

diff --git a/docker/inference/Dockerfile.server b/docker/inference/Dockerfile.server
new file mode 100644
index 00000000..34d8e0c9
--- /dev/null
+++ b/docker/inference/Dockerfile.server
@@ -0,0 +1,68 @@
+# syntax=docker/dockerfile:1
+
+ARG MODULE="inference"
+ARG SERVICE="server"
+
+ARG APP_USER="${MODULE}-${SERVICE}"
+ARG APP_RELATIVE_PATH="${MODULE}/${SERVICE}"
+
+
+FROM python:3-slim as build
+ARG APP_RELATIVE_PATH
+
+WORKDIR /build
+
+COPY ./${APP_RELATIVE_PATH}/requirements.txt .
+
+RUN --mount=type=cache,target=/var/cache/pip \
+    pip install                  \
+      --cache-dir=/var/cache/pip \
+      --target=lib               \
+      -r requirements.txt
+
+
+
+FROM python:3.10-alpine3.17 as dev
+ARG APP_USER
+ARG APP_RELATIVE_PATH
+ARG MODULE
+ARG SERVICE
+
+ENV APP_BASE="/opt/${MODULE}"
+ENV APP_ROOT="${APP_BASE}/${SERVICE}"
+ENV APP_LIBS="/var/opt/${APP_RELATIVE_PATH}/lib"
+ENV SHARED_LIBS_BASE="${APP_BASE}/lib"
+
+ENV PATH="${PATH}:${APP_LIBS}/bin"
+ENV PYTHONPATH="${PYTHONPATH}:${APP_LIBS}"
+
+ENV PORT="8000"
+
+
+RUN adduser               \
+      --disabled-password \
+      --no-create-home    \
+      "${APP_USER}"
+
+
+WORKDIR ${APP_ROOT}
+
+
+COPY --chown="${APP_USER}:${APP_USER}" ./oasst-shared ${SHARED_LIBS_BASE}/oasst-shared
+RUN --mount=type=cache,target=/var/cache/pip,from=build \
+    pip install                  \
+      --cache-dir=/var/cache/pip \
+      -e "${SHARED_LIBS_BASE}/oasst-shared"
+
+
+USER ${APP_USER}
+
+
+COPY --chown="${APP_USER}:${APP_USER}" --from=build /build/lib                     ${APP_LIBS}
+COPY --chown="${APP_USER}:${APP_USER}"              ./${APP_RELATIVE_PATH}/main.py .
+
+
+VOLUME [ "${APP_BASE}/lib/oasst-shared" ]
+
+
+CMD uvicorn main:app --reload --host 0.0.0.0 --port "${PORT}"

From fef871161c8fb14572ccb3fe753b98858ce0a7cc Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Mon, 30 Jan 2023 22:45:38 +0400
Subject: [PATCH 02/10] =?UTF-8?q?feat(inference):=20add=20Dockerfile=20for?=
 =?UTF-8?q?=20the=20inference=20stack's=20worker=20=F0=9F=90=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Initial definition of the inference stack's worker container image.

Notes:
- The oasst-shared package is installed as root as a workaround: it seems
  that python 3.10+ doesn't pick up on a .egg-link file installed on the
  PYTHONPATH anywhere other than at a global site-package directory.
  This quirk is similar to the behaviour described here
  https://groups.google.com/g/python-virtualenv/c/sKVq_6gG5z4.

- pydantic 1.9.1 (specified by oasst-shared) is incompatible with python 3.11,
  and needs to be upgraded. For now, this image uses python 3.10, as
  specified by the .python-version file
---
 docker/inference/Dockerfile.worker | 66 ++++++++++++++++++++++++++++++
 1 file changed, 66 insertions(+)
 create mode 100644 docker/inference/Dockerfile.worker

diff --git a/docker/inference/Dockerfile.worker b/docker/inference/Dockerfile.worker
new file mode 100644
index 00000000..85bd9ebf
--- /dev/null
+++ b/docker/inference/Dockerfile.worker
@@ -0,0 +1,66 @@
+# syntax=docker/dockerfile:1
+
+ARG MODULE="inference"
+ARG SERVICE="worker"
+
+ARG APP_USER="${MODULE}-${SERVICE}"
+ARG APP_RELATIVE_PATH="${MODULE}/${SERVICE}"
+
+
+FROM python:3.10-alpine3.17 as build
+ARG APP_RELATIVE_PATH
+
+WORKDIR /build
+
+COPY ./${APP_RELATIVE_PATH}/requirements.txt .
+
+RUN --mount=type=cache,target=/var/cache/pip \
+    pip install                  \
+      --cache-dir=/var/cache/pip \
+      --target=lib               \
+      -r requirements.txt
+
+
+
+FROM python:3.10-alpine3.17 as dev
+ARG APP_USER
+ARG APP_RELATIVE_PATH
+ARG MODULE
+ARG SERVICE
+
+ENV APP_BASE="/opt/${MODULE}"
+ENV APP_ROOT="${APP_BASE}/${SERVICE}"
+ENV APP_LIBS="/var/opt/${APP_RELATIVE_PATH}/lib"
+ENV SHARED_LIBS_BASE="${APP_BASE}/lib"
+
+ENV PATH="${PATH}:${APP_LIBS}/bin"
+ENV PYTHONPATH="${PYTHONPATH}:${APP_LIBS}"
+
+
+RUN adduser               \
+      --disabled-password \
+      --no-create-home    \
+      "${APP_USER}"
+
+
+WORKDIR ${APP_ROOT}
+
+
+COPY --chown="${APP_USER}:${APP_USER}" ./oasst-shared ${SHARED_LIBS_BASE}/oasst-shared
+RUN --mount=type=cache,target=/var/cache/pip,from=build \
+    pip install                  \
+      --cache-dir=/var/cache/pip \
+      -e "${SHARED_LIBS_BASE}/oasst-shared"
+
+
+USER ${APP_USER}
+
+
+COPY --chown="${APP_USER}:${APP_USER}" --from=build /build/lib                         ${APP_LIBS}
+COPY --chown="${APP_USER}:${APP_USER}"              ./${APP_RELATIVE_PATH}/__main__.py .
+
+
+VOLUME [ "${APP_BASE}/lib/oasst-shared" ]
+
+
+CMD python3 __main__.py --backend-url "${BACKEND_URL}" --inference-server-url "${INFERENCE_SERVER_URL}"

From 6b3fe78693f6f483b6657da546a82408795bd802 Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Mon, 30 Jan 2023 23:00:47 +0400
Subject: [PATCH 03/10] =?UTF-8?q?feat(inference):=20add=20Dockerfile=20for?=
 =?UTF-8?q?=20the=20inference=20stack's=20text=20client=20=F0=9F=90=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Initial definition of the inference stack's text client container image.
---
 docker/inference/Dockerfile.text-client | 46 +++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 docker/inference/Dockerfile.text-client

diff --git a/docker/inference/Dockerfile.text-client b/docker/inference/Dockerfile.text-client
new file mode 100644
index 00000000..1a4c4ae0
--- /dev/null
+++ b/docker/inference/Dockerfile.text-client
@@ -0,0 +1,46 @@
+# syntax=docker/dockerfile:1
+
+ARG APP_USER="text-client"
+ARG APP_RELATIVE_PATH="inference/text-client"
+
+
+FROM python:3.10-alpine3.17 as build
+ARG APP_RELATIVE_PATH
+
+WORKDIR /build
+
+COPY ./${APP_RELATIVE_PATH}/requirements.txt .
+
+RUN --mount=type=cache,target=/var/cache/pip \
+    pip install                  \
+      --cache-dir=/var/cache/pip \
+      --target=lib               \
+      -r requirements.txt
+
+
+
+FROM python:3.10-alpine3.17 as dev
+ARG APP_USER
+ARG APP_RELATIVE_PATH
+
+ENV APP_ROOT="/opt/${APP_RELATIVE_PATH}"
+ENV APP_LIBS="/var/opt/${APP_RELATIVE_PATH}/lib"
+
+ENV PATH="${PATH}:${APP_LIBS}/bin"
+ENV PYTHONPATH="${PYTHONPATH}:${APP_LIBS}"
+
+
+RUN adduser               \
+      --disabled-password \
+      --no-create-home    \
+      "${APP_USER}"
+
+USER ${APP_USER}
+
+WORKDIR ${APP_ROOT}
+
+COPY --chown="${APP_USER}:${APP_USER}" --from=build /build/lib                         ${APP_LIBS}
+COPY --chown="${APP_USER}:${APP_USER}"              ./${APP_RELATIVE_PATH}/__main__.py .
+
+
+CMD python3 __main__.py --backend-url "${BACKEND_URL}"

From 6d6b3504ac3a0c6d62c7759a9b91ae6372227f92 Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Mon, 30 Jan 2023 23:15:56 +0400
Subject: [PATCH 04/10] =?UTF-8?q?feat(inference):=20add=20compose=20file?=
 =?UTF-8?q?=20for=20the=20inference=20stack=20=F0=9F=90=B3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Initial inference stack definition.
---
 docker-compose.inference.yaml | 63 +++++++++++++++++++++++++++++++++++
 1 file changed, 63 insertions(+)
 create mode 100644 docker-compose.inference.yaml

diff --git a/docker-compose.inference.yaml b/docker-compose.inference.yaml
new file mode 100644
index 00000000..38609b36
--- /dev/null
+++ b/docker-compose.inference.yaml
@@ -0,0 +1,63 @@
+version: "3.7"
+
+services:
+  inference-server:
+    build:
+      dockerfile: docker/inference/Dockerfile.server
+      context: .
+    image: oasst-inference-server
+    environment:
+      - "PORT=8000"
+      - "REDIS_HOST=redis"
+    volumes:
+      - "./oasst-shared:/opt/inference/lib/oasst-shared"
+      - "./inference/server:/opt/inference/server"
+    restart: unless-stopped
+    depends_on:
+      redis:
+        condition: service_healthy
+
+  inference-worker:
+    build:
+      dockerfile: docker/inference/Dockerfile.worker
+      context: .
+    image: oasst-inference-worker
+    environment:
+      - "BACKEND_URL=ws://inference-server:8000"
+      - "INFERENCE_SERVER_URL=http://inference-text-generation-server"
+    volumes:
+      - "./oasst-shared:/opt/inference/lib/oasst-shared"
+      - "./inference/worker:/opt/inference/worker"
+    depends_on:
+      - inference-server
+    deploy:
+      replicas: 1
+
+  inference-text-client:
+    build:
+      dockerfile: docker/inference/Dockerfile.text-client
+      context: .
+    image: oasst-inference-text-client
+    environment:
+      - "BACKEND_URL=http://inference-server:8000"
+    tty: true
+    stdin_open: true
+    volumes:
+      - "./inference/text-client:/opt/inference/text-client"
+    restart: unless-stopped
+    depends_on:
+      - inference-server
+
+  inference-text-generation-server:
+    image: ykilcher/text-generation-inference
+    environment:
+      - "MODEL_ID=distilgpt2"
+
+  redis:
+    image: redis
+    restart: always
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
+      interval: 2s
+      timeout: 2s
+      retries: 10

From 0ab56041b6e08035ec535db644cf9598337835fc Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Mon, 30 Jan 2023 23:30:05 +0400
Subject: [PATCH 05/10] =?UTF-8?q?feat(inference):=20integrate=20inference?=
 =?UTF-8?q?=20stack=20with=20the=20unified=20compose=20descriptor=20?=
 =?UTF-8?q?=F0=9F=90=B3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docker-compose.inference.yaml | 63 -----------------------------------
 docker-compose.yaml           | 56 +++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 63 deletions(-)
 delete mode 100644 docker-compose.inference.yaml

diff --git a/docker-compose.inference.yaml b/docker-compose.inference.yaml
deleted file mode 100644
index 38609b36..00000000
--- a/docker-compose.inference.yaml
+++ /dev/null
@@ -1,63 +0,0 @@
-version: "3.7"
-
-services:
-  inference-server:
-    build:
-      dockerfile: docker/inference/Dockerfile.server
-      context: .
-    image: oasst-inference-server
-    environment:
-      - "PORT=8000"
-      - "REDIS_HOST=redis"
-    volumes:
-      - "./oasst-shared:/opt/inference/lib/oasst-shared"
-      - "./inference/server:/opt/inference/server"
-    restart: unless-stopped
-    depends_on:
-      redis:
-        condition: service_healthy
-
-  inference-worker:
-    build:
-      dockerfile: docker/inference/Dockerfile.worker
-      context: .
-    image: oasst-inference-worker
-    environment:
-      - "BACKEND_URL=ws://inference-server:8000"
-      - "INFERENCE_SERVER_URL=http://inference-text-generation-server"
-    volumes:
-      - "./oasst-shared:/opt/inference/lib/oasst-shared"
-      - "./inference/worker:/opt/inference/worker"
-    depends_on:
-      - inference-server
-    deploy:
-      replicas: 1
-
-  inference-text-client:
-    build:
-      dockerfile: docker/inference/Dockerfile.text-client
-      context: .
-    image: oasst-inference-text-client
-    environment:
-      - "BACKEND_URL=http://inference-server:8000"
-    tty: true
-    stdin_open: true
-    volumes:
-      - "./inference/text-client:/opt/inference/text-client"
-    restart: unless-stopped
-    depends_on:
-      - inference-server
-
-  inference-text-generation-server:
-    image: ykilcher/text-generation-inference
-    environment:
-      - "MODEL_ID=distilgpt2"
-
-  redis:
-    image: redis
-    restart: always
-    healthcheck:
-      test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
-      interval: 2s
-      timeout: 2s
-      retries: 10
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 908457cd..238e8679 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -131,3 +131,59 @@ services:
     ports:
       - "3000:3000"
     command: bash wait-for-postgres.sh node server.js
+
+  inference-server:
+    build:
+      dockerfile: docker/inference/Dockerfile.server
+      context: .
+    image: oasst-inference-server
+    environment:
+      - "PORT=8000"
+      - "REDIS_HOST=redis"
+    volumes:
+      - "./oasst-shared:/opt/inference/lib/oasst-shared"
+      - "./inference/server:/opt/inference/server"
+    restart: unless-stopped
+    depends_on:
+      redis:
+        condition: service_healthy
+    profiles: ["inference"]
+
+  inference-worker:
+    build:
+      dockerfile: docker/inference/Dockerfile.worker
+      context: .
+    image: oasst-inference-worker
+    environment:
+      - "BACKEND_URL=ws://inference-server:8000"
+      - "INFERENCE_SERVER_URL=http://inference-text-generation-server"
+    volumes:
+      - "./oasst-shared:/opt/inference/lib/oasst-shared"
+      - "./inference/worker:/opt/inference/worker"
+    depends_on:
+      - inference-server
+    deploy:
+      replicas: 1
+    profiles: ["inference"]
+
+  inference-text-client:
+    build:
+      dockerfile: docker/inference/Dockerfile.text-client
+      context: .
+    image: oasst-inference-text-client
+    environment:
+      - "BACKEND_URL=http://inference-server:8000"
+    tty: true
+    stdin_open: true
+    volumes:
+      - "./inference/worker:/opt/inference/worker"
+    restart: unless-stopped
+    depends_on:
+      - inference-server
+    profiles: ["inference"]
+
+  inference-text-generation-server:
+    image: ykilcher/text-generation-inference
+    environment:
+      - "MODEL_ID=distilgpt2"
+    profiles: ["inference"]

From 7961fe3b71867771ddf9b7f2a909b8a947ee6c47 Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Tue, 31 Jan 2023 00:12:11 +0400
Subject: [PATCH 06/10] =?UTF-8?q?chore(inference):=20add=20docs=20about=20?=
 =?UTF-8?q?the=20containerized=20inference=20stack=20=F0=9F=93=9D?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fixes #913
---
 README.md            |  4 ++++
 docs/docs/faq/faq.md | 27 +++++++++++++++++++++
 inference/README.md  | 56 +++++++++++++++++++++++++++++++++++++++++---
 3 files changed, 84 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 7cac0788..927314ff 100644
--- a/README.md
+++ b/README.md
@@ -51,6 +51,10 @@ docker compose up --build
 Then, navigate to `http://localhost:3000` (It may take some time to boot up) and
 interact with the website.
 
+> **Note:** If an issue occurs with the build, please head to the
+> [FAQ](https://projects.laion.ai/Open-Assistant/docs/faq) and check out the
+> entries about Docker.
+
 > **Note:** When logging in via email, navigate to `http://localhost:1080` to
 > get the magic email login link.
 
diff --git a/docs/docs/faq/faq.md b/docs/docs/faq/faq.md
index 0db57d30..025ae02b 100644
--- a/docs/docs/faq/faq.md
+++ b/docs/docs/faq/faq.md
@@ -9,6 +9,33 @@ For more details and information check out
 [this SO thread](https://stackoverflow.com/questions/66514436/difference-between-docker-compose-and-docker-compose)
 that explains it all in detail.
 
+### Enable Docker's BuildKit Backend
+
+[BuildKit](https://docs.docker.com/build/buildkit/) is Docker's new and improved
+builder backend. In addition to being faster and more efficient, it supports
+many new features, among which is the ability to provide a persistent cache,
+which outlives builds, to compilers and package managers. This is very useful to
+speed up consecutive builds, and is used by some container images of
+OpenAssistant's stack.
+
+The BuildKit backend is used by
+[default by Compose V2](https://www.docker.com/blog/announcing-compose-v2-general-availability/)
+(see above). <br/> But if you want to build an image with `docker build` instead
+of `docker compose build`, you might need to enable BuildKit.
+
+To do so, just add `DOCKER_BUILDKIT=1` to your environment.
+
+For instance:
+
+```shell
+export DOCKER_BUILDKIT=1
+```
+
+You could also, more conveniently,
+[enable BuildKit by default](https://docs.docker.com/build/buildkit/#:~:text=To%20enable%20docker%20BuildKit%20by%20default),
+or use
+[Docker Buildx](https://docs.docker.com/build/#:~:text=The%20new%20client%20Docker%20Buildx).
+
 ### Pre-commit
 
 We are using pre-commit to ensure the quality of the code as well as the same
diff --git a/inference/README.md b/inference/README.md
index 6e1da2c7..0475c876 100644
--- a/inference/README.md
+++ b/inference/README.md
@@ -1,14 +1,64 @@
-# OpenAssitant Inference
+# OpenAssistant Inference
 
 Preliminary implementation of the inference engine for OpenAssistant.
 
-## Development Variant 1 (you'll need tmux)
+## Development Variant 1 (docker compose)
+
+The services of the inference stack are prefixed with "inference-" in the
+[unified compose descriptor](../docker-compose.yaml). <br/> Prior to building
+those, please ensure that you have Docker's new
+[BuildKit](https://docs.docker.com/build/buildkit/) backend enabled. See the
+[FAQ](https://projects.laion.ai/Open-Assistant/docs/faq#enable-dockers-buildkit-backend)
+for more info.
+
+To build the services, run:
+
+```shell
+docker compose --profile inference build
+```
+
+Spin up the stack:
+
+```shell
+docker compose --profile inference up -d
+```
+
+Tail the logs:
+
+```shell
+docker compose logs -f    \
+    inference-server      \
+    inference-worker      \
+    inference-text-client \
+    inference-text-generation-server
+```
+
+Attach to the text-client, and start chatting:
+
+```shell
+docker attach open-assistant-inference-text-client-1
+```
+
+> **Note:** In the last step, `open-assistant-inference-text-client-1` refers to
+> the name of the `text-client` container started in step 2.
+
+> **Note:** The compose file contains the bind mounts enabling you to develop on
+> the modules of the inference stack, and the `oasst-shared` package, without
+> rebuilding.
+
+> **Note:** You can spin up any number of workers by adjusting the number of
+> replicas of the `inference-worker` service to your liking.
+
+> **Note:** Please wait for the `inference-text-generation-server` service to
+> output `{"message":"Connected"}` before starting to chat.
+
+## Development Variant 2 (you'll need tmux)
 
 Run `./full-dev-setup.sh` to start the full development setup. Make sure to wait
 until the 2nd terminal is ready and says `{"message":"Connected"}` before
 entering input into the last terminal.
 
-## Development Variant 2 (you'll need multiple terminals)
+## Development Variant 3 (you'll need multiple terminals)
 
 Run a redis container (or use the one of the general docker compose file):
 

From 6ae6ff51d512f64d8b825ca5e0788d12b05d7ea2 Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Tue, 31 Jan 2023 19:30:12 +0400
Subject: [PATCH 07/10] =?UTF-8?q?feat(inference):=20add=20a=20prod=20stage?=
 =?UTF-8?q?=20to=20the=20server's=20dockerfile=20=F0=9F=90=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docker/inference/Dockerfile.server | 44 ++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 11 deletions(-)

diff --git a/docker/inference/Dockerfile.server b/docker/inference/Dockerfile.server
index 34d8e0c9..0838a21e 100644
--- a/docker/inference/Dockerfile.server
+++ b/docker/inference/Dockerfile.server
@@ -22,7 +22,7 @@ RUN --mount=type=cache,target=/var/cache/pip \
 
 
 
-FROM python:3.10-alpine3.17 as dev
+FROM python:3.10-alpine3.17 as base-env
 ARG APP_USER
 ARG APP_RELATIVE_PATH
 ARG MODULE
@@ -44,25 +44,47 @@ RUN adduser               \
       --no-create-home    \
       "${APP_USER}"
 
+USER ${APP_USER}
 
 WORKDIR ${APP_ROOT}
 
 
-COPY --chown="${APP_USER}:${APP_USER}" ./oasst-shared ${SHARED_LIBS_BASE}/oasst-shared
-RUN --mount=type=cache,target=/var/cache/pip,from=build \
-    pip install                  \
-      --cache-dir=/var/cache/pip \
-      -e "${SHARED_LIBS_BASE}/oasst-shared"
-
-
-USER ${APP_USER}
-
-
 COPY --chown="${APP_USER}:${APP_USER}" --from=build /build/lib                     ${APP_LIBS}
 COPY --chown="${APP_USER}:${APP_USER}"              ./${APP_RELATIVE_PATH}/main.py .
 
 
+
+FROM base-env as dev
+ARG APP_USER
+
+
+COPY --chown="${APP_USER}:${APP_USER}" ./oasst-shared ${SHARED_LIBS_BASE}/oasst-shared
+
+USER root
+RUN --mount=type=cache,target=/var/cache/pip,from=build \
+    pip install                  \
+      --cache-dir=/var/cache/pip \
+      -e "${SHARED_LIBS_BASE}/oasst-shared"
+USER ${APP_USER}
+
+
 VOLUME [ "${APP_BASE}/lib/oasst-shared" ]
 
 
 CMD uvicorn main:app --reload --host 0.0.0.0 --port "${PORT}"
+
+
+
+FROM base-env as prod
+ARG APP_USER
+
+
+COPY --chown="${APP_USER}:${APP_USER}" ./oasst-shared /tmp/lib/oasst-shared
+RUN --mount=type=cache,target=/var/cache/pip,from=dev \
+    pip install                  \
+      --cache-dir=/var/cache/pip \
+      --target="${APP_LIBS}"     \
+      /tmp/lib/oasst-shared
+
+
+CMD uvicorn main:app --host 0.0.0.0 --port "${PORT}"

From d9dac3fe1a9e3b92d000f71bc55d4d406b45ee4b Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Tue, 31 Jan 2023 19:45:15 +0400
Subject: [PATCH 08/10] =?UTF-8?q?feat(inference):=20add=20a=20prod=20stage?=
 =?UTF-8?q?=20to=20the=20worker's=20dockerfile=20=F0=9F=90=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docker/inference/Dockerfile.worker | 43 +++++++++++++++++++++---------
 1 file changed, 31 insertions(+), 12 deletions(-)

diff --git a/docker/inference/Dockerfile.worker b/docker/inference/Dockerfile.worker
index 85bd9ebf..06f040ab 100644
--- a/docker/inference/Dockerfile.worker
+++ b/docker/inference/Dockerfile.worker
@@ -22,7 +22,7 @@ RUN --mount=type=cache,target=/var/cache/pip \
 
 
 
-FROM python:3.10-alpine3.17 as dev
+FROM python:3.10-alpine3.17 as base-env
 ARG APP_USER
 ARG APP_RELATIVE_PATH
 ARG MODULE
@@ -42,25 +42,44 @@ RUN adduser               \
       --no-create-home    \
       "${APP_USER}"
 
+USER ${APP_USER}
 
 WORKDIR ${APP_ROOT}
 
 
-COPY --chown="${APP_USER}:${APP_USER}" ./oasst-shared ${SHARED_LIBS_BASE}/oasst-shared
-RUN --mount=type=cache,target=/var/cache/pip,from=build \
-    pip install                  \
-      --cache-dir=/var/cache/pip \
-      -e "${SHARED_LIBS_BASE}/oasst-shared"
-
-
-USER ${APP_USER}
-
-
 COPY --chown="${APP_USER}:${APP_USER}" --from=build /build/lib                         ${APP_LIBS}
 COPY --chown="${APP_USER}:${APP_USER}"              ./${APP_RELATIVE_PATH}/__main__.py .
 
 
+CMD python3 __main__.py --backend-url "${BACKEND_URL}" --inference-server-url "${INFERENCE_SERVER_URL}"
+
+
+
+FROM base-env as dev
+ARG APP_USER
+
+
+COPY --chown="${APP_USER}:${APP_USER}" ./oasst-shared ${SHARED_LIBS_BASE}/oasst-shared
+
+USER root
+RUN --mount=type=cache,target=/var/cache/pip,from=build \
+    pip install                  \
+      --cache-dir=/var/cache/pip \
+      -e "${SHARED_LIBS_BASE}/oasst-shared"
+USER ${APP_USER}
+
+
 VOLUME [ "${APP_BASE}/lib/oasst-shared" ]
 
 
-CMD python3 __main__.py --backend-url "${BACKEND_URL}" --inference-server-url "${INFERENCE_SERVER_URL}"
+
+FROM base-env as prod
+ARG APP_USER
+
+
+COPY --chown="${APP_USER}:${APP_USER}" ./oasst-shared /tmp/lib/oasst-shared
+RUN --mount=type=cache,target=/var/cache/pip,from=dev \
+    pip install                  \
+      --cache-dir=/var/cache/pip \
+      --target="${APP_LIBS}"     \
+      /tmp/lib/oasst-shared

From 8d6908d663c30058fda0026261d06696ff20b7ec Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Tue, 31 Jan 2023 19:56:13 +0400
Subject: [PATCH 09/10] =?UTF-8?q?chore(inference):=20follow=20the=20patter?=
 =?UTF-8?q?n=20of=20splitting=20base-env=20and=20prod=20stage=20?=
 =?UTF-8?q?=F0=9F=90=8B?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docker/inference/Dockerfile.text-client | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docker/inference/Dockerfile.text-client b/docker/inference/Dockerfile.text-client
index 1a4c4ae0..23a54abe 100644
--- a/docker/inference/Dockerfile.text-client
+++ b/docker/inference/Dockerfile.text-client
@@ -19,7 +19,7 @@ RUN --mount=type=cache,target=/var/cache/pip \
 
 
 
-FROM python:3.10-alpine3.17 as dev
+FROM python:3.10-alpine3.17 as base-env
 ARG APP_USER
 ARG APP_RELATIVE_PATH
 
@@ -43,4 +43,8 @@ COPY --chown="${APP_USER}:${APP_USER}" --from=build /build/lib
 COPY --chown="${APP_USER}:${APP_USER}"              ./${APP_RELATIVE_PATH}/__main__.py .
 
 
+
+FROM base-env as prod
+
+
 CMD python3 __main__.py --backend-url "${BACKEND_URL}"

From e59b73d942e0678373609d0f311c249e722f629d Mon Sep 17 00:00:00 2001
From: Alan Jean <afinegameofnil@gmail.com>
Date: Tue, 31 Jan 2023 20:01:29 +0400
Subject: [PATCH 10/10] =?UTF-8?q?chore(inference):=20target=20dev=20stage?=
 =?UTF-8?q?=20at=20build=20time=20=F0=9F=90=B3?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docker-compose.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yaml b/docker-compose.yaml
index 238e8679..b9c9c82f 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -136,7 +136,8 @@ services:
     build:
       dockerfile: docker/inference/Dockerfile.server
       context: .
-    image: oasst-inference-server
+      target: dev
+    image: oasst-inference-server:dev
     environment:
       - "PORT=8000"
       - "REDIS_HOST=redis"
@@ -153,7 +154,8 @@ services:
     build:
       dockerfile: docker/inference/Dockerfile.worker
       context: .
-    image: oasst-inference-worker
+      target: dev
+    image: oasst-inference-worker:dev
     environment:
       - "BACKEND_URL=ws://inference-server:8000"
       - "INFERENCE_SERVER_URL=http://inference-text-generation-server"