From c056a31d2f79f7e5e49a2cde75ad527efe251cd3 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Tue, 20 Dec 2022 20:28:53 +0900 Subject: [PATCH 1/4] Ensuring the website can be built and deployed fully in docker. This includes an end to end docker-compose configuration as a simple demonstration. --- docker/Dockerfile.website | 58 ++++++++++++ scripts/endtoend-demo/README.md | 18 ++++ scripts/endtoend-demo/docker-compose.yaml | 89 +++++++++++++++++++ .../frontend-development/docker-compose.yaml | 5 +- website/next.config.js | 1 + website/package.json | 4 - website/src/components/CallToAction.tsx | 4 +- website/src/components/Header.tsx | 22 ++--- website/src/pages/api/auth/[...nextauth].ts | 3 +- website/src/pages/api/new_task/[task_type].ts | 5 +- website/src/pages/api/update_task.ts | 5 +- website/src/pages/grading/grade-output.tsx | 2 +- website/tsconfig.json | 6 +- 13 files changed, 194 insertions(+), 28 deletions(-) create mode 100644 docker/Dockerfile.website create mode 100644 scripts/endtoend-demo/README.md create mode 100644 scripts/endtoend-demo/docker-compose.yaml diff --git a/docker/Dockerfile.website b/docker/Dockerfile.website new file mode 100644 index 00000000..c43d44e0 --- /dev/null +++ b/docker/Dockerfile.website @@ -0,0 +1,58 @@ +# Install dependencies only when needed +FROM node:16.19 AS deps +# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed. +# RUN apk add --no-cache libc6-compat +WORKDIR /app + +# Install dependencies based on the preferred package manager +COPY ./website/package.json ./website/package-lock.json ./ +RUN \ + if [ -f package-lock.json ]; then npm ci; \ + else echo "Lockfile not found." && exit 1; \ + fi + +# Rebuild the source code only when needed +FROM node:16.19 AS builder +WORKDIR /app +COPY --from=deps /app/node_modules ./node_modules +COPY ./website/ . + +# Next.js collects completely anonymous telemetry data about general usage. +# Learn more here: https://nextjs.org/telemetry +# Uncomment the following line in case you want to disable telemetry during the build. +# ENV NEXT_TELEMETRY_DISABLED 1 + +# RUN yarn build +RUN npx prisma generate +RUN npm run build + +# Production image, copy all the files and run next +FROM node:16.19 AS runner +WORKDIR /app + +ENV NODE_ENV production +# Uncomment the following line in case you want to disable telemetry during runtime. +# ENV NEXT_TELEMETRY_DISABLED 1 + +RUN addgroup --system --gid 1001 nodejs +RUN adduser --system --uid 1001 nextjs + +COPY --from=builder /app/public ./public + +# Copy over the schema file so we can do a db push on startup. +# TODO: Delete when used for real production. +COPY ./website/prisma/schema.prisma ./ +COPY ./website/wait-for-it.sh ./ + +# Automatically leverage output traces to reduce image size +# https://nextjs.org/docs/advanced-features/output-file-tracing +COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ +COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static + +USER nextjs + +EXPOSE 3000 + +ENV PORT 3000 + +CMD ["node", "server.js"] diff --git a/scripts/endtoend-demo/README.md b/scripts/endtoend-demo/README.md new file mode 100644 index 00000000..49b193a5 --- /dev/null +++ b/scripts/endtoend-demo/README.md @@ -0,0 +1,18 @@ +# End to End Demo + +This sets up an entire stack needed to run Open Assistant, including the website, backend, and associated dependent services. + +To start the service, do the following: + +```sh +docker compose up --build +cd ../../website +npx prisma db push +``` + +NOTE: we're working on making the prisma step integrated into the docker +initialization process. + +Then, navigate to `http://localhost:3000` and interact with the website. When +logging in, navigate to `http://localhost:1080` to get the magic email login +link. diff --git a/scripts/endtoend-demo/docker-compose.yaml b/scripts/endtoend-demo/docker-compose.yaml new file mode 100644 index 00000000..b17a7b74 --- /dev/null +++ b/scripts/endtoend-demo/docker-compose.yaml @@ -0,0 +1,89 @@ +version: "3.7" + +services: + # This DB is for the FastAPI Backend. + db: + image: postgres + restart: always + ports: + - 5432:5432 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 2s + timeout: 2s + retries: 10 + + # This DB is for Web Authentication and data caching. + webdb: + image: postgres + restart: always + ports: + - 5433:5432 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres"] + interval: 2s + timeout: 2s + retries: 10 + + # This lets you manually inspect the web and backend databases. + adminer: + image: adminer + restart: always + ports: + - 8089:8080 + + # This fakes an SMTP email server used by website authentication. + # User registration emails can be found by going to localhost:1080 and + # opening the emails listed. + maildev: + image: maildev/maildev + restart: always + environment: + - MAILDEV_WEB_PORT=1080 + - MAILDEV_SMTP_PORT=1025 + ports: + - "1080:1080" + - "1025:1025" + + # The oassist backend service. + backend: + build: + dockerfile: docker/Dockerfile.backend + context: ../../ + image: oasst-backend + environment: + - POSTGRES_HOST=db + - ALLOW_ANY_API_KEY=True + - MAX_WORKERS=1 + depends_on: + db: + condition: service_healthy + ports: + - "8080:8080" + + # The oassist web service. + web: + build: + dockerfile: docker/Dockerfile.website + context: ../../ + image: oasst-web + environment: + - DATABASE_URL=postgres://postgres:postgres@webdb/ocgpt_website + - FASTAPI_URL=http://backend:8080 + - FASTAPI_KEY=1234 + - NEXTAUTH_SECRET=O/M2uIbGj+lDD2oyNa8ax4jEOJqCPJzO53UbWShmq98= + - EMAIL_SERVER_HOST=maildev + - EMAIL_SERVER_PORT=1025 + - EMAIL_FROM=info@example.com + - NEXTAUTH_URL=http://localhost:3000 + depends_on: + webdb: + condition: service_healthy + ports: + - "3000:3000" diff --git a/scripts/frontend-development/docker-compose.yaml b/scripts/frontend-development/docker-compose.yaml index 6f9c01f4..99f30f62 100644 --- a/scripts/frontend-development/docker-compose.yaml +++ b/scripts/frontend-development/docker-compose.yaml @@ -47,8 +47,9 @@ services: ports: - "8080:8080" - # This fakes and SMTP email server. User registration emails can be found by going to - # localhost:1080 and opening the emails listed. + # This fakes an SMTP email server used by website authentication. + # User registration emails can be found by going to localhost:1080 and + # opening the emails listed. maildev: image: maildev/maildev restart: always diff --git a/website/next.config.js b/website/next.config.js index c1c1519d..1cbdb0e0 100644 --- a/website/next.config.js +++ b/website/next.config.js @@ -1,5 +1,6 @@ /** @type {import('next').NextConfig} */ const nextConfig = { + output: "standalone", reactStrictMode: true, experimental: { scrollRestoration: true, diff --git a/website/package.json b/website/package.json index b7680c79..10708782 100644 --- a/website/package.json +++ b/website/package.json @@ -17,10 +17,6 @@ "@heroicons/react": "^2.0.13", "@next-auth/prisma-adapter": "^1.0.5", "@prisma/client": "^4.7.1", - "@supabase/auth-helpers-nextjs": "^0.5.2", - "@supabase/auth-helpers-react": "^0.3.1", - "@supabase/auth-ui-react": "^0.2.6", - "@supabase/supabase-js": "^2.1.4", "@tailwindcss/forms": "^0.5.3", "autoprefixer": "^10.4.13", "axios": "^1.2.1", diff --git a/website/src/components/CallToAction.tsx b/website/src/components/CallToAction.tsx index 61f11540..35949645 100644 --- a/website/src/components/CallToAction.tsx +++ b/website/src/components/CallToAction.tsx @@ -16,7 +16,7 @@ export function CallToAction() { here:

- + + ); } return ( - + ); } diff --git a/website/src/pages/api/auth/[...nextauth].ts b/website/src/pages/api/auth/[...nextauth].ts index 6b2c0ae7..86d8bdc4 100644 --- a/website/src/pages/api/auth/[...nextauth].ts +++ b/website/src/pages/api/auth/[...nextauth].ts @@ -1,3 +1,4 @@ +import type { AuthOptions } from "next-auth"; import NextAuth from "next-auth"; import DiscordProvider from "next-auth/providers/discord"; import EmailProvider from "next-auth/providers/email"; @@ -31,7 +32,7 @@ if (process.env.DISCORD_CLIENT_ID) { ); } -export const authOptions = { +export const authOptions: AuthOptions = { // Ensure we can store user data in a database. adapter: PrismaAdapter(prisma), providers, diff --git a/website/src/pages/api/new_task/[task_type].ts b/website/src/pages/api/new_task/[task_type].ts index 0d45c5f3..3943536c 100644 --- a/website/src/pages/api/new_task/[task_type].ts +++ b/website/src/pages/api/new_task/[task_type].ts @@ -1,5 +1,6 @@ import { getToken } from "next-auth/jwt"; +import prisma from "src/lib/prismadb"; import { authOptions } from "src/pages/api/auth/[...nextauth]"; /** @@ -10,7 +11,7 @@ import { authOptions } from "src/pages/api/auth/[...nextauth]"; * 3) Send and Ack to the Task Backend with our local id for the task. * 4) Return everything to the client. */ -export default async (req, res) => { +const handler = async (req, res) => { const { task_type } = req.query; const token = await getToken({ req }); @@ -69,3 +70,5 @@ export default async (req, res) => { // Send the results to the client. res.status(200).json(registeredTask); }; + +export default handler; diff --git a/website/src/pages/api/update_task.ts b/website/src/pages/api/update_task.ts index b7919c5a..6760623c 100644 --- a/website/src/pages/api/update_task.ts +++ b/website/src/pages/api/update_task.ts @@ -1,5 +1,6 @@ import { getToken } from "next-auth/jwt"; +import prisma from "src/lib/prismadb"; import { authOptions } from "src/pages/api/auth/[...nextauth]"; /** @@ -11,7 +12,7 @@ import { authOptions } from "src/pages/api/auth/[...nextauth]"; * 3) (TODO) Acks the new task with our local task ID to the Task Backend. * 4) Returns the newly created task to the client. */ -export default async (req, res) => { +const handler = async (req, res) => { const token = await getToken({ req }); // Return nothing if the user isn't registered. @@ -76,3 +77,5 @@ export default async (req, res) => { // Send the next task in the sequence to the client. res.status(200).json(newRegisteredTask); }; + +export default handler; diff --git a/website/src/pages/grading/grade-output.tsx b/website/src/pages/grading/grade-output.tsx index 5fabe7ba..c04a99c3 100644 --- a/website/src/pages/grading/grade-output.tsx +++ b/website/src/pages/grading/grade-output.tsx @@ -134,7 +134,7 @@ const GradeOutput = () => { diff --git a/website/tsconfig.json b/website/tsconfig.json index 1563f3e8..8a5ec4fe 100644 --- a/website/tsconfig.json +++ b/website/tsconfig.json @@ -13,7 +13,11 @@ "moduleResolution": "node", "resolveJsonModule": true, "isolatedModules": true, - "jsx": "preserve" + "jsx": "preserve", + "baseUrl": ".", + "paths": { + "@/*": ["src/*"] + } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], "exclude": ["node_modules"] From 98fe656f7dc94d0531d2673d95b47b253cf84a11 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Tue, 20 Dec 2022 20:30:49 +0900 Subject: [PATCH 2/4] Dropping some unused copies from the website dockerfile --- docker/Dockerfile.website | 5 ----- 1 file changed, 5 deletions(-) diff --git a/docker/Dockerfile.website b/docker/Dockerfile.website index c43d44e0..cd723833 100644 --- a/docker/Dockerfile.website +++ b/docker/Dockerfile.website @@ -39,11 +39,6 @@ RUN adduser --system --uid 1001 nextjs COPY --from=builder /app/public ./public -# Copy over the schema file so we can do a db push on startup. -# TODO: Delete when used for real production. -COPY ./website/prisma/schema.prisma ./ -COPY ./website/wait-for-it.sh ./ - # Automatically leverage output traces to reduce image size # https://nextjs.org/docs/advanced-features/output-file-tracing COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ From c6b00365fd0f0d7192ba9fd56ab14b97b91e8d27 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Wed, 21 Dec 2022 12:17:08 +0900 Subject: [PATCH 3/4] Simplifying the end to end docker setup to ensure it prepares the web database properly before the website starts. --- docker/Dockerfile.website | 9 ++++++++ scripts/endtoend-demo/docker-compose.yaml | 1 + website/wait-for-postgres.sh | 25 +++++++++++++++++++++++ 3 files changed, 35 insertions(+) create mode 100755 website/wait-for-postgres.sh diff --git a/docker/Dockerfile.website b/docker/Dockerfile.website index cd723833..afb18baf 100644 --- a/docker/Dockerfile.website +++ b/docker/Dockerfile.website @@ -39,6 +39,15 @@ RUN adduser --system --uid 1001 nextjs COPY --from=builder /app/public ./public +# Copy over the prisma schema so we can to `npx prisma db push` and ensure the +# database exists on startup. +COPY --chown=nextjs:nodejs ./website/prisma/schema.prisma ./ +# Copy over a startup script that'll run `npx prisma db push` before starting +# the webserver. This ensures the webserver can actually check user accounts. +# This is a prisma variant of the postgres solution suggested in +# https://docs.docker.com/compose/startup-order/ +COPY --chown=nextjs:nodejs ./website/wait-for-postgres.sh ./ + # Automatically leverage output traces to reduce image size # https://nextjs.org/docs/advanced-features/output-file-tracing COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ diff --git a/scripts/endtoend-demo/docker-compose.yaml b/scripts/endtoend-demo/docker-compose.yaml index b17a7b74..8aa2b0fa 100644 --- a/scripts/endtoend-demo/docker-compose.yaml +++ b/scripts/endtoend-demo/docker-compose.yaml @@ -87,3 +87,4 @@ services: condition: service_healthy ports: - "3000:3000" + command: bash wait-for-postgres.sh node server.js diff --git a/website/wait-for-postgres.sh b/website/wait-for-postgres.sh new file mode 100755 index 00000000..6ff1a024 --- /dev/null +++ b/website/wait-for-postgres.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# wait-for-postgres.sh +# +# This script tries to connect to a postgres database and then runs +# `npx prisma db push` +# It repeats until that setup completes (running multiple times will be a +# noop). Then, it'll run the next command provided. +# +# You don't need to provide the connection URL. This reads it from the +# DATABASE_URL environment variable which should be set. +# +# To run we suggest: +# ./wait-for-postgres.sh npm run dev + +set -e + +until npx prisma db push --skip-generate; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 +done + +>&2 echo "Postgres is up - executing command" +# Print and execute all other arguments starting with `$1` +# So `exec "$1" "$2" "$3" ...` +exec "$@" From 4ad46a45e434dbf883468641f4bbf67303a84d53 Mon Sep 17 00:00:00 2001 From: Keith Stevens Date: Wed, 21 Dec 2022 13:15:44 +0900 Subject: [PATCH 4/4] Updating the README files to be a bit more comprehensive --- scripts/endtoend-demo/README.md | 8 +-- website/README.md | 120 +++++++++++++++++++++++++------- 2 files changed, 97 insertions(+), 31 deletions(-) diff --git a/scripts/endtoend-demo/README.md b/scripts/endtoend-demo/README.md index 49b193a5..2d738453 100644 --- a/scripts/endtoend-demo/README.md +++ b/scripts/endtoend-demo/README.md @@ -1,18 +1,14 @@ # End to End Demo -This sets up an entire stack needed to run Open Assistant, including the website, backend, and associated dependent services. +This sets up an entire stack needed to run Open Assistant, including the +website, backend, and associated dependent services. To start the service, do the following: ```sh docker compose up --build -cd ../../website -npx prisma db push ``` -NOTE: we're working on making the prisma step integrated into the docker -initialization process. - Then, navigate to `http://localhost:3000` and interact with the website. When logging in, navigate to `http://localhost:1080` to get the magic email login link. diff --git a/website/README.md b/website/README.md index d88f6bed..7d57ceaa 100644 --- a/website/README.md +++ b/website/README.md @@ -1,31 +1,109 @@ -This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app). +# Open-Assistant NextJS Website -## Required Environment Variables +## Purpose -This requires: +This provides a comprehensive webapp interface for LAION's Open Assistant +project. Initially it will support: -- `NEXT_PUBLIC_SUPABASE_URL`: A supabase Authorization URL -- `NEXT_PUBLIC_SUPABASE_ANON_KEY`: A public supabase key for Authorization +1. User registration using either Discord or Email. +1. Adding responses to incomplete Open Assistant tasks. +1. Rating or Ranking responses to Open Assistant tasks. +1. Viewing an activity leaderboard. +1. Tracking community wide updates. -These can both be optained from the Supabase project settings page. +This interface compliments the Discord bot and will give access to the same +underlying tasks. -## Getting Started +## Contributing -First, run the development server: +### Major Dependencies -```bash -npm run dev -# or -yarn dev -``` +This website is built using: -Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. +1. [npm](https://www.npmjs.com/): The node package manager for building. +1. [React](https://reactjs.org/): The core frontend framework. +1. [Next.js](https://nextjs.org/): A React scaffolding framework to streamline + development. +1. [Prisma](https://www.prisma.io/): An ORM to interact with a web specific + [Postgres](https://www.postgresql.org/) database. +1. [NextAuth.js](https://next-auth.js.org/): A user authentication framework + to ensure we handle accounts with best practices. +1. [TailwindCSS](https://tailwindcss.com/): A general purpose framework for + styling any component. +1. [Chakra-UI](https://chakra-ui.com/): A wide collection of pre-built UI + components that generally look pretty good. -You can start editing the page by modifying `pages/index.js`. The page auto-updates as you edit the file. +### Set up your environment -[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.js`. +To contribute to the website, make sure you have the following setup and +installed: -The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages. +1. [NVM](https://github.com/nvm-sh/nvm): The Node Version Manager makes it + easy to ensure you have the right NodeJS version installed. Once installed, + run `nvm use 16` to use Node 16.x. The website is known to be stable with + NodeJS version 16.x. This will install both Node and NPM. +1. [Docker](https://www.docker.com/): We use docker to simplify running + dependent services. + +### Getting everything up and running + +If you're doing active development we suggest the following workflow: + +1. In one tab, navigate to + `${OPEN_ASSISTANT_ROOT}/scripts/frontend-development`. +1. Run `docker compose up --build`. You can optionally include `-d` to detach and + later track the logs if desired. +1. In another tab navigate to `${OPEN_ASSISTANT_ROOT/website`. +1. Run `npm install` +1. Run `npx prisma db push` (This is also needed when you restart the docker + stack from scratch). +1. Run `npm run dev`. Now the website is up and running locally at + `http://localhost:3000`. +1. To create an account, login via the user using email authentication and + navigate to `http://localhost:1080`. Check the email listed and click the + log in link. You're now logged in and authenticated. + +## Code Layout + +### React Code + +All react code is under `src/` with a few sub directories: + +1. `pages/`: All pages a user could navigate too and API URLs which are under `pages/api/`. +1. `components/`: All re-usable React components. If something gets used + twice we should create a component and put it here. +1. `lib/`: A generic place to store library files that are used anywhere. + This doesn't have much structure yet. + +NOTE: `styles/` can be ignored for now. + +### Database + +All database configurations are stored in `prisma/schema.prisma`. + +### Static Content + +All static images, fonts, svgs, etc are stored in `public/`. + +### Styles + +We're not really using CSS styles. `styles/` can be ignored. + +## Best Practices + +When writing code for the website, we have a few best practices: + +1. When importing packages import external dependencies first then local + dependencies. Order them alphabetically according to the package name. +1. When trying to implement something new, check if + [Chakra-UI](https://chakra-ui.com/) has components that are close enough to + your need. For example Sliders, Radio Buttons, Progress indicators, etc. They + have a lot and we can save time by re-using what they have and tweaking the + style as needed. +1. Format everything with [Prettier](https://prettier.io/). This is done by + default with pre-submits. We currently don't have any custom settings. +1. Define functional React components (with types for all properties when + feasible). ## Learn More @@ -33,11 +111,3 @@ To learn more about Next.js, take a look at the following resources: - [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. - -You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome! - -## Deploy on Vercel - -The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. - -Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.