Forced merge of remote-tracking branch 'origin/24-web-deploy-aws' into main

.pre-commit-config.yaml

@@ -10,6 +10,11 @@ repos:
       - id: trailing-whitespace
       - id: check-ast
       - id: check-yaml
+        # Always check YAML but skip a few YAML files that are auto-generated
+        # and which break the standard YAML check.  The alternative would be to
+        # skip any unsafe errors (and thus break YAML compatibility) or use
+        # some other checker that may not work in general.
+        exclude: copilot/web/addons/*
       - id: check-json
       - id: check-case-conflict
       - id: detect-private-key

copilot/.workspace

@@ -0,0 +1 @@
+application: open-assistant

copilot/README.md

@@ -0,0 +1,94 @@
+# Deploying on AWS
+
+## Introduction
+
+[Copilot](https://aws.github.io/copilot-cli/) helps simplify AWS resources and
+automate deploymnents for projects.
+
+This sample configuration runs the Open Assistant web app as an ECS Fargate
+services backed by a Serverless Aurora Postgres database.
+
+## To Setup
+
+Setup requires a few steps:
+
+```sh
+copilot app init --domain your_domain.com
+```
+
+This will initialize and register a variety of URLs with your
+`your_domain.com`. Replace with a proper domain to setup SSL certificates.
+
+```sh
+copilot env deploy
+```
+
+This will create a variety of aws roles and services needed for deployment.
+
+```sh
+copilot deploy
+```
+
+This will depoy the services but it won't be 100% ready for usage. Before
+being ready, we have to inspect the AWS Secrets manager and extract out the
+database credentials. Read those credentials then put them, and a few other
+secrets, in a `secrets.yml` file like the following:
+
+```yaml
+DATABASE_URL:
+  staging: postgres://postgres:${db_password}@${db_host}:${db_port}/${db_name}
+DISCORD_CLIENT_ID:
+  staging: ...
+DISCORD_CLIENT_SECRET:
+  staging: ...
+EMAIL_SERVER_HOST:
+  staging: ...
+EMAIL_SERVER_PORT:
+  staging: ...
+EMAIL_SERVER_USER:
+  staging: ...
+EMAIL_SERVER_PASSWORD:
+  staging: ...
+EMAIL_FROM:
+  staging: ...
+FASTAPI_URL:
+  staging: ...
+FASTAPI_KEY:
+  staging: ...
+NEXTAUTH_SECRET:
+  staging: ...
+```
+
+Then, upload the secrets to AWS with:
+
+```sh
+copilot secret init --cli-input-yaml secrets.yml
+```
+
+Now, finally deploy:
+
+```sh
+copilot deploy
+```
+
+If we documented everything correctly, the site should work properly.
+
+## To Update Manually
+
+First, make sure the database is updated with any schema changes:
+
+```sh
+copilot task run \
+    --app open-assistant --env staging \
+    -n prisma-push \
+    --dockerfile docker/Dockerfile.prisma --build-context "./" \
+    --secrets DATABASE_URL=/copilot/open-assistant/staging/secrets/DATABASE_URL
+```
+
+Next, deploy everything:
+
+```sh
+copilot deploy
+```
+
+TODO: Make this a pipeline once github and aws are fully connected.

copilot/environments/staging/manifest.yml

@@ -0,0 +1,10 @@
+# The manifest for the "staging" environment.
+# Read the full specification for the "Environment" type at:
+#  https://aws.github.io/copilot-cli/docs/manifest/environment/
+
+name: staging
+type: Environment
+
+# Configure observability for your environment resources.
+observability:
+  container_insights: false

copilot/web/addons/web-cluster.yml

@@ -0,0 +1,144 @@
+Parameters:
+  App:
+    Type: String
+    Description: Your application's name.
+  Env:
+    Type: String
+    Description: The environment name your service, job, or workflow is being deployed to.
+  Name:
+    Type: String
+    Description: The name of the service, job, or workflow being deployed.
+  # Customize your Aurora Serverless cluster by setting the default value of the following parameters.
+  webclusterDBName:
+    Type: String
+    Description: The name of the initial database to be created in the Aurora Serverless v2 cluster.
+    Default: oassist_web
+    # Cannot have special characters
+    # Naming constraints: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints
+Mappings:
+  webclusterEnvScalingConfigurationMap:
+    staging:
+      "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
+      "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128
+
+    All:
+      "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128
+      "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128
+
+Resources:
+  webclusterDBSubnetGroup:
+    Type: "AWS::RDS::DBSubnetGroup"
+    Properties:
+      DBSubnetGroupDescription: Group of Copilot private subnets for Aurora Serverless v2 cluster.
+      SubnetIds:
+        !Split [",", { "Fn::ImportValue": !Sub "${App}-${Env}-PrivateSubnets" }]
+  webclusterSecurityGroup:
+    Metadata:
+      "aws:copilot:description": "A security group for your workload to access the Aurora Serverless v2 cluster webcluster"
+    Type: "AWS::EC2::SecurityGroup"
+    Properties:
+      GroupDescription: !Sub "The Security Group for ${Name} to access Aurora Serverless v2 cluster webcluster."
+      VpcId:
+        Fn::ImportValue: !Sub "${App}-${Env}-VpcId"
+      Tags:
+        - Key: Name
+          Value: !Sub "copilot-${App}-${Env}-${Name}-Aurora"
+  webclusterDBClusterSecurityGroup:
+    Metadata:
+      "aws:copilot:description": "A security group for your Aurora Serverless v2 cluster webcluster"
+    Type: AWS::EC2::SecurityGroup
+    Properties:
+      GroupDescription: The Security Group for the Aurora Serverless v2 cluster.
+      SecurityGroupIngress:
+        - ToPort: 5432
+          FromPort: 5432
+          IpProtocol: tcp
+          Description: !Sub "From the Aurora Security Group of the workload ${Name}."
+          SourceSecurityGroupId: !Ref webclusterSecurityGroup
+      VpcId:
+        Fn::ImportValue: !Sub "${App}-${Env}-VpcId"
+  webclusterAuroraSecret:
+    Metadata:
+      "aws:copilot:description": "A Secrets Manager secret to store your DB credentials"
+    Type: AWS::SecretsManager::Secret
+    Properties:
+      Description: !Sub Aurora main user secret for ${AWS::StackName}
+      GenerateSecretString:
+        SecretStringTemplate: '{"username": "postgres"}'
+        GenerateStringKey: "password"
+        ExcludePunctuation: true
+        IncludeSpace: false
+        PasswordLength: 16
+  webclusterDBClusterParameterGroup:
+    Metadata:
+      "aws:copilot:description": "A DB parameter group for engine configuration values"
+    Type: "AWS::RDS::DBClusterParameterGroup"
+    Properties:
+      Description: !Ref "AWS::StackName"
+      Family: "aurora-postgresql14"
+      Parameters:
+        client_encoding: "UTF8"
+  webclusterDBCluster:
+    Metadata:
+      "aws:copilot:description": "The webcluster Aurora Serverless v2 database cluster"
+    Type: "AWS::RDS::DBCluster"
+    Properties:
+      MasterUsername:
+        !Join [
+          "",
+          [
+            "{{resolve:secretsmanager:",
+            !Ref webclusterAuroraSecret,
+            ":SecretString:username}}",
+          ],
+        ]
+      MasterUserPassword:
+        !Join [
+          "",
+          [
+            "{{resolve:secretsmanager:",
+            !Ref webclusterAuroraSecret,
+            ":SecretString:password}}",
+          ],
+        ]
+      DatabaseName: !Ref webclusterDBName
+      Engine: "aurora-postgresql"
+      EngineVersion: "14.4"
+      DBClusterParameterGroupName: !Ref webclusterDBClusterParameterGroup
+      DBSubnetGroupName: !Ref webclusterDBSubnetGroup
+      Port: 5432
+      VpcSecurityGroupIds:
+        - !Ref webclusterDBClusterSecurityGroup
+      ServerlessV2ScalingConfiguration:
+        # Replace "All" below with "!Ref Env" to set different autoscaling limits per environment.
+        MinCapacity:
+          !FindInMap [webclusterEnvScalingConfigurationMap, All, DBMinCapacity]
+        MaxCapacity:
+          !FindInMap [webclusterEnvScalingConfigurationMap, All, DBMaxCapacity]
+  webclusterDBWriterInstance:
+    Metadata:
+      "aws:copilot:description": "The webcluster Aurora Serverless v2 writer instance"
+    Type: "AWS::RDS::DBInstance"
+    Properties:
+      DBClusterIdentifier: !Ref webclusterDBCluster
+      DBInstanceClass: db.serverless
+      Engine: "aurora-postgresql"
+      PromotionTier: 1
+      AvailabilityZone: !Select
+        - 0
+        - !GetAZs
+          Ref: AWS::Region
+
+  webclusterSecretAuroraClusterAttachment:
+    Type: AWS::SecretsManager::SecretTargetAttachment
+    Properties:
+      SecretId: !Ref webclusterAuroraSecret
+      TargetId: !Ref webclusterDBCluster
+      TargetType: AWS::RDS::DBCluster
+Outputs:
+  webclusterSecret: # injected as WEBCLUSTER_SECRET environment variable by Copilot.
+    Description: "The JSON secret that holds the database username and password. Fields are 'host', 'port', 'dbname', 'username', 'password', 'dbClusterIdentifier' and 'engine'"
+    Value: !Ref webclusterAuroraSecret
+  webclusterSecurityGroup:
+    Description: "The security group to attach to the workload."
+    Value: !Ref webclusterSecurityGroup

copilot/web/manifest.yml

@@ -0,0 +1,41 @@
+# The manifest for the "web" service.
+# Read the full specification for the "Load Balanced Web Service" type at:
+#  https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/
+
+name: web
+type: Load Balanced Web Service
+
+http:
+  path: "/"
+
+image:
+  build:
+    dockerfile: docker/Dockerfile.website
+    context: ./
+  port: 3000
+
+cpu: 256
+memory: 512
+platform: linux/amd64
+count: 1
+exec: true
+network:
+  connect: true
+
+environments:
+  staging:
+    variables:
+      NEXTAUTH_URL: https://web.staging.open-assistant.surfacedata.org
+
+secrets:
+  DATABASE_URL: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/DATABASE_URL
+  DISCORD_CLIENT_ID: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/DISCORD_CLIENT_ID
+  DISCORD_CLIENT_SECRET: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/DISCORD_CLIENT_SECRET
+  EMAIL_SERVER_HOST: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/EMAIL_SERVER_HOST
+  EMAIL_SERVER_PASSWORD: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/EMAIL_SERVER_PASSWORD
+  EMAIL_SERVER_PORT: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/EMAIL_SERVER_PORT
+  EMAIL_SERVER_USER: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/EMAIL_SERVER_USER
+  EMAIL_FROM: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/EMAIL_FROM
+  FASTAPI_KEY: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/FASTAPI_KEY
+  FASTAPI_URL: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/FASTAPI_URL
+  NEXTAUTH_SECRET: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/NEXTAUTH_SECRET

docker/Dockerfile.prisma

@@ -0,0 +1,10 @@
+# This docker file will update the web database with schema changes.  Later it
+# will be updated to do prisma migrations.
+
+FROM --platform=linux/amd64 node:16.18.0 AS runner
+
+WORKDIR /app
+
+COPY ./website/ .
+
+CMD ["npx", "prisma", "db", "push"]

docker/Dockerfile.website

@@ -1,7 +1,8 @@
+# Note: node:16.18.0 works most reliably when using different platforms (namely
+# Mac M1) and avoids recent Prisma docker bugs that lead to segfaults.
+
 # Install dependencies only when needed
-FROM node:16.19 AS deps
-# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
-# RUN apk add --no-cache libc6-compat
+FROM node:16.18.0 AS deps
 WORKDIR /app
 
 # Install dependencies based on the preferred package manager
@@ -12,27 +13,19 @@ RUN \
   fi
 
 # Rebuild the source code only when needed
-FROM node:16.19 AS builder
+FROM node:16.18.0 AS builder
 WORKDIR /app
 COPY --from=deps /app/node_modules ./node_modules
 COPY ./website/ .
 
-# Next.js collects completely anonymous telemetry data about general usage.
-# Learn more here: https://nextjs.org/telemetry
-# Uncomment the following line in case you want to disable telemetry during the build.
-# ENV NEXT_TELEMETRY_DISABLED 1
-
-# RUN yarn build
 RUN npx prisma generate
 RUN npm run build
 
 # Production image, copy all the files and run next
-FROM node:16.19 AS runner
+FROM node:16.18.0 AS runner
 WORKDIR /app
 
 ENV NODE_ENV production
-# Uncomment the following line in case you want to disable telemetry during runtime.
-# ENV NEXT_TELEMETRY_DISABLED 1
 
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
@@ -48,8 +41,6 @@ COPY --chown=nextjs:nodejs ./website/prisma/schema.prisma ./
 #   https://docs.docker.com/compose/startup-order/
 COPY --chown=nextjs:nodejs ./website/wait-for-postgres.sh ./
 
-# Automatically leverage output traces to reduce image size
-# https://nextjs.org/docs/advanced-features/output-file-tracing
 COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
 COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
 

scripts/endtoend-demo/docker-compose.yaml

@@ -12,6 +12,8 @@ services:
     extends:
       file: ../frontend-development/docker-compose.yaml
       service: webdb
+    environment:
+      POSTGRES_DB: oasst_web
 
   # This lets you manually inspect the web and backend databases.
   adminer:
@@ -40,7 +42,7 @@ services:
       context: ../../
     image: oasst-web
     environment:
-      - DATABASE_URL=postgres://postgres:postgres@webdb/ocgpt_website
+      - DATABASE_URL=postgres://postgres:postgres@webdb/oasst_web
       - FASTAPI_URL=http://backend:8080
       - FASTAPI_KEY=1234
       - NEXTAUTH_SECRET=O/M2uIbGj+lDD2oyNa8ax4jEOJqCPJzO53UbWShmq98=

website/package.json

@@ -33,6 +33,7 @@
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-icons": "^4.7.1",
+    "sharp": "^0.31.3",
     "swr": "^2.0.0",
     "tailwindcss": "^3.2.4",
     "use-debounce": "^9.0.2"

website/prisma/migrations/20221215064332_initialized/migration.sql

@@ -1,66 +0,0 @@
--- CreateTable
-CREATE TABLE "Account" (
-    "id" TEXT NOT NULL,
-    "userId" TEXT NOT NULL,
-    "type" TEXT NOT NULL,
-    "provider" TEXT NOT NULL,
-    "providerAccountId" TEXT NOT NULL,
-    "refresh_token" TEXT,
-    "access_token" TEXT,
-    "expires_at" INTEGER,
-    "token_type" TEXT,
-    "scope" TEXT,
-    "id_token" TEXT,
-    "session_state" TEXT,
-
-    CONSTRAINT "Account_pkey" PRIMARY KEY ("id")
-);
-
--- CreateTable
-CREATE TABLE "Session" (
-    "id" TEXT NOT NULL,
-    "sessionToken" TEXT NOT NULL,
-    "userId" TEXT NOT NULL,
-    "expires" TIMESTAMP(3) NOT NULL,
-
-    CONSTRAINT "Session_pkey" PRIMARY KEY ("id")
-);
-
--- CreateTable
-CREATE TABLE "User" (
-    "id" TEXT NOT NULL,
-    "name" TEXT,
-    "email" TEXT,
-    "emailVerified" TIMESTAMP(3),
-    "image" TEXT,
-
-    CONSTRAINT "User_pkey" PRIMARY KEY ("id")
-);
-
--- CreateTable
-CREATE TABLE "VerificationToken" (
-    "identifier" TEXT NOT NULL,
-    "token" TEXT NOT NULL,
-    "expires" TIMESTAMP(3) NOT NULL
-);
-
--- CreateIndex
-CREATE UNIQUE INDEX "Account_provider_providerAccountId_key" ON "Account"("provider", "providerAccountId");
-
--- CreateIndex
-CREATE UNIQUE INDEX "Session_sessionToken_key" ON "Session"("sessionToken");
-
--- CreateIndex
-CREATE UNIQUE INDEX "User_email_key" ON "User"("email");
-
--- CreateIndex
-CREATE UNIQUE INDEX "VerificationToken_token_key" ON "VerificationToken"("token");
-
--- CreateIndex
-CREATE UNIQUE INDEX "VerificationToken_identifier_token_key" ON "VerificationToken"("identifier", "token");
-
--- AddForeignKey
-ALTER TABLE "Account" ADD CONSTRAINT "Account_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-
--- AddForeignKey
-ALTER TABLE "Session" ADD CONSTRAINT "Session_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;

website/prisma/migrations/20221215101838_record_labeler_id/migration.sql

@@ -1,2 +0,0 @@
--- AlterTable
-ALTER TABLE "User" ADD COLUMN     "labelerId" INTEGER;

website/prisma/migrations/migration_lock.toml

@@ -1,3 +0,0 @@
-# Please do not edit this file manually
-# It should be added in your version-control system (i.e. Git)
-provider = "postgresql"

website/prisma/schema.prisma

@@ -5,6 +5,7 @@ datasource db {
 
 generator client {
   provider        = "prisma-client-js"
+  binaryTargets = ["native", "linux-musl"]
 }
 
 model Account {

website/src/pages/auth/signup.tsx

@@ -1,9 +1,10 @@
 import { Button, Input, Stack } from "@chakra-ui/react";
 import Head from "next/head";
+import Link from "next/link";
 import { FaDiscord, FaEnvelope, FaBug, FaGithub } from "react-icons/fa";
 import { getCsrfToken, getProviders, signIn } from "next-auth/react";
 import { useRef } from "react";
-import Link from "next/link";
+import { FaDiscord, FaEnvelope, FaGithub } from "react-icons/fa";
 
 import { AuthLayout } from "src/components/AuthLayout";
 
@@ -12,11 +13,19 @@ export default function Signin({ csrfToken, providers }) {
   const emailEl = useRef(null);
   const debugUsernameEl = useRef(null);
 
+  const signinWithDiscord = () => {
+    signIn(discord.id, { callbackUrl: "/" });
+  };
+
   const signinWithEmail = (ev: React.FormEvent) => {
     ev.preventDefault();
     signIn(email.id, { callbackUrl: "/", email: emailEl.current.value });
   };
 
+  const signinWithGithub = () => {
+    signIn(github.id, { callbackUrl: "/" });
+  };
+
   function signinWithDebugCredentials(ev: React.FormEvent) {
     ev.preventDefault();
     signIn(credentials.id, { callbackUrl: "/", username: debugUsernameEl.current.value });
@@ -61,8 +70,7 @@ export default function Signin({ csrfToken, providers }) {
               size="lg"
               leftIcon={<FaDiscord />}
               color="white"
-              onClick={() => signIn(discord, { callbackUrl: "/" })}
-              // isDisabled="false"
+              onClick={signinWithDiscord}
             >
               Continue with Discord
             </Button>
@@ -77,7 +85,7 @@ export default function Signin({ csrfToken, providers }) {
               size={"lg"}
               leftIcon={<FaGithub />}
               colorScheme="blue"
-              // isDisabled="false"
+              onClick={signinWithGithub}
             >
               Continue with Github
             </Button>