diff --git a/copilot/README.md b/copilot/README.md new file mode 100644 index 00000000..b4748c0a --- /dev/null +++ b/copilot/README.md @@ -0,0 +1,93 @@ +# Deploying on AWS + +## Introduction + +[Copilot](https://aws.github.io/copilot-cli/) helps simplify AWS resources and +automate deploymnents for projects. + +This sample configuration runs the Open Assistant web app as an ECS Fargate +services backed by a Serverless Aurora Postgres database. + +## To Setup + +Setup requires a few steps: + +```sh +copilot app init --domain your_domain.tls +``` + +This will initialize and register a variety of URls with your domain. + +```sh +copilot env deploy +``` + +This will create a variety of aws roles and services needed for deployment. + +```sh +copilot deploy +``` + +This will depoy the services but it won't be 100% ready for usage. Before +being ready, we have to inspect the AWS Secrets manager and extract out the +database credentials. Read those credentials then put them, and a few other +secrets, in a `secrets.yml` file like the following: + +```yaml +DATABASE_URL: + staging: postgres://postgres:${db_password}@${db_host}:${db_port}/${db_name} +DISCORD_CLIENT_ID: + staging: ... +DISCORD_CLIENT_SECRET: + staging: ... +EMAIL_SERVER_HOST: + staging: ... +EMAIL_SERVER_PORT: + staging: ... +EMAIL_SERVER_USER: + staging: ... +EMAIL_SERVER_PASSWORD: + staging: ... +EMAIL_FROM: + staging: ... +FASTAPI_URL: + staging: ... +FASTAPI_KEY: + staging: ... +NEXTAUTH_SECRET: + staging: ... +``` + +Then, upload the secrets to AWS with: + +```sh +copilot secret init --cli-input-yaml secrets.yml +``` + +Now, finally deploy: + +```sh +copilot deploy +``` + +If we documented everything correctly, the site should work properly. + +## To Update Manually + +First, make sure the database is updated with any schema changes: + +```sh +copilot task run \ + --app open-assistant --env staging \ + -n prisma-push \ + --dockerfile docker/Dockerfile.prisma --build-context "./" \ + --secrets DATABASE_URL=/copilot/open-assistant/staging/secrets/DATABASE_URL +``` + +Next, deploy everything: + +```sh +copilot deploy +``` + +TODO: Make this a pipeline once github and aws are fully connected. diff --git a/copilot/pipelines/open-assistant-main/buildspec.yml b/copilot/pipelines/open-assistant-main/buildspec.yml new file mode 100644 index 00000000..e1cab4d5 --- /dev/null +++ b/copilot/pipelines/open-assistant-main/buildspec.yml @@ -0,0 +1,71 @@ +# Buildspec runs in the build stage of your pipeline. +version: 0.2 +phases: + install: + runtime-versions: + docker: 19 + ruby: 2.6 + commands: + - echo "cd into $CODEBUILD_SRC_DIR" + - cd $CODEBUILD_SRC_DIR + # Download the copilot linux binary. + - wget -q https://ecs-cli-v2-release.s3.amazonaws.com/copilot-linux-v1.24.0 + - mv ./copilot-linux-v1.24.0 ./copilot-linux + - chmod +x ./copilot-linux + build: + commands: + - echo "Run your tests" + # - make test + post_build: + commands: + - ls -l + - export COLOR="false" + - pipeline=$(cat $CODEBUILD_SRC_DIR/copilot/pipelines/open-assistant-main/manifest.yml | ruby -ryaml -rjson -e 'puts JSON.pretty_generate(YAML.load(ARGF))') + - pl_envs=$(echo $pipeline | jq -r '.stages[].name') + # Find all the local services in the workspace. + - svc_ls_result=$(./copilot-linux svc ls --local --json) + - svc_list=$(echo $svc_ls_result | jq '.services') + - > + if [ ! "$svc_list" = null ]; then + svcs=$(echo $svc_ls_result | jq -r '.services[].name'); + fi + # Find all the local jobs in the workspace. + - job_ls_result=$(./copilot-linux job ls --local --json) + - job_list=$(echo $job_ls_result | jq '.jobs') + - > + if [ ! "$job_list" = null ]; then + jobs=$(echo $job_ls_result | jq -r '.jobs[].name'); + fi + # Raise error if no services or jobs are found. + - > + if [ "$svc_list" = null ] && [ "$job_list" = null ]; then + echo "No services or jobs found for the pipeline to deploy. Please create at least one service or job and push the manifest to the remote." 1>&2; + exit 1; + fi + # Generate the cloudformation templates. + # The tag is the build ID but we replaced the colon ':' with a dash '-'. + # We truncate the tag (from the front) to 128 characters, the limit for Docker tags + # (https://docs.docker.com/engine/reference/commandline/tag/) + # Check if the `svc package` commanded exited with a non-zero status. If so, echo error msg and exit. + - > + for env in $pl_envs; do + tag=$(echo ${CODEBUILD_BUILD_ID##*:}-$env | sed 's/:/-/g' | rev | cut -c 1-128 | rev) + for svc in $svcs; do + ./copilot-linux svc package -n $svc -e $env --output-dir './infrastructure' --tag $tag --upload-assets; + if [ $? -ne 0 ]; then + echo "Cloudformation stack and config files were not generated. Please check build logs to see if there was a manifest validation error." 1>&2; + exit 1; + fi + done; + for job in $jobs; do + ./copilot-linux job package -n $job -e $env --output-dir './infrastructure' --tag $tag --upload-assets; + if [ $? -ne 0 ]; then + echo "Cloudformation stack and config files were not generated. Please check build logs to see if there was a manifest validation error." 1>&2; + exit 1; + fi + done; + done; + - ls -lah ./infrastructure +artifacts: + files: + - "infrastructure/*" diff --git a/copilot/pipelines/open-assistant-main/manifest.yml b/copilot/pipelines/open-assistant-main/manifest.yml new file mode 100644 index 00000000..25be4c75 --- /dev/null +++ b/copilot/pipelines/open-assistant-main/manifest.yml @@ -0,0 +1,17 @@ +# The manifest for the "open-assistant-main" pipeline. +# This YAML file defines your pipeline: the source repository it tracks and the order of the environments to deploy to. +# For more info: https://aws.github.io/copilot-cli/docs/manifest/pipeline/ + +name: open-assistant-main + +version: 1 + +source: + provider: GitHub + properties: + branch: 24-web-deploy-aws + repository: https://github.com/LAION-AI/Open-Assistant + +stages: + - name: staging + test_commands: [npx prisma db push --schema website/prisma/schema.prisma] diff --git a/docker/Dockerfile.prisma b/docker/Dockerfile.prisma new file mode 100644 index 00000000..b6a0def0 --- /dev/null +++ b/docker/Dockerfile.prisma @@ -0,0 +1,7 @@ +FROM --platform=linux/amd64 node:16.18.0 AS runner + +WORKDIR /app + +COPY ./website/ . + +CMD ["npx", "prisma", "db", "push"]