mirror of
https://github.com/wassname/Open-Assistant.git
synced 2026-07-10 00:20:06 +08:00
Merge remote-tracking branch 'origin/main' into user_menu_fix
This commit is contained in:
notmd
@@ -0,0 +1,52 @@
|
||||
name: Deploy to dev machine
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
stack-name:
|
||||
required: false
|
||||
type: string
|
||||
default: dev
|
||||
image-tag:
|
||||
required: false
|
||||
type: string
|
||||
default: latest
|
||||
backend-port:
|
||||
required: false
|
||||
type: string
|
||||
default: 8080
|
||||
website-port:
|
||||
required: false
|
||||
type: string
|
||||
default: 3000
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
WEB_ADMIN_USERS: ${{ secrets.DEV_WEB_ADMIN_USERS }}
|
||||
WEB_DISCORD_CLIENT_ID: ${{ secrets.DEV_WEB_DISCORD_CLIENT_ID }}
|
||||
WEB_DISCORD_CLIENT_SECRET: ${{ secrets.DEV_WEB_DISCORD_CLIENT_SECRET }}
|
||||
WEB_EMAIL_SERVER_HOST: ${{ secrets.DEV_WEB_EMAIL_SERVER_HOST }}
|
||||
WEB_EMAIL_SERVER_PASSWORD: ${{ secrets.DEV_WEB_EMAIL_SERVER_PASSWORD }}
|
||||
WEB_EMAIL_SERVER_PORT: ${{ secrets.DEV_WEB_EMAIL_SERVER_PORT }}
|
||||
WEB_EMAIL_SERVER_USER: ${{ secrets.DEV_WEB_EMAIL_SERVER_USER }}
|
||||
WEB_NEXTAUTH_SECRET: ${{ secrets.DEV_WEB_NEXTAUTH_SECRET }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Run playbook
|
||||
uses: dawidd6/action-ansible-playbook@v2
|
||||
with:
|
||||
# Required, playbook filepath
|
||||
playbook: deploy-dev.yaml
|
||||
# Optional, directory where playbooks live
|
||||
directory: ansible
|
||||
# Optional, SSH private key
|
||||
key: ${{secrets.DEV_NODE_PRIVATE_KEY}}
|
||||
# Optional, literal inventory file contents
|
||||
inventory: |
|
||||
[dev]
|
||||
dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team
|
||||
options: |
|
||||
--extra-vars "stack_name=${{inputs.stack-name}} image_tag=${{inputs.image-tag}} backend_port=${{inputs.backend-port}} website_port=${{inputs.website-port}}"
|
||||
@@ -46,8 +46,9 @@ jobs:
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ inputs.image-name }}
|
||||
tags: |
|
||||
type=sha,prefix=${{ env.TAG_PREFIX }},format=short
|
||||
type=ref,event=tag
|
||||
type=raw,value=latest,enable=${{ github.ref_name == 'main' }}
|
||||
type=sha,prefix=${{ env.TAG_PREFIX }},format=short,enable=${{ github.ref_name != 'main' }}
|
||||
type=ref,event=tag,enable=${{ github.ref_name != 'main' }}
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v3.2.0
|
||||
with:
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
name: pre-commit
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
workflow_call:
|
||||
pull_request_target:
|
||||
|
||||
jobs:
|
||||
@@ -18,7 +16,7 @@ jobs:
|
||||
|
||||
# in case of push, check out the main branch
|
||||
- uses: actions/checkout@v3
|
||||
if: github.event_name == 'push'
|
||||
if: github.event_name != 'pull_request_target'
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
|
||||
@@ -1,12 +1,19 @@
|
||||
name: Release
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
release:
|
||||
types: [released]
|
||||
types:
|
||||
- released
|
||||
|
||||
jobs:
|
||||
pre-commit:
|
||||
uses: ./.github/workflows/pre-commit.yaml
|
||||
build-backend:
|
||||
uses: ./.github/workflows/docker-build.yaml
|
||||
needs: pre-commit
|
||||
with:
|
||||
image-name: oasst-backend
|
||||
context: .
|
||||
@@ -14,6 +21,7 @@ jobs:
|
||||
build-args: ""
|
||||
build-web:
|
||||
uses: ./.github/workflows/docker-build.yaml
|
||||
needs: pre-commit
|
||||
with:
|
||||
image-name: oasst-web
|
||||
context: .
|
||||
@@ -21,6 +29,7 @@ jobs:
|
||||
build-args: ""
|
||||
build-bot:
|
||||
uses: ./.github/workflows/docker-build.yaml
|
||||
needs: pre-commit
|
||||
with:
|
||||
image-name: oasst-discord-bot
|
||||
context: .
|
||||
@@ -28,29 +37,12 @@ jobs:
|
||||
build-args: ""
|
||||
deploy-dev:
|
||||
needs: [build-backend, build-web, build-bot]
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
WEB_ADMIN_USERS: ${{ secrets.DEV_WEB_ADMIN_USERS }}
|
||||
WEB_DISCORD_CLIENT_ID: ${{ secrets.DEV_WEB_DISCORD_CLIENT_ID }}
|
||||
WEB_DISCORD_CLIENT_SECRET: ${{ secrets.DEV_WEB_DISCORD_CLIENT_SECRET }}
|
||||
WEB_EMAIL_SERVER_HOST: ${{ secrets.DEV_WEB_EMAIL_SERVER_HOST }}
|
||||
WEB_EMAIL_SERVER_PASSWORD: ${{ secrets.DEV_WEB_EMAIL_SERVER_PASSWORD }}
|
||||
WEB_EMAIL_SERVER_PORT: ${{ secrets.DEV_WEB_EMAIL_SERVER_PORT }}
|
||||
WEB_EMAIL_SERVER_USER: ${{ secrets.DEV_WEB_EMAIL_SERVER_USER }}
|
||||
WEB_NEXTAUTH_SECRET: ${{ secrets.DEV_WEB_NEXTAUTH_SECRET }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v2
|
||||
- name: Run playbook
|
||||
uses: dawidd6/action-ansible-playbook@v2
|
||||
with:
|
||||
# Required, playbook filepath
|
||||
playbook: dev.yaml
|
||||
# Optional, directory where playbooks live
|
||||
directory: ansible
|
||||
# Optional, SSH private key
|
||||
key: ${{secrets.DEV_NODE_PRIVATE_KEY}}
|
||||
# Optional, literal inventory file contents
|
||||
inventory: |
|
||||
[dev]
|
||||
dev01 ansible_host=${{secrets.DEV_NODE_IP}} ansible_connection=ssh ansible_user=web-team
|
||||
uses: ./.github/workflows/deploy-dev.yaml
|
||||
secrets: inherit
|
||||
with:
|
||||
stack-name: ${{ github.event_name == 'release' && 'staging' || 'dev' }}
|
||||
image-tag:
|
||||
${{ github.event_name == 'release' && github.event.release.tag_name ||
|
||||
'latest' }}
|
||||
backend-port: ${{ github.event_name == 'release' && '8180' || '8080' }}
|
||||
website-port: ${{ github.event_name == 'release' && '3100' || '3000' }}
|
||||
|
||||
@@ -44,3 +44,9 @@ jobs:
|
||||
run: ./scripts/frontend-development/run-contract-test.sh
|
||||
|
||||
- run: ./scripts/backend-development/stop-mock-server.sh
|
||||
|
||||
#- uses: stefanzweifel/git-auto-commit-action@v4
|
||||
# with:
|
||||
# file_pattern: "docs/docs/api/openapi.json"
|
||||
# commit_message:
|
||||
# update docs/docs/api/openapi.json by run ${{ github.run_id }}
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
#
|
||||
# /WARNING!
|
||||
|
||||
exclude: build|stubs|^bot/templates/$|openassistant/templates
|
||||
exclude: build|stubs|^bot/templates/$|openassistant/templates|docs/docs/api/openapi.json
|
||||
|
||||
repos:
|
||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||
|
||||
+28
-2
@@ -37,20 +37,28 @@ contributions smoothly we recommend the following:
|
||||
1. Before working on any changes, try to
|
||||
[sync the forked repository](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork)
|
||||
to keep it up-to-date with the upstream repository.
|
||||
1. Work on a small focused change that only touches on a few files.
|
||||
1. On a
|
||||
[new branch](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-and-deleting-branches-within-your-repository)
|
||||
in your fork (aka a "feature branch" and not `main`) work on a small focused
|
||||
change that only touches on a few files.
|
||||
1. Run `pre-commit` and make sure all files have formatting fixed. This
|
||||
simplifies life for reviewers.
|
||||
1. Package up a small bit of work that solves part of the problem
|
||||
[into a Pull Request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request-from-a-fork)
|
||||
and
|
||||
[send it out for review](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review).
|
||||
[Here](https://github.com/LAION-AI/Open-Assistant/pull/658) is an example PR
|
||||
for this project to illustrate this flow.
|
||||
1. If you're lucky, we can merge your change into `main` without any problems.
|
||||
If there's changes to files you're working on, resolve them by:
|
||||
1. First try rebase as suggested
|
||||
[in these instructions](https://timwise.co.uk/2019/10/14/merge-vs-rebase/#should-you-rebase).
|
||||
1. If rebase feels too painful, merge as suggested
|
||||
[in these instructions](https://timwise.co.uk/2019/10/14/merge-vs-rebase/#should-you-merge).
|
||||
1. Once you've resolved any conflicts, finish the review and merge into `main`.
|
||||
1. Once you've resolved any conflicts, finish the review and
|
||||
[squash and merge](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/incorporating-changes-from-a-pull-request/about-pull-request-merges#squash-and-merge-your-commits)
|
||||
your PR (when squashing try to clean up or update the individual commit
|
||||
messages to be one sensible single one).
|
||||
1. Merge in your change and move onto a new issue or the second step of your
|
||||
current issue.
|
||||
|
||||
@@ -59,6 +67,24 @@ need help on it or would like suggestions on how to approach the issue. If so,
|
||||
share wildly. If they seem to have a good handle on it, let them work on their
|
||||
solution until a challenge comes up.
|
||||
|
||||
#### Tips
|
||||
|
||||
- At any point you can compare your feature branch to the upstream/main of
|
||||
`LAION-AI/Open-Assistant` but using a URL like this:
|
||||
https://github.com/LAION-AI/Open-Assistant/compare/main...andrewm4894:Open-Assistant:my-example-feature-branch.
|
||||
Obviously just replace `andrewm4894` with your own GitHub user name and
|
||||
`my-example-feature-branch` with whatever you called the feature branch you
|
||||
are working on, so something like
|
||||
`https://github.com/LAION-AI/Open-Assistant/compare/main...<your_github_username>:Open-Assistant:<your_branch_name>`.
|
||||
This will show the changes that would appear in a PR, so you can check this to
|
||||
make sure it just looks like only the files you have changed or added will be
|
||||
part of the PR.
|
||||
- Try not to work on the `main` branch in your fork - ideally you can keep this
|
||||
as just a updated copy of `main` from `LAION-AI/Open-Assistant`.
|
||||
- If your feature branch gets messed up, just update the `main` branch in your
|
||||
fork and create a fresh new clean "feature branch" you can try again on by
|
||||
adding your changes one by one in separate commits or all as a single commit.
|
||||
|
||||
### When does a review finish
|
||||
|
||||
A review finishes when all blocking comments are addressed and at least one
|
||||
|
||||
@@ -1,29 +1,37 @@
|
||||
# ansible playbook to set up some docker containers
|
||||
|
||||
- name: Set up a dev node
|
||||
- name: Deploy to dev node
|
||||
hosts: dev
|
||||
gather_facts: true
|
||||
vars:
|
||||
stack_name: "dev"
|
||||
image_tag: latest
|
||||
backend_port: 8080
|
||||
website_port: 3000
|
||||
tasks:
|
||||
- name: Create network
|
||||
community.docker.docker_network:
|
||||
name: oasst
|
||||
name: "oasst-{{ stack_name }}"
|
||||
state: present
|
||||
driver: bridge
|
||||
|
||||
- name: Create stack files directory
|
||||
ansible.builtin.file:
|
||||
path: "./{{ stack_name }}"
|
||||
state: directory
|
||||
|
||||
- name: Copy redis.conf to managed node
|
||||
ansible.builtin.copy:
|
||||
src: ./redis.conf
|
||||
dest: ./redis.conf
|
||||
dest: "./{{ stack_name }}/redis.conf"
|
||||
|
||||
- name: Set up Redis
|
||||
community.docker.docker_container:
|
||||
name: oasst-redis
|
||||
name: "oasst-{{ stack_name }}-redis"
|
||||
image: redis
|
||||
state: started
|
||||
restart_policy: always
|
||||
network_mode: oasst
|
||||
ports:
|
||||
- 6379:6379
|
||||
network_mode: "oasst-{{ stack_name }}"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
|
||||
interval: 2s
|
||||
@@ -31,73 +39,64 @@
|
||||
retries: 10
|
||||
command: redis-server /usr/local/etc/redis/redis.conf
|
||||
volumes:
|
||||
- "./redis.conf:/usr/local/etc/redis/redis.conf"
|
||||
|
||||
- name: Set up Redis Insights
|
||||
community.docker.docker_container:
|
||||
name: oasst-redis-insights
|
||||
image: redislabs/redisinsight:latest
|
||||
state: started
|
||||
restart_policy: always
|
||||
network_mode: oasst
|
||||
ports:
|
||||
- 8001:8001
|
||||
- "./{{ stack_name }}/redis.conf:/usr/local/etc/redis/redis.conf"
|
||||
|
||||
- name: Create postgres containers
|
||||
community.docker.docker_container:
|
||||
name: "{{ item.name }}"
|
||||
name: "oasst-{{ stack_name }}-postgres-{{ item.name }}"
|
||||
image: postgres:15
|
||||
state: started
|
||||
restart_policy: always
|
||||
network_mode: oasst
|
||||
network_mode: "oasst-{{ stack_name }}"
|
||||
env:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: postgres
|
||||
volumes:
|
||||
- "{{ item.name }}:/var/lib/postgresql/data"
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "postgres"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 10
|
||||
loop:
|
||||
- name: oasst-postgres
|
||||
- name: oasst-postgres-web
|
||||
- name: backend
|
||||
- name: web
|
||||
|
||||
- name: Run the oasst oasst-backend
|
||||
community.docker.docker_container:
|
||||
name: oasst-backend
|
||||
image: ghcr.io/laion-ai/open-assistant/oasst-backend
|
||||
name: "oasst-{{ stack_name }}-backend"
|
||||
image: "ghcr.io/laion-ai/open-assistant/oasst-backend:{{ image_tag }}"
|
||||
state: started
|
||||
recreate: true
|
||||
pull: true
|
||||
restart_policy: always
|
||||
network_mode: oasst
|
||||
network_mode: "oasst-{{ stack_name }}"
|
||||
env:
|
||||
POSTGRES_HOST: oasst-postgres
|
||||
REDIS_HOST: oasst-redis
|
||||
POSTGRES_HOST: "oasst-{{ stack_name }}-postgres-backend"
|
||||
REDIS_HOST: "oasst-{{ stack_name }}-redis"
|
||||
DEBUG_ALLOW_ANY_API_KEY: "true"
|
||||
DEBUG_USE_SEED_DATA: "true"
|
||||
DEBUG_ALLOW_SELF_LABELING: "true"
|
||||
MAX_WORKERS: "1"
|
||||
RATE_LIMIT: "false"
|
||||
DEBUG_SKIP_EMBEDDING_COMPUTATION: "true"
|
||||
DEBUG_SKIP_TOXICITY_CALCULATION: "true"
|
||||
ports:
|
||||
- 8080:8080
|
||||
- "{{ backend_port }}:8080"
|
||||
|
||||
- name: Run the oasst oasst-web frontend
|
||||
community.docker.docker_container:
|
||||
name: oasst-web
|
||||
image: ghcr.io/laion-ai/open-assistant/oasst-web
|
||||
name: "oasst-{{ stack_name }}-web"
|
||||
image: "ghcr.io/laion-ai/open-assistant/oasst-web:{{ image_tag }}"
|
||||
state: started
|
||||
recreate: true
|
||||
pull: true
|
||||
restart_policy: always
|
||||
network_mode: oasst
|
||||
network_mode: "oasst-{{ stack_name }}"
|
||||
env:
|
||||
ADMIN_USERS: "{{ lookup('ansible.builtin.env', 'WEB_ADMIN_USERS') }}"
|
||||
DATABASE_URL: postgres://postgres:postgres@oasst-postgres-web/postgres
|
||||
DATABASE_URL:
|
||||
"postgres://postgres:postgres@oasst-{{ stack_name
|
||||
}}-postgres-web/postgres"
|
||||
DEBUG_LOGIN: "true"
|
||||
DISCORD_CLIENT_ID:
|
||||
"{{ lookup('ansible.builtin.env', 'WEB_DISCORD_CLIENT_ID') }}"
|
||||
@@ -112,11 +111,11 @@
|
||||
"{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_PORT') }}"
|
||||
EMAIL_SERVER_USER:
|
||||
"{{ lookup('ansible.builtin.env', 'WEB_EMAIL_SERVER_USER') }}"
|
||||
FASTAPI_URL: http://oasst-backend:8080
|
||||
FASTAPI_URL: "http://oasst-{{ stack_name }}-backend:8080"
|
||||
FASTAPI_KEY: "1234"
|
||||
NEXTAUTH_SECRET:
|
||||
"{{ lookup('ansible.builtin.env', 'WEB_NEXTAUTH_SECRET') }}"
|
||||
NEXTAUTH_URL: http://web.dev.open-assistant.io/
|
||||
NEXTAUTH_URL: http://web.{{ stack_name }}.open-assistant.io/
|
||||
ports:
|
||||
- 3000:3000
|
||||
- "{{ website_port }}:3000"
|
||||
command: bash wait-for-postgres.sh node server.js
|
||||
@@ -59,3 +59,6 @@ without having to actually set up and run a development backend.
|
||||
# save openapi.json to docs/docs/api
|
||||
wget localhost:8080/api/v1/openapi.json -O docs/docs/api/openapi.json
|
||||
```
|
||||
|
||||
Note: The api docs should be automatically updated by the
|
||||
`test-api-contract.yaml` workflow.
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
"""MessageToxicity
|
||||
|
||||
Revision ID: bcc2fe18d214
|
||||
Revises: 20cd871f4ec7
|
||||
Create Date: 2023-01-08 22:00:43.297719
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "bcc2fe18d214"
|
||||
down_revision = "846cc08ac79f"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"message_toxicity",
|
||||
sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False),
|
||||
sa.Column("toxicity", sa.Float(), nullable=True),
|
||||
sa.Column("created_date", sa.DateTime(), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False),
|
||||
sa.Column("model", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["message_id"],
|
||||
["message.id"],
|
||||
),
|
||||
sa.PrimaryKeyConstraint("message_id", "model"),
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_table("message_toxicity")
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,90 @@
|
||||
"""add rank to message table
|
||||
|
||||
Revision ID: 619255ae9076
|
||||
Revises: bcc2fe18d214
|
||||
Create Date: 2023-01-14 15:09:03.462482
|
||||
|
||||
"""
|
||||
import sqlalchemy as sa
|
||||
import sqlmodel
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = "619255ae9076"
|
||||
down_revision = "bcc2fe18d214"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column("message", sa.Column("rank", sa.Integer(), nullable=True))
|
||||
op.add_column("message_toxicity", sa.Column("score", sa.Float(), nullable=True))
|
||||
op.add_column("message_toxicity", sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=256), nullable=False))
|
||||
op.drop_column("message_toxicity", "toxicity")
|
||||
op.add_column("user_stats", sa.Column("time_frame", sqlmodel.sql.sqltypes.AutoString(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("prompts", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("replies_assistant", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("replies_prompter", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("labels_simple", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("labels_full", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("rankings_total", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("rankings_good", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("accepted_prompts", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("accepted_replies_assistant", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("accepted_replies_prompter", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("reply_assistant_ranked_1", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("reply_assistant_ranked_2", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("reply_assistant_ranked_3", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("reply_prompter_ranked_1", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("reply_prompter_ranked_2", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("reply_prompter_ranked_3", sa.Integer(), nullable=False))
|
||||
op.add_column("user_stats", sa.Column("streak_last_day_date", sa.DateTime(), nullable=True))
|
||||
op.add_column("user_stats", sa.Column("streak_days", sa.Integer(), nullable=True))
|
||||
op.drop_column("user_stats", "messages")
|
||||
op.drop_column("user_stats", "upvotes")
|
||||
op.drop_column("user_stats", "task_reward")
|
||||
op.drop_column("user_stats", "compare_wins")
|
||||
op.drop_column("user_stats", "compare_losses")
|
||||
op.drop_column("user_stats", "downvotes")
|
||||
op.drop_column("user_stats", "reactions")
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column("user_stats", sa.Column("reactions", sa.INTEGER(), autoincrement=False, nullable=False))
|
||||
op.add_column("user_stats", sa.Column("downvotes", sa.INTEGER(), autoincrement=False, nullable=False))
|
||||
op.add_column("user_stats", sa.Column("compare_losses", sa.INTEGER(), autoincrement=False, nullable=False))
|
||||
op.add_column("user_stats", sa.Column("compare_wins", sa.INTEGER(), autoincrement=False, nullable=False))
|
||||
op.add_column("user_stats", sa.Column("task_reward", sa.INTEGER(), autoincrement=False, nullable=False))
|
||||
op.add_column("user_stats", sa.Column("upvotes", sa.INTEGER(), autoincrement=False, nullable=False))
|
||||
op.add_column("user_stats", sa.Column("messages", sa.INTEGER(), autoincrement=False, nullable=False))
|
||||
op.drop_column("user_stats", "streak_days")
|
||||
op.drop_column("user_stats", "streak_last_day_date")
|
||||
op.drop_column("user_stats", "reply_prompter_ranked_3")
|
||||
op.drop_column("user_stats", "reply_prompter_ranked_2")
|
||||
op.drop_column("user_stats", "reply_prompter_ranked_1")
|
||||
op.drop_column("user_stats", "reply_assistant_ranked_3")
|
||||
op.drop_column("user_stats", "reply_assistant_ranked_2")
|
||||
op.drop_column("user_stats", "reply_assistant_ranked_1")
|
||||
op.drop_column("user_stats", "accepted_replies_prompter")
|
||||
op.drop_column("user_stats", "accepted_replies_assistant")
|
||||
op.drop_column("user_stats", "accepted_prompts")
|
||||
op.drop_column("user_stats", "rankings_good")
|
||||
op.drop_column("user_stats", "rankings_total")
|
||||
op.drop_column("user_stats", "labels_full")
|
||||
op.drop_column("user_stats", "labels_simple")
|
||||
op.drop_column("user_stats", "replies_prompter")
|
||||
op.drop_column("user_stats", "replies_assistant")
|
||||
op.drop_column("user_stats", "prompts")
|
||||
op.drop_column("user_stats", "time_frame")
|
||||
op.add_column(
|
||||
"message_toxicity",
|
||||
sa.Column("toxicity", postgresql.DOUBLE_PRECISION(precision=53), autoincrement=False, nullable=True),
|
||||
)
|
||||
op.drop_column("message_toxicity", "label")
|
||||
op.drop_column("message_toxicity", "score")
|
||||
op.drop_column("message", "rank")
|
||||
# ### end Alembic commands ###
|
||||
@@ -4,7 +4,7 @@ from fastapi import APIRouter, Depends
|
||||
from oasst_backend.api import deps
|
||||
from oasst_backend.models import ApiClient
|
||||
from oasst_backend.schemas.hugging_face import ToxicityClassification
|
||||
from oasst_backend.utils.hugging_face import HfUrl, HuggingFaceAPI
|
||||
from oasst_backend.utils.hugging_face import HfClassificationModel, HfUrl, HuggingFaceAPI
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -25,7 +25,8 @@ async def get_text_toxicity(
|
||||
ToxicityClassification: the score of toxicity of the message.
|
||||
"""
|
||||
|
||||
api_url: str = HfUrl.HUGGINGFACE_TOXIC_ROBERTA.value
|
||||
api_url: str = HfUrl.HUGGINGFACE_TOXIC_CLASSIFICATION.value + "/" + HfClassificationModel.TOXIC_ROBERTA.value
|
||||
|
||||
hugging_face_api = HuggingFaceAPI(api_url)
|
||||
response = await hugging_face_api.post(msg)
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ from starlette.status import HTTP_204_NO_CONTENT
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/users/{user_id}", response_model=protocol.User)
|
||||
@router.get("/users/{user_id}", response_model=protocol.FrontEndUser)
|
||||
def get_user(
|
||||
user_id: UUID,
|
||||
api_client_id: UUID = None,
|
||||
|
||||
@@ -20,28 +20,28 @@ class TreeManagerConfiguration(BaseModel):
|
||||
"""Maximum number of reply messages per tree node."""
|
||||
|
||||
goal_tree_size: int = 15
|
||||
"""Total number of messages to gather per tree"""
|
||||
"""Total number of messages to gather per tree."""
|
||||
|
||||
num_reviews_initial_prompt: int = 3
|
||||
"""Number of peer review checks to collect in INITIAL_PROMPT_REVIEW state."""
|
||||
|
||||
num_reviews_reply: int = 3
|
||||
"""Number of peer review checks to collect per reply (other than initial_prompt)"""
|
||||
"""Number of peer review checks to collect per reply (other than initial_prompt)."""
|
||||
|
||||
p_full_labeling_review_prompt: float = 0.1
|
||||
"""Probability of full text-labeling (instead of mandatory only) for initial prompts"""
|
||||
"""Probability of full text-labeling (instead of mandatory only) for initial prompts."""
|
||||
|
||||
p_full_labeling_review_reply_assistant: float = 0.1
|
||||
"""Probability of full text-labeling (instead of mandatory only) for assistant replies"""
|
||||
"""Probability of full text-labeling (instead of mandatory only) for assistant replies."""
|
||||
|
||||
p_full_labeling_review_reply_prompter: float = 0.1
|
||||
"""Probability of full text-labeling (instead of mandatory only) for prompter replies"""
|
||||
"""Probability of full text-labeling (instead of mandatory only) for prompter replies."""
|
||||
|
||||
acceptance_threshold_initial_prompt: float = 0.6
|
||||
"""Threshold for accepting an initial prompt"""
|
||||
"""Threshold for accepting an initial prompt."""
|
||||
|
||||
acceptance_threshold_reply: float = 0.6
|
||||
"""Threshold for accepting a reply"""
|
||||
"""Threshold for accepting a reply."""
|
||||
|
||||
num_required_rankings: int = 3
|
||||
"""Number of rankings in which the message participated."""
|
||||
@@ -50,7 +50,7 @@ class TreeManagerConfiguration(BaseModel):
|
||||
"""Mandatory labels in text-labeling tasks for initial prompts."""
|
||||
|
||||
mandatory_labels_assistant_reply: Optional[list[protocol_schema.TextLabel]] = [protocol_schema.TextLabel.spam]
|
||||
"""Mandatory labels in text-labeling tasks for assistant reylies."""
|
||||
"""Mandatory labels in text-labeling tasks for assistant replies."""
|
||||
|
||||
mandatory_labels_prompter_reply: Optional[list[protocol_schema.TextLabel]] = [protocol_schema.TextLabel.spam]
|
||||
"""Mandatory labels in text-labeling tasks for prompter replies."""
|
||||
@@ -79,6 +79,7 @@ class Settings(BaseSettings):
|
||||
)
|
||||
DEBUG_ALLOW_SELF_LABELING: bool = False # allow users to label their own messages
|
||||
DEBUG_SKIP_EMBEDDING_COMPUTATION: bool = False
|
||||
DEBUG_SKIP_TOXICITY_CALCULATION: bool = False
|
||||
|
||||
HUGGING_FACE_API_KEY: str = ""
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from .journal import Journal, JournalIntegration
|
||||
from .message import Message
|
||||
from .message_embedding import MessageEmbedding
|
||||
from .message_reaction import MessageReaction
|
||||
from .message_toxicity import MessageToxicity
|
||||
from .message_tree_state import MessageTreeState
|
||||
from .task import Task
|
||||
from .text_labels import TextLabels
|
||||
@@ -17,6 +18,7 @@ __all__ = [
|
||||
"MessageEmbedding",
|
||||
"MessageReaction",
|
||||
"MessageTreeState",
|
||||
"MessageToxicity",
|
||||
"Task",
|
||||
"TextLabels",
|
||||
"Journal",
|
||||
|
||||
@@ -45,6 +45,8 @@ class Message(SQLModel, table=True):
|
||||
review_result: bool = Field(sa_column=sa.Column(sa.Boolean, default=False, server_default=false(), nullable=False))
|
||||
ranking_count: int = Field(sa_column=sa.Column(sa.Integer, default=0, server_default=sa.text("0"), nullable=False))
|
||||
|
||||
rank: Optional[int] = Field(nullable=True)
|
||||
|
||||
def ensure_is_message(self) -> None:
|
||||
if not self.payload or not isinstance(self.payload.payload, MessagePayload):
|
||||
raise OasstError("Invalid message", OasstErrorCode.INVALID_MESSAGE, HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
import sqlalchemy as sa
|
||||
import sqlalchemy.dialects.postgresql as pg
|
||||
from sqlmodel import Field, Float, SQLModel
|
||||
|
||||
|
||||
class MessageToxicity(SQLModel, table=True):
|
||||
__tablename__ = "message_toxicity"
|
||||
__table_args__ = (sa.PrimaryKeyConstraint("message_id", "model"),)
|
||||
|
||||
message_id: UUID = Field(sa_column=sa.Column(pg.UUID(as_uuid=True), sa.ForeignKey("message.id"), nullable=False))
|
||||
model: str = Field(max_length=256, nullable=False)
|
||||
|
||||
# Storing the score and the label of the message
|
||||
score: float = Field(sa_column=sa.Column(Float), nullable=False)
|
||||
label: str = Field(max_length=256, nullable=False)
|
||||
|
||||
# In the case that the Message Embedding is created afterwards
|
||||
created_date: Optional[datetime] = Field(
|
||||
sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp())
|
||||
)
|
||||
@@ -1,4 +1,5 @@
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
from uuid import UUID
|
||||
|
||||
@@ -7,21 +8,46 @@ import sqlalchemy.dialects.postgresql as pg
|
||||
from sqlmodel import Field, SQLModel
|
||||
|
||||
|
||||
class UserStatsTimeFrame(str, Enum):
|
||||
day = "day"
|
||||
week = "week"
|
||||
month = "month"
|
||||
total = "total"
|
||||
|
||||
|
||||
class UserStats(SQLModel, table=True):
|
||||
__tablename__ = "user_stats"
|
||||
|
||||
user_id: Optional[UUID] = Field(
|
||||
sa_column=sa.Column(pg.UUID(as_uuid=True), sa.ForeignKey("user.id"), primary_key=True)
|
||||
)
|
||||
time_frame: Optional[str] = Field(nullable=False, primary_key=True)
|
||||
|
||||
leader_score: int = 0
|
||||
modified_date: Optional[datetime] = Field(
|
||||
sa_column=sa.Column(sa.DateTime(), nullable=False, server_default=sa.func.current_timestamp())
|
||||
)
|
||||
|
||||
reactions: int = 0 # reactions sent by user
|
||||
messages: int = 0 # messages sent by user
|
||||
upvotes: int = 0 # received upvotes (form other users)
|
||||
downvotes: int = 0 # received downvotes (from other users)
|
||||
task_reward: int = 0 # reward for task completions
|
||||
compare_wins: int = 0 # num times user's message won compare tasks
|
||||
compare_losses: int = 0 # num times users's message lost compare tasks
|
||||
prompts: int = 0
|
||||
replies_assistant: int = 0
|
||||
replies_prompter: int = 0
|
||||
labels_simple: int = 0
|
||||
labels_full: int = 0
|
||||
rankings_total: int = 0
|
||||
rankings_good: int = 0
|
||||
|
||||
accepted_prompts: int = 0
|
||||
accepted_replies_assistant: int = 0
|
||||
accepted_replies_prompter: int = 0
|
||||
|
||||
reply_assistant_ranked_1: int = 0
|
||||
reply_assistant_ranked_2: int = 0
|
||||
reply_assistant_ranked_3: int = 0
|
||||
|
||||
reply_prompter_ranked_1: int = 0
|
||||
reply_prompter_ranked_2: int = 0
|
||||
reply_prompter_ranked_3: int = 0
|
||||
|
||||
# only used for time span "total"
|
||||
streak_last_day_date: Optional[datetime] = Field(nullable=True)
|
||||
streak_days: Optional[int] = Field(nullable=True)
|
||||
|
||||
@@ -14,6 +14,7 @@ from oasst_backend.models import (
|
||||
Message,
|
||||
MessageEmbedding,
|
||||
MessageReaction,
|
||||
MessageToxicity,
|
||||
MessageTreeState,
|
||||
Task,
|
||||
TextLabels,
|
||||
@@ -293,6 +294,25 @@ class PromptRepository:
|
||||
|
||||
return reaction, task
|
||||
|
||||
def insert_toxicity(self, message_id: UUID, model: str, score: float, label: str) -> MessageToxicity:
|
||||
"""Save the toxicity score of a new message in the database.
|
||||
Args:
|
||||
message_id (UUID): the identifier of the message we want to save its toxicity score
|
||||
model (str): the model used for creating the toxicity score
|
||||
score (float): the toxicity score that we obtained from the model
|
||||
label (str): the final classification in toxicity of the model
|
||||
Raises:
|
||||
OasstError: if misses some of the before params
|
||||
Returns:
|
||||
MessageToxicity: the instance in the database of the score saved for that message
|
||||
"""
|
||||
|
||||
message_toxicity = MessageToxicity(message_id=message_id, model=model, score=score, label=label)
|
||||
self.db.add(message_toxicity)
|
||||
self.db.commit()
|
||||
self.db.refresh(message_toxicity)
|
||||
return message_toxicity
|
||||
|
||||
def insert_message_embedding(self, message_id: UUID, model: str, embedding: List[float]) -> MessageEmbedding:
|
||||
"""Insert the embedding of a new message in the database.
|
||||
|
||||
@@ -308,9 +328,6 @@ class PromptRepository:
|
||||
MessageEmbedding: the instance in the database of the embedding saved for that message
|
||||
"""
|
||||
|
||||
if None in (message_id, model, embedding):
|
||||
raise OasstError("Paramters missing to add embedding", OasstErrorCode.GENERIC_ERROR)
|
||||
|
||||
message_embedding = MessageEmbedding(message_id=message_id, model=model, embedding=embedding)
|
||||
self.db.add(message_embedding)
|
||||
self.db.commit()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import random
|
||||
from enum import Enum
|
||||
from http import HTTPStatus
|
||||
from typing import Optional, Tuple
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
from uuid import UUID
|
||||
|
||||
import numpy as np
|
||||
@@ -11,7 +11,7 @@ from oasst_backend.api.v1.utils import prepare_conversation, prepare_conversatio
|
||||
from oasst_backend.config import TreeManagerConfiguration, settings
|
||||
from oasst_backend.models import Message, MessageReaction, MessageTreeState, TextLabels, message_tree_state
|
||||
from oasst_backend.prompt_repository import PromptRepository
|
||||
from oasst_backend.utils.hugging_face import HfEmbeddingModel, HfUrl, HuggingFaceAPI
|
||||
from oasst_backend.utils.hugging_face import HfClassificationModel, HfEmbeddingModel, HfUrl, HuggingFaceAPI
|
||||
from oasst_shared.exceptions.oasst_api_error import OasstError, OasstErrorCode
|
||||
from oasst_shared.schemas import protocol as protocol_schema
|
||||
from sqlalchemy.sql import text
|
||||
@@ -363,6 +363,25 @@ class TreeManager:
|
||||
f"Could not fetch embbeddings for text reply to {interaction.message_id=} with {interaction.text=} by {interaction.user=}."
|
||||
)
|
||||
|
||||
if not settings.DEBUG_SKIP_TOXICITY_CALCULATION:
|
||||
try:
|
||||
model_name: str = HfClassificationModel.TOXIC_ROBERTA.value
|
||||
hugging_face_api: HuggingFaceAPI = HuggingFaceAPI(
|
||||
f"{HfUrl.HUGGINGFACE_FEATURE_EXTRACTION.value}/{model_name}"
|
||||
)
|
||||
|
||||
toxicity: List[List[Dict[str, Any]]] = await hugging_face_api.post(interaction.text)
|
||||
toxicity = toxicity[0][0]
|
||||
|
||||
pr.insert_toxicity(
|
||||
message_id=message.id, model=model_name, score=toxicity["score"], label=toxicity["label"]
|
||||
)
|
||||
|
||||
except OasstError:
|
||||
logger.error(
|
||||
f"Could not compute toxicity for text reply to {interaction.message_id=} with {interaction.text=} by {interaction.user=}."
|
||||
)
|
||||
|
||||
case protocol_schema.MessageRating:
|
||||
logger.info(
|
||||
f"Frontend reports rating of {interaction.message_id=} with {interaction.rating=} by {interaction.user=}."
|
||||
|
||||
@@ -8,10 +8,14 @@ from oasst_shared.exceptions import OasstError, OasstErrorCode
|
||||
|
||||
|
||||
class HfUrl(str, Enum):
|
||||
HUGGINGFACE_TOXIC_ROBERTA = ("https://api-inference.huggingface.co/models/unitary/multilingual-toxic-xlm-roberta",)
|
||||
HUGGINGFACE_TOXIC_CLASSIFICATION = "https://api-inference.huggingface.co/models"
|
||||
HUGGINGFACE_FEATURE_EXTRACTION = "https://api-inference.huggingface.co/pipeline/feature-extraction"
|
||||
|
||||
|
||||
class HfClassificationModel(str, Enum):
|
||||
TOXIC_ROBERTA = "unitary/multilingual-toxic-xlm-roberta"
|
||||
|
||||
|
||||
class HfEmbeddingModel(str, Enum):
|
||||
MINILM = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
aiosqlite # database
|
||||
hikari # discord framework
|
||||
hikari-lightbulb # command handler
|
||||
hikari-miru # modals and buttons
|
||||
hikari[speedups]
|
||||
loguru
|
||||
pydantic[dotenv]
|
||||
aiosqlite == 0.18.0 # database
|
||||
hikari-lightbulb == 2.3.1 # command handler
|
||||
hikari-miru == 2.0.2 # modals and buttons
|
||||
hikari[speedups] == 2.0.0.dev115 # discord framework
|
||||
loguru == 0.6.0
|
||||
pydantic[dotenv] == 1.10.4
|
||||
|
||||
uvloop; os_name != 'nt' # Faster drop-in replacement for asyncio event loop
|
||||
uvloop == 0.17.0; os_name != 'nt' # Faster drop-in replacement for asyncio event loop
|
||||
|
||||
@@ -101,6 +101,7 @@ services:
|
||||
- DEBUG_USE_SEED_DATA=True
|
||||
- DEBUG_ALLOW_SELF_LABELING=True
|
||||
- MAX_WORKERS=1
|
||||
- DEBUG_SKIP_TOXICITY_CALCULATION=True
|
||||
- DEBUG_SKIP_EMBEDDING_COMPUTATION=True
|
||||
depends_on:
|
||||
db:
|
||||
|
||||
+1689
-304
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,8 @@
|
||||
# Tasks
|
||||
|
||||
Understand a bit more about each type of Open Assistant labeling task.
|
||||
|
||||
- [Label Assistant Reply](label_assistant_reply.md)
|
||||
- [Label Prompter Reply](label_prompter_reply.md)
|
||||
- [Reply as Assistant](reply_as_assistant.md)
|
||||
- [Reply as User](reply_as_user.md)
|
||||
@@ -0,0 +1,3 @@
|
||||
# Label Assistant Reply
|
||||
|
||||
Given the following discussion, provide labels for the final prompt.
|
||||
@@ -0,0 +1,3 @@
|
||||
# Label Prompter Reply
|
||||
|
||||
Given the following discussion, provide labels for the final prompt.
|
||||
@@ -0,0 +1,3 @@
|
||||
# Reply as Assistant
|
||||
|
||||
Given the following conversation, provide an adequate reply.
|
||||
@@ -0,0 +1,3 @@
|
||||
# Reply as User
|
||||
|
||||
Given the following conversation, provide an adequate reply.
|
||||
@@ -32,12 +32,15 @@ const config = {
|
||||
|
||||
presets: [
|
||||
[
|
||||
"classic",
|
||||
"docusaurus-preset-openapi",
|
||||
/** @type {import('@docusaurus/preset-classic').Options} */
|
||||
({
|
||||
docs: {
|
||||
sidebarPath: require.resolve("./sidebars.js"),
|
||||
},
|
||||
api: {
|
||||
path: "docs/api/openapi.json",
|
||||
},
|
||||
blog: false,
|
||||
theme: {
|
||||
customCss: require.resolve("./src/css/custom.css"),
|
||||
@@ -62,11 +65,7 @@ const config = {
|
||||
position: "left",
|
||||
label: "Docs",
|
||||
},
|
||||
{
|
||||
href: "https://editor.swagger.io/?url=https://raw.githubusercontent.com/LAION-AI/Open-Assistant/main/docs/docs/api/openapi.json",
|
||||
label: "API",
|
||||
position: "left",
|
||||
},
|
||||
{ to: "/api", label: "API", position: "left" },
|
||||
{
|
||||
href: "https://github.com/LAION-AI/Open-Assistant",
|
||||
label: "GitHub",
|
||||
|
||||
+3
-1
@@ -19,9 +19,11 @@
|
||||
"@docusaurus/preset-classic": "2.2.0",
|
||||
"@mdx-js/react": "^1.6.22",
|
||||
"clsx": "^1.2.1",
|
||||
"docusaurus-preset-openapi": "^0.6.3",
|
||||
"prism-react-renderer": "^1.3.5",
|
||||
"react": "^17.0.2",
|
||||
"react-dom": "^17.0.2"
|
||||
"react-dom": "^17.0.2",
|
||||
"url": "^0.11.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@docusaurus/module-type-aliases": "2.2.0",
|
||||
|
||||
@@ -24,6 +24,20 @@ const sidebars = {
|
||||
},
|
||||
items: ["guides/prompting"],
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
label: "Tasks",
|
||||
link: {
|
||||
type: "doc",
|
||||
id: "tasks/README",
|
||||
},
|
||||
items: [
|
||||
"tasks/label_assistant_reply",
|
||||
"tasks/label_prompter_reply",
|
||||
"tasks/reply_as_assistant",
|
||||
"tasks/reply_as_user",
|
||||
],
|
||||
},
|
||||
{
|
||||
type: "category",
|
||||
label: "Data",
|
||||
|
||||
+972
-37
File diff suppressed because it is too large
Load Diff
@@ -58,6 +58,8 @@ the end to trigger deepspeed
|
||||
python trainer.py --configs defaults your-model-name --deepspeed
|
||||
```
|
||||
|
||||
## Dataset choices
|
||||
|
||||
## Results
|
||||
|
||||
Experimental results in wandb
|
||||
|
||||
@@ -6,7 +6,7 @@ defaults:
|
||||
per_device_eval_batch_size: 2
|
||||
weight_decay: 0.00
|
||||
warmup_steps: 600
|
||||
eval_steps: 100
|
||||
eval_steps: 500
|
||||
save_steps: 500
|
||||
max_length: 512
|
||||
num_train_epochs: 3
|
||||
@@ -18,7 +18,19 @@ defaults:
|
||||
datasets:
|
||||
- webgpt
|
||||
- prompt_dialogue
|
||||
cache_dir: ~/.cache
|
||||
- squad_v2
|
||||
- adversarial_qa
|
||||
- trivia_qa_nocontext
|
||||
- xsum
|
||||
- cnn_dailymail
|
||||
- prompt_dialogue
|
||||
- multi_news
|
||||
- scitldr
|
||||
- soda
|
||||
- joke
|
||||
- gsm8k
|
||||
- samsum
|
||||
cache_dir: .cache
|
||||
loss_fn: CrossEntropyLoss
|
||||
eval_size:
|
||||
log_dir: "base"
|
||||
@@ -48,14 +60,14 @@ gpt-jt:
|
||||
per_device_eval_batch_size: 4
|
||||
|
||||
codegen:
|
||||
learning_rate: 2e-6
|
||||
learning_rate: 8e-6
|
||||
model_name: Salesforce/codegen-2B-multi
|
||||
weight_decay: 0.01
|
||||
max_length: 812
|
||||
warmup_steps: 600
|
||||
max_length: 520
|
||||
warmup_steps: 1000
|
||||
gradient_checkpointing: false
|
||||
gradient_accumulation_steps: 5
|
||||
per_device_train_batch_size: 4
|
||||
gradient_accumulation_steps: 9
|
||||
per_device_train_batch_size: 2
|
||||
per_device_eval_batch_size: 4
|
||||
|
||||
debug:
|
||||
|
||||
@@ -1,136 +1,11 @@
|
||||
import numpy as np
|
||||
from datasets import load_dataset
|
||||
from custom_datasets.prompt_dialogue import PromptGeneratedDataset
|
||||
from custom_datasets.qa_datasets import SODA, JokeExplaination, QADataset, WebGPT
|
||||
from custom_datasets.summarization import SummarizationDataset
|
||||
from sklearn.model_selection import train_test_split
|
||||
from torch.utils.data import Dataset, Subset
|
||||
from torch.utils.data import Subset
|
||||
|
||||
from .prompt_dialogue import PromptGeneratedDataset
|
||||
|
||||
QA_SPECIAL_TOKENS = {"Question": "<question>", "Answer": "<answer>"}
|
||||
SUMMARIZATION_SPECIAL_TOKENS = {"Text": "", "Summary": "TL;DR:"}
|
||||
|
||||
summarization_name_mapping = {
|
||||
"cnn_dailymail": ("article", "highlights"),
|
||||
"samsum": ("dialogue", "summary"),
|
||||
"xsum": ("document", "summary"),
|
||||
"multi_news": ("document", "summary"),
|
||||
"scitldr": ("source", "target"),
|
||||
"billsum": ("text", "summary"),
|
||||
"reddit": ("content", "summary"),
|
||||
}
|
||||
summarization_config_mapping = {
|
||||
"cnn_dailymail": ("3.0.0",),
|
||||
"samsum": (),
|
||||
"xsum": (),
|
||||
"multi_news": (),
|
||||
"scitldr": ("AIC",),
|
||||
"billsum": (),
|
||||
"reddit": (),
|
||||
}
|
||||
|
||||
QA_DATASETS = ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_noconext"]
|
||||
SUMMARIZATION_DATASETS = ["xsum", "cnn_dailymail", "samsum", "multi_news"]
|
||||
|
||||
|
||||
def index_squad_v2(example):
|
||||
return example["title"] + ". " + example["context"] + " " + example["question"], example["answers"]["text"][0]
|
||||
|
||||
|
||||
def index_trivia_qa_nocontext(example):
|
||||
# dummy return one randomly
|
||||
return example["question"], example["answer"]["aliases"][np.random.randint(len(example["answer"]["aliases"]))]
|
||||
|
||||
|
||||
def index_trivia_qa_context(example):
|
||||
question = example["question"]
|
||||
title = example["title"][np.random.randint(len(example["title"]))]
|
||||
context = example["search_context"][np.random.randint(len(example["search_context"]))]
|
||||
answer = example["answer"]["aliases"][np.random.randint(len(example["answer"]["aliases"]))]
|
||||
|
||||
return title + ". " + context + " " + question, answer
|
||||
|
||||
|
||||
def index_adversarial_qa(example):
|
||||
return example["title"] + ". " + example["context"] + " " + example["question"], example["answers"]["text"][0]
|
||||
|
||||
|
||||
class QADataset(Dataset):
|
||||
def __init__(self, dataset, cache_dir, split):
|
||||
if dataset == "squad_v2":
|
||||
self.index_fn = index_squad_v2
|
||||
self.dataset = load_dataset("squad_v2", cache_dir=cache_dir, split=split)
|
||||
elif dataset == "trivia_qa_nocontext":
|
||||
self.index_fn = index_trivia_qa_nocontext
|
||||
self.dataset = load_dataset("trivia_qa", "rc.nocontext")
|
||||
elif dataset == "trivia_qa_context":
|
||||
self.index_fn = index_trivia_qa_context
|
||||
self.dataset = load_dataset("trivia_qa", "rc")
|
||||
elif dataset == "adversarial_qa":
|
||||
self.index_fn = index_adversarial_qa
|
||||
self.dataset = load_dataset("adversarial_qa", "adversarialQA")
|
||||
else:
|
||||
raise ValueError("Unknown dataset : " + dataset)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.dataset)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
data = self.dataset[idx]
|
||||
return self.index_fn(data)
|
||||
|
||||
|
||||
def index_summary_default(text, summary):
|
||||
return text, summary
|
||||
|
||||
|
||||
def index_summary_merge(text, summary):
|
||||
return " ".join(text), " ".join(summary)
|
||||
|
||||
|
||||
class SummarizationDataset(Dataset):
|
||||
def __init__(self, dataset, cache_dir, split):
|
||||
self.dataset = load_dataset(dataset, *summarization_config_mapping[dataset], cache_dir=cache_dir, split=split)
|
||||
self.summary_column, self.text_column = summarization_name_mapping[dataset]
|
||||
self.preprocess_fn = index_summary_merge if dataset == "scitdlr" else index_summary_merge
|
||||
|
||||
def __len__(self):
|
||||
return len(self.dataset)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
data = self.dataset[idx]
|
||||
text, summary = data[self.text_column], data[self.summary_column]
|
||||
text, summary = self.preprocess_fn(text, summary)
|
||||
|
||||
return "".join(
|
||||
SUMMARIZATION_SPECIAL_TOKENS["Text"], text, " ", SUMMARIZATION_SPECIAL_TOKENS["Summary"], summary
|
||||
)
|
||||
|
||||
|
||||
class WebGPT(Dataset):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
dataset = load_dataset("openai/webgpt_comparisons")
|
||||
questions = {}
|
||||
# using prompt as our index will allows us
|
||||
# to add additional generated prompt later
|
||||
self.index2question = {}
|
||||
for row in dataset["train"]:
|
||||
question = row["question"]["full_text"]
|
||||
if question not in self.index2question:
|
||||
self.index2question[len(self.index2question)] = question
|
||||
|
||||
# only keep the best answer
|
||||
questions[question] = row["answer_0" if row["score_0"] > row["score_1"] else "answer_1"]
|
||||
|
||||
self.questions = questions
|
||||
|
||||
def __len__(self):
|
||||
return len(self.index2question)
|
||||
|
||||
def __getitem__(self, index):
|
||||
question = self.index2question[index]
|
||||
answer = self.questions[question]
|
||||
return [question, answer]
|
||||
QA_DATASETS = ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_nocontext", "gsm8k"]
|
||||
SUMMARIZATION_DATASETS = ["xsum", "cnn_dailymail", "samsum", "multi_news", "scitldr", "billsum"]
|
||||
|
||||
|
||||
def train_val_dataset(dataset, val_split=0.2):
|
||||
@@ -143,19 +18,26 @@ def train_val_dataset(dataset, val_split=0.2):
|
||||
def get_one_dataset(conf, dataset_name):
|
||||
dataset_name = dataset_name.lower()
|
||||
|
||||
if dataset_name in ["squad_v2", "adversarial_qa", "trivia_qa_context", "trivia_qa_noconext"]:
|
||||
if dataset_name in QA_DATASETS:
|
||||
train = QADataset(dataset_name, conf.cache_dir, "train")
|
||||
eval = QADataset(dataset_name, conf.cache_dir, "validation")
|
||||
val_name = "validation" if dataset_name not in ["gsm8k"] else "test"
|
||||
eval = QADataset(dataset_name, conf.cache_dir, val_name)
|
||||
|
||||
elif dataset_name in ["xsum", "cnn_dailymail", "samsum", "multi_news", "scitldr", "billsum", "reddit"]:
|
||||
elif dataset_name in SUMMARIZATION_DATASETS:
|
||||
train = SummarizationDataset(dataset_name, conf.cache_dir, "train")
|
||||
eval = SummarizationDataset(dataset_name, conf.cache_dir, "validation")
|
||||
|
||||
val_name = "validation" if dataset_name not in ["billsum"] else "test"
|
||||
eval = SummarizationDataset(dataset_name, conf.cache_dir, val_name)
|
||||
elif dataset_name == "webgpt":
|
||||
dataset = WebGPT()
|
||||
train, eval = train_val_dataset(dataset, val_split=0.2)
|
||||
elif dataset_name == "prompt_dialogue":
|
||||
dataset = PromptGeneratedDataset()
|
||||
dataset = PromptGeneratedDataset(conf.cache_dir)
|
||||
train, eval = train_val_dataset(dataset, val_split=0.2)
|
||||
elif dataset_name == "soda":
|
||||
dataset = SODA(conf.cache_dir)
|
||||
train, eval = train_val_dataset(dataset, val_split=0.1)
|
||||
elif dataset_name == "joke":
|
||||
dataset = JokeExplaination(conf.cache_dir)
|
||||
train, eval = train_val_dataset(dataset, val_split=0.2)
|
||||
else:
|
||||
raise ValueError(f"Unknown dataset {dataset_name}")
|
||||
|
||||
@@ -3,11 +3,10 @@ from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from custom_datasets.qa_datasets import QA_SPECIAL_TOKENS
|
||||
from torch.nn import functional as F
|
||||
from transformers.tokenization_utils_base import PaddingStrategy, PreTrainedTokenizerBase
|
||||
|
||||
from . import QA_SPECIAL_TOKENS
|
||||
|
||||
|
||||
@dataclass
|
||||
class DialogueDataCollator:
|
||||
@@ -35,7 +34,7 @@ class DialogueDataCollator:
|
||||
|
||||
# Add a way for the model to terminate generation
|
||||
# When we predict the start of a new expected question, we want to be able to stop generation
|
||||
messages.append(QA_SPECIAL_TOKENS["Question"])
|
||||
messages.append(self.tokenizer.eos_token)
|
||||
|
||||
flatten_message = self.tokenizer(
|
||||
"".join(messages),
|
||||
|
||||
@@ -16,10 +16,10 @@ class PromptGeneratedDataset(Dataset):
|
||||
|
||||
url = "https://github.com/Rallio67/language-model-agents/raw/main/chat_dialogue_v2_c.txt"
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, cache_dir) -> None:
|
||||
super().__init__()
|
||||
os.makedirs("datasets", exist_ok=True)
|
||||
chat_dialogue = os.path.join("datasets", "chat_dialogue_v2_c.txt")
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
chat_dialogue = os.path.join(cache_dir, "chat_dialogue_v2_c.txt")
|
||||
if not os.path.exists(chat_dialogue):
|
||||
with urlopen(self.url) as file:
|
||||
content = file.read().decode()
|
||||
@@ -49,18 +49,3 @@ class PromptGeneratedDataset(Dataset):
|
||||
def __getitem__(self, index):
|
||||
question, answer = self.pairs[index]
|
||||
return question, answer
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from torch.utils.data import DataLoader
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from .dialogue_collator import DialogueDataCollator
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("Salesforce/codegen-2B-multi")
|
||||
tokenizer.add_special_tokens({"pad_token": "<|endoftext|>", "sep_token": "<|endoftext|>"})
|
||||
dataset = PromptGeneratedDataset()
|
||||
collate_fn = DialogueDataCollator(tokenizer, padding=True, max_length=128)
|
||||
dataloader = DataLoader(dataset, collate_fn=collate_fn, batch_size=5)
|
||||
for batch in dataloader:
|
||||
print(batch["input_ids"].shape)
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import json
|
||||
import os
|
||||
from urllib.request import urlopen
|
||||
|
||||
import numpy as np
|
||||
from datasets import load_dataset
|
||||
from torch.utils.data import Dataset
|
||||
|
||||
QA_SPECIAL_TOKENS = {"Question": "<human>", "Answer": "<bot>", "StartPrefix": "<prefix>", "EndPrefix": "</prefix>"}
|
||||
|
||||
|
||||
def index_squad_v2(example):
|
||||
if len(example["answers"]["text"]):
|
||||
answer = example["answers"]["text"][0]
|
||||
else:
|
||||
answer = "I do not have answer for that"
|
||||
return example["context"] + " " + example["question"], answer
|
||||
|
||||
|
||||
def index_trivia_qa_nocontext(example):
|
||||
# dummy return one randomly
|
||||
return example["question"], example["answer"]["aliases"][np.random.randint(len(example["answer"]["aliases"]))]
|
||||
|
||||
|
||||
def index_trivia_qa_context(example):
|
||||
question = example["question"]
|
||||
if len(example["search_results"]["search_context"]):
|
||||
context = example["search_results"]["search_context"][
|
||||
np.random.randint(len(example["search_results"]["search_context"]))
|
||||
]
|
||||
else:
|
||||
context = ""
|
||||
answer = example["answer"]["aliases"][np.random.randint(len(example["answer"]["aliases"]))]
|
||||
|
||||
return context + " " + question, answer
|
||||
|
||||
|
||||
def index_adversarial_qa(example):
|
||||
return example["title"] + ". " + example["context"] + " " + example["question"], example["answers"]["text"][0]
|
||||
|
||||
|
||||
def index_gsm8k(example):
|
||||
return example["question"], example["answer"]
|
||||
|
||||
|
||||
class QADataset(Dataset):
|
||||
def __init__(self, dataset, cache_dir, split):
|
||||
if dataset == "squad_v2":
|
||||
self.index_fn = index_squad_v2
|
||||
self.dataset = load_dataset("squad_v2", cache_dir=cache_dir, split=split)
|
||||
elif dataset == "trivia_qa_nocontext":
|
||||
self.index_fn = index_trivia_qa_nocontext
|
||||
self.dataset = load_dataset("trivia_qa", "rc.nocontext", split=split, cache_dir=cache_dir)
|
||||
elif dataset == "trivia_qa_context":
|
||||
self.index_fn = index_trivia_qa_context
|
||||
self.dataset = load_dataset("trivia_qa", "rc", split=split, cache_dir=cache_dir)
|
||||
elif dataset == "adversarial_qa":
|
||||
self.index_fn = index_adversarial_qa
|
||||
self.dataset = load_dataset("adversarial_qa", "adversarialQA", split=split, cache_dir=cache_dir)
|
||||
elif dataset == "gsm8k":
|
||||
self.index_fn = index_gsm8k
|
||||
self.dataset = load_dataset("gsm8k", "main", split=split, cache_dir=cache_dir)
|
||||
elif dataset == "adversarial_qa":
|
||||
self.index_fn = index_adversarial_qa
|
||||
self.dataset = load_dataset("adversarial_qa", "adversarialQA", split=split, cache_dir=cache_dir)
|
||||
else:
|
||||
raise ValueError("Unknown dataset : " + dataset)
|
||||
|
||||
def __len__(self):
|
||||
return len(self.dataset)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
data = self.dataset[idx]
|
||||
return self.index_fn(data)
|
||||
|
||||
|
||||
class WebGPT(Dataset):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
dataset = load_dataset("openai/webgpt_comparisons")
|
||||
questions = {}
|
||||
# using prompt as our index will allows us
|
||||
# to add additional generated prompt later
|
||||
self.index2question = {}
|
||||
for row in dataset["train"]:
|
||||
question = row["question"]["full_text"]
|
||||
if question not in self.index2question:
|
||||
self.index2question[len(self.index2question)] = question
|
||||
|
||||
# only keep the best answer
|
||||
questions[question] = row["answer_0" if row["score_0"] > row["score_1"] else "answer_1"]
|
||||
|
||||
self.questions = questions
|
||||
|
||||
def __len__(self):
|
||||
return len(self.index2question)
|
||||
|
||||
def __getitem__(self, index):
|
||||
question = self.index2question[index]
|
||||
answer = self.questions[question]
|
||||
return [question, answer]
|
||||
|
||||
|
||||
class SODA(Dataset):
|
||||
def process_soda_convo(self, data):
|
||||
pairs = []
|
||||
play_as = data["speakers"][1]
|
||||
prefix = "<prefix>{}. {}</prefix>".format(data["narrative"], "your name {}".format(play_as))
|
||||
question, answer = "", ""
|
||||
prefix, postfix = "", ""
|
||||
previous_chat = []
|
||||
|
||||
for idx, convo in enumerate(data["dialogue"]):
|
||||
if idx % 2 == 0:
|
||||
question = convo
|
||||
prefix = data["speakers"][idx]
|
||||
else:
|
||||
answer = convo
|
||||
postfix = data["speakers"][idx]
|
||||
if len(question) and len(answer) and prefix != postfix and postfix == play_as:
|
||||
history = "<sep>".join(["{}<bot>{}".format(*p) for p in previous_chat])
|
||||
if len(history):
|
||||
history += "<sep>"
|
||||
pairs.append((prefix + history + question, answer))
|
||||
previous_chat.append((question, answer))
|
||||
return pairs
|
||||
|
||||
def __init__(self, cache_dir, max_sample_size=10000, input_max_length=1024) -> None:
|
||||
super().__init__()
|
||||
|
||||
self.pairs = []
|
||||
dataset = load_dataset("allenai/soda", cache_dir=cache_dir)["train"]
|
||||
for data in dataset:
|
||||
data_pair = self.process_soda_convo(data)
|
||||
for (prompt, answer) in data_pair:
|
||||
if len(prompt) < input_max_length:
|
||||
self.pairs.append((prompt, answer))
|
||||
|
||||
if len(self.pairs) > max_sample_size:
|
||||
break
|
||||
|
||||
def __len__(self):
|
||||
return len(self.pairs)
|
||||
|
||||
def __getitem__(self, index):
|
||||
question, answer = self.pairs[index]
|
||||
return question, answer
|
||||
|
||||
|
||||
class JokeExplaination(Dataset):
|
||||
""" """
|
||||
|
||||
url = "https://gist.github.com/theblackcat102/42b697e24a13fdb499e20edfbf618361/raw/1834dca207898c15f93b809d1195f6f6e47c9e1e/joke_explained.jsonl"
|
||||
|
||||
def __init__(self, cache_dir) -> None:
|
||||
super().__init__()
|
||||
os.makedirs(cache_dir, exist_ok=True)
|
||||
joke_explain_filename = os.path.join(cache_dir, "joke_explaination.jsonl")
|
||||
if not os.path.exists(joke_explain_filename):
|
||||
with urlopen(self.url) as file:
|
||||
content = file.read().decode()
|
||||
with open(joke_explain_filename, "w") as fout:
|
||||
fout.write(content)
|
||||
|
||||
question = ""
|
||||
answer = ""
|
||||
self.pairs = []
|
||||
with open(joke_explain_filename, "r") as f:
|
||||
for line in f:
|
||||
data = json.loads(line)
|
||||
joke = data["joke"]
|
||||
explanation = data["explaination"]
|
||||
self.pairs.append((joke, explanation))
|
||||
|
||||
if len(question) > 0 and len(answer) > 0:
|
||||
self.pairs.append((question, answer))
|
||||
|
||||
def __len__(self):
|
||||
return len(self.pairs)
|
||||
|
||||
def __getitem__(self, index):
|
||||
question, answer = self.pairs[index]
|
||||
return question, answer
|
||||
@@ -0,0 +1,62 @@
|
||||
import random
|
||||
|
||||
from datasets import load_dataset
|
||||
from torch.utils.data import Dataset
|
||||
|
||||
SUMMARIZATION_SPECIAL_TOKENS = {"Text": "", "Summary": ["TL;DR:", "Summarize this", "Give me the summary"]}
|
||||
|
||||
SUMMARY_SPECIAL_PROMPT = {
|
||||
"multi_news": ["Summarize in bullet points", "Generate summary in list of points"],
|
||||
"xsum": ["Give me summary in one sentence", "Short TLDR", "Give me a concise summary"],
|
||||
"samsum": ["TLDR;", "Summarize this dialogue", "Summarize dialogue"],
|
||||
}
|
||||
|
||||
summarization_config_mapping = {
|
||||
"cnn_dailymail": ("3.0.0",),
|
||||
"samsum": (),
|
||||
"xsum": (),
|
||||
"multi_news": (),
|
||||
"scitldr": ("AIC",),
|
||||
"billsum": (),
|
||||
"reddit": (),
|
||||
}
|
||||
|
||||
summarization_name_mapping = {
|
||||
"cnn_dailymail": ("article", "highlights"),
|
||||
"samsum": ("dialogue", "summary"),
|
||||
"xsum": ("document", "summary"),
|
||||
"multi_news": ("document", "summary"),
|
||||
"scitldr": ("source", "target"),
|
||||
"billsum": ("text", "summary"),
|
||||
"reddit": ("content", "summary"),
|
||||
}
|
||||
|
||||
|
||||
def index_summary_default(text, summary):
|
||||
return text.replace("\n\n", "\n"), summary
|
||||
|
||||
|
||||
def index_summary_merge(text, summary):
|
||||
return " ".join(text), " ".join(summary)
|
||||
|
||||
|
||||
class SummarizationDataset(Dataset):
|
||||
def __init__(self, dataset, cache_dir, split):
|
||||
self.name = dataset
|
||||
self.dataset = load_dataset(dataset, *summarization_config_mapping[dataset], cache_dir=cache_dir, split=split)
|
||||
self.text_column, self.summary_column = summarization_name_mapping[dataset]
|
||||
self.preprocess_fn = index_summary_merge if dataset == "scitldr" else index_summary_default
|
||||
|
||||
def __len__(self):
|
||||
return len(self.dataset)
|
||||
|
||||
def __getitem__(self, idx):
|
||||
data = self.dataset[idx]
|
||||
text, summary = data[self.text_column], data[self.summary_column]
|
||||
text, summary = self.preprocess_fn(text, summary)
|
||||
if self.name in SUMMARY_SPECIAL_PROMPT:
|
||||
prompt = random.choice(SUMMARIZATION_SPECIAL_TOKENS["Summary"])
|
||||
else:
|
||||
prompt = random.choice(SUMMARIZATION_SPECIAL_TOKENS["Summary"])
|
||||
|
||||
return ("".join([SUMMARIZATION_SPECIAL_TOKENS["Text"], " ".join(text.split(" ")[:256]), prompt]), summary)
|
||||
@@ -0,0 +1,54 @@
|
||||
from argparse import Namespace
|
||||
|
||||
from custom_datasets import QA_DATASETS, SUMMARIZATION_DATASETS, get_one_dataset
|
||||
from custom_datasets.dialogue_collator import DialogueDataCollator
|
||||
|
||||
|
||||
def test_all_datasets():
|
||||
qa_base = QA_DATASETS
|
||||
summarize_base = SUMMARIZATION_DATASETS
|
||||
others = ["prompt_dialogue", "webgpt", "soda", "joke"]
|
||||
|
||||
config = Namespace(cache_dir=".cache")
|
||||
for dataset_name in others + qa_base + summarize_base:
|
||||
print(dataset_name)
|
||||
train, eval = get_one_dataset(config, dataset_name)
|
||||
# sanity check
|
||||
for idx in range(min(len(train), 1000)):
|
||||
train[idx]
|
||||
for idx in range(min(len(eval), 1000)):
|
||||
eval[idx]
|
||||
|
||||
|
||||
def test_collate_fn():
|
||||
from torch.utils.data import ConcatDataset, DataLoader
|
||||
from utils import get_tokenizer
|
||||
|
||||
config = Namespace(cache_dir=".cache", model_name="Salesforce/codegen-2B-multi")
|
||||
tokenizer = get_tokenizer(config)
|
||||
collate_fn = DialogueDataCollator(tokenizer, max_length=512)
|
||||
qa_base = QA_DATASETS
|
||||
summarize_base = SUMMARIZATION_DATASETS
|
||||
others = ["prompt_dialogue", "webgpt", "soda", "joke", "gsm8k"]
|
||||
|
||||
trains, evals = [], []
|
||||
for dataset_name in others + qa_base + summarize_base:
|
||||
print(dataset_name)
|
||||
train, eval = get_one_dataset(config, dataset_name)
|
||||
trains.append(train)
|
||||
evals.append(eval)
|
||||
|
||||
dataloader = DataLoader(ConcatDataset(trains), collate_fn=collate_fn, batch_size=128)
|
||||
for batch in dataloader:
|
||||
# print(batch.keys())
|
||||
# print(tokenizer.decode(batch['input_ids'][0]))
|
||||
# print('-----')
|
||||
# print(tokenizer.decode(batch['targets'][0][batch['label_masks'][0]]))
|
||||
assert batch["targets"].shape[1] <= 512
|
||||
dataloader = DataLoader(ConcatDataset(evals), collate_fn=collate_fn, batch_size=128)
|
||||
for batch in dataloader:
|
||||
assert batch["targets"].shape[1] <= 512
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_collate_fn()
|
||||
@@ -0,0 +1,9 @@
|
||||
from argparse import Namespace
|
||||
|
||||
from utils import get_tokenizer
|
||||
|
||||
|
||||
def test_tokenizer():
|
||||
get_tokenizer(Namespace(model_name="Salesforce/codegen-2B-multi", cache_dir=".cache"))
|
||||
get_tokenizer(Namespace(model_name="facebook/galactica-1.3b", cache_dir=".cache"))
|
||||
get_tokenizer(Namespace(model_name="", cache_dir=".cache"))
|
||||
@@ -1,13 +1,15 @@
|
||||
from functools import partial
|
||||
# from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
import evaluate
|
||||
import nltk
|
||||
import numpy as np
|
||||
|
||||
# import nltk
|
||||
# import numpy as np
|
||||
import transformers
|
||||
import yaml
|
||||
from custom_datasets import QA_DATASETS, QA_SPECIAL_TOKENS, SUMMARIZATION_DATASETS, get_one_dataset
|
||||
from custom_datasets import get_one_dataset
|
||||
from custom_datasets.dialogue_collator import DialogueDataCollator
|
||||
from custom_datasets.qa_datasets import QA_SPECIAL_TOKENS
|
||||
from losses import CrossEntropyLoss, PolyLoss
|
||||
from models import freeze_top_n_layers, get_specific_model
|
||||
from sklearn.model_selection import train_test_split
|
||||
@@ -51,25 +53,25 @@ def preprocess_qa(eval_pred):
|
||||
return (eval_pred.predictions, eval_pred.label_ids)
|
||||
|
||||
|
||||
def postprocess_summarization(preds, labels):
|
||||
preds = [pred.strip() for pred in preds]
|
||||
labels = [label.strip() for label in labels]
|
||||
# def postprocess_summarization(preds, labels):
|
||||
# preds = [pred.strip() for pred in preds]
|
||||
# labels = [label.strip() for label in labels]
|
||||
|
||||
preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in preds]
|
||||
labels = ["\n".join(nltk.sent_tokenize(label)) for label in labels]
|
||||
# preds = ["\n".join(nltk.sent_tokenize(pred)) for pred in preds]
|
||||
# labels = ["\n".join(nltk.sent_tokenize(label)) for label in labels]
|
||||
|
||||
return preds, labels
|
||||
# return preds, labels
|
||||
|
||||
|
||||
def preprocess_summarization(eval_pred, tokenizer, ignore_pad_token_for_loss=True):
|
||||
preds, labels = eval_pred
|
||||
decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
|
||||
if ignore_pad_token_for_loss:
|
||||
labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
|
||||
decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
|
||||
# def preprocess_summarization(eval_pred, tokenizer, ignore_pad_token_for_loss=True):
|
||||
# preds, labels = eval_pred
|
||||
# decoded_preds = tokenizer.batch_decode(preds, skip_special_tokens=True)
|
||||
# if ignore_pad_token_for_loss:
|
||||
# labels = np.where(labels != -100, labels, tokenizer.pad_token_id)
|
||||
# decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)
|
||||
|
||||
decoded_preds, decoded_labels = postprocess_summarization(decoded_preds, decoded_labels)
|
||||
return decoded_preds, decoded_labels
|
||||
# decoded_preds, decoded_labels = postprocess_summarization(decoded_preds, decoded_labels)
|
||||
# return decoded_preds, decoded_labels
|
||||
|
||||
|
||||
def get_metrics(conf, tokenizer):
|
||||
@@ -77,16 +79,16 @@ def get_metrics(conf, tokenizer):
|
||||
# metrics in the future for more thorough evaluation
|
||||
metrics, preprocess_fns = [evaluate.load("accuracy")], [default_preprocess]
|
||||
|
||||
if any(dataset in QA_DATASETS for dataset in conf.datasets):
|
||||
raise ValueError("TODO")
|
||||
metrics.append(evaluate.load("squad_v2"))
|
||||
preprocess_fns.append(preprocess_qa)
|
||||
if any(dataset in SUMMARIZATION_DATASETS for dataset in conf.datasets):
|
||||
raise ValueError("TODO")
|
||||
metrics.append(evaluate.load("rouge"))
|
||||
preprocess_fns.append(
|
||||
partial(preprocess_summarization, tokenizer, ignore_pad_token_for_loss=conf.ignore_pad_token_for_loss)
|
||||
)
|
||||
# if any(dataset in QA_DATASETS for dataset in conf.datasets):
|
||||
# raise ValueError("TODO")
|
||||
# metrics.append(evaluate.load("squad_v2"))
|
||||
# preprocess_fns.append(preprocess_qa)
|
||||
# if any(dataset in SUMMARIZATION_DATASETS for dataset in conf.datasets):
|
||||
# raise ValueError("TODO")
|
||||
# metrics.append(evaluate.load("rouge"))
|
||||
# preprocess_fns.append(
|
||||
# partial(preprocess_summarization, tokenizer, ignore_pad_token_for_loss=conf.ignore_pad_token_for_loss)
|
||||
# )
|
||||
|
||||
return metrics, preprocess_fns
|
||||
|
||||
|
||||
+8
-7
@@ -1,17 +1,18 @@
|
||||
# Generate Topics, Questions, and Answers from a text
|
||||
# Generate Topics, Questions, and Answers from a paragraph of text
|
||||
|
||||
This python code can be used to generate topics, questions, and answers from a
|
||||
paragraph of text. This is a good way to generate ground truth knowledge about a
|
||||
topic from a trusted source.
|
||||
|
||||
The output of this is a dictionary with:
|
||||
The output of this is a dictionary with the following information:
|
||||
|
||||
1. submitted paragraph
|
||||
1. generated topics
|
||||
1. generated questions
|
||||
1. generated topic prefixes that can be prepended to the questions
|
||||
1. open book answer based only on the provided paragraph
|
||||
1. closed book answers generated by FLAN-T5-11B
|
||||
2. generated topics
|
||||
3. generated questions
|
||||
4. generated topic prefixes that can be prepended to the questions
|
||||
5. open book answer based only on the provided paragraph
|
||||
6. closed book answers generated by FLAN-T5-11B (uses only question and
|
||||
optionally question prefix to generate the answer)
|
||||
|
||||
## Contributing
|
||||
|
||||
+48
-79
@@ -1,4 +1,4 @@
|
||||
# This notebook will run on a system with a single RTX3090 (24 GB vram).
|
||||
# This notebook will run on a system with a single RTX3090 (24 GB vram) GPU.
|
||||
# You need to install accelerate, bitsandbytes, and transformers
|
||||
|
||||
import math
|
||||
@@ -10,8 +10,7 @@ import torch
|
||||
# load all needed libraries
|
||||
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
|
||||
|
||||
# This device map will work a GPU with > 24GB vram.
|
||||
# It uses nearly all the memory.
|
||||
# This device map will work a GPU with > 24GB vram. It uses nearly all the memory.
|
||||
device_map_T5_13B = {
|
||||
"shared": 0,
|
||||
"decoder.embed_tokens": 0,
|
||||
@@ -71,31 +70,32 @@ device_map_T5_13B = {
|
||||
"lm_head": 0,
|
||||
}
|
||||
|
||||
|
||||
# Load the model in bfloat16. Make sure to use bfloat16
|
||||
# if you are doing inference with 16bit precision.
|
||||
# Load the model in bfloat16. Make sure to use bfloat16 if you are doing inference with 16bit precision.
|
||||
tokenizer = AutoTokenizer.from_pretrained("flan-t5-xxl")
|
||||
model = AutoModelForSeq2SeqLM.from_pretrained(
|
||||
"flan-t5-xxl",
|
||||
device_map=device_map_T5_13B,
|
||||
torch_dtype=torch.bfloat16,
|
||||
load_in_8bit=False,
|
||||
"flan-t5-xxl", device_map=device_map_T5_13B, torch_dtype=torch.bfloat16, load_in_8bit=False
|
||||
)
|
||||
|
||||
|
||||
# Load strings as knowledge sources for QA generation.
|
||||
# You can do this with a pickle.
|
||||
# Load an array of strings that are are reference or verified knowledge sources for QA generation. You can do this with a pickle.
|
||||
objects = []
|
||||
with (open("paragraphs.pkl", "rb")) as openfile:
|
||||
while True:
|
||||
try:
|
||||
objects.append(pickle.load(openfile))
|
||||
except EOFError:
|
||||
print("Problem laoding your pickle file, using the default array")
|
||||
pickle_fail = True
|
||||
break
|
||||
paragraphs = objects[0]
|
||||
|
||||
# Make sure no paragraphs are too long for T5.
|
||||
# It handles up to 512 tokens context length.
|
||||
# If you don't know how to get an array of paragraphs, you can uncomment the next line.
|
||||
if pickle_fail:
|
||||
paragraphs = [
|
||||
"Like for me, this thing is like a little side hobby, but it's also one that's deeply fulfilling. So not just from a business perspective, which is not the way I think about it. I just think from a life human perspective, it's I probably wouldn't have this kind of conversation with you off mic, like this long, this deep, this attentive. There's something really fulfilling about these conversations. So what advice would you have for me? What advice do you have for yourself? Oh, have you not introspected this that deeply? Oh, I have advice. I think the first advice I would give to you is I think you should have me on more often. Yeah. Yeah. That's first and foremost. And second is go on your podcast and have a conversation. Well, I would say you come on my podcast when you're ready. Yeah. When you feel like the product that I'm putting out would benefit from your presence and vice versa, not as a favor to a bro, but at the right time.",
|
||||
"Well, we really are looking through a two dimensional screen until it's what we intuit to be a three dimensional world and also inferring dynamic stuff, making it 4D. Anyway, is it possible to visualize some pretty pictures that give us a deeper sense of the truth of reality? I think that we will incrementally be able to do that. I think that, for example, the picture that we have of electrons and photons interacting and scattering, it may have not been possible until Faraday did all of his experiments and then Maxwell wrote down his equations. And we were then sort of forced by his equations to think in a new way. And then when Planck in 1900, desperate to try to solve the problem of black body radiation, what they call the ultraviolet catastrophe where Newton was predicting infinite energies where there weren't infinite energies in black body radiation. And he in desperation proposed packets of energy. Then once you've done that, and then you have an Einstein come along five years later and show how that explains the photoelectric effect.",
|
||||
"But man, I don't know how I would feel about just bacteria everywhere. Well, it would be depressing if it was true. I suppose depressing, I don't think, I don't. I don't know what's more depressing, bacteria everywhere or nothing everywhere. Yes, either of them are chilling. Yeah. But whether it's chilling or not, I don't think should force us to change our view about whether it's real or not. Yes. And what I'm saying may or may not be true. So how would you feel if we discovered life on Mars? Absolutely. It sounds like you would be less excited than some others because you're like, well. What I would be most interested in is how similar to life on Earth it would be. It would actually turn into quite a subtle problem because the likelihood of life having gone to and fro between Mars and the Earth is quite, I wouldn't say high, but it's not low, it's quite feasible. And so if we found life on Mars and it had very similar genetic code, but it was slightly different, most people would interpret that immediately as evidence that they've been transit one way or the other and that it was a common origin of life on Mars or on the Earth and it went one way or the other way.",
|
||||
]
|
||||
|
||||
# Make sure no paragraphs are too long for T5. It handles up to 512 tokens context length.
|
||||
fixed_paragraphs = []
|
||||
for k in paragraphs:
|
||||
if len(k) > 1100:
|
||||
@@ -107,16 +107,13 @@ print("Length filtered number of paragraphs:", len(fixed_paragraphs))
|
||||
paragraphs = fixed_paragraphs
|
||||
|
||||
|
||||
# Sort_Tuple sorts a list of tuples
|
||||
# by the second element.
|
||||
# Sort_Tuple sorts a list of tuples where the first element is text and the second element is logits e.g. ("text",-1.5)
|
||||
def Sort_Tuple(tup):
|
||||
tup.sort(key=lambda x: x[1], reverse=True)
|
||||
return tup
|
||||
|
||||
|
||||
# ask_flan_T5 takes a text input and returns the
|
||||
# response of FLAN_T5 and a normalized logits
|
||||
# score for the generation.
|
||||
# ask_flan_T5 is a function that takes an input text and returns the response of FLAN_T5 and a normalized logits score for the generation.
|
||||
def ask_flan_T5(input_text):
|
||||
inputs = tokenizer.encode(input_text, return_tensors="pt").cuda(0)
|
||||
outputs = model.generate(
|
||||
@@ -135,17 +132,20 @@ def ask_flan_T5(input_text):
|
||||
for i in outputs.sequences:
|
||||
logprobs = 0
|
||||
counter = 0
|
||||
output_scores = ""
|
||||
for k in i[1:]:
|
||||
word_piece = tokenizer.decode(k.item())
|
||||
word_prob = (round(probs[0][counter][k.item()].item(), 2)) + 0.001
|
||||
word_logprob = round(math.log(word_prob), 2)
|
||||
logprobs = logprobs + math.log(word_prob)
|
||||
next_piece = word_piece + "(" + str(word_prob) + " " + str(word_logprob) + ")"
|
||||
output_scores = output_scores + " " + next_piece
|
||||
counter += 1
|
||||
out_tuple = (out_text, round(logprobs, 2))
|
||||
return out_tuple
|
||||
|
||||
|
||||
# ask_flan_T5D is a function that takes an input text and
|
||||
# returns the deterministic(do_sample=False) output of
|
||||
# FLAN_T5 and logits.
|
||||
# ask_flan_T5D is a function that takes an input text and returns the deterministic(do_sample=False) output of FLAN_T5 and a normalized logits score for the generation.
|
||||
def ask_flan_T5D(input_text):
|
||||
inputs = tokenizer.encode(input_text, return_tensors="pt").cuda(0)
|
||||
outputs = model.generate(
|
||||
@@ -162,9 +162,14 @@ def ask_flan_T5D(input_text):
|
||||
for i in outputs.sequences:
|
||||
logprobs = 0
|
||||
counter = 0
|
||||
output_scores = ""
|
||||
for k in i[1:]:
|
||||
word_piece = tokenizer.decode(k.item())
|
||||
word_prob = (round(probs[0][counter][k.item()].item(), 2)) + 0.001
|
||||
word_logprob = round(math.log(word_prob), 2)
|
||||
logprobs = logprobs + math.log(word_prob)
|
||||
next_piece = word_piece + "(" + str(word_prob) + " " + str(word_logprob) + ")"
|
||||
output_scores = output_scores + " " + next_piece
|
||||
counter += 1
|
||||
out_tuple = (out_text, round(logprobs, 2))
|
||||
return out_tuple
|
||||
@@ -173,12 +178,7 @@ def ask_flan_T5D(input_text):
|
||||
# Generate a topic classifier for a paragraph of text
|
||||
def generate_topic(paragraph):
|
||||
results = set()
|
||||
input_text = (
|
||||
"Task: Create a topic classifier for the provided \
|
||||
paragraph.\nParagraph:\n"
|
||||
+ paragraph
|
||||
+ "\nTopic: "
|
||||
)
|
||||
input_text = "Task: Create a topic classifier for the provided paragraph.\nParagraph:\n" + paragraph + "\nTopic: "
|
||||
for k in range(0, 20):
|
||||
result = ask_flan_T5(input_text)
|
||||
if result[1] > -4:
|
||||
@@ -196,11 +196,7 @@ def generate_topic_prefix(topic_set):
|
||||
for entry in topic_set:
|
||||
topic = entry[0]
|
||||
input_text = (
|
||||
"Task: Create a prepositional phrase about the topic.\n\
|
||||
Example 1\n Topic: climbing mount everest\nPrepositional \
|
||||
Phrase: With regards to climbing mount everest,\nExample \
|
||||
2\nTopic: United States Air Force\nPrepositional Phrase: \
|
||||
On the topic of the United States Air Force,\n Example 3\nTopic: "
|
||||
"Task: Create a prepositional phrase about the topic.\nExample 1\nTopic: climbing mount everest\nPrepositional Phrase: With regards to climbing mount everest,\nExample 2\nTopic: United States Air Force\nPrepositional Phrase: On the topic of the United States Air Force,\nExample 3\nTopic: "
|
||||
+ topic
|
||||
+ "\nPrepositional Phrase: "
|
||||
)
|
||||
@@ -210,12 +206,10 @@ def generate_topic_prefix(topic_set):
|
||||
return sorted_results[0:5]
|
||||
|
||||
|
||||
# Generate who/what/where/when/why questions from a paragraph.
|
||||
# Number of questions variable is an integer which indicates how
|
||||
# many of each question type to try to generate.
|
||||
# Generate who/what/where/when/why questions from a paragraph. Number of questions variable is an integer which indicates how many of each question type to try to generate.
|
||||
def generate_questions(paragraph, number_of_questions):
|
||||
if len(tokenizer.encode(paragraph)) > 480:
|
||||
print("Warning, the context length is too long.")
|
||||
print("Warning, the context length is too long and could give bad results.")
|
||||
question_set = set()
|
||||
question_types = [
|
||||
"What",
|
||||
@@ -239,17 +233,12 @@ def generate_questions(paragraph, number_of_questions):
|
||||
return question_set
|
||||
|
||||
|
||||
# Generate answers for a set of questions.
|
||||
# Input is the paragraph of text and a set of questions where each question
|
||||
# is a tuple generated from the generate_questions() function.
|
||||
# Generate answers for a set of questions. Input is the paragraph of text and a set of questions where each question is a tuple generated from the generate_questions() function.
|
||||
def generate_answers(paragraph, question_set):
|
||||
possible_answers = set()
|
||||
for question in question_set:
|
||||
input_text = (
|
||||
"Please read the following paragraph and \
|
||||
then answer the question using only data \
|
||||
found in the text. If no answer is possible, respond \
|
||||
'NA'.\nText:\n"
|
||||
"Please read the following paragraph and then answer the question using only data found in the text. If no answer is possible, respond 'NA'.\nText:\n"
|
||||
+ paragraph
|
||||
+ "\nQuestion:\n"
|
||||
+ question[1][0]
|
||||
@@ -263,16 +252,13 @@ def generate_answers(paragraph, question_set):
|
||||
return possible_answers
|
||||
|
||||
|
||||
# Generate questions from a paragraph and set of answers.
|
||||
# Input is the paragraph of text and a set of answers where each question
|
||||
# is a tuple generated from the generate_answers() function.
|
||||
# Generate questions from a paragraph and set of answers. Input is the paragraph of text and a set of answers where each question is a tuple generated from the generate_answers() function.
|
||||
def generate_question2(paragraph, qa_set):
|
||||
qaq_results = set()
|
||||
for qa_item in qa_set:
|
||||
answer = qa_item[2][0]
|
||||
input_text = (
|
||||
"Please read the following paragraph and \
|
||||
then generate a question whose answer is: "
|
||||
"Please read the following paragraph and then generate a question whose answer is: "
|
||||
+ answer
|
||||
+ "\nParagraph:\n"
|
||||
+ paragraph
|
||||
@@ -283,23 +269,20 @@ def generate_question2(paragraph, qa_set):
|
||||
return qaq_results
|
||||
|
||||
|
||||
# Generate answers from a paragraph and set of questions.
|
||||
# Input is the paragraph of text and a set of questions where each answer
|
||||
# is a tuple generated from the generate_questions2() function.
|
||||
# Generate answers from a paragraph and set of questions. Input is the paragraph of text and a set of questions where each answer is a tuple generated from the generate_questions2() function.
|
||||
def generate_answers2(paragraph, question_set):
|
||||
possible_answers = set()
|
||||
for question in question_set:
|
||||
input_text = (
|
||||
"Please read the following paragraph and \
|
||||
then answer the question using only data \
|
||||
found in the text. If no answer is possible, respond \
|
||||
'NA'.\nText:\n"
|
||||
"Please read the following paragraph and then answer the question using only data found in the text. If no answer is possible, respond 'NA'.\nText:\n"
|
||||
+ paragraph
|
||||
+ "\nQuestion:\n"
|
||||
+ question
|
||||
+ "\nAnswer:\n"
|
||||
)
|
||||
answer = ask_flan_T5D(input_text)
|
||||
# print(question)
|
||||
# print(answer)
|
||||
possible_answers.add((question, answer))
|
||||
return possible_answers
|
||||
|
||||
@@ -314,10 +297,7 @@ def generate_declarative(qaq_set):
|
||||
pass
|
||||
else:
|
||||
input_text = (
|
||||
"Generate a declarative statement based on the \
|
||||
given question and answer pair.\nQ: What is \
|
||||
sitting on the couch?\nA: poodle\nA poodle is \
|
||||
sitting on the couch.\nQ: "
|
||||
"Generate a declarative statement based on the given question and answer pair.\nQ: What is sitting on the couch?\nA: poodle\nA poodle is sitting on the couch.\nQ: "
|
||||
+ question
|
||||
+ "\nA: "
|
||||
+ answer
|
||||
@@ -339,20 +319,7 @@ def generate_closed_answer(qaqd_set):
|
||||
pass
|
||||
else:
|
||||
input_text = (
|
||||
"Task: Answer the question in a detailed fashion. \
|
||||
If the question cannot be answered without more \
|
||||
information, please answer NA.\nExample 1:\nQuestion: \
|
||||
Why does Shala like cookies?\nAnswer: It is not possible \
|
||||
to know why Shala likes cookies without more information, \
|
||||
but many people that like cookies enjoy their taste or \
|
||||
some of their ingredients (e.g. chocolate chips or \
|
||||
peanut butter).\nExample 2:\nQuestion: Why would someone \
|
||||
vote in an election?\nAnswer: There are many reasons \
|
||||
someone might vote in an election, for instance to have \
|
||||
their voice heard or to help a candidate they like win the \
|
||||
race.\nExample 3\nQuestion: What decoration goes on top of \
|
||||
a Christmas tree?\nAnswer: Usually a star is placed at the \
|
||||
top of a Christmas tree.\nExample 4:\nQuestion: "
|
||||
"Task: Answer the question in a detailed fashion. If the question cannot be answered without more information, please answer NA.\nExample 1:\nQuestion: Why does Shala like cookies?\nAnswer: It is not possible to know why Shala likes cookies without more information, but many people that like cookies enjoy their taste or some of their ingredients (e.g. chocolate chips or peanut butter).\nExample 2:\nQuestion: Why would someone vote in an election?\nAnswer: There are many reasons someone might vote in an election, for instance to have their voice heard or to help a candidate they like win the race.\nExample 3\nQuestion: What decoration goes on top of a Christmas tree?\nAnswer: Usually a star is placed at the top of a Christmas tree.\nExample 4:\nQuestion: "
|
||||
+ question
|
||||
+ "\nAnswer: "
|
||||
)
|
||||
@@ -361,8 +328,7 @@ def generate_closed_answer(qaqd_set):
|
||||
return qaqd_results
|
||||
|
||||
|
||||
# Create a dictionary of questions and answers from a list of paragraphs.
|
||||
# Takes about 20 seconds per paragraph to process.
|
||||
# Create a dictionary of questions and answers from a list of paragraphs. Takes about 20 seconds per paragraph to process.
|
||||
start_time = time.perf_counter()
|
||||
questions_dict = {}
|
||||
uniq_id = 100000
|
||||
@@ -399,8 +365,11 @@ generation_time = stop_time - start_time
|
||||
print(questions_dict[uniq_id - 1])
|
||||
print(generation_time)
|
||||
|
||||
|
||||
# create a binary pickle file to save your dictionary
|
||||
f = open("questions_dict.pkl", "wb")
|
||||
|
||||
# write the python object (dict) to pickle file
|
||||
pickle.dump(questions_dict, f)
|
||||
|
||||
# close file
|
||||
f.close()
|
||||
@@ -0,0 +1,5 @@
|
||||
# Data Augmentation
|
||||
|
||||
This folder contains subfolders of notebooks broadly relating to data
|
||||
augmentation. Each subfolder contains a README.md file explaining what the
|
||||
notebooks in that folder do.
|
||||
+4
-4
@@ -5,7 +5,7 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-argumentation/EssayInstructions.ipynb)"
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-augmentation/essay-instructions/essay-instructions.ipynb)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -210,7 +210,7 @@
|
||||
"provenance": []
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.8.10 64-bit",
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
@@ -224,11 +224,11 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.10"
|
||||
"version": "3.7.4 (tags/v3.7.4:e09359112e, Jul 8 2019, 20:34:20) [MSC v.1916 64 bit (AMD64)]"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
|
||||
"hash": "25d5c2324055587ceaeef27650c79ce8358ea61d7689f2e0b8ada5d53f85bce4"
|
||||
}
|
||||
}
|
||||
},
|
||||
+12
-4
@@ -5,16 +5,24 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-argumentation/EssayRevision.ipynb)"
|
||||
"# Essay Revision"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-augmentation/essay-revision/essay-revision.ipynb)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "o0lAqmWhsiUe"
|
||||
},
|
||||
"source": [
|
||||
"#Essay Revision\n",
|
||||
"The goal of this notebook is to use data argumentation to have data on improving essays. The way this is done is by taking a template \"good\" essay and making step by step changes that make it worse and add intructions on how to fix it."
|
||||
]
|
||||
},
|
||||
@@ -319,11 +327,11 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.4"
|
||||
"version": "3.7.4 (tags/v3.7.4:e09359112e, Jul 8 2019, 20:34:20) [MSC v.1916 64 bit (AMD64)]"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "492d89208e1af30f4727fd53e254ea56e6b1a843b376782bfa5f6ce13d676265"
|
||||
"hash": "25d5c2324055587ceaeef27650c79ce8358ea61d7689f2e0b8ada5d53f85bce4"
|
||||
}
|
||||
}
|
||||
},
|
||||
+18
-3
@@ -5,16 +5,24 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-argumentation/StackExchangeBuilder.ipynb)"
|
||||
"# Ingest StackExchange data dumps"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-augmentation/stackexchange-builder/stackexchange-builder.ipynb)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {
|
||||
"id": "TB7CEfs8F-8u"
|
||||
},
|
||||
"source": [
|
||||
"# Ingest StackExchange data dumps\n",
|
||||
"This notebook takes a StackExchange Data dump \"Posts.xml\" file and ingests it into a Pandas Dataframe. Outputs of the file can be JSON, JSONL, Parquet, or CSV. "
|
||||
]
|
||||
},
|
||||
@@ -1842,10 +1850,17 @@
|
||||
},
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
"name": "python",
|
||||
"version": "3.7.4 (tags/v3.7.4:e09359112e, Jul 8 2019, 20:34:20) [MSC v.1916 64 bit (AMD64)]"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "25d5c2324055587ceaeef27650c79ce8358ea61d7689f2e0b8ada5d53f85bce4"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
+9
-3
@@ -9,11 +9,12 @@
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"id": "b2e3c95c",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/data-argumentation/UnifiedQA.ipynb)"
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/data-augmentation/unified-qa/unified-qa.ipynb)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -493,7 +494,7 @@
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
@@ -507,7 +508,12 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.9"
|
||||
"version": "3.7.4 (tags/v3.7.4:e09359112e, Jul 8 2019, 20:34:20) [MSC v.1916 64 bit (AMD64)]"
|
||||
},
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "25d5c2324055587ceaeef27650c79ce8358ea61d7689f2e0b8ada5d53f85bce4"
|
||||
}
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
+12
-5
@@ -5,7 +5,15 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb)"
|
||||
"# Detoxify evaluation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/detoxify-evaluation/detoxify-evaluation.ipynb)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -23,7 +31,6 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Detoxify evaluation\n",
|
||||
"[Detoxify](https://github.com/unitaryai/detoxify) is a open source model used to identify prompts as toxic\n",
|
||||
"\n",
|
||||
"<img src=\"https://raw.githubusercontent.com/unitaryai/detoxify/master/examples.png\" alt=\"Image from detoxify github that shows the example input/output of their model\" />\n",
|
||||
@@ -472,7 +479,7 @@
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "DetoxifyEvaluation",
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
@@ -486,12 +493,12 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.8"
|
||||
"version": "3.7.4 (tags/v3.7.4:e09359112e, Jul 8 2019, 20:34:20) [MSC v.1916 64 bit (AMD64)]"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "aeda4fe49bddd52f429be231bf767df53f2b167abae0a465a8ef142aa6b97b8a"
|
||||
"hash": "25d5c2324055587ceaeef27650c79ce8358ea61d7689f2e0b8ada5d53f85bce4"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1,4 +1,4 @@
|
||||
# OpenBuggerNotebook
|
||||
# OpenBugger
|
||||
|
||||
https://github.com/furlat/OpenBugger/blob/main/README.md is a Python package
|
||||
that allows you to inject syntax and logic errors into your code. This can be
|
||||
+11
-3
@@ -5,7 +5,15 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/code-bugger/openbugger_example.ipynb)"
|
||||
"# OpenBugger Example"
|
||||
]
|
||||
},
|
||||
{
|
||||
"attachments": {},
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"[](https://colab.research.google.com/github/LAION-AI/Open-Assistant/blob/main/notebooks/openbugger/openbugger_example.ipynb)"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -272,12 +280,12 @@
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.10.6 (tags/v3.10.6:9c7b4bd, Aug 1 2022, 21:53:49) [MSC v.1932 64 bit (AMD64)]"
|
||||
"version": "3.7.4 (tags/v3.7.4:e09359112e, Jul 8 2019, 20:34:20) [MSC v.1916 64 bit (AMD64)]"
|
||||
},
|
||||
"orig_nbformat": 4,
|
||||
"vscode": {
|
||||
"interpreter": {
|
||||
"hash": "ceba285e8b4e6478fe8ad229bc63940a90ad5cf3d143521e7c38823a2e915b21"
|
||||
"hash": "25d5c2324055587ceaeef27650c79ce8358ea61d7689f2e0b8ada5d53f85bce4"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -6,6 +6,7 @@ pushd "$parent_path/../../backend"
|
||||
|
||||
export DEBUG_SKIP_API_KEY_CHECK=True
|
||||
export DEBUG_USE_SEED_DATA=True
|
||||
export DEBUG_SKIP_TOXICITY_CALCULATION=True
|
||||
export DEBUG_ALLOW_SELF_LABELING=True
|
||||
export DEBUG_SKIP_EMBEDDING_COMPUTATION=True
|
||||
|
||||
|
||||
@@ -11,6 +11,9 @@ echo "Generating OpenAPI schema..."
|
||||
python -m main --print-openapi-schema > $OPENAPI_JSON_FILE_NAME
|
||||
echo "Done!"
|
||||
|
||||
echo "Formatting & Copying OpenAPI schema to docs directory..."
|
||||
jq . $OPENAPI_JSON_FILE_NAME > ../docs/docs/api/openapi.json
|
||||
|
||||
# If oasst-mock-backend docker container is already running,
|
||||
# just restart it
|
||||
if [ "$(docker ps -q -f name=oasst-mock-backend)" ]; then
|
||||
|
||||
Generated
+616
-29
File diff suppressed because it is too large
Load Diff
@@ -58,6 +58,7 @@
|
||||
"react-dom": "18.2.0",
|
||||
"react-feature-flags": "^1.0.0",
|
||||
"react-icons": "^4.7.1",
|
||||
"sharp": "^0.31.3",
|
||||
"swr": "^2.0.0",
|
||||
"tailwindcss": "^3.2.4",
|
||||
"use-debounce": "^9.0.2"
|
||||
|
||||
@@ -41,6 +41,7 @@ model User {
|
||||
email String? @unique
|
||||
emailVerified DateTime?
|
||||
image String?
|
||||
isNew Boolean @default(true)
|
||||
role String @default("general")
|
||||
|
||||
accounts Account[]
|
||||
|
||||
@@ -3,7 +3,6 @@ import {
|
||||
Button,
|
||||
Checkbox,
|
||||
Flex,
|
||||
Grid,
|
||||
Popover,
|
||||
PopoverAnchor,
|
||||
PopoverArrow,
|
||||
@@ -15,7 +14,6 @@ import {
|
||||
SliderFilledTrack,
|
||||
SliderThumb,
|
||||
SliderTrack,
|
||||
Spacer,
|
||||
Tooltip,
|
||||
useBoolean,
|
||||
useColorMode,
|
||||
@@ -23,6 +21,7 @@ import {
|
||||
useId,
|
||||
} from "@chakra-ui/react";
|
||||
import { QuestionMarkCircleIcon } from "@heroicons/react/20/solid";
|
||||
import clsx from "clsx";
|
||||
import { useEffect, useReducer } from "react";
|
||||
import { FiAlertCircle } from "react-icons/fi";
|
||||
import { get, post } from "src/lib/api";
|
||||
@@ -146,24 +145,25 @@ export const FlaggableElement = (props: FlaggableElementProps) => {
|
||||
isLazy
|
||||
lazyBehavior="keepMounted"
|
||||
>
|
||||
<Grid display="flex" alignItems="center" gap="2">
|
||||
<Box display="flex" alignItems="center" gap="2">
|
||||
<PopoverAnchor>{props.children}</PopoverAnchor>
|
||||
|
||||
<Tooltip label="Report" bg="red.500" aria-label="A tooltip">
|
||||
<div>
|
||||
<Box>
|
||||
<PopoverTrigger>
|
||||
<Box as="button" display="flex" alignItems="center" justifyContent="center" borderRadius="full" p="1">
|
||||
<FiAlertCircle size="20" className="text-red-400" aria-hidden="true" />
|
||||
</Box>
|
||||
</PopoverTrigger>
|
||||
</div>
|
||||
</Box>
|
||||
</Tooltip>
|
||||
</Grid>
|
||||
</Box>
|
||||
|
||||
<PopoverContent width="fit-content" p="3">
|
||||
<PopoverContent width="auto" p="3" m="4" maxWidth="calc(100vw - 2rem)">
|
||||
<PopoverArrow />
|
||||
<div className="relative h-4">
|
||||
<Box className="relative h-4">
|
||||
<PopoverCloseButton />
|
||||
</div>
|
||||
</Box>
|
||||
<PopoverBody>
|
||||
{report.label_values.map(({ label, checked, value }, i) => (
|
||||
<FlagCheckbox
|
||||
@@ -207,9 +207,9 @@ export function FlagCheckbox(props: FlagCheckboxProps): JSX.Element {
|
||||
let AdditionalExplanation = null;
|
||||
if (props.label.help_text) {
|
||||
AdditionalExplanation = (
|
||||
<a href="#" className="group flex items-center space-x-2.5 text-sm ">
|
||||
<a href="#" className="text-sm inline group leading-4">
|
||||
<QuestionMarkCircleIcon
|
||||
className="flex h-5 w-5 ml-3 text-gray-400 group-hover:text-gray-500"
|
||||
className="h-5 w-5 ml-1 text-gray-400 group-hover:text-gray-500 inline"
|
||||
aria-hidden="true"
|
||||
/>
|
||||
</a>
|
||||
@@ -221,23 +221,30 @@ export function FlagCheckbox(props: FlagCheckboxProps): JSX.Element {
|
||||
|
||||
const labelTextClass =
|
||||
colorMode === "light"
|
||||
? `text-${colors.light.text} hover:text-blue-700 float-left`
|
||||
: `text-${colors.dark.text} hover:text-blue-400 float-left`;
|
||||
? `text-${colors.light.text} hover:text-blue-700`
|
||||
: `text-${colors.dark.text} hover:text-blue-400`;
|
||||
|
||||
return (
|
||||
<Flex gap="2">
|
||||
<Checkbox
|
||||
id={id}
|
||||
isChecked={props.checked}
|
||||
onChange={(e) => {
|
||||
props.checkboxHandler(e.target.checked, props.idx);
|
||||
}}
|
||||
/>
|
||||
<label className="text-sm form-check-label" htmlFor={id}>
|
||||
<span className={labelTextClass}>{props.label.display_text}</span>
|
||||
{AdditionalExplanation}
|
||||
</label>
|
||||
<Spacer />
|
||||
<Flex gap="4" justifyContent="space-between" className="my-2">
|
||||
<div className="flex items-start align-middle">
|
||||
<Checkbox
|
||||
id={id}
|
||||
isChecked={props.checked}
|
||||
onChange={(e) => {
|
||||
props.checkboxHandler(e.target.checked, props.idx);
|
||||
}}
|
||||
/>
|
||||
<label
|
||||
className={clsx(
|
||||
"text-sm form-check-label ml-2 break-all inline align-middle first-line:leading-4",
|
||||
labelTextClass
|
||||
)}
|
||||
htmlFor={id}
|
||||
>
|
||||
{props.label.display_text}
|
||||
{AdditionalExplanation}
|
||||
</label>
|
||||
</div>
|
||||
<div
|
||||
onClick={() => {
|
||||
if (!props.checked) {
|
||||
|
||||
@@ -54,6 +54,7 @@ export const getDashboardLayout = (page: React.ReactElement) => (
|
||||
>
|
||||
{page}
|
||||
</SideMenuLayout>
|
||||
<Footer />
|
||||
</div>
|
||||
);
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ export function MessageTableEntry(props: MessageTableEntryProps) {
|
||||
|
||||
return (
|
||||
<FlaggableElement message={item}>
|
||||
<HStack w="100%" gap={2}>
|
||||
<HStack w={["full", "full", "full", "fit-content"]} gap={2}>
|
||||
<Box borderRadius="full" border="solid" borderWidth="1px" borderColor={borderColor} bg={avatarColor}>
|
||||
<Avatar
|
||||
size="sm"
|
||||
@@ -28,21 +28,20 @@ export function MessageTableEntry(props: MessageTableEntryProps) {
|
||||
/>
|
||||
</Box>
|
||||
{props.enabled ? (
|
||||
<Box maxWidth="xl">
|
||||
<Box width={["full", "full", "full", "fit-content"]} maxWidth={["full", "full", "full", "2xl"]}>
|
||||
<Link href={`/messages/${item.id}`}>
|
||||
<LinkBox
|
||||
bg={item.is_assistant ? backgroundColor : backgroundColor2}
|
||||
className={`p-4 rounded-md whitespace-pre-wrap w-full`}
|
||||
>
|
||||
<LinkBox bg={item.is_assistant ? backgroundColor : backgroundColor2} p="4" borderRadius="md">
|
||||
{item.text}
|
||||
</LinkBox>
|
||||
</Link>
|
||||
</Box>
|
||||
) : (
|
||||
<Box
|
||||
maxWidth="xl"
|
||||
width={["full", "full", "full", "fit-content"]}
|
||||
maxWidth={["full", "full", "full", "2xl"]}
|
||||
bg={item.is_assistant ? backgroundColor : backgroundColor2}
|
||||
className={`p-4 rounded-md whitespace-pre-wrap w-full`}
|
||||
p="4"
|
||||
borderRadius="md"
|
||||
>
|
||||
{item.text}
|
||||
</Box>
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
import Link from "next/link";
|
||||
import { useState } from "react";
|
||||
import { get } from "src/lib/api";
|
||||
import type { User } from "src/types/Users";
|
||||
import useSWR from "swr";
|
||||
|
||||
/**
|
||||
@@ -22,7 +23,7 @@ import useSWR from "swr";
|
||||
*/
|
||||
const UsersCell = () => {
|
||||
const [pageIndex, setPageIndex] = useState(0);
|
||||
const [users, setUsers] = useState([]);
|
||||
const [users, setUsers] = useState<User[]>([]);
|
||||
|
||||
// Fetch and save the users.
|
||||
// This follows useSWR's recommendation for simple pagination:
|
||||
@@ -53,21 +54,23 @@ const UsersCell = () => {
|
||||
<Thead>
|
||||
<Tr>
|
||||
<Th>Id</Th>
|
||||
<Th>Email</Th>
|
||||
<Th>Auth Id</Th>
|
||||
<Th>Auth Method</Th>
|
||||
<Th>Name</Th>
|
||||
<Th>Role</Th>
|
||||
<Th>Update</Th>
|
||||
</Tr>
|
||||
</Thead>
|
||||
<Tbody>
|
||||
{users.map((user, index) => (
|
||||
<Tr key={index}>
|
||||
<Td>{user.id}</Td>
|
||||
<Td>{user.email}</Td>
|
||||
<Td>{user.name}</Td>
|
||||
<Td>{user.role}</Td>
|
||||
{users.map(({ id, user_id, auth_method, display_name, role }) => (
|
||||
<Tr key={user_id}>
|
||||
<Td>{user_id}</Td>
|
||||
<Td>{id}</Td>
|
||||
<Td>{auth_method}</Td>
|
||||
<Td>{display_name}</Td>
|
||||
<Td>{role}</Td>
|
||||
<Td>
|
||||
<Link href={`/admin/manage_user/${user.id}`}>Manage</Link>
|
||||
<Link href={`/admin/manage_user/${user_id}`}>Manage</Link>
|
||||
</Td>
|
||||
</Tr>
|
||||
))}
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { JWT } from "next-auth/jwt";
|
||||
import type { Message } from "src/types/Conversation";
|
||||
import type { BackendUser } from "src/types/Users";
|
||||
|
||||
export class OasstError {
|
||||
message: string;
|
||||
@@ -43,6 +45,32 @@ export class OasstApiClient {
|
||||
return await resp.json();
|
||||
}
|
||||
|
||||
private async put(path: string): Promise<any> {
|
||||
const resp = await fetch(`${this.oasstApiUrl}${path}`, {
|
||||
method: "PUT",
|
||||
headers: {
|
||||
"X-API-Key": this.oasstApiKey,
|
||||
},
|
||||
});
|
||||
|
||||
if (resp.status === 204) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (resp.status >= 300) {
|
||||
const errorText = await resp.text();
|
||||
let error: any;
|
||||
try {
|
||||
error = JSON.parse(errorText);
|
||||
} catch (e) {
|
||||
throw new OasstError(errorText, 0, resp.status);
|
||||
}
|
||||
throw new OasstError(error.message ?? error, error.error_code, resp.status);
|
||||
}
|
||||
|
||||
return await resp.json();
|
||||
}
|
||||
|
||||
private async get(path: string): Promise<any> {
|
||||
const resp = await fetch(`${this.oasstApiUrl}${path}`, {
|
||||
method: "GET",
|
||||
@@ -121,6 +149,34 @@ export class OasstApiClient {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the `BackendUser` associated with `user_id`
|
||||
*/
|
||||
async fetch_user(user_id: string): Promise<BackendUser> {
|
||||
return this.get(`/api/v1/users/users/${user_id}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the `max_count` `BackendUser`s stored by the backend.
|
||||
*/
|
||||
async fetch_users(max_count: number): Promise<BackendUser[]> {
|
||||
return this.get(`/api/v1/frontend_users/?max_count=${max_count}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the `Message`s associated with `user_id` in the backend.
|
||||
*/
|
||||
async fetch_user_messages(user_id: string): Promise<Message[]> {
|
||||
return this.get(`/api/v1/users/${user_id}/messages`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates the backend's knowledge about the `user_id`.
|
||||
*/
|
||||
async set_user_status(user_id: string, is_enabled: boolean, notes): Promise<void> {
|
||||
return this.put(`/api/v1/users/users/${user_id}?enabled=${is_enabled}¬es=${notes}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the valid labels for messages.
|
||||
*/
|
||||
|
||||
@@ -7,6 +7,7 @@ import { useEffect } from "react";
|
||||
import { getAdminLayout } from "src/components/Layout";
|
||||
import { UserMessagesCell } from "src/components/UserMessagesCell";
|
||||
import { post } from "src/lib/api";
|
||||
import { oasstApiClient } from "src/lib/oasst_api_client";
|
||||
import prisma from "src/lib/prismadb";
|
||||
import useSWRMutation from "swr/mutation";
|
||||
|
||||
@@ -68,24 +69,17 @@ const ManageUser = ({ user }) => {
|
||||
}}
|
||||
>
|
||||
<Form>
|
||||
<Field name="user_id" type="hidden" />
|
||||
<Field name="id" type="hidden" />
|
||||
<Field name="name">
|
||||
<Field name="auth_method" type="hidden" />
|
||||
<Field name="display_name">
|
||||
{({ field }) => (
|
||||
<FormControl>
|
||||
<FormLabel>Username</FormLabel>
|
||||
<FormLabel>Display Name</FormLabel>
|
||||
<Input {...field} isDisabled />
|
||||
</FormControl>
|
||||
)}
|
||||
</Field>
|
||||
<Field name="email">
|
||||
{({ field }) => (
|
||||
<FormControl>
|
||||
<FormLabel>Email</FormLabel>
|
||||
<Input {...field} isDisabled />
|
||||
</FormControl>
|
||||
)}
|
||||
</Field>
|
||||
|
||||
<Field name="role">
|
||||
{({ field }) => (
|
||||
<FormControl>
|
||||
@@ -98,13 +92,21 @@ const ManageUser = ({ user }) => {
|
||||
</FormControl>
|
||||
)}
|
||||
</Field>
|
||||
<Field name="notes">
|
||||
{({ field }) => (
|
||||
<FormControl>
|
||||
<FormLabel>Notes</FormLabel>
|
||||
<Input {...field} />
|
||||
</FormControl>
|
||||
)}
|
||||
</Field>
|
||||
<Button mt={4} type="submit">
|
||||
Update
|
||||
</Button>
|
||||
</Form>
|
||||
</Formik>
|
||||
</Container>
|
||||
<UserMessagesCell path={`/api/admin/user_messages?user=${user.id}`} />
|
||||
<UserMessagesCell path={`/api/admin/user_messages?user=${user.user_id}`} />
|
||||
</Stack>
|
||||
</>
|
||||
);
|
||||
@@ -114,15 +116,17 @@ const ManageUser = ({ user }) => {
|
||||
* Fetch the user's data on the server side when rendering.
|
||||
*/
|
||||
export async function getServerSideProps({ query }) {
|
||||
const user = await prisma.user.findUnique({
|
||||
where: { id: query.id },
|
||||
const backend_user = await oasstApiClient.fetch_user(query.id);
|
||||
const local_user = await prisma.user.findUnique({
|
||||
where: { id: backend_user.id },
|
||||
select: {
|
||||
id: true,
|
||||
name: true,
|
||||
email: true,
|
||||
role: true,
|
||||
},
|
||||
});
|
||||
const user = {
|
||||
...backend_user,
|
||||
role: local_user?.role || "general",
|
||||
};
|
||||
return {
|
||||
props: {
|
||||
user,
|
||||
|
||||
@@ -1,22 +1,28 @@
|
||||
import { withRole } from "src/lib/auth";
|
||||
import { oasstApiClient } from "src/lib/oasst_api_client";
|
||||
import prisma from "src/lib/prismadb";
|
||||
|
||||
/**
|
||||
* Update's the user's data in the database. Accessible only to admins.
|
||||
*/
|
||||
const handler = withRole("admin", async (req, res) => {
|
||||
const { id, role } = req.body;
|
||||
const { id, auth_method, user_id, notes, role } = req.body;
|
||||
|
||||
await prisma.user.update({
|
||||
where: {
|
||||
id,
|
||||
},
|
||||
data: {
|
||||
role,
|
||||
},
|
||||
});
|
||||
// If the user is authorized by the web, update their role.
|
||||
if (auth_method === "local") {
|
||||
await prisma.user.update({
|
||||
where: {
|
||||
id,
|
||||
},
|
||||
data: {
|
||||
role,
|
||||
},
|
||||
});
|
||||
}
|
||||
// Tell the backend the user's enabled or not enabled status.
|
||||
await oasstApiClient.set_user_status(user_id, role !== "banned", notes);
|
||||
|
||||
res.status(200).end();
|
||||
res.status(200).json({});
|
||||
});
|
||||
|
||||
export default handler;
|
||||
|
||||
@@ -1,15 +1,13 @@
|
||||
import { withRole } from "src/lib/auth";
|
||||
import { oasstApiClient } from "src/lib/oasst_api_client";
|
||||
import type { Message } from "src/types/Conversation";
|
||||
|
||||
/**
|
||||
* Returns the messages recorded by the backend for a user.
|
||||
*/
|
||||
const handler = withRole("admin", async (req, res) => {
|
||||
const { user } = req.query;
|
||||
|
||||
const messagesRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/frontend_users/local/${user}/messages`, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
"X-API-Key": process.env.FASTAPI_KEY,
|
||||
},
|
||||
});
|
||||
const messages = await messagesRes.json();
|
||||
const messages: Message[] = await oasstApiClient.fetch_user_messages(user as string);
|
||||
res.status(200).json(messages);
|
||||
});
|
||||
|
||||
|
||||
@@ -1,31 +1,44 @@
|
||||
import { withRole } from "src/lib/auth";
|
||||
import { oasstApiClient } from "src/lib/oasst_api_client";
|
||||
import prisma from "src/lib/prismadb";
|
||||
|
||||
// The number of users to fetch in any request.
|
||||
const PAGE_SIZE = 20;
|
||||
|
||||
/**
|
||||
* Returns a list of user results from the database when the requesting user is
|
||||
* a logged in admin.
|
||||
*/
|
||||
const handler = withRole("admin", async (req, res) => {
|
||||
// Figure out the pagination index and skip that number of users.
|
||||
//
|
||||
// Note: with Prisma this isn't the most efficient but it's the only possible
|
||||
// option with cuid based User IDs.
|
||||
const { pageIndex } = req.query;
|
||||
const skip = parseInt(pageIndex as string) * PAGE_SIZE || 0;
|
||||
// TODO(#673): Update this to support pagination.
|
||||
|
||||
// Fetch 20 users.
|
||||
const users = await prisma.user.findMany({
|
||||
// First, get all the users according to the backend.
|
||||
const all_users = await oasstApiClient.fetch_users(20);
|
||||
|
||||
// Next, get all the users stored in the web's auth datbase to fetch their role.
|
||||
const local_user_ids = all_users.map(({ id }) => id);
|
||||
const local_users = await prisma.user.findMany({
|
||||
where: {
|
||||
id: {
|
||||
in: local_user_ids,
|
||||
},
|
||||
},
|
||||
select: {
|
||||
id: true,
|
||||
role: true,
|
||||
name: true,
|
||||
email: true,
|
||||
},
|
||||
skip,
|
||||
take: PAGE_SIZE,
|
||||
});
|
||||
|
||||
// Combine the information by updating the set of full users with their role.
|
||||
// Default any users without a role set locally as "general".
|
||||
const local_user_map = local_users.reduce((result, user) => {
|
||||
result.set(user.id, user.role);
|
||||
return result;
|
||||
}, new Map());
|
||||
|
||||
const users = all_users.map((user) => {
|
||||
const role = local_user_map.get(user.id) || "general";
|
||||
return {
|
||||
...user,
|
||||
role,
|
||||
};
|
||||
});
|
||||
|
||||
res.status(200).json(users);
|
||||
|
||||
@@ -50,7 +50,7 @@ if (boolean(process.env.DEBUG_LOGIN) || process.env.NODE_ENV === "development")
|
||||
where: {
|
||||
id: user.id,
|
||||
},
|
||||
update: {},
|
||||
update: user,
|
||||
create: user,
|
||||
});
|
||||
return user;
|
||||
@@ -86,6 +86,7 @@ export const authOptions: AuthOptions = {
|
||||
*/
|
||||
async session({ session, token }) {
|
||||
session.user.role = token.role;
|
||||
session.user.isNew = token.isNew;
|
||||
return session;
|
||||
},
|
||||
/**
|
||||
@@ -93,11 +94,12 @@ export const authOptions: AuthOptions = {
|
||||
* This let's use forward the role to the session object.
|
||||
*/
|
||||
async jwt({ token }) {
|
||||
const { role } = await prisma.user.findUnique({
|
||||
const { isNew, role } = await prisma.user.findUnique({
|
||||
where: { id: token.sub },
|
||||
select: { role: true },
|
||||
select: { role: true, isNew: true },
|
||||
});
|
||||
token.role = role;
|
||||
token.isNew = isNew;
|
||||
return token;
|
||||
},
|
||||
},
|
||||
|
||||
@@ -17,6 +17,9 @@ const handler = withoutRole("banned", async (req, res, token) => {
|
||||
// Parse out the local task ID and the interaction contents.
|
||||
const { id: frontendId, content, update_type } = req.body;
|
||||
|
||||
// Record that the user has done meaningful work and is no longer new.
|
||||
await prisma.user.update({ where: { id: token.sub }, data: { isNew: false } });
|
||||
|
||||
// Accept the task so that we can complete it, this will probably go away soon.
|
||||
const registeredTask = await prisma.registeredTask.findUniqueOrThrow({ where: { id: frontendId } });
|
||||
const task = registeredTask.task as Prisma.JsonObject;
|
||||
|
||||
@@ -94,7 +94,14 @@ function Signin({ csrfToken, providers }) {
|
||||
{email && (
|
||||
<form onSubmit={signinWithEmail}>
|
||||
<Stack>
|
||||
<Input data-cy="email-address" variant="outline" size="lg" placeholder="Email Address" ref={emailEl} />
|
||||
<Input
|
||||
type="email"
|
||||
data-cy="email-address"
|
||||
variant="outline"
|
||||
size="lg"
|
||||
placeholder="Email Address"
|
||||
ref={emailEl}
|
||||
/>
|
||||
<Button
|
||||
data-cy="signin-email-button"
|
||||
size={"lg"}
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
import Head from "next/head";
|
||||
import { useSession } from "next-auth/react";
|
||||
import { LeaderboardTable, TaskOption } from "src/components/Dashboard";
|
||||
import { getDashboardLayout } from "src/components/Layout";
|
||||
import { TaskCategory } from "src/components/Tasks/TaskTypes";
|
||||
|
||||
const Dashboard = () => {
|
||||
const { data: session } = useSession();
|
||||
|
||||
// TODO(#670): Do something more meaningful when the user is new.
|
||||
console.log(session?.user?.isNew);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Head>
|
||||
|
||||
@@ -27,6 +27,6 @@ const RandomTask = () => {
|
||||
);
|
||||
};
|
||||
|
||||
RandomTask.getLayout = getDashboardLayout;
|
||||
RandomTask.getLayout = (page) => getDashboardLayout(page);
|
||||
|
||||
export default RandomTask;
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* Reports the Backend's knowledge of a user.
|
||||
*/
|
||||
export interface BackendUser {
|
||||
/**
|
||||
* The user's unique ID according to the `auth_method`.
|
||||
*/
|
||||
id: string;
|
||||
|
||||
/**
|
||||
* The user's set name
|
||||
*/
|
||||
display_name: string;
|
||||
|
||||
/**
|
||||
* The authorization method. One of:
|
||||
* - discord
|
||||
* - local
|
||||
*/
|
||||
auth_method: string;
|
||||
|
||||
/**
|
||||
* The backend's UUID for this user.
|
||||
*/
|
||||
user_id: string;
|
||||
|
||||
/**
|
||||
* Arbitrary notes about the user.
|
||||
*/
|
||||
notes: string;
|
||||
|
||||
/**
|
||||
* True when the user is able to access the platform. False otherwise.
|
||||
*/
|
||||
enabled: boolean;
|
||||
|
||||
/**
|
||||
* True when the user is marked for deletion. False otherwise.
|
||||
*/
|
||||
deleted: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* An expanded User for the web.
|
||||
*/
|
||||
export interface User extends BackendUser {
|
||||
/**
|
||||
* The user's roles within the webapp.
|
||||
*/
|
||||
role: string;
|
||||
}
|
||||
Vendored
+4
@@ -6,6 +6,8 @@ declare module "next-auth" {
|
||||
user: {
|
||||
/** The user's role. */
|
||||
role: string;
|
||||
/** True when the user is new. */
|
||||
isNew: boolean;
|
||||
} & DefaultSession["user"];
|
||||
}
|
||||
}
|
||||
@@ -14,5 +16,7 @@ declare module "next-auth/jwt" {
|
||||
interface JWT {
|
||||
/** The user's role. */
|
||||
role?: string;
|
||||
/** True when the user is new. */
|
||||
isNew?: boolean;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user