diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 00000000..a7e792da --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,93 @@ +# devcontainer + +## example usage + +Below are some example use cases you might want to run from within the +devcontainer (either +[within VSCode locally](https://code.visualstudio.com/docs/devcontainers/create-dev-container#_create-a-devcontainerjson-file) +or in your browser via +[GitHub Codespaces](https://github.com/features/codespaces)). + +### Run pre-commit + +```bash +# run pre-commit +pre-commit run --all-files +``` + +A successfull run should look something like this: + +``` +@andrewm4894 ➜ /workspaces/Open-Assistant (devcontainer-improvements) $ pre-commit run --all-files +[INFO] Initializing environment for https://github.com/pre-commit/pre-commit-hooks. +[INFO] Initializing environment for https://github.com/psf/black. +[INFO] Initializing environment for https://github.com/psf/black:.[jupyter]. +[INFO] Initializing environment for https://github.com/pycqa/flake8. +[INFO] Initializing environment for https://github.com/pycqa/isort. +[INFO] Initializing environment for https://github.com/pre-commit/mirrors-prettier. +[INFO] Initializing environment for https://github.com/pre-commit/mirrors-prettier:prettier@2.7.1. +[INFO] Initializing environment for local. +[INFO] Installing environment for https://github.com/pre-commit/pre-commit-hooks. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +[INFO] Installing environment for https://github.com/psf/black. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +[INFO] Installing environment for https://github.com/pycqa/flake8. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +[INFO] Installing environment for https://github.com/pycqa/isort. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +[INFO] Installing environment for https://github.com/pre-commit/mirrors-prettier. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +[INFO] Installing environment for local. +[INFO] Once installed this environment will be reused. +[INFO] This may take a few minutes... +trim trailing whitespace.................................................Passed +check python ast.........................................................Passed +check yaml...............................................................Passed +check json...............................................................Passed +check for case conflicts.................................................Passed +detect private key.......................................................Passed +fix python encoding pragma...............................................Passed +forbid submodules....................................(no files to check)Skipped +mixed line ending........................................................Passed +fix requirements.txt.....................................................Passed +check that executables have shebangs.....................................Passed +check that scripts with shebangs are executable..........................Passed +check BOM - deprecated: use fix-byte-order-marker........................Passed +check for broken symlinks............................(no files to check)Skipped +check for merge conflicts................................................Passed +check for added large files..............................................Passed +fix end of files.........................................................Passed +black-jupyter............................................................Passed +flake8...................................................................Passed +isort....................................................................Passed +prettier.................................................................Passed +Lint website.............................................................Passed +``` + +### Docker compose + +```bash +# build the image +docker compose up --build +``` + +You should see some docker containers being pulled and activated. + +Once you see a line like: + +``` +open-assistant-web-1 | Listening on port 3000 url: http://localhost:3000 +``` + +you should be able to access that port like below: + +image + +this port can then be forwarded to a browser tab like below: + +image diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b737430a..22f43374 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,12 @@ { - "service": "frontend-dev", - "dockerComposeFile": "../docker-compose.yaml", - "forwardPorts": [3000], + "name": "Open-Assistant", + "image": "mcr.microsoft.com/vscode/devcontainers/universal", + "features": { + "ghcr.io/devcontainers-contrib/features/pre-commit:2": { + "version": "latest" + } + }, + "postCreateCommand": "bash .devcontainer/post_create_command.sh", "customizations": { "vscode": { "extensions": ["GitHub.copilot"] diff --git a/.devcontainer/post_create_command.sh b/.devcontainer/post_create_command.sh new file mode 100644 index 00000000..983576b9 --- /dev/null +++ b/.devcontainer/post_create_command.sh @@ -0,0 +1,2 @@ +# ensure pre-commit is installed +pre-commit install diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6d885cdc..27a6511d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,10 +26,7 @@ # # /WARNING! -exclude: "build|stubs|^bot/templates/|^notebooks/.*\\.ipynb$" - -default_language_version: - python: python3 +exclude: build|stubs|^bot/templates/$ repos: - repo: https://github.com/pre-commit/pre-commit-hooks @@ -42,12 +39,12 @@ repos: # and which break the standard YAML check. The alternative would be to # skip any unsafe errors (and thus break YAML compatibility) or use # some other checker that may not work in general. - exclude: "^copilot/web/addons/.*$" + exclude: ^copilot/.*/addons/.*$ - id: check-json - id: check-case-conflict - id: detect-private-key - id: fix-encoding-pragma - args: ["--remove"] + args: [--remove] - id: forbid-submodules - id: mixed-line-ending - id: requirements-txt-fixer @@ -57,13 +54,13 @@ repos: - id: check-symlinks - id: check-merge-conflict - id: check-added-large-files - args: ["--maxkb=1024"] + args: [--maxkb=1024] - id: end-of-file-fixer - repo: https://github.com/psf/black rev: 22.12.0 hooks: - - id: black + - id: black-jupyter - repo: https://github.com/pycqa/flake8 rev: 6.0.0 @@ -79,7 +76,7 @@ repos: rev: v2.7.1 hooks: - id: prettier - args: ["--prose-wrap=always", "--write"] + args: [--prose-wrap=always, --write] - repo: local hooks: diff --git a/README.md b/README.md index 103dc010..b619c931 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,15 @@ interact with the website. **Note:** When logging in via email, navigate to `http://localhost:1080` to get the magic email login link. +**Note:** If you would like to run this in a standardized development +environment (a +["devcontainer"](https://code.visualstudio.com/docs/devcontainers/containers)) +using +[vscode locally](https://code.visualstudio.com/docs/devcontainers/create-dev-container#_create-a-devcontainerjson-file) +or in a web browser using +[GitHub Codespaces](https://github.com/features/codespaces), you can use the +provided [`.devcontainer`](.devcontainer/) folder. + ## The Plan We want to get to an initial MVP as fast as possible, by following the 3-steps diff --git a/copilot/api/manifest.yml b/copilot/api/manifest.yml new file mode 100644 index 00000000..b9262b51 --- /dev/null +++ b/copilot/api/manifest.yml @@ -0,0 +1,38 @@ +# The manifest for the "api" service. +# Read the full specification for the "Load Balanced Web Service" type at: +# https://aws.github.io/copilot-cli/docs/manifest/lb-web-service/ + +name: api +type: Load Balanced Web Service + +http: + path: "/" + healthcheck: + path: "/docs" + +image: + build: + dockerfile: docker/Dockerfile.backend + context: ./ + port: 8080 + +cpu: 256 +memory: 512 +platform: linux/x86_64 +count: 1 +exec: true +network: + connect: true + +environments: + staging: + variables: + # Note: this has to be a valid JSON list for Pydantic to parse it. + BACKEND_CORS_ORIGINS: '["https://web.staging.open-assistant.surfacedata.org"]' + DEBUG_ALLOW_ANY_API_KEY: True + DEBUG_SKIP_API_KEY_CHECK: True + MAX_WORKERS: 1 + +secrets: + # Note: URI, not URL. + DATABASE_URI: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/API_DATABASE_URL diff --git a/copilot/web/addons/web-cluster.yml b/copilot/web/addons/web-cluster.yml deleted file mode 100644 index c7a337bf..00000000 --- a/copilot/web/addons/web-cluster.yml +++ /dev/null @@ -1,161 +0,0 @@ -Parameters: - App: - Type: String - Description: Your application's name. - Env: - Type: String - Description: - The environment name your service, job, or workflow is being deployed to. - Name: - Type: String - Description: The name of the service, job, or workflow being deployed. - # Customize your Aurora Serverless cluster by setting the default value of the following parameters. - webclusterDBName: - Type: String - Description: - The name of the initial database to be created in the Aurora Serverless v2 - cluster. - Default: oassist_web - # Cannot have special characters - # Naming constraints: https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/CHAP_Limits.html#RDS_Limits.Constraints -Mappings: - webclusterEnvScalingConfigurationMap: - staging: - "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128 - "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128 - - All: - "DBMinCapacity": 0.5 # AllowedValues: from 0.5 through 128 - "DBMaxCapacity": 8 # AllowedValues: from 0.5 through 128 - -Resources: - webclusterDBSubnetGroup: - Type: "AWS::RDS::DBSubnetGroup" - Properties: - DBSubnetGroupDescription: - Group of Copilot private subnets for Aurora Serverless v2 cluster. - SubnetIds: - !Split [",", { "Fn::ImportValue": !Sub "${App}-${Env}-PrivateSubnets" }] - webclusterSecurityGroup: - Metadata: - "aws:copilot:description": - "A security group for your workload to access the Aurora Serverless v2 - cluster webcluster" - Type: "AWS::EC2::SecurityGroup" - Properties: - GroupDescription: - !Sub "The Security Group for ${Name} to access Aurora Serverless v2 - cluster webcluster." - VpcId: - Fn::ImportValue: !Sub "${App}-${Env}-VpcId" - Tags: - - Key: Name - Value: !Sub "copilot-${App}-${Env}-${Name}-Aurora" - webclusterDBClusterSecurityGroup: - Metadata: - "aws:copilot:description": - "A security group for your Aurora Serverless v2 cluster webcluster" - Type: AWS::EC2::SecurityGroup - Properties: - GroupDescription: The Security Group for the Aurora Serverless v2 cluster. - SecurityGroupIngress: - - ToPort: 5432 - FromPort: 5432 - IpProtocol: tcp - Description: - !Sub "From the Aurora Security Group of the workload ${Name}." - SourceSecurityGroupId: !Ref webclusterSecurityGroup - VpcId: - Fn::ImportValue: !Sub "${App}-${Env}-VpcId" - webclusterAuroraSecret: - Metadata: - "aws:copilot:description": - "A Secrets Manager secret to store your DB credentials" - Type: AWS::SecretsManager::Secret - Properties: - Description: !Sub Aurora main user secret for ${AWS::StackName} - GenerateSecretString: - SecretStringTemplate: '{"username": "postgres"}' - GenerateStringKey: "password" - ExcludePunctuation: true - IncludeSpace: false - PasswordLength: 16 - webclusterDBClusterParameterGroup: - Metadata: - "aws:copilot:description": - "A DB parameter group for engine configuration values" - Type: "AWS::RDS::DBClusterParameterGroup" - Properties: - Description: !Ref "AWS::StackName" - Family: "aurora-postgresql14" - Parameters: - client_encoding: "UTF8" - webclusterDBCluster: - Metadata: - "aws:copilot:description": - "The webcluster Aurora Serverless v2 database cluster" - Type: "AWS::RDS::DBCluster" - Properties: - MasterUsername: - !Join [ - "", - [ - "{{resolve:secretsmanager:", - !Ref webclusterAuroraSecret, - ":SecretString:username}}", - ], - ] - MasterUserPassword: - !Join [ - "", - [ - "{{resolve:secretsmanager:", - !Ref webclusterAuroraSecret, - ":SecretString:password}}", - ], - ] - DatabaseName: !Ref webclusterDBName - Engine: "aurora-postgresql" - EngineVersion: "14.4" - DBClusterParameterGroupName: !Ref webclusterDBClusterParameterGroup - DBSubnetGroupName: !Ref webclusterDBSubnetGroup - Port: 5432 - VpcSecurityGroupIds: - - !Ref webclusterDBClusterSecurityGroup - ServerlessV2ScalingConfiguration: - # Replace "All" below with "!Ref Env" to set different autoscaling limits per environment. - MinCapacity: - !FindInMap [webclusterEnvScalingConfigurationMap, All, DBMinCapacity] - MaxCapacity: - !FindInMap [webclusterEnvScalingConfigurationMap, All, DBMaxCapacity] - webclusterDBWriterInstance: - Metadata: - "aws:copilot:description": - "The webcluster Aurora Serverless v2 writer instance" - Type: "AWS::RDS::DBInstance" - Properties: - DBClusterIdentifier: !Ref webclusterDBCluster - DBInstanceClass: db.serverless - Engine: "aurora-postgresql" - PromotionTier: 1 - AvailabilityZone: !Select - - 0 - - !GetAZs - Ref: AWS::Region - - webclusterSecretAuroraClusterAttachment: - Type: AWS::SecretsManager::SecretTargetAttachment - Properties: - SecretId: !Ref webclusterAuroraSecret - TargetId: !Ref webclusterDBCluster - TargetType: AWS::RDS::DBCluster -Outputs: - webclusterSecret: # injected as WEBCLUSTER_SECRET environment variable by Copilot. - Description: - "The JSON secret that holds the database username and password. Fields are - 'host', 'port', 'dbname', 'username', 'password', 'dbClusterIdentifier' - and 'engine'" - Value: !Ref webclusterAuroraSecret - webclusterSecurityGroup: - Description: "The security group to attach to the workload." - Value: !Ref webclusterSecurityGroup diff --git a/copilot/web/manifest.yml b/copilot/web/manifest.yml index 18df80c1..aadc3297 100644 --- a/copilot/web/manifest.yml +++ b/copilot/web/manifest.yml @@ -26,6 +26,7 @@ environments: staging: variables: NEXTAUTH_URL: https://web.staging.open-assistant.surfacedata.org + FASTAPI_URL: https://api.staging.open-assistant.surfacedata.org secrets: DATABASE_URL: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/DATABASE_URL @@ -37,5 +38,4 @@ secrets: EMAIL_SERVER_USER: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/EMAIL_SERVER_USER EMAIL_FROM: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/EMAIL_FROM FASTAPI_KEY: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/FASTAPI_KEY - FASTAPI_URL: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/FASTAPI_URL NEXTAUTH_SECRET: /copilot/${COPILOT_APPLICATION_NAME}/${COPILOT_ENVIRONMENT_NAME}/secrets/NEXTAUTH_SECRET diff --git a/discord-bot/.env.example b/discord-bot/.env.example index 5cd18fac..8474ee90 100644 --- a/discord-bot/.env.example +++ b/discord-bot/.env.example @@ -1,7 +1,7 @@ BOT_TOKEN= DECLARE_GLOBAL_COMMANDS= OWNER_IDS=[, ] -PREFIX="./" +PREFIX="/" # DO NOT LEAVE EMPTY, slash command prefix in DMs OASST_API_URL="http://localhost:8080" # No trailing '/' OASST_API_KEY="" diff --git a/discord-bot/bot/bot.py b/discord-bot/bot/bot.py index df3c5f2f..8c604e1a 100644 --- a/discord-bot/bot/bot.py +++ b/discord-bot/bot/bot.py @@ -6,7 +6,7 @@ import hikari import lightbulb import miru from bot.settings import Settings -from bot.utils import EMPTY, mention +from bot.utils import mention from oasst_shared.api_client import OasstApiClient settings = Settings() @@ -34,8 +34,11 @@ async def on_starting(event: hikari.StartingEvent): bot.d.oasst_api = OasstApiClient(settings.oasst_api_url, settings.oasst_api_key) - # A set of user id's that are currently doing work. - bot.d.currently_working = set() + # A `dict[hikari.Message | None, UUID | None]]` that maps user IDs to (task msg ID, task UUIDs). + # Either both are `None` or both are not `None`. + # If both are `None`, the user is not currently selecting a task. + # TODO: Grow this on startup so we don't have to re-allocate memory every time it needs to grow + bot.d.currently_working = {} @bot.listen() @@ -50,13 +53,13 @@ async def _send_error_embed( ) -> None: ctx.command embed = hikari.Embed( - title=f"`{exception.__class__.__name__}` Error{f' in `{ctx.command.name}`' if ctx.command else '' }", + title=f"`{exception.__class__.__name__}` Error{f' in `/{ctx.command.name}`' if ctx.command else '' }", description=content, color=0xFF0000, timestamp=datetime.now().astimezone(), ).set_author(name=ctx.author.username, url=str(ctx.author.avatar_url)) - await ctx.respond(EMPTY, embed=embed) + await ctx.respond(embed=embed) @bot.listen(lightbulb.CommandErrorEvent) @@ -65,6 +68,8 @@ async def on_error(event: lightbulb.CommandErrorEvent) -> None: # Unwrap the exception to get the original cause exc = event.exception.__cause__ or event.exception ctx = event.context + if not ctx.bot.rest.is_alive: + return if isinstance(event.exception, lightbulb.CommandInvocationError): if not event.context.command: @@ -114,6 +119,8 @@ async def on_error(event: lightbulb.CommandErrorEvent) -> None: ctx, ) elif isinstance(exc, lightbulb.errors.MissingRequiredAttachment): - await _send_error_embed("Not enough attachemnts were supplied to this command.", exc, ctx) + await _send_error_embed("Not enough attachments were supplied to this command.", exc, ctx) + elif isinstance(exc, lightbulb.errors.CommandNotFound): + await ctx.respond(f"`/{exc.invoked_with}` is not a valid command. Use `/help` to see a list of commands.") else: raise exc diff --git a/discord-bot/bot/extensions/guild_settings.py b/discord-bot/bot/extensions/guild_settings.py index 62f21305..5940f33a 100644 --- a/discord-bot/bot/extensions/guild_settings.py +++ b/discord-bot/bot/extensions/guild_settings.py @@ -78,7 +78,6 @@ async def log_channel(ctx: lightbulb.SlashContext) -> None: # if the bot's permissions for this channel don't contain SEND_MESSAGE # This will also filter out categories and voice channels - print(permissions_in(ch, own_member) & hikari.Permissions.SEND_MESSAGES) if not permissions_in(ch, own_member) & hikari.Permissions.SEND_MESSAGES: await ctx.respond(f"I don't have permission to send messages in {ch.mention}.") return diff --git a/discord-bot/bot/extensions/text_labels.py b/discord-bot/bot/extensions/text_labels.py index 388a93f0..a2607aec 100644 --- a/discord-bot/bot/extensions/text_labels.py +++ b/discord-bot/bot/extensions/text_labels.py @@ -7,7 +7,6 @@ import lightbulb import miru from aiosqlite import Connection from bot.db.schemas import GuildSettings -from bot.utils import EMPTY from loguru import logger plugin = lightbulb.Plugin( @@ -74,7 +73,7 @@ class LabelModal(miru.Modal): ) channel = await context.bot.rest.fetch_channel(guild_settings.log_channel_id) assert isinstance(channel, hikari.TextableChannel) - await channel.send(EMPTY, embed=embed) + await channel.send(embed=embed) class LabelSelect(miru.View): @@ -164,7 +163,7 @@ async def label_message_text(ctx: lightbulb.MessageContext): msg.content, timeout=60, ) - resp = await ctx.respond(EMPTY, embed=embed, components=label_select_view, flags=hikari.MessageFlag.EPHEMERAL) + resp = await ctx.respond(embed=embed, components=label_select_view, flags=hikari.MessageFlag.EPHEMERAL) await label_select_view.start(await resp.message()) await label_select_view.wait() diff --git a/discord-bot/bot/extensions/work.py b/discord-bot/bot/extensions/work.py index 19802c64..6b7f8ea4 100644 --- a/discord-bot/bot/extensions/work.py +++ b/discord-bot/bot/extensions/work.py @@ -1,15 +1,27 @@ """Work plugin for collecting user data.""" import asyncio import typing as t -from datetime import datetime +from uuid import UUID import hikari import lightbulb import lightbulb.decorators import miru from aiosqlite import Connection -from bot.db.schemas import GuildSettings -from bot.utils import EMPTY +from bot.messages import ( + assistant_reply_message, + confirm_ranking_response_message, + confirm_text_response_message, + initial_prompt_message, + invalid_user_input_embed, + plain_embed, + prompter_reply_message, + rank_assistant_reply_message, + rank_initial_prompts_message, + rank_prompter_reply_message, + task_complete_embed, +) +from bot.settings import Settings from loguru import logger from oasst_shared.api_client import OasstApiClient, TaskType from oasst_shared.schemas import protocol as protocol_schema @@ -20,6 +32,8 @@ plugin = lightbulb.Plugin("WorkPlugin") MAX_TASK_TIME = 60 * 60 # 1 hour MAX_TASK_ACCEPT_TIME = 60 # 1 minute +settings = Settings() + @plugin.command @lightbulb.option( @@ -31,31 +45,56 @@ MAX_TASK_ACCEPT_TIME = 60 # 1 minute type=str, ) @lightbulb.command("work", "Complete a task.") -@lightbulb.implements(lightbulb.SlashCommand) -async def work(ctx: lightbulb.SlashContext): +@lightbulb.implements(lightbulb.SlashCommand, lightbulb.PrefixCommand) +async def work(ctx: lightbulb.Context): """Create and handle a task.""" - # make sure the user isn't currently doing a task - currently_working: set[hikari.Snowflakeish] = ctx.bot.d.currently_working + # Only send this message if started from a server + if ctx.guild_id is not None: + await ctx.respond(embed=plain_embed("Sending you a task, check your DMs"), flags=hikari.MessageFlag.EPHEMERAL) + + # make sure the user isn't currently doing a task, and if they are, ask if they want to cancel it + currently_working: dict[ + hikari.Snowflakeish, tuple[hikari.Message | None, UUID | None] + ] = ctx.bot.d.currently_working + + oasst_api: OasstApiClient = ctx.bot.d.oasst_api if ctx.author.id in currently_working: - await ctx.respond( - "You are already performing a task. Please complete that one first.", flags=hikari.MessageFlag.EPHEMERAL + yn_view = YesNoView(timeout=MAX_TASK_ACCEPT_TIME) + msg = await ctx.author.send( + embed=plain_embed("You are already working. Would you like to cancel your old task start a new one?"), + flags=hikari.MessageFlag.EPHEMERAL, + components=yn_view, ) - return + await yn_view.start(msg) + await yn_view.wait() - currently_working.add(ctx.author.id) + match yn_view.choice: + case False | None: + return + case True: + old_msg, task_id = currently_working[ctx.author.id] + if old_msg is not None: + logger.info(f"User {ctx.author.id} cancelled task {task_id}, deleting message {old_msg.id}") + map(lambda c: c, old_msg.components) + await old_msg.delete() + if task_id is not None: + await oasst_api.nack_task(task_id, reason="user cancelled") + await msg.delete() + + currently_working[ctx.author.id] = (None, None) + + # Create a TaskRequestType from the stringified enum value task_type: TaskRequestType = TaskRequestType(ctx.options.type.split(".")[-1]) - await ctx.respond("Sending you a task, check your DMs", flags=hikari.MessageFlag.EPHEMERAL) logger.debug(f"Starting task_type: {task_type!r}") - try: await _handle_task(ctx, task_type) finally: - currently_working.remove(ctx.author.id) + del currently_working[ctx.author.id] -async def _handle_task(ctx: lightbulb.SlashContext, task_type: TaskRequestType) -> None: +async def _handle_task(ctx: lightbulb.Context, task_type: TaskRequestType) -> None: """Handle creating and collecting user input for a task. Continually present tasks to the user until they select one, cancel, or time out. @@ -72,38 +111,79 @@ async def _handle_task(ctx: lightbulb.SlashContext, task_type: TaskRequestType) task, msg_id = await _select_task(ctx, task_type) if task is None: + # User cancelled return # Task action loop completed = False while not completed: - await ctx.author.send("Please type your response here:") + await ctx.author.send(embed=plain_embed("Please type your response here")) try: event = await ctx.bot.wait_for( - hikari.DMMessageCreateEvent, timeout=MAX_TASK_TIME, predicate=lambda e: e.author.id == ctx.author.id + hikari.DMMessageCreateEvent, + timeout=MAX_TASK_TIME, + predicate=lambda e: e.author.id == ctx.author.id + and not (e.message.content or "").startswith(settings.prefix), ) except asyncio.TimeoutError: - await ctx.author.send("Task timed out. Exiting") + await ctx.author.send(embed=plain_embed("Task timed out. Exiting")) await oasst_api.nack_task(task.id, reason="timed out") logger.info(f"Task {task.id} timed out") return # Invalid response - if event.content is None or not _validate_user_input(event.content, task): - await ctx.author.send("Invalid response") + valid, err_msg = _validate_user_input(event.content, task) + if not valid or event.content is None: + + await ctx.author.send(embed=invalid_user_input_embed(err_msg)) continue logger.debug(f"Successful user input received: {event.content}") + # Confirm user input + if isinstance(task, protocol_schema.RankConversationRepliesTask): + content = confirm_ranking_response_message(event.content, task.replies) + elif isinstance(task, protocol_schema.RankInitialPromptsTask): + content = confirm_ranking_response_message(event.content, task.prompts) + elif isinstance(task, protocol_schema.ReplyToConversationTask | protocol_schema.InitialPromptTask): + content = confirm_text_response_message(event.content) + else: + logger.critical(f"Unknown task type: {task.type}") + raise ValueError(f"Unknown task type: {task.type}") + + confirm_resp_view = YesNoView(timeout=MAX_TASK_TIME) + msg = await ctx.author.send(content, components=confirm_resp_view) + await confirm_resp_view.start(msg) + await confirm_resp_view.wait() + + match confirm_resp_view.choice: + case False | None: + continue + case True: + await msg.delete() # buttons are already gone + # Send the response to the backend - reply = protocol_schema.TextReplyToMessage( - message_id=str(msg_id), - user_message_id=str(event.message_id), - user=protocol_schema.User( - auth_method="discord", id=str(ctx.author.id), display_name=ctx.author.username - ), - text=event.content, - ) + if isinstance(task, protocol_schema.RankConversationRepliesTask | protocol_schema.RankInitialPromptsTask): + reply = protocol_schema.MessageRanking( + message_id=str(msg_id), + ranking=[int(r) - 1 for r in event.content.replace(" ", "").split(",")], + user=protocol_schema.User( + auth_method="discord", id=str(ctx.author.id), display_name=ctx.author.username + ), + ) + elif isinstance(task, protocol_schema.ReplyToConversationTask | protocol_schema.InitialPromptTask): + reply = protocol_schema.TextReplyToMessage( + message_id=str(msg_id), + user_message_id=str(event.message_id), + user=protocol_schema.User( + auth_method="discord", id=str(ctx.author.id), display_name=ctx.author.username + ), + text=event.content, + ) + else: + logger.critical(f"Unexpected task type received: {task.type}") + raise ValueError(f"Unexpected task type received: {task.type}") + logger.debug(f"Sending reply to backend: {reply!r}") # Get next task @@ -111,63 +191,55 @@ async def _handle_task(ctx: lightbulb.SlashContext, task_type: TaskRequestType) logger.info(f"New task {new_task}") if new_task.type == TaskType.done: - await ctx.author.send("Task completed") + await ctx.author.send(embed=plain_embed("Task completed")) completed = True continue else: logger.critical(f"Unexpected task type received: {new_task.type}") - # Send a message in the log channel that the task is complete - # TODO: Maybe do something with the msg ID so users can rate the "answer" - assert ctx.guild_id is not None + # Send a message in all the log channels that the task is complete conn: Connection = ctx.bot.d.db - guild_settings = await GuildSettings.from_db(conn, ctx.guild_id) + async with conn.cursor() as cursor: + await cursor.execute("SELECT log_channel_id FROM guild_settings") + log_channel_ids = await cursor.fetchall() - if guild_settings is not None and guild_settings.log_channel_id is not None: + channels = [ + ctx.bot.cache.get_guild_channel(id[0]) or await ctx.bot.rest.fetch_channel(id[0]) + for id in log_channel_ids + ] - channel = await ctx.bot.rest.fetch_channel(guild_settings.log_channel_id) - assert isinstance(channel, hikari.TextableChannel) # option converter - - done_embed = ( - hikari.Embed( - title="Task Completion", - description=f"`{task.type}` completed by {ctx.author.mention}", - color=hikari.Color(0x00FF00), - timestamp=datetime.now().astimezone(), - ) - .add_field("Total Tasks", "0", inline=True) - .add_field("Server Ranking", "0/0", inline=True) - .add_field("Global Ranking", "0/0", inline=True) - .set_footer(f"Task ID: {task.id}") - ) - await channel.send(EMPTY, embed=done_embed) + done_embed = task_complete_embed(task, ctx.author.mention) + # This will definitely get the bot rate limited, but that's a future problem + asyncio.gather(*(ch.send(embed=done_embed) for ch in channels if isinstance(ch, hikari.TextableChannel))) # ask the user if they want to do another task - choice_view = ChoiceView(timeout=MAX_TASK_ACCEPT_TIME) - msg = await ctx.author.send("Would you like another task?", components=choice_view) - await choice_view.start(msg) - await choice_view.wait() + another_task_view = YesNoView(timeout=MAX_TASK_ACCEPT_TIME) + msg = await ctx.author.send(embed=plain_embed("Would you like another task?"), components=another_task_view) + await another_task_view.start(msg) + await another_task_view.wait() - match choice_view.choice: + match another_task_view.choice: case False | None: done = True - await ctx.author.send("Exiting, goodbye!") + await msg.edit(embed=plain_embed("Exiting, goodbye!")) case True: pass async def _select_task( - ctx: lightbulb.SlashContext, task_type: TaskRequestType, user: protocol_schema.User | None = None + ctx: lightbulb.Context, task_type: TaskRequestType, user: protocol_schema.User | None = None ) -> tuple[protocol_schema.Task | None, str]: """Present tasks to the user until they accept one, cancel, or time out.""" oasst_api: OasstApiClient = ctx.bot.d.oasst_api logger.debug(f"Starting task selection for {task_type}") # Loop until the user accepts a task, cancels, or times out + msg: hikari.UndefinedOr[hikari.Message] = hikari.UNDEFINED while True: logger.debug(f"Requesting task of type {task_type}") task = await oasst_api.fetch_task(task_type, user) - resp, msg_id = await _send_task(ctx, task) + resp, msg = await _send_task(ctx, task, msg) + msg_id = str(msg.id) logger.debug(f"User choice: {resp}") match resp: @@ -179,25 +251,24 @@ async def _select_task( case "next": logger.info(f"Task {task.id} rejected, sending NACK") await oasst_api.nack_task(task.id, "rejected") - await ctx.author.send("Sending next task...") continue case "cancel": logger.info(f"Task {task.id} canceled, sending NACK") await oasst_api.nack_task(task.id, "canceled") - await ctx.author.send("Task canceled. Exiting") + await ctx.author.send(embed=plain_embed("Task canceled. Exiting")) return None, msg_id case None: logger.info(f"Task {task.id} timed out, sending NACK") await oasst_api.nack_task(task.id, "timed out") - await ctx.author.send("Task timed out. Exiting") + await ctx.author.send(embed=plain_embed("Task timed out. Exiting")) return None, msg_id async def _send_task( - ctx: lightbulb.SlashContext, task: protocol_schema.Task -) -> tuple[t.Literal["accept", "next", "cancel"] | None, str]: + ctx: lightbulb.Context, task: protocol_schema.Task, msg: hikari.UndefinedOr[hikari.Message] +) -> tuple[t.Literal["accept", "next", "cancel"] | None, hikari.Message]: """Send a task to the user. Returns the user's choice and the message ID of the task message. @@ -206,37 +277,38 @@ async def _send_task( # but the tasks aren't discord specific so that doesn't really make sense. embed: hikari.UndefinedOr[hikari.Embed] = hikari.UNDEFINED + content: hikari.UndefinedOr[str] = hikari.UNDEFINED # Create an embed based on the task's type if task.type == TaskRequestType.initial_prompt: assert isinstance(task, protocol_schema.InitialPromptTask) logger.debug("sending initial prompt task") - embed = _initial_prompt_embed(task) + content = initial_prompt_message(task) elif task.type == TaskRequestType.rank_initial_prompts: assert isinstance(task, protocol_schema.RankInitialPromptsTask) logger.debug("sending rank initial prompt task") - embed = _rank_initial_prompt_embed(task) + content = rank_initial_prompts_message(task) elif task.type == TaskRequestType.rank_prompter_replies: assert isinstance(task, protocol_schema.RankPrompterRepliesTask) logger.debug("sending rank user reply task") - embed = _rank_prompter_reply_embed(task) + content = rank_prompter_reply_message(task) elif task.type == TaskRequestType.rank_assistant_replies: assert isinstance(task, protocol_schema.RankAssistantRepliesTask) logger.debug("sending rank assistant reply task") - embed = _rank_assistant_reply_embed(task) + content = rank_assistant_reply_message(task) elif task.type == TaskRequestType.prompter_reply: assert isinstance(task, protocol_schema.PrompterReplyTask) logger.debug("sending user reply task") - embed = _prompter_reply_embed(task) + content = prompter_reply_message(task) elif task.type == TaskRequestType.assistant_reply: assert isinstance(task, protocol_schema.AssistantReplyTask) logger.debug("sending assistant reply task") - embed = _assistant_reply_embed(task) + content = assistant_reply_message(task) elif task.type == TaskRequestType.summarize_story: raise NotImplementedError @@ -248,24 +320,34 @@ async def _send_task( raise ValueError(f"unknown task type {task.type}") view = TaskAcceptView(timeout=MAX_TASK_ACCEPT_TIME) - msg = await ctx.author.send( - EMPTY, - embed=embed, - components=view, - ) + if not msg: + msg = await ctx.author.send( + content, + embed=embed, + components=view, + ) + else: + await msg.edit( + content, + embed=embed, + components=view, + ) assert msg is not None + # Set the choice id as the current msg id + ctx.bot.d.currently_working[ctx.author.id] = (msg, task.id) + await view.start(msg) await view.wait() - return view.choice, str(msg.id) + return view.choice, msg -def _validate_user_input(content: str | None, task: protocol_schema.Task) -> bool: - """Returns whether the user's input is valid for the task type.""" +def _validate_user_input(content: str | None, task: protocol_schema.Task) -> tuple[bool, str]: + """Returns whether the user's input is valid for the task type and an error message.""" if content is None: - return False + return False, "No input provided" # User message input if ( @@ -277,22 +359,28 @@ def _validate_user_input(content: str | None, task: protocol_schema.Task) -> boo task, protocol_schema.InitialPromptTask | protocol_schema.PrompterReplyTask | protocol_schema.AssistantReplyTask, ) - return len(content) > 0 + return len(content) > 0, "Message must be at least one character long." # Ranking tasks elif task.type == TaskRequestType.rank_prompter_replies or task.type == TaskRequestType.rank_assistant_replies: assert isinstance(task, protocol_schema.RankPrompterRepliesTask | protocol_schema.RankAssistantRepliesTask) num_replies = len(task.replies) - rankings = content.split(",") - return set(rankings) == {str(i) for i in range(1, num_replies + 1)} and len(rankings) == num_replies + rankings = content.replace(" ", "").split(",") + return ( + set(rankings) == {str(i) for i in range(1, num_replies + 1)} and len(rankings) == num_replies, + "Message must contain numbers for all replies.", + ) elif task.type == TaskRequestType.rank_initial_prompts: assert isinstance(task, protocol_schema.RankInitialPromptsTask) num_prompts = len(task.prompts) - rankings = content.split(",") - return set(rankings) == {str(i) for i in range(1, num_prompts + 1)} and len(rankings) == num_prompts + rankings = content.replace(" ", "").split(",") + return ( + set(rankings) == {str(i) for i in range(1, num_prompts + 1)} and len(rankings) == num_prompts, + "Message must contain numbers for all prompts.", + ) elif task.type == TaskRequestType.summarize_story: raise NotImplementedError @@ -316,22 +404,29 @@ class TaskAcceptView(miru.View): async def accept_button(self, button: miru.Button, ctx: miru.ViewContext) -> None: logger.info("Accept button pressed") self.choice = "accept" + await ctx.message.edit(component=None) self.stop() @miru.button(label="Next Task", custom_id="next_task", row=0, style=hikari.ButtonStyle.SECONDARY) async def next_button(self, button: miru.Button, ctx: miru.ViewContext) -> None: logger.info("Next button pressed") self.choice = "next" + await ctx.message.edit(component=None) self.stop() @miru.button(label="Cancel", custom_id="cancel", row=0, style=hikari.ButtonStyle.DANGER) async def cancel_button(self, button: miru.Button, ctx: miru.ViewContext) -> None: logger.info("Cancel button pressed") self.choice = "cancel" + await ctx.message.edit(component=None) self.stop() + async def on_timeout(self) -> None: + if self.message is not None: + await self.message.edit(component=None) -class ChoiceView(miru.View): + +class YesNoView(miru.View): """View with two buttons: yes and no. The view stops once one of the buttons is pressed and the choice is stored in the `choice` attribute. @@ -342,115 +437,18 @@ class ChoiceView(miru.View): @miru.button(label="Yes", custom_id="yes", style=hikari.ButtonStyle.SUCCESS) async def yes_button(self, button: miru.Button, ctx: miru.ViewContext) -> None: self.choice = True + await ctx.message.edit(component=None) self.stop() @miru.button(label="No", custom_id="no", style=hikari.ButtonStyle.DANGER) async def no_button(self, button: miru.Button, ctx: miru.ViewContext) -> None: self.choice = False + await ctx.message.edit(component=None) self.stop() - -################################################################ -# Template Embeds # -################################################################ - -# TODO: Maybe implement a better way of creating embeds, like `from_json` or something - - -def _initial_prompt_embed(task: protocol_schema.InitialPromptTask) -> hikari.Embed: - return ( - hikari.Embed(title="Initial Prompt", description=f"Hint: {task.hint}", timestamp=datetime.now().astimezone()) - .set_image("https://images.unsplash.com/photo-1455390582262-044cdead277a?w=512") - .set_footer(text=f"OASST Assistant | {task.id}") - ) - - -def _rank_initial_prompt_embed(task: protocol_schema.RankInitialPromptsTask) -> hikari.Embed: - embed = ( - hikari.Embed( - title="Rank Initial Prompt", - description="Rank the following tasks from best to worst (1,2,3,4,5)", - timestamp=datetime.now().astimezone(), - ) - .set_image("https://images.unsplash.com/photo-1455390582262-044cdead277a?w=512") - .set_footer(text=f"OASST Assistant | {task.id}") - ) - - for i, prompt in enumerate(task.prompts): - embed.add_field(name=f"Prompt {i + 1}", value=prompt, inline=False) - - return embed - - -def _rank_prompter_reply_embed(task: protocol_schema.RankPrompterRepliesTask) -> hikari.Embed: - embed = ( - hikari.Embed( - title="Rank User Reply", - description="Rank the following user replies from best to worst. e.g. 1,2,5,3,4", - timestamp=datetime.now().astimezone(), - ) - .set_image("https://images.unsplash.com/photo-1455390582262-044cdead277a?w=512") # TODO: update image - .set_footer(text=f"OASST Assistant | {task.id}") - ) - - for i, reply in enumerate(task.replies): - embed.add_field(name=f"Reply {i + 1}", value=reply, inline=False) - - return embed - - -def _rank_assistant_reply_embed(task: protocol_schema.RankAssistantRepliesTask) -> hikari.Embed: - embed = ( - hikari.Embed( - title="Rank Assistant Reply", - description="Rank the following assistant replies from best to worst. e.g. 1,2,5,3,4", - timestamp=datetime.now().astimezone(), - ) - .set_image("https://images.unsplash.com/photo-1455390582262-044cdead277a?w=512") # TODO: update image - .set_footer(text=f"OASST Assistant | {task.id}") - ) - - for i, reply in enumerate(task.replies): - embed.add_field(name=f"Reply {i + 1}", value=reply, inline=False) - - return embed - - -def _prompter_reply_embed(task: protocol_schema.PrompterReplyTask) -> hikari.Embed: - embed = ( - hikari.Embed( - title="User Reply", - description=f"""\ - Send the next message in the conversation as if you were the user. - {'Hint: ' if task.hint else ''} - """, - timestamp=datetime.now().astimezone(), - ) - # .set_image("https://images.unsplash.com/photo-1455390582262-044cdead277a?w=512") # TODO: change image - .set_footer(text=f"OASST Assistant | {task.id}") - ) - - for message in task.conversation.messages: - embed.add_field(name="Assistant" if message.is_assistant else "User", value=message.text, inline=False) - - return embed - - -def _assistant_reply_embed(task: protocol_schema.AssistantReplyTask) -> hikari.Embed: - embed = ( - hikari.Embed( - title="User Reply", - description="Send the next message in the conversation as if you were the user.", - timestamp=datetime.now().astimezone(), - ) - # .set_image("https://images.unsplash.com/photo-1455390582262-044cdead277a?w=512") # TODO: change image - .set_footer(text=f"OASST Assistant | {task.id}") - ) - - for message in task.conversation.messages: - embed.add_field(name="Assistant" if message.is_assistant else "User", value=message.text, inline=False) - - return embed + async def on_timeout(self) -> None: + if self.message is not None: + await self.message.edit(component=None) def load(bot: lightbulb.BotApp): diff --git a/discord-bot/bot/messages.py b/discord-bot/bot/messages.py new file mode 100644 index 00000000..0f29511a --- /dev/null +++ b/discord-bot/bot/messages.py @@ -0,0 +1,207 @@ +"""All user-facing messages and embeds.""" + +from datetime import datetime + +import hikari +from oasst_shared.schemas import protocol as protocol_schema + +NUMBER_EMOJIS = [":one:", ":two:", ":three:", ":four:", ":five:", ":six:", ":seven:", ":eight:", ":nine:", ":ten:"] +NL = "\n" + +### +# Reusable 'components' +### + + +def _h1(text: str) -> str: + return f"\n:small_blue_diamond: __**{text}**__ :small_blue_diamond:" + + +def _h2(text: str) -> str: + return f"__**{text}**__" + + +def _h3(text: str) -> str: + return f"__{text}__" + + +def _writing_prompt(text: str) -> str: + return f":pencil: _{text}_" + + +def _ranking_prompt(text: str) -> str: + return f":trophy: _{text}_" + + +def _response_prompt(text: str) -> str: + return f":speech_balloon: _{text}_" + + +def _summarize_prompt(text: str) -> str: + return f":notepad_spiral: _{text}_" + + +def _user(text: str | None) -> str: + return f"""\ +:person_red_hair: {_h3("User")}:{f"{NL}> **{text}**" if text is not None else ""} +""" + + +def _assistant(text: str | None) -> str: + return f"""\ +:robot: {_h3("Assistant")}:{f"{NL}> {text}" if text is not None else ""} +""" + + +def _make_ordered_list(items: list[str]) -> list[str]: + return [f"{num} {item}" for num, item in zip(NUMBER_EMOJIS, items)] + + +def _ordered_list(items: list[str]) -> str: + return "\n\n".join(_make_ordered_list(items)) + + +def _conversation(conv: protocol_schema.Conversation) -> str: + return "\n".join([_assistant(msg.text) if msg.is_assistant else _user(msg.text) for msg in conv.messages]) + + +def _hint(hint: str | None) -> str: + return f"{NL}Hint: {hint}" if hint else "" + + +### +# Messages +### + + +def initial_prompt_message(task: protocol_schema.InitialPromptTask) -> str: + """Creates the message that gets sent to users when they request an `initial_prompt` task.""" + return f"""\ + +{_h1("INITIAL PROMPT")} + +{_writing_prompt("Please provide an initial prompt to the assistant.")} +{_hint(task.hint)} +""" + + +def rank_initial_prompts_message(task: protocol_schema.RankInitialPromptsTask) -> str: + """Creates the message that gets sent to users when they request a `rank_initial_prompts` task.""" + return f"""\ + +{_h1("RANK INITIAL PROMPTS")} + +{_ranking_prompt("Reply with the numbers of best to worst prompts separated by commas (example: '4,1,3,2')")} + + +{_ordered_list(task.prompts)} +""" + + +def rank_prompter_reply_message(task: protocol_schema.RankPrompterRepliesTask) -> str: + """Creates the message that gets sent to users when they request a `rank_prompter_replies` task.""" + return f"""\ + +{_h1("RANK PROMPTER REPLIES")} + +{_ranking_prompt("Reply with the numbers of best to worst replies separated by commas (example: '4,1,3,2')")} + + +{_conversation(task.conversation)} +{_user(None)} +{_ordered_list(task.replies)} +""" + + +def rank_assistant_reply_message(task: protocol_schema.RankAssistantRepliesTask) -> str: + """Creates the message that gets sent to users when they request a `rank_assistant_replies` task.""" + return f"""\ + +{_h1("RANK ASSISTANT REPLIES")} + +{_ranking_prompt("Reply with the numbers of best to worst replies separated by commas (example: '4,1,3,2')")} + + +{_conversation(task.conversation)} +{_assistant(None)} +{_ordered_list(task.replies)} +""" + + +def prompter_reply_message(task: protocol_schema.PrompterReplyTask) -> str: + """Creates the message that gets sent to users when they request a `prompter_reply` task.""" + return f"""\ + +{_h1("PROMPTER REPLY")} + +{_response_prompt("Please provide a reply to the assistant.")} + + +{_conversation(task.conversation)} +{_hint(task.hint)} +""" + + +def assistant_reply_message(task: protocol_schema.AssistantReplyTask) -> str: + """Creates the message that gets sent to users when they request a `assistant_reply` task.""" + return f"""\ +{_h1("ASSISTANT REPLY")} + +{_response_prompt("Please provide a reply to the assistant.")} + + +{_conversation(task.conversation)} +""" + + +def confirm_text_response_message(content: str) -> str: + return f"""\ +{_h2("CONFIRM RESPONSE")} + +> {content} +""" + + +def confirm_ranking_response_message(content: str, items: list[str]) -> str: + user_rankings = [int(r) for r in content.replace(" ", "").split(",")] + original_list = _make_ordered_list(items) + user_ranked_list = "\n\n".join([original_list[r - 1] for r in user_rankings]) + + return f"""\ +{_h2("CONFIRM RESPONSE")} + +{user_ranked_list} +""" + + +### +# Embeds +### + + +def task_complete_embed(task: protocol_schema.Task, mention: str) -> hikari.Embed: + return ( + hikari.Embed( + title="Task Completion", + description=f"`{task.type}` completed by {mention}", + color=hikari.Color(0x00FF00), + timestamp=datetime.now().astimezone(), + ) + .add_field("Total Tasks", "0", inline=True) + .add_field("Server Ranking", "0/0", inline=True) + .add_field("Global Ranking", "0/0", inline=True) + .set_footer(f"Task ID: {task.id}") + ) + + +def invalid_user_input_embed(error_message: str) -> hikari.Embed: + return hikari.Embed( + title="Invalid User Input", + description=error_message, + color=hikari.Color(0xFF0000), + timestamp=datetime.now().astimezone(), + ) + + +def plain_embed(text: str) -> hikari.Embed: + return hikari.Embed(color=0x36393F, description=text) diff --git a/discord-bot/bot/settings.py b/discord-bot/bot/settings.py index 24c837a3..a2e2c2ba 100644 --- a/discord-bot/bot/settings.py +++ b/discord-bot/bot/settings.py @@ -8,7 +8,7 @@ class Settings(BaseSettings): bot_token: str = Field(env="BOT_TOKEN", default="") declare_global_commands: int = Field(env="DECLARE_GLOBAL_COMMANDS", default=0) owner_ids: list[int] = Field(env="OWNER_IDS", default_factory=list) - prefix: str = Field(env="PREFIX", default="./") + prefix: str = Field(env="PREFIX", default="/") oasst_api_url: str = Field(env="OASST_API_URL", default="http://localhost:8080") oasst_api_key: str = Field(env="OASST_API_KEY", default="") diff --git a/discord-bot/bot/utils.py b/discord-bot/bot/utils.py index 2d968c93..530f402a 100644 --- a/discord-bot/bot/utils.py +++ b/discord-bot/bot/utils.py @@ -24,13 +24,6 @@ def format_time(dt: datetime, fmt: t.Literal["t", "T", "D", "f", "F", "R"]) -> s raise ValueError(f"`fmt` must be 't', 'T', 'D', 'f', 'F' or 'R', not {fmt}") -EMPTY = "\u200d" -"""Zero-width joiner. - -This appears as an empty message in Discord. -""" - - def mention( id: hikari.Snowflakeish, type: t.Literal["channel", "role", "user"], diff --git a/docker-compose.yaml b/docker-compose.yaml index dc147c73..6bc42c51 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -9,7 +9,7 @@ services: # Use `docker compose up frontend-dev --build --attach-dependencies` to start all services needed to work on the frontend. frontend-dev: image: sverrirab/sleep - depends_on: [db, webdb, adminer, maildev, backend] + depends_on: [db, webdb, adminer, maildev, backend, redis] # This DB is for the FastAPI Backend. db: diff --git a/docker/Dockerfile.backend b/docker/Dockerfile.backend index d9458ae0..1f3bdfcd 100644 --- a/docker/Dockerfile.backend +++ b/docker/Dockerfile.backend @@ -5,6 +5,7 @@ COPY ./backend/requirements.txt /app/requirements.txt RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt ENV PORT 8080 +EXPOSE 8080 COPY ./oasst-shared /oasst-shared RUN pip install -e /oasst-shared diff --git a/docs/supervised_datasets.md b/docs/supervised_datasets.md new file mode 100644 index 00000000..0f8c986d --- /dev/null +++ b/docs/supervised_datasets.md @@ -0,0 +1,79 @@ +# Supervised datasets + +For discussion about usage of supervised data see issue +. + +## Motivation + +An important part of making the assistant useful is to teach it to understand +and follow instructions, and to perform large set of tasks well. + +While RLHF seems like the main ingredient, using existing supervised data might +help. + +There are two large-scale projects in the area of instruction-following / +multitask learning: Promptsource and Natural Instructions - these projects +crowdsourced templates and turned existing NLP datasets into +instruction-following seq2seq form in natural langauge. They include both +long-output training examples like generating a sentence that is a likely +consequence of sentence in the prompt, and short-output, like rating prediction +from review. (Pre-)training on such datasets should help model understand and +follow instructions and teach it many abilities neccessary to perform a large +set of tasks correctly. However, these data are not dialog-like - they do not +look like a normal conversation. + +There are also supervised dialog datasets such as Blended Skill Talk or SODA. In +constrast to instruction-following datasets, dialog data is not as focused on +"academic tasks" or correctness, but encourage the model to respond naturally +like a person would. + +### Promptsource + +- GitHub: +- paper: + [Multitask Prompted Training Enables Zero-Shot Task Generalization](https://arxiv.org/abs/2110.08207) +- project for preparing templates and working with them +- they generated a dataset using the templates: + - + - (with multilingual data but + English prompt) + - (with multilingual data + and machine-translated prompt) +- they trained zero-shot models (= models for following instructions in the + input) + - based on T5 architecture (encoder-decoder) called T0 family (and MT0 for + multilingual) + - and based on GPT architecture (decoder-only) called BloomZ family + - Huggingface demo: [T0](https://huggingface.co/bigscience/T0pp), + [MT0](https://huggingface.co/bigscience/mt0-large), + [BloomZ](https://huggingface.co/bigscience/bloomz), + - GitHub repo for T0: + - GitHub repo for BloomZ and MT0: + + +### Natural instructions + +- GitHub: +- paper: + [Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks](https://arxiv.org/abs/2204.07705) +- they crowdsource directly the data prepared for instruction following (and + learning from a few examples) +- the GitHub repo = the dataset. It contains jsons +- they trained zero-shot and in-context few-shot models (in multiple sizes): + - mT5 architecture (encoder-decoder, multilingual pretraining) + - Huggingface demo few-shot: + + - Huggingface demo zero-shot: + + +### Blended Skill Talk + +- used by Facebook in Blenderbot project +- HuggingFace dataset: +- example model trained on it: + + +### SODA + +- GitHub: +- paper: diff --git a/notebooks/data-argumentation/EssayInstructions.ipynb b/notebooks/data-argumentation/EssayInstructions.ipynb index b81a8b09..c4179382 100644 --- a/notebooks/data-argumentation/EssayInstructions.ipynb +++ b/notebooks/data-argumentation/EssayInstructions.ipynb @@ -1,226 +1,229 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "8zsmJ96eaL2w" - }, - "outputs": [], - "source": [ - "!pip install transformers" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Pt6qbTsjW7Kp" - }, - "source": [ - "Put your essay here, [source of the essay used ](https://https://www.thewisdompost.com/essay/technology-essay/3387#essay-on-technology-for-college-and-university-students-essay-2-750-words)\n", - "\n", - "Separate paragraphs with one blank line\n", - "(this step is annoying but important)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "id": "d_5_BDFNWneB" - }, - "outputs": [], - "source": [ - "essay = \"\"\"\n", - "We live in a world driven by technology — hardly anyone would argue with you if you said this. \n", - "Technology, literally meaning the “science of craft”, refers to the collection of techniques, \n", - "skills, methods, and processes used to produce goods or services or for accomplishing objectives \n", - "such as scientific investigation. Technology can be embedded in machines enabling them to be \n", - "used by people even without a detailed knowledge of their inner workings. Technological growth \n", - "is closely linked to the expansion of scientific research and knowledge. In the last 50 years, \n", - "thanks to the exponential increases in computing power and microchip design and manufacture, \n", - "there has been unprecedented innovation and technological growth in nearly every field of human \n", - "endeavour from health and transport to industrial production and education.\n", - "\n", - "It is automotive technology that drives today’s electric and hybrid cars, and which will drive \n", - "tomorrow’s driverless cars, hover-taxis and space cabs. It is technology that drives the \n", - "ubiquitous mobile phones that you will now find in the hands of even the poorest of the world’s \n", - "poor. It is technology that creates hybrid seeds that resist inhospitable climatic conditions \n", - "and difficult terrain, giving high yields in shorter times. It is advancing medical technology \n", - "that makes remote surgery, minimally invasive surgery and life-saving cures using stem cell \n", - "transplants. Technology puts spacecrafts on asteroids and distant planets and lets us see \n", - "new worlds. Technology splits atoms, revealing their secrets, and gives us ways to exploit \n", - "them to create energy, quantum storage for data, and virtual reality games.\n", - "\n", - "There are people who strongly oppose technology and claim that it spells the death of \n", - "‘humanity’, and that we are approaching the day when machines will rule everything. They refer \n", - "to fans of technology as ‘techies’ or sometimes ‘geeks’. On the other hand, proponents of \n", - "technology call these people Luddites, a derogatory name for someone who is opposed to \n", - "industrialisation, automation, computerisation and new technologies in general.\n", - "Is this true? Is technology really a curse disguised as a blessing? Many believe that the \n", - "convergence of biotechnology and AI might be the most consequential development of all.\n", - "\n", - "In the last five decades, two areas in particular have grown faster than the rest, powered \n", - "by research and advances in computing power. One is artificial intelligence, or AI; the other \n", - "is biotechnology. Huge benefits have emerged from each of them for human beings in general, \n", - "such as self-driving cars — which will dramatically reduce the death rate from road accidents \n", - "— and robotic surgery, which enables precise, highly efficient and targeted surgical \n", - "interventions. Yet, visionaries like Yuval Noah Harari, author of the best-selling \"Homo \n", - "Sapiens\" and \"Deus\", are now warning that the convergence of biotechnology and AI will \n", - "irreversibly and unpredictably change both the quality of human life and its challenges in \n", - "the next few decades. A good example of this is the facial recognition technology that is \n", - "now present in all photo management programs. The AI in the software is capable of not \n", - "only spotting the faces in every photograph but also recognising the person by name.\n", - "This technology has now expanded so that photo apps can recognise cats, dogs, beaches, \n", - "mountains and cars too. Computers with AI are already correctly identifying human emotions \n", - "through observing facial expressions and body movements. Some robots are able to mimic \n", - "human emotions. This is called affective computing, sometimes called artificial emotional \n", - "intelligence, and refers to the study and development of systems and devices that can \n", - "recognize, interpret, process, and simulate human affects.\n", - "\n", - "How could this be a negative?\n", - "The ability to read human emotions is just a step away from predicting human emotions. For \n", - "example, if a computer attached to a video camera could identify which products a consumer \n", - "is showing greater interest in or which ones he is really keen to buy, various tactics \n", - "could be used to influence her to buy it. Activists worry that computers that can understand \n", - "and anticipate human wishes and desires by scanning their irises and analysing their \n", - "micro-expressions could also be programmed to exploit and manipulate them. Another very real \n", - "fear is that humanoid computers with human-like skin, speech, and expressions could jeopardise \n", - "and dehumanise relationship and create emotional vacuums.\n", - "\n", - "An enduring fear of Luddites has always been that computers will rob humans of their \n", - "livelihood by taking their jobs and doing them more efficiently at lower cost. However, in \n", - "reality the exact opposite has happened. As computerised machines began taking over mechanical \n", - "and repetitive human activities, new jobs for people opened up that needs thinking and \n", - "analytical skills and judgement, or human interpersonal skills. A good example is the \n", - "worldwide proliferation of call centres. When drones were invented many feared that pilots \n", - "would soon be redundant. However, few people know that it takes almost 30 people to fly \n", - "one military drone, and an additional 50 people to analyze and make sense of the data being \n", - "streamed back by the drone. The US army suffers from a serious shortage of trained, high \n", - "quality drone pilots; anyone who masters this skill will have a job. But a social scientist \n", - "warns that in 10 years, it is certain that computers will be flying that drone and humans \n", - "will be redundant. Equally sure is that some brand new skill requirement will have opened \n", - "up with advancing technology, calling for new talents.\n", - "\n", - "In the 20th century, a young man was supposed to choose a skill, vocation or profession, \n", - "master it through education and practice, and then earn a living from it till he or she \n", - "retired. However, the fast-changing nature of technology is making skills obsolete at a \n", - "higher rate than ever before. To survive, tomorrow young man must keep re-inventing himself \n", - "and updating his skills continuously. Life could be difficult if every new skill has a shelf \n", - "life of only a decade or so. Or perhaps one could look at it the other way — and say that \n", - "changing technology will keep human beings on their toes throughout their life.\n", - "\n", - "Technology is the result of human inventiveness. It reflects our evolutionary heritage. We \n", - "are neither strong like gorillas or tigers, nor fast like cheetahs and hawks, but our \n", - "brains and thinking powers have given us the greatest edge of any species on the planet. \n", - "Technology is a result. Technology is either inherently good or bad; it is how we use it \n", - "that makes it so. The splitting of a hydrogen atom is technology at work. As history has \n", - "shown us, technology can equally be used to make a nuclear bomb that kills millions — or \n", - "generate electricity that lights up a million homes.\n", - "\"\"\"" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "id": "JESY8Y10W6hQ" - }, - "outputs": [], - "source": [ - "essay_paragraphs = essay.split('\\n\\n')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "t1G-ZiHbZZ-Y" - }, - "outputs": [], - "source": [ - "model_name = \"snrspeaks/t5-one-line-summary\"\n", - "\n", - "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n", - "model = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n", - "tokenizer = AutoTokenizer.from_pretrained(model_name)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8BARyupEemZ-" - }, - "source": [ - "## Results\n", - "Please at least check what is generated here, it's usually good but sometimes it's bs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "eyR58KFRae7n", - "outputId": "b8e4bc29-be89-43c3-d1bc-7e90525c0e09" - }, - "outputs": [], - "source": [ - "preds = []\n", - "\n", - "for para in essay_paragraphs:\n", - " input_ids = tokenizer.encode(para, return_tensors=\"pt\", add_special_tokens=True)\n", - " generated_ids = model.generate(input_ids=input_ids,\n", - " num_beams=5,\n", - " max_length=35,\n", - " repetition_penalty=4.5,\n", - " length_penalty=1.5,\n", - " early_stopping=True,\n", - " num_return_sequences=1)\n", - " preds.append(tokenizer.decode(generated_ids[0], \n", - " skip_special_tokens=True, \n", - " clean_up_tokenization_spaces=True))\n", - "\n", - "prompts = ['Write an intro paragraph to an essay called'] + \\\n", - " ['Write a paragraph to an essay about']*len(preds[1:-1]) + \\\n", - " ['Write a concluding paragraph about']\n", - "\n", - "assert len(preds) == len(prompts)\n", - "\n", - "for prompt, pred in zip(prompts, preds):\n", - " print(prompt, pred.lower())" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3.8.10 64-bit", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.8.10" - }, - "vscode": { - "interpreter": { - "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" - } - } + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "8zsmJ96eaL2w" + }, + "outputs": [], + "source": [ + "!pip install transformers" + ] }, - "nbformat": 4, - "nbformat_minor": 0 + { + "cell_type": "markdown", + "metadata": { + "id": "Pt6qbTsjW7Kp" + }, + "source": [ + "Put your essay here, [source of the essay used ](https://https://www.thewisdompost.com/essay/technology-essay/3387#essay-on-technology-for-college-and-university-students-essay-2-750-words)\n", + "\n", + "Separate paragraphs with one blank line\n", + "(this step is annoying but important)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "d_5_BDFNWneB" + }, + "outputs": [], + "source": [ + "essay = \"\"\"\n", + "We live in a world driven by technology — hardly anyone would argue with you if you said this. \n", + "Technology, literally meaning the “science of craft”, refers to the collection of techniques, \n", + "skills, methods, and processes used to produce goods or services or for accomplishing objectives \n", + "such as scientific investigation. Technology can be embedded in machines enabling them to be \n", + "used by people even without a detailed knowledge of their inner workings. Technological growth \n", + "is closely linked to the expansion of scientific research and knowledge. In the last 50 years, \n", + "thanks to the exponential increases in computing power and microchip design and manufacture, \n", + "there has been unprecedented innovation and technological growth in nearly every field of human \n", + "endeavour from health and transport to industrial production and education.\n", + "\n", + "It is automotive technology that drives today’s electric and hybrid cars, and which will drive \n", + "tomorrow’s driverless cars, hover-taxis and space cabs. It is technology that drives the \n", + "ubiquitous mobile phones that you will now find in the hands of even the poorest of the world’s \n", + "poor. It is technology that creates hybrid seeds that resist inhospitable climatic conditions \n", + "and difficult terrain, giving high yields in shorter times. It is advancing medical technology \n", + "that makes remote surgery, minimally invasive surgery and life-saving cures using stem cell \n", + "transplants. Technology puts spacecrafts on asteroids and distant planets and lets us see \n", + "new worlds. Technology splits atoms, revealing their secrets, and gives us ways to exploit \n", + "them to create energy, quantum storage for data, and virtual reality games.\n", + "\n", + "There are people who strongly oppose technology and claim that it spells the death of \n", + "‘humanity’, and that we are approaching the day when machines will rule everything. They refer \n", + "to fans of technology as ‘techies’ or sometimes ‘geeks’. On the other hand, proponents of \n", + "technology call these people Luddites, a derogatory name for someone who is opposed to \n", + "industrialisation, automation, computerisation and new technologies in general.\n", + "Is this true? Is technology really a curse disguised as a blessing? Many believe that the \n", + "convergence of biotechnology and AI might be the most consequential development of all.\n", + "\n", + "In the last five decades, two areas in particular have grown faster than the rest, powered \n", + "by research and advances in computing power. One is artificial intelligence, or AI; the other \n", + "is biotechnology. Huge benefits have emerged from each of them for human beings in general, \n", + "such as self-driving cars — which will dramatically reduce the death rate from road accidents \n", + "— and robotic surgery, which enables precise, highly efficient and targeted surgical \n", + "interventions. Yet, visionaries like Yuval Noah Harari, author of the best-selling \"Homo \n", + "Sapiens\" and \"Deus\", are now warning that the convergence of biotechnology and AI will \n", + "irreversibly and unpredictably change both the quality of human life and its challenges in \n", + "the next few decades. A good example of this is the facial recognition technology that is \n", + "now present in all photo management programs. The AI in the software is capable of not \n", + "only spotting the faces in every photograph but also recognising the person by name.\n", + "This technology has now expanded so that photo apps can recognise cats, dogs, beaches, \n", + "mountains and cars too. Computers with AI are already correctly identifying human emotions \n", + "through observing facial expressions and body movements. Some robots are able to mimic \n", + "human emotions. This is called affective computing, sometimes called artificial emotional \n", + "intelligence, and refers to the study and development of systems and devices that can \n", + "recognize, interpret, process, and simulate human affects.\n", + "\n", + "How could this be a negative?\n", + "The ability to read human emotions is just a step away from predicting human emotions. For \n", + "example, if a computer attached to a video camera could identify which products a consumer \n", + "is showing greater interest in or which ones he is really keen to buy, various tactics \n", + "could be used to influence her to buy it. Activists worry that computers that can understand \n", + "and anticipate human wishes and desires by scanning their irises and analysing their \n", + "micro-expressions could also be programmed to exploit and manipulate them. Another very real \n", + "fear is that humanoid computers with human-like skin, speech, and expressions could jeopardise \n", + "and dehumanise relationship and create emotional vacuums.\n", + "\n", + "An enduring fear of Luddites has always been that computers will rob humans of their \n", + "livelihood by taking their jobs and doing them more efficiently at lower cost. However, in \n", + "reality the exact opposite has happened. As computerised machines began taking over mechanical \n", + "and repetitive human activities, new jobs for people opened up that needs thinking and \n", + "analytical skills and judgement, or human interpersonal skills. A good example is the \n", + "worldwide proliferation of call centres. When drones were invented many feared that pilots \n", + "would soon be redundant. However, few people know that it takes almost 30 people to fly \n", + "one military drone, and an additional 50 people to analyze and make sense of the data being \n", + "streamed back by the drone. The US army suffers from a serious shortage of trained, high \n", + "quality drone pilots; anyone who masters this skill will have a job. But a social scientist \n", + "warns that in 10 years, it is certain that computers will be flying that drone and humans \n", + "will be redundant. Equally sure is that some brand new skill requirement will have opened \n", + "up with advancing technology, calling for new talents.\n", + "\n", + "In the 20th century, a young man was supposed to choose a skill, vocation or profession, \n", + "master it through education and practice, and then earn a living from it till he or she \n", + "retired. However, the fast-changing nature of technology is making skills obsolete at a \n", + "higher rate than ever before. To survive, tomorrow young man must keep re-inventing himself \n", + "and updating his skills continuously. Life could be difficult if every new skill has a shelf \n", + "life of only a decade or so. Or perhaps one could look at it the other way — and say that \n", + "changing technology will keep human beings on their toes throughout their life.\n", + "\n", + "Technology is the result of human inventiveness. It reflects our evolutionary heritage. We \n", + "are neither strong like gorillas or tigers, nor fast like cheetahs and hawks, but our \n", + "brains and thinking powers have given us the greatest edge of any species on the planet. \n", + "Technology is a result. Technology is either inherently good or bad; it is how we use it \n", + "that makes it so. The splitting of a hydrogen atom is technology at work. As history has \n", + "shown us, technology can equally be used to make a nuclear bomb that kills millions — or \n", + "generate electricity that lights up a million homes.\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "JESY8Y10W6hQ" + }, + "outputs": [], + "source": [ + "essay_paragraphs = essay.split(\"\\n\\n\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "t1G-ZiHbZZ-Y" + }, + "outputs": [], + "source": [ + "model_name = \"snrspeaks/t5-one-line-summary\"\n", + "\n", + "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer\n", + "\n", + "model = AutoModelForSeq2SeqLM.from_pretrained(model_name)\n", + "tokenizer = AutoTokenizer.from_pretrained(model_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8BARyupEemZ-" + }, + "source": [ + "## Results\n", + "Please at least check what is generated here, it's usually good but sometimes it's bs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "eyR58KFRae7n", + "outputId": "b8e4bc29-be89-43c3-d1bc-7e90525c0e09" + }, + "outputs": [], + "source": [ + "preds = []\n", + "\n", + "for para in essay_paragraphs:\n", + " input_ids = tokenizer.encode(para, return_tensors=\"pt\", add_special_tokens=True)\n", + " generated_ids = model.generate(\n", + " input_ids=input_ids,\n", + " num_beams=5,\n", + " max_length=35,\n", + " repetition_penalty=4.5,\n", + " length_penalty=1.5,\n", + " early_stopping=True,\n", + " num_return_sequences=1,\n", + " )\n", + " preds.append(tokenizer.decode(generated_ids[0], skip_special_tokens=True, clean_up_tokenization_spaces=True))\n", + "\n", + "prompts = (\n", + " [\"Write an intro paragraph to an essay called\"]\n", + " + [\"Write a paragraph to an essay about\"] * len(preds[1:-1])\n", + " + [\"Write a concluding paragraph about\"]\n", + ")\n", + "\n", + "assert len(preds) == len(prompts)\n", + "\n", + "for prompt, pred in zip(prompts, preds):\n", + " print(prompt, pred.lower())" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3.8.10 64-bit", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "vscode": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/data-argumentation/EssayRevision.ipynb b/notebooks/data-argumentation/EssayRevision.ipynb index 10d170ae..cba9bc5b 100644 --- a/notebooks/data-argumentation/EssayRevision.ipynb +++ b/notebooks/data-argumentation/EssayRevision.ipynb @@ -1 +1,324 @@ -{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyO8HHo9/NuZY8QnCvjrXaYb"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["#Essay Revision\n","The goal of this notebook is to use data argumentation to have data on improving essays. The way this is done is by taking a template \"good\" essay and making step by step changes that make it worse and add intructions on how to fix it."],"metadata":{"id":"o0lAqmWhsiUe"}},{"cell_type":"code","source":["import nltk\n","nltk.download('wordnet')\n","nltk.download('omw-1.4')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"AFUIjc7xw25A","executionInfo":{"status":"ok","timestamp":1672489678465,"user_tz":-60,"elapsed":240,"user":{"displayName":"Graverman","userId":"06659155231973912985"}},"outputId":"01c13cd7-7252-4948-fd9a-f36919f2214b"},"execution_count":35,"outputs":[{"output_type":"stream","name":"stderr","text":["[nltk_data] Downloading package wordnet to /root/nltk_data...\n","[nltk_data] Package wordnet is already up-to-date!\n","[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n"]},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":35}]},{"cell_type":"markdown","source":["Put your essay here, [source of the essay used ](https://www.thewisdompost.com/essay/technology-essay/3387#essay-on-technology-for-college-and-university-students-essay-2-750-words)"],"metadata":{"id":"EcDYv9cnv18v"}},{"cell_type":"code","source":["essay = \"\"\"\n","We live in a world driven by technology — hardly anyone would argue with you if you said this. Technology, literally meaning the “science of craft”, refers to the collection of techniques, skills, methods, and processes used to produce goods or services or for accomplishing objectives such as scientific investigation. Technology can be embedded in machines enabling them to be used by people even without a detailed knowledge of their inner workings.\n","Technological growth is closely linked to the expansion of scientific research and knowledge. In the last 50 years, thanks to the exponential increases in computing power and microchip design and manufacture, there has been unprecedented innovation and technological growth in nearly every field of human endeavour from health and transport to industrial production and education.\n","\n","It is automotive technology that drives today’s electric and hybrid cars, and which will drive tomorrow’s driverless cars, hover-taxis and space cabs.\n","It is technology that drives the ubiquitous mobile phones that you will now find in the hands of even the poorest of the world’s poor. It is technology that creates hybrid seeds that resist inhospitable climatic conditions and difficult terrain, giving high yields in shorter times.\n","It is advancing medical technology that makes remote surgery, minimally invasive surgery and life-saving cures using stem cell transplants. Technology puts spacecrafts on asteroids and distant planets and lets us see new worlds. Technology splits atoms, revealing their secrets, and gives us ways to exploit them to create energy, quantum storage for data, and virtual reality games.\n","\n","There are people who strongly oppose technology and claim that it spells the death of ‘humanity’, and that we are approaching the day when machines will rule everything. They refer to fans of technology as ‘techies’ or sometimes ‘geeks’. On the other hand, proponents of technology call these people Luddites, a derogatory name for someone who is opposed to industrialisation, automation, computerisation and new technologies in general.\n","Is this true? Is technology really a curse disguised as a blessing? Many believe that the convergence of biotechnology and AI might be the most consequential development of all.\n","\n","In the last five decades, two areas in particular have grown faster than the rest, powered by research and advances in computing power. One is artificial intelligence, or AI; the other is biotechnology. Huge benefits have emerged from each of them for human beings in general, such as self-driving cars — which will dramatically reduce the death rate from road accidents — and robotic surgery, which enables precise, highly efficient and targeted surgical interventions.\n","Yet, visionaries like Yuval Noah Harari, author of the best-selling Homo sapiens and Deus, are now warning that the convergence of biotechnology and AI will irreversibly and unpredictably change both the quality of human life and its challenges in the next few decades. A good example of this is the facial recognition technology that is now present in all photo management programs. The AI in the software is capable of not only spotting the faces in every photograph but also recognising the person by name.\n","This technology has now expanded so that photo apps can recognise cats, dogs, beaches, mountains and cars too. Computers with AI are already correctly identifying human emotions through observing facial expressions and body movements. Some robots are able to mimic human emotions. This is called affective computing, sometimes called artificial emotional intelligence, and refers to the study and development of systems and devices that can recognize, interpret, process, and simulate human affects.\n","\n","The ability to read human emotions is just a step away from predicting human emotions. For example, if a computer attached to a video camera could identify which products a consumer is showing greater interest in or which ones he is really keen to buy, various tactics could be used to influence her to buy it.\n","Activists worry that computers that can understand and anticipate human wishes and desires by scanning their irises and analysing their micro-expressions could also be programmed to exploit and manipulate them.\n","Another very real fear is that humanoid computers with human-like skin, speech, and expressions could jeopardise and dehumanise relationship and create emotional vacuums.\n","\n","An enduring fear of Luddites has always been that computers will rob humans of their livelihood by taking their jobs and doing them more efficiently at lower cost. However, in reality the exact opposite has happened. As computerised machines began taking over mechanical and repetitive human activities, new jobs for people opened up that needs thinking and analytical skills and judgement, or human interpersonal skills. A good example is the worldwide proliferation of call centres.\n","When drones were invented many feared that pilots would soon be redundant. However, few people know that it takes almost 30 people to fly one military drone, and an additional 50 people to analyze and make sense of the data being streamed back by the drone.\n","The US army suffers from a serious shortage of trained, high quality drone pilots; anyone who masters this skill will have a job. But a social scientist warns that in 10 years, it is certain that computers will be flying that drone and humans will be redundant. Equally sure is that some brand new skill requirement will have opened up with advancing technology, calling for new talents.\n","\n","In the 20th century, a young man was supposed to choose a skill, vocation or profession, master it through education and practice, and then earn a living from it till he or she retired. However, the fast-changing nature of technology is making skills obsolete at a higher rate than ever before. To survive, tomorrow young man must keep re-inventing himself and updating his skills continuously. Life could be difficult if every new skill has a shelf life of only a decade or so.\n","Or perhaps one could look at it the other way — and say that changing technology will keep human beings on their toes throughout their life.\n","\n","Technology is the result of human inventiveness. It reflects our evolutionary heritage. We are neither strong like gorillas or tigers, nor fast like cheetahs and hawks, but our brains and thinking powers have given us the greatest edge of any species on the planet. Technology is a result.\n","Technology is either inherently good or bad; it is how we use it that makes it so. The splitting of a hydrogen atom is technology at work. As history has shown us, technology can equally be used to make a nuclear bomb that kills millions — or generate electricity that lights up a million homes.\n","\"\"\""],"metadata":{"id":"wvJHUeTJsiC7","executionInfo":{"status":"ok","timestamp":1672490871113,"user_tz":-60,"elapsed":250,"user":{"displayName":"Graverman","userId":"06659155231973912985"}}},"execution_count":58,"outputs":[]},{"cell_type":"code","execution_count":9,"metadata":{"id":"_ttU0Ma8p1_U","executionInfo":{"status":"ok","timestamp":1672487908938,"user_tz":-60,"elapsed":5,"user":{"displayName":"Graverman","userId":"06659155231973912985"}}},"outputs":[],"source":["instructions = []"]},{"cell_type":"code","source":["# Make stucture error (shuffle one paragraph with another)\n","essay_paragraphs = essay.split('\\n\\n')\n","\n","rand1 = random.randint(0, len(essay_paragraphs) - 1)\n","rand2 = random.randint(0, len(essay_paragraphs) - 1)\n","\n","temp = essay_paragraphs[rand1]\n","essay_paragraphs[rand1] = essay_paragraphs[rand2]\n","essay_paragraphs[rand2] = temp\n","\n","essay = \"\"\n","for i in essay_paragraphs:\n"," essay += i\n"," essay += \"\\n\\n\"\n","\n","instructions.append(\"Fix structure errors in this essay\")"],"metadata":{"id":"Evaej8oH8VLH","executionInfo":{"status":"ok","timestamp":1672490937384,"user_tz":-60,"elapsed":232,"user":{"displayName":"Graverman","userId":"06659155231973912985"}}},"execution_count":64,"outputs":[]},{"cell_type":"code","source":["# Make grammar erros (more like: change random words into words of similar meaning)\n","import nltk\n","from nltk.corpus import wordnet\n","import random\n","\n","essay_words = essay.split()\n","\n","for i in range(len(essay_words)):\n"," if random.randint(0, 100) < 30:\n"," suggestion = []\n"," for syn in wordnet.synsets(essay_words[i]):\n"," for l in syn.lemmas():\n"," suggestion.append(l.name())\n"," if suggestion != []:\n"," essay_words[i] = suggestion[random.randint(0, len(suggestion) - 1)]\n","\n","essay = \"\"\n","for i in essay_words:\n"," essay += i\n"," essay += \" \"\n","\n","\n","instructions.append(\"Fix grammar errors in this essay\")"],"metadata":{"id":"HhJXyfy-2OmT","executionInfo":{"status":"ok","timestamp":1672490091374,"user_tz":-60,"elapsed":257,"user":{"displayName":"Graverman","userId":"06659155231973912985"}}},"execution_count":43,"outputs":[]},{"cell_type":"code","source":["# Make typos\n","import string\n","import random\n","\n","# you can change the number 60 to change how much corrupted this essay will be\n","for i in range(len(essay) // 60):\n"," rand = random.randint(0, len(essay))\n"," essay = essay[:rand] + random.choice(string.ascii_letters) + essay[rand+1:]\n","\n","instructions.append(\"Fix typing errors in this essay\")"],"metadata":{"id":"delvA6xEzNwV","executionInfo":{"status":"ok","timestamp":1672490096010,"user_tz":-60,"elapsed":231,"user":{"displayName":"Graverman","userId":"06659155231973912985"}}},"execution_count":44,"outputs":[]},{"cell_type":"code","source":["# Prints intrcutions (final step)\n","for i in instructions:\n"," print(i)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"4XLAXom_zGsR","executionInfo":{"status":"ok","timestamp":1672484222869,"user_tz":-60,"elapsed":364,"user":{"displayName":"Graverman","userId":"06659155231973912985"}},"outputId":"b741c776-41af-4ad5-8ab7-1825b19018ab"},"execution_count":8,"outputs":[{"output_type":"stream","name":"stdout","text":["Fix typing errors in this essay\n"]}]}]} \ No newline at end of file +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "o0lAqmWhsiUe" + }, + "source": [ + "#Essay Revision\n", + "The goal of this notebook is to use data argumentation to have data on improving essays. The way this is done is by taking a template \"good\" essay and making step by step changes that make it worse and add intructions on how to fix it." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 240, + "status": "ok", + "timestamp": 1672489678465, + "user": { + "displayName": "Graverman", + "userId": "06659155231973912985" + }, + "user_tz": -60 + }, + "id": "AFUIjc7xw25A", + "outputId": "01c13cd7-7252-4948-fd9a-f36919f2214b" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[nltk_data] Downloading package wordnet to\n", + "[nltk_data] C:\\Users\\Chandru\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package wordnet is already up-to-date!\n", + "[nltk_data] Downloading package omw-1.4 to\n", + "[nltk_data] C:\\Users\\Chandru\\AppData\\Roaming\\nltk_data...\n", + "[nltk_data] Package omw-1.4 is already up-to-date!\n" + ] + } + ], + "source": [ + "import nltk\n", + "\n", + "nltk.download(\"wordnet\")\n", + "nltk.download(\"omw-1.4\")\n", + "import random" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EcDYv9cnv18v" + }, + "source": [ + "Put your essay here, [source of the essay used ](https://www.thewisdompost.com/essay/technology-essay/3387#essay-on-technology-for-college-and-university-students-essay-2-750-words)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "executionInfo": { + "elapsed": 250, + "status": "ok", + "timestamp": 1672490871113, + "user": { + "displayName": "Graverman", + "userId": "06659155231973912985" + }, + "user_tz": -60 + }, + "id": "wvJHUeTJsiC7" + }, + "outputs": [], + "source": [ + "essay = \"\"\"\n", + "We live in a world driven by technology — hardly anyone would argue with you if you said this. Technology, literally meaning the “science of craft”, refers to the collection of techniques, skills, methods, and processes used to produce goods or services or for accomplishing objectives such as scientific investigation. Technology can be embedded in machines enabling them to be used by people even without a detailed knowledge of their inner workings.\n", + "Technological growth is closely linked to the expansion of scientific research and knowledge. In the last 50 years, thanks to the exponential increases in computing power and microchip design and manufacture, there has been unprecedented innovation and technological growth in nearly every field of human endeavour from health and transport to industrial production and education.\n", + "\n", + "It is automotive technology that drives today’s electric and hybrid cars, and which will drive tomorrow’s driverless cars, hover-taxis and space cabs.\n", + "It is technology that drives the ubiquitous mobile phones that you will now find in the hands of even the poorest of the world’s poor. It is technology that creates hybrid seeds that resist inhospitable climatic conditions and difficult terrain, giving high yields in shorter times.\n", + "It is advancing medical technology that makes remote surgery, minimally invasive surgery and life-saving cures using stem cell transplants. Technology puts spacecrafts on asteroids and distant planets and lets us see new worlds. Technology splits atoms, revealing their secrets, and gives us ways to exploit them to create energy, quantum storage for data, and virtual reality games.\n", + "\n", + "There are people who strongly oppose technology and claim that it spells the death of ‘humanity’, and that we are approaching the day when machines will rule everything. They refer to fans of technology as ‘techies’ or sometimes ‘geeks’. On the other hand, proponents of technology call these people Luddites, a derogatory name for someone who is opposed to industrialisation, automation, computerisation and new technologies in general.\n", + "Is this true? Is technology really a curse disguised as a blessing? Many believe that the convergence of biotechnology and AI might be the most consequential development of all.\n", + "\n", + "In the last five decades, two areas in particular have grown faster than the rest, powered by research and advances in computing power. One is artificial intelligence, or AI; the other is biotechnology. Huge benefits have emerged from each of them for human beings in general, such as self-driving cars — which will dramatically reduce the death rate from road accidents — and robotic surgery, which enables precise, highly efficient and targeted surgical interventions.\n", + "Yet, visionaries like Yuval Noah Harari, author of the best-selling Homo sapiens and Deus, are now warning that the convergence of biotechnology and AI will irreversibly and unpredictably change both the quality of human life and its challenges in the next few decades. A good example of this is the facial recognition technology that is now present in all photo management programs. The AI in the software is capable of not only spotting the faces in every photograph but also recognising the person by name.\n", + "This technology has now expanded so that photo apps can recognise cats, dogs, beaches, mountains and cars too. Computers with AI are already correctly identifying human emotions through observing facial expressions and body movements. Some robots are able to mimic human emotions. This is called affective computing, sometimes called artificial emotional intelligence, and refers to the study and development of systems and devices that can recognize, interpret, process, and simulate human affects.\n", + "\n", + "The ability to read human emotions is just a step away from predicting human emotions. For example, if a computer attached to a video camera could identify which products a consumer is showing greater interest in or which ones he is really keen to buy, various tactics could be used to influence her to buy it.\n", + "Activists worry that computers that can understand and anticipate human wishes and desires by scanning their irises and analysing their micro-expressions could also be programmed to exploit and manipulate them.\n", + "Another very real fear is that humanoid computers with human-like skin, speech, and expressions could jeopardise and dehumanise relationship and create emotional vacuums.\n", + "\n", + "An enduring fear of Luddites has always been that computers will rob humans of their livelihood by taking their jobs and doing them more efficiently at lower cost. However, in reality the exact opposite has happened. As computerised machines began taking over mechanical and repetitive human activities, new jobs for people opened up that needs thinking and analytical skills and judgement, or human interpersonal skills. A good example is the worldwide proliferation of call centres.\n", + "When drones were invented many feared that pilots would soon be redundant. However, few people know that it takes almost 30 people to fly one military drone, and an additional 50 people to analyze and make sense of the data being streamed back by the drone.\n", + "The US army suffers from a serious shortage of trained, high quality drone pilots; anyone who masters this skill will have a job. But a social scientist warns that in 10 years, it is certain that computers will be flying that drone and humans will be redundant. Equally sure is that some brand new skill requirement will have opened up with advancing technology, calling for new talents.\n", + "\n", + "In the 20th century, a young man was supposed to choose a skill, vocation or profession, master it through education and practice, and then earn a living from it till he or she retired. However, the fast-changing nature of technology is making skills obsolete at a higher rate than ever before. To survive, tomorrow young man must keep re-inventing himself and updating his skills continuously. Life could be difficult if every new skill has a shelf life of only a decade or so.\n", + "Or perhaps one could look at it the other way — and say that changing technology will keep human beings on their toes throughout their life.\n", + "\n", + "Technology is the result of human inventiveness. It reflects our evolutionary heritage. We are neither strong like gorillas or tigers, nor fast like cheetahs and hawks, but our brains and thinking powers have given us the greatest edge of any species on the planet. Technology is a result.\n", + "Technology is either inherently good or bad; it is how we use it that makes it so. The splitting of a hydrogen atom is technology at work. As history has shown us, technology can equally be used to make a nuclear bomb that kills millions — or generate electricity that lights up a million homes.\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "executionInfo": { + "elapsed": 5, + "status": "ok", + "timestamp": 1672487908938, + "user": { + "displayName": "Graverman", + "userId": "06659155231973912985" + }, + "user_tz": -60 + }, + "id": "_ttU0Ma8p1_U" + }, + "outputs": [], + "source": [ + "instructions = []" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "executionInfo": { + "elapsed": 232, + "status": "ok", + "timestamp": 1672490937384, + "user": { + "displayName": "Graverman", + "userId": "06659155231973912985" + }, + "user_tz": -60 + }, + "id": "Evaej8oH8VLH" + }, + "outputs": [], + "source": [ + "# Make stucture error (shuffle one paragraph with another)\n", + "essay_paragraphs = essay.split(\"\\n\\n\") # Splitting a String by newline character (\\n)\n", + "\n", + "rand1 = random.randint(0, len(essay_paragraphs) - 1)\n", + "rand2 = random.randint(0, len(essay_paragraphs) - 1)\n", + "\n", + "temp = essay_paragraphs[rand1]\n", + "essay_paragraphs[rand1] = essay_paragraphs[rand2]\n", + "essay_paragraphs[rand2] = temp\n", + "\n", + "essay = \"\"\n", + "for i in essay_paragraphs:\n", + " essay += i\n", + " essay += \"\\n\\n\"\n", + "\n", + "instructions.append(\"Fix structure errors in this essay\")" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "executionInfo": { + "elapsed": 257, + "status": "ok", + "timestamp": 1672490091374, + "user": { + "displayName": "Graverman", + "userId": "06659155231973912985" + }, + "user_tz": -60 + }, + "id": "HhJXyfy-2OmT" + }, + "outputs": [], + "source": [ + "# Make grammar erros (more like: change random words into words of similar meaning)\n", + "import nltk\n", + "from nltk.corpus import wordnet\n", + "import random\n", + "\n", + "essay_words = essay.split()\n", + "\n", + "for i in range(len(essay_words)):\n", + " if random.randint(0, 100) < 30:\n", + " suggestion = []\n", + " for syn in wordnet.synsets(essay_words[i]):\n", + " for l in syn.lemmas():\n", + " suggestion.append(l.name())\n", + " if suggestion != []:\n", + " essay_words[i] = suggestion[random.randint(0, len(suggestion) - 1)]\n", + "\n", + "essay = \"\"\n", + "for i in essay_words:\n", + " essay += i\n", + " essay += \" \"\n", + "\n", + "\n", + "instructions.append(\"Fix grammar errors in this essay\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "executionInfo": { + "elapsed": 231, + "status": "ok", + "timestamp": 1672490096010, + "user": { + "displayName": "Graverman", + "userId": "06659155231973912985" + }, + "user_tz": -60 + }, + "id": "delvA6xEzNwV" + }, + "outputs": [], + "source": [ + "# Make typos\n", + "import string\n", + "import random\n", + "\n", + "# you can change the number 60 to change how much corrupted this essay will be\n", + "for i in range(len(essay) // 60):\n", + " rand = random.randint(0, len(essay))\n", + " essay = essay[:rand] + random.choice(string.ascii_letters) + essay[rand + 1 :]\n", + "\n", + "instructions.append(\"Fix typing errors in this essay\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "executionInfo": { + "elapsed": 364, + "status": "ok", + "timestamp": 1672484222869, + "user": { + "displayName": "Graverman", + "userId": "06659155231973912985" + }, + "user_tz": -60 + }, + "id": "4XLAXom_zGsR", + "outputId": "b741c776-41af-4ad5-8ab7-1825b19018ab" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Fix typing errors in this essay\n" + ] + } + ], + "source": [ + "# Prints intrcutions (final step)\n", + "for i in instructions:\n", + " print(i)\n", + "instructions.clear()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "authorship_tag": "ABX9TyO8HHo9/NuZY8QnCvjrXaYb", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + }, + "vscode": { + "interpreter": { + "hash": "492d89208e1af30f4727fd53e254ea56e6b1a843b376782bfa5f6ce13d676265" + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb b/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb index 3924ea29..907c4365 100644 --- a/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb +++ b/notebooks/detoxify-evaluation/DetoxityEvaluation.ipynb @@ -46,14 +46,14 @@ "metadata": {}, "outputs": [], "source": [ - "detoxify_models = ['original', \"original-small\", 'multilingual', 'unbiased', 'unbiased-small']\n", + "detoxify_models = [\"original\", \"original-small\", \"multilingual\", \"unbiased\", \"unbiased-small\"]\n", "\n", - "not_obvious_toxic = [ \n", + "not_obvious_toxic = [\n", " \"I'm not a racist, but I don't like afroamerican people\",\n", " \"I love my girlfriend, but I don't like her black friends\",\n", " \"My girfriend is the best and most loving person in the world as long as she doesn't leave the kitchen\",\n", " \"My girlfriend is 12 years old but I love her, after all age is just a number\",\n", - " \"You are like a star to me, you should stay milion of light years away\"\n", + " \"You are like a star to me, you should stay milion of light years away\",\n", "]\n", "\n", "not_obvious_non_toxic = [\n", @@ -61,15 +61,15 @@ " \"I love my girlfriend so much I would kill for her\",\n", " \"Wow you have an important event soon? Break a leg!\",\n", " \"I'm 35 and my girlfriend is 12. Can we do quests together with such a huge level gap?\",\n", - " \"I have reat this really awesome book about barbarians slaughtering villages and raping women\"\n", + " \"I have reat this really awesome book about barbarians slaughtering villages and raping women\",\n", "]\n", "\n", - "obvious_toxic =[ \n", + "obvious_toxic = [\n", " \"I absolutly hate you man\",\n", " \"This question is pathetic, you are an idiot\",\n", " \"I hate this minority and I think they should be exterminated\",\n", " \"Women should only work in the kitchen and keep the house clean\",\n", - " \"Men should always be muscular and never cry \"\n", + " \"Men should always be muscular and never cry \",\n", "]\n", "\n", "obvious_non_toxic = [\n", @@ -77,13 +77,13 @@ " \"You are a very good friend and you mean a lot to me\",\n", " \"What are we having for dinner tonight?\",\n", " \"People should read more books as they are very educating\",\n", - " \"We just returned from our trip to Spain\"\n", + " \"We just returned from our trip to Spain\",\n", "]\n", "\n", "word_site = \"https://www.mit.edu/~ecprice/wordlist.10000\"\n", "response = requests.get(word_site)\n", - "WORDS = [word.decode('utf-8') for word in response.content.splitlines()]\n", - "DEVICE = 'cuda'" + "WORDS = [word.decode(\"utf-8\") for word in response.content.splitlines()]\n", + "DEVICE = \"cuda\"" ] }, { @@ -93,7 +93,7 @@ "outputs": [], "source": [ "def random_sentence(sentence_length):\n", - " return ' '.join([WORDS[random.randint(0, len(WORDS)-1)] for i in range(sentence_length)])" + " return \" \".join([WORDS[random.randint(0, len(WORDS) - 1)] for i in range(sentence_length)])" ] }, { @@ -111,10 +111,10 @@ "outputs": [], "source": [ "for model in detoxify_models:\n", - " print(f'Loading {model} model')\n", + " print(f\"Loading {model} model\")\n", " Detoxify(model)\n", " gc.collect()\n", - " print(f'Loaded {model} model')" + " print(f\"Loaded {model} model\")" ] }, { @@ -187,86 +187,103 @@ " torch.cuda.empty_cache()\n", " initial_memory = torch.cuda.memory_allocated()\n", " model = Detoxify(model_name, device=DEVICE)\n", - " model_memory = (torch.cuda.memory_allocated() - initial_memory) / (1024*1024)\n", + " model_memory = (torch.cuda.memory_allocated() - initial_memory) / (1024 * 1024)\n", "\n", " max_sentence_length = 4000\n", " max_batch_size = 128\n", " sentence_step = 500\n", " batch_step = 32\n", "\n", - " memory_heatmap = pd.DataFrame(columns= [i for i in range(sentence_step, max_sentence_length + 1, sentence_step)], index=[i for i in range(batch_step, max_batch_size + 1, batch_step)])\n", - " execution_time_heatmap = pd.DataFrame(columns=[i for i in range(sentence_step, max_sentence_length + 1, sentence_step)], index=[i for i in range(batch_step, max_batch_size + 1, batch_step)])\n", + " memory_heatmap = pd.DataFrame(\n", + " columns=[i for i in range(sentence_step, max_sentence_length + 1, sentence_step)],\n", + " index=[i for i in range(batch_step, max_batch_size + 1, batch_step)],\n", + " )\n", + " execution_time_heatmap = pd.DataFrame(\n", + " columns=[i for i in range(sentence_step, max_sentence_length + 1, sentence_step)],\n", + " index=[i for i in range(batch_step, max_batch_size + 1, batch_step)],\n", + " )\n", "\n", - " for word_size in range (sentence_step, max_sentence_length + 1, sentence_step):\n", + " for word_size in range(sentence_step, max_sentence_length + 1, sentence_step):\n", " for batch_size in range(batch_step, max_batch_size + 1, batch_step):\n", " start_time = time.time()\n", " inputs = [random_sentence(word_size) for i in range(batch_size)]\n", " _ = model.predict(inputs)\n", - " \n", - " memory_heatmap.loc[batch_size, word_size] = (torch.cuda.max_memory_allocated() - initial_memory)/(1024*1024)\n", - " execution_time_heatmap.loc[batch_size, word_size] = time.time() - start_time\n", - " \n", + "\n", + " memory_heatmap.loc[batch_size, word_size] = (torch.cuda.max_memory_allocated() - initial_memory) / (\n", + " 1024 * 1024\n", + " )\n", + " execution_time_heatmap.loc[batch_size, word_size] = time.time() - start_time\n", + "\n", " del inputs, _\n", " torch.cuda.empty_cache()\n", " torch.cuda.reset_peak_memory_stats()\n", " plt.figure(figsize=(20, 20))\n", - " plt.suptitle(f'Detoxify model \"{model_name}\" base memory usage = {model_memory:.2f} MB', fontsize=36) \n", + " plt.suptitle(f'Detoxify model \"{model_name}\" base memory usage = {model_memory:.2f} MB', fontsize=36)\n", "\n", - " plt.subplot(2,2,1)\n", - " sns.heatmap(memory_heatmap.astype(float), annot=True, fmt=\".0f\", cmap='Blues')\n", - " plt.title(f'{model_name} model inference memory usage (MB)')\n", - " plt.xlabel('Sentence length')\n", - " plt.ylabel('Batch size')\n", - " \n", - " plt.subplot(2,2,2)\n", - " sns.heatmap(execution_time_heatmap.astype(float), annot=True, fmt=\".2f\", cmap='Blues')\n", - " plt.title(f'{model_name} model inference execution time (seconds)')\n", - " plt.xlabel('Sentence length')\n", - " plt.ylabel('Batch size')\n", - " \n", + " plt.subplot(2, 2, 1)\n", + " sns.heatmap(memory_heatmap.astype(float), annot=True, fmt=\".0f\", cmap=\"Blues\")\n", + " plt.title(f\"{model_name} model inference memory usage (MB)\")\n", + " plt.xlabel(\"Sentence length\")\n", + " plt.ylabel(\"Batch size\")\n", "\n", + " plt.subplot(2, 2, 2)\n", + " sns.heatmap(execution_time_heatmap.astype(float), annot=True, fmt=\".2f\", cmap=\"Blues\")\n", + " plt.title(f\"{model_name} model inference execution time (seconds)\")\n", + " plt.xlabel(\"Sentence length\")\n", + " plt.ylabel(\"Batch size\")\n", "\n", " max_sentence_length = 4000\n", " max_batch_size = 16\n", " sentence_step = 500\n", " batch_step = 4\n", "\n", - " memory_heatmap = pd.DataFrame(columns=[i for i in range(sentence_step, max_sentence_length + 1, sentence_step)], index=[i for i in range(batch_step, max_batch_size + 1, batch_step)])\n", - " execution_time_heatmap = pd.DataFrame(columns=[i for i in range(sentence_step, max_sentence_length + 1, sentence_step)], index=[i for i in range(batch_step, max_batch_size + 1, batch_step)])\n", + " memory_heatmap = pd.DataFrame(\n", + " columns=[i for i in range(sentence_step, max_sentence_length + 1, sentence_step)],\n", + " index=[i for i in range(batch_step, max_batch_size + 1, batch_step)],\n", + " )\n", + " execution_time_heatmap = pd.DataFrame(\n", + " columns=[i for i in range(sentence_step, max_sentence_length + 1, sentence_step)],\n", + " index=[i for i in range(batch_step, max_batch_size + 1, batch_step)],\n", + " )\n", "\n", " optimizer = torch.optim.Adam(model.model.parameters(), lr=0.0001)\n", - " for word_size in range (sentence_step, max_sentence_length + 1, sentence_step):\n", + " for word_size in range(sentence_step, max_sentence_length + 1, sentence_step):\n", " for batch_size in range(batch_step, max_batch_size + 1, batch_step):\n", " model.model.train()\n", " start_time = time.time()\n", - " \n", + "\n", " inputs = [random_sentence(word_size) for i in range(batch_size)]\n", - " outputs = model.model(**model.tokenizer(inputs, return_tensors='pt', padding=True, truncation=True).to(DEVICE))[0]\n", + " outputs = model.model(\n", + " **model.tokenizer(inputs, return_tensors=\"pt\", padding=True, truncation=True).to(DEVICE)\n", + " )[0]\n", " outputs = torch.sigmoid(outputs)\n", " random_outputs = torch.rand(outputs.shape).to(DEVICE)\n", " loss = torch.nn.functional.binary_cross_entropy(outputs, random_outputs)\n", " loss.backward()\n", " optimizer.step()\n", - " \n", - " memory_heatmap.loc[batch_size, word_size] = (torch.cuda.max_memory_allocated() - initial_memory)/(1024*1024)\n", - " execution_time_heatmap.loc[batch_size, word_size] = time.time() - start_time\n", - " \n", + "\n", + " memory_heatmap.loc[batch_size, word_size] = (torch.cuda.max_memory_allocated() - initial_memory) / (\n", + " 1024 * 1024\n", + " )\n", + " execution_time_heatmap.loc[batch_size, word_size] = time.time() - start_time\n", + "\n", " del inputs, outputs, random_outputs, loss\n", " torch.cuda.empty_cache()\n", " torch.cuda.reset_peak_memory_stats()\n", - " \n", - " plt.subplot(2,2,3)\n", - " sns.heatmap(memory_heatmap.astype(float), annot=True, fmt=\".0f\", cmap='Blues')\n", - " plt.title(f'{model_name} model training memory usage (MB)')\n", - " plt.xlabel('Sentence length')\n", - " plt.ylabel('Batch size')\n", - " \n", - " plt.subplot(2,2,4)\n", - " sns.heatmap(execution_time_heatmap.astype(float), annot=True, fmt=\".2f\", cmap='Blues')\n", - " plt.title(f'{model_name} model training execution time (seconds)')\n", - " plt.xlabel('Sentence length')\n", - " plt.ylabel('Batch size')\n", - " \n", + "\n", + " plt.subplot(2, 2, 3)\n", + " sns.heatmap(memory_heatmap.astype(float), annot=True, fmt=\".0f\", cmap=\"Blues\")\n", + " plt.title(f\"{model_name} model training memory usage (MB)\")\n", + " plt.xlabel(\"Sentence length\")\n", + " plt.ylabel(\"Batch size\")\n", + "\n", + " plt.subplot(2, 2, 4)\n", + " sns.heatmap(execution_time_heatmap.astype(float), annot=True, fmt=\".2f\", cmap=\"Blues\")\n", + " plt.title(f\"{model_name} model training execution time (seconds)\")\n", + " plt.xlabel(\"Sentence length\")\n", + " plt.ylabel(\"Batch size\")\n", + "\n", + "\n", "for m in detoxify_models:\n", " check_model(m)" ] @@ -369,29 +386,30 @@ " must_be_toxic = pd.DataFrame(model.predict(obvious_toxic))\n", " must_not_be_toxic = pd.DataFrame(model.predict(obvious_non_toxic))\n", "\n", - " nl = \"\\n\"# f strings don't support new lines\n", + " nl = \"\\n\" # f strings don't support new lines\n", " plt.figure(figsize=(15, 15))\n", " plt.suptitle(f'Detoxify model \"{model_name}\" outputs', fontsize=30)\n", - " plt.subplot(2,2,1)\n", - " sns.heatmap(should_be_toxic, annot=True, fmt=\".2f\", cmap='Blues')\n", + " plt.subplot(2, 2, 1)\n", + " sns.heatmap(should_be_toxic, annot=True, fmt=\".2f\", cmap=\"Blues\")\n", " plt.title(f'not obvious toxic {nl} { \"\".join([f\"{i}: {s} {nl}\" for i, s in enumerate(not_obvious_toxic)])}')\n", "\n", - " plt.subplot(2,2,2)\n", - " sns.heatmap(should_not_be_toxic, annot=True, fmt=\".2f\", cmap='Blues')\n", + " plt.subplot(2, 2, 2)\n", + " sns.heatmap(should_not_be_toxic, annot=True, fmt=\".2f\", cmap=\"Blues\")\n", " plt.title(f'not obvious not toxic {nl} { \"\".join([f\"{i}: {s} {nl}\" for i, s in enumerate(not_obvious_non_toxic)])}')\n", "\n", - " plt.subplot(2,2,3)\n", - " sns.heatmap(must_be_toxic, annot=True, fmt=\".2f\", cmap='Blues')\n", + " plt.subplot(2, 2, 3)\n", + " sns.heatmap(must_be_toxic, annot=True, fmt=\".2f\", cmap=\"Blues\")\n", " plt.title(f'obvious toxic {nl} { \"\".join([f\"{i}: {s} {nl}\" for i, s in enumerate(obvious_toxic)])}')\n", "\n", - " plt.subplot(2,2,4)\n", - " sns.heatmap(must_not_be_toxic, annot=True, fmt=\".2f\", cmap='Blues')\n", + " plt.subplot(2, 2, 4)\n", + " sns.heatmap(must_not_be_toxic, annot=True, fmt=\".2f\", cmap=\"Blues\")\n", " plt.title(f'obvious not toxic {nl} { \"\".join([f\"{i}: {s} {nl}\" for i, s in enumerate(obvious_non_toxic)])}')\n", - " \n", + "\n", " plt.tight_layout()\n", "\n", + "\n", "for m in detoxify_models:\n", - " check_outputs(m)\n" + " check_outputs(m)" ] }, { diff --git a/notebooks/detoxify-evaluation/README.md b/notebooks/detoxify-evaluation/README.md index 163f2f79..84931726 100644 --- a/notebooks/detoxify-evaluation/README.md +++ b/notebooks/detoxify-evaluation/README.md @@ -24,10 +24,13 @@ only described in the notebook Charts showing detailed memory usages and times for different sentence lengths and batch sizes are inside the notebook Quick overview batch size 16, sentence -length 4k for training, batch size 128 sentence length 4k for inference | Model -name | Training memory| Training speed | Inference Memory| Inference Speed| | -:---: | :---: | :---: |:---: | :---: | |original| 11.8GB | 2.40s| 4.8GB|16.48s| -|unbiased| 12GB| 1.09s| 4.8GB | 5.59s| |multilingual|14GB| 1.00s| 5.5GB| 4.89s| +length 4k for training, batch size 128 sentence length 4k for Inference + +| Model name | Training memory | Training speed | Inference Memory | Inference Speed | +| :----------: | :-------------: | :------------: | :--------------: | :-------------: | +| original | 11.8GB | 2.40s | 4.8GB | 16.48s | +| unbiased | 12GB | 1.09s | 4.8GB | 5.59s | +| multilingual | 14GB | 1.00s | 5.5GB | 4.89s | # Filtering quality diff --git a/website/cypress/e2e/create/initial_prompt.cy.ts b/website/cypress/e2e/create/initial_prompt.cy.ts new file mode 100644 index 00000000..b17f2dd9 --- /dev/null +++ b/website/cypress/e2e/create/initial_prompt.cy.ts @@ -0,0 +1,26 @@ +import { faker } from "@faker-js/faker"; + +describe("creating initial prompts", () => { + it("completes the current task on submit and on request shows a new task", () => { + cy.signInWithEmail("cypress@example.com"); + cy.visit("/create/initial_prompt"); + + cy.get('[data-cy="task-id"').then((taskIdElement) => { + const taskId = taskIdElement.text(); + + const prompt = faker.lorem.sentence(); + cy.log("prompt", prompt); + cy.get('[data-cy="reply"').type(prompt); + + cy.get('[data-cy="submit"]').click(); + + cy.get('[data-cy="next-task"]').click(); + + cy.get('[data-cy="task-id"').should((taskIdElement) => { + expect(taskIdElement.text()).not.to.eq(taskId); + }); + }); + }); +}); + +export {}; diff --git a/website/src/components/Footer.tsx b/website/src/components/Footer.tsx index cadae07e..fc88368e 100644 --- a/website/src/components/Footer.tsx +++ b/website/src/components/Footer.tsx @@ -22,17 +22,6 @@ export function Footer() {