diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..eb3a37f7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.venv +*.pyc diff --git a/backend/app/api/v1/api.py b/backend/app/api/v1/api.py index 7c97c9cb..5a704c2d 100644 --- a/backend/app/api/v1/api.py +++ b/backend/app/api/v1/api.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- -from app.api.v1 import labelers, prompts +from app.api.v1 import labelers, prompts, tasks from fastapi import APIRouter api_router = APIRouter() api_router.include_router(labelers.router, prefix="/labelers", tags=["labelers"]) api_router.include_router(prompts.router, prefix="/prompts", tags=["prompts"]) +api_router.include_router(tasks.router, prefix="/tasks", tags=["tasks"]) diff --git a/backend/app/api/v1/tasks.py b/backend/app/api/v1/tasks.py new file mode 100644 index 00000000..2a5397e5 --- /dev/null +++ b/backend/app/api/v1/tasks.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +from typing import Any, List +from uuid import UUID + +from app.api import deps +from app.schemas import protocol as protocol_schema +from fastapi import APIRouter, Depends, HTTPException +from fastapi.security.api_key import APIKey +from loguru import logger +from sqlmodel import Session +from starlette.status import HTTP_400_BAD_REQUEST + +router = APIRouter() + + +@router.post("/", response_model=List[protocol_schema.SummarizeStoryTask]) # work with Union once more types are added +def request_task( + *, + db: Session = Depends(deps.get_db), + api_key: APIKey = Depends(deps.get_api_key), + request: protocol_schema.GenericTaskRequest, # work with Union once more types are added +) -> Any: + """ + Create new task. + """ + deps.api_auth(api_key, db, create=True) + + # TODO: Create a task and store it in the database. + + match (request.type): + case "generic": + # here we create a task at random (and store it in the database) + logger.info("Frontend requested a generic task.") + task = protocol_schema.SummarizeStoryTask( + story="This is a story. A very long story. So long, it needs to be summarized.", + ) + + case _: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail="Invalid request type.", + ) + if request.user_id is not None: + task.addressed_users = [request.user_id] + + return [task] + + +@router.post("/{task_id}/ack") +def acknowledge_task( + *, + db: Session = Depends(deps.get_db), + api_key: APIKey = Depends(deps.get_api_key), + task_id: UUID, + response: protocol_schema.PostCreatedTaskResponse, +) -> Any: + """ + The frontend acknowledges a task. + """ + deps.api_auth(api_key, db, create=True) + + match (response.type): + case "post_created": + logger.info(f"Frontend acknowledged {task_id=} and created {response.post_id=}.") + # here we would store the post id in the database for the task + case _: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail="Invalid response type.", + ) + + return {} + + +@router.post("/interaction") +def post_interaction( + *, + db: Session = Depends(deps.get_db), + api_key: APIKey = Depends(deps.get_api_key), + interaction: protocol_schema.TextReplyToPost, +) -> Any: + """ + The frontend reports an interaction. + """ + deps.api_auth(api_key, db, create=True) + + response = [] + match (interaction.type): + case "text_reply_to_post": + logger.info( + f"Frontend reports text reply to {interaction.post_id=} with {interaction.text=} by {interaction.user_id=}." + ) + # here we would store the text reply in the database + response.append( + protocol_schema.TaskDone( + reply_to_post_id=interaction.user_post_id, + addressed_users=[interaction.user_id], + ) + ) + case _: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail="Invalid response type.", + ) + + return response diff --git a/backend/app/config.py b/backend/app/config.py index 85cd8631..f0218bee 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -# touch -from typing import List, Optional, Union +from typing import Any, Dict, List, Optional, Union from pydantic import AnyHttpUrl, BaseSettings, PostgresDsn, validator @@ -8,8 +7,25 @@ from pydantic import AnyHttpUrl, BaseSettings, PostgresDsn, validator class Settings(BaseSettings): PROJECT_NAME: str = "open-chatGPT backend" API_V1_STR: str = "/api/v1" + + POSTGRES_SERVER: str = "localhost:5432" + POSTGRES_USER: str = "postgres" + POSTGRES_PASSWORD: str = "postgres" + POSTGRES_DB: str = "postgres" DATABASE_URI: Optional[PostgresDsn] = None + @validator("DATABASE_URI", pre=True) + def assemble_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any: + if isinstance(v, str): + return v + return PostgresDsn.build( + scheme="postgresql", + user=values.get("POSTGRES_USER"), + password=values.get("POSTGRES_PASSWORD"), + host=values.get("POSTGRES_SERVER"), + path=f"/{values.get('POSTGRES_DB') or ''}", + ) + BACKEND_CORS_ORIGINS: List[AnyHttpUrl] = [] UPDATE_ALEMBIC: bool = True diff --git a/backend/app/schemas/protocol.py b/backend/app/schemas/protocol.py new file mode 100644 index 00000000..554c49e1 --- /dev/null +++ b/backend/app/schemas/protocol.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +from typing import Literal, Optional +from uuid import UUID + +import pydantic +from pydantic import BaseModel + + +class TaskRequest(BaseModel): + """The frontend asks the backend for a task.""" + + type: str + user_id: Optional[str] = None + + +class GenericTaskRequest(TaskRequest): + type: Literal["generic"] = "generic" + + +class Task(BaseModel): + """A task is a unit of work that the backend gives to the frontend.""" + + id: UUID = pydantic.Field(default_factory=UUID) + type: str + addressed_users: Optional[list[str]] = None + + +class TaskResponse(BaseModel): + """A task response is a message from the frontend to acknowledge the given task.""" + + type: str + status: Literal["success", "failure"] + + +class PostCreatedTaskResponse(TaskResponse): + type: Literal["post_created"] = "post_created" + post_id: UUID + + +class SummarizeStoryTask(Task): + type: Literal["summarize_story"] = "summarize_story" + story: str + + +class TaskDone(Task): + type: Literal["task_done"] = "task_done" + reply_to_post_id: UUID + + +class Interaction(BaseModel): + """An interaction is a message from the frontend to the backend.""" + + type: str + user_id: str + + +class TextReplyToPost(Interaction): + """A user has replied to a post with text.""" + + type: Literal["text_reply_to_post"] = "text_reply_to_post" + post_id: UUID + user_post_id: UUID + text: str diff --git a/backend/postprocessing/rankings.py b/backend/postprocessing/rankings.py new file mode 100644 index 00000000..38686f67 --- /dev/null +++ b/backend/postprocessing/rankings.py @@ -0,0 +1,141 @@ +# -*- coding: utf-8 -*- +from typing import List + +import numpy as np + + +def head_to_head_votes(ranks: List[List[int]]): + tallies = np.zeros((len(ranks[0]), len(ranks[0]))) + names = sorted(ranks[0]) + ranks = np.array(ranks) + # we want the sorted indices + ranks = np.argsort(ranks, axis=1) + for i in range(ranks.shape[1]): + for j in range(i + 1, ranks.shape[1]): + # now count the cases someone voted for i over j + over_j = np.sum(ranks[:, i] < ranks[:, j]) + over_i = np.sum(ranks[:, j] < ranks[:, i]) + tallies[i, j] = over_j + # tallies[i,j] = over_i + tallies[j, i] = over_i + # tallies[j,i] = over_j + return tallies, names + + +def cycle_detect(pairs): + """Recursively detect cylces by removing condorcet losers until either only one pair is left or condorcet loosers no longer exist + This method upholds the invariant that in a ranking for all a,b either a>b or b>a for all a,b. + + + Returns + ------- + out : False if the pairs do not contain a cycle, True if the pairs contain a cycle + + + """ + # get all condorcet losers (pairs that loose to all other pairs) + # idea: filter all losers that are never winners + # print("pairs", pairs) + if len(pairs) <= 1: + return False + losers = [c_lose for c_lose in np.unique(pairs[:, 1]) if c_lose not in pairs[:, 0]] + if len(losers) == 0: + # if we recursively removed pairs, and at some point we did not have + # a condorcet loser, that means everything is both a winner and loser, + # yielding at least one (winner,loser), (loser,winner) pair + return True + + new = [] + for p in pairs: + if p[1] not in losers: + new.append(p) + return cycle_detect(np.array(new)) + + +def get_winner(pairs): + """ + This returns _one_ concordant winner. + It could be that there are multiple concordant winners, but in our case + since we are interested in a ranking, we have to choose one at random. + """ + losers = np.unique(pairs[:, 1]).astype(int) + winners = np.unique(pairs[:, 0]).astype(int) + for w in winners: + if w not in losers: + return w + + +def get_ranking(pairs): + """ + Abuses concordance property to get a (not necessarily unqiue) ranking. + The lack of uniqueness is due to the potential existance of multiple + equally ranked winners. We have to pick one, which is where + the non-uniqueness comes from + """ + if len(pairs) == 1: + return list(pairs[0]) + w = get_winner(pairs) + # now remove the winner from the list of pairs + p_new = np.array([(a, b) for a, b in pairs if a != w]) + return [w] + get_ranking(p_new) + + +def ranked_pairs(ranks: List[List[int]]): + """ + Expects a list of rankings for an item like: + [("w","x","z","y") for _ in range(3)] + + [("w","y","x","z") for _ in range(2)] + + [("x","y","z","w") for _ in range(4)] + + [("x","z","w","y") for _ in range(5)] + + [("y","w","x","z") for _ in range(1)] + This code is quite brain melting, but the idea is the following: + 1. create a head-to-head matrix that tallies up all win-lose combinations of preferences + 2. take all combinations that win more than they loose and sort those by how often they win + 3. use that to create an (implicit) directed graph + 4. recursively extract nodes from the graph that do not have incoming edges + 5. said recursive list is the ranking + """ + tallies, names = head_to_head_votes(ranks) + tallies = tallies - tallies.T + # print(tallies) + # note: the resulting tally matrix should be skew-symmetric + # order by strenght of victory (using tideman's original method, don't think it would make a difference for us) + sorted_majorities = [] + for i in range(len(ranks[0])): + for j in range(len(ranks[i])): + if tallies[i, j] > 0: + sorted_majorities.append((i, j, tallies[i, j])) + # we don't explicitly deal with tied majorities here + sorted_majorities = np.array(sorted(sorted_majorities, key=lambda x: x[2], reverse=True)) + # now do lock ins + lock_ins = [] + for (x, y, _) in sorted_majorities: + # invariant: lock_ins has no cycles here + lock_ins.append((x, y)) + # print("lock ins are now",np.array(lock_ins)) + if cycle_detect(np.array(lock_ins)): + # print("backup: cycle detected") + # if there's a cycle, delete the new addition and continue + lock_ins = lock_ins[:-1] + # now simply return all winners in order, and attach the losers + # to the back. This is because the overall loser might not be unique + # and (by concordance property) may never exist in any winning set to begin with. + # (otherwise he would either not be the loser, or cycles exist!) + # Since there could be multiple overall losers, we just return them in any order + # as we are unable to find a closer ranking + numerical_ranks = np.array(get_ranking(np.array(lock_ins))).astype(int) + conversion = [names[n] for n in numerical_ranks] + return conversion + + +if __name__ == "__main__": + ranks = ( + [("w", "x", "z", "y") for _ in range(1)] + + [("w", "y", "x", "z") for _ in range(2)] + # + [("x","y","z","w") for _ in range(4)] + + [("x", "z", "w", "y") for _ in range(5)] + + [("y", "w", "x", "z") for _ in range(1)] + # [("y","z","w","x") for _ in range(1000)] + ) + rp = ranked_pairs(ranks) + print(rp) diff --git a/backend/requirements.txt b/backend/requirements.txt index 92668609..b882d594 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,6 +1,7 @@ alembic==1.8.1 fastapi==0.88.0 loguru==0.6.0 +numpy==1.22.4 psycopg2-binary==2.9.5 pydantic==1.9.1 python-dotenv==0.21.0 diff --git a/backend/scripts/run-local.sh b/backend/scripts/run-local.sh index 6e89ddf0..d8bd86c6 100755 --- a/backend/scripts/run-local.sh +++ b/backend/scripts/run-local.sh @@ -1,5 +1,3 @@ #!/usr/bin/env bash -export DATABASE_URI=postgresql://fozziethebeat@localhost:5432/ocgpt_backend - uvicorn app.main:app --reload diff --git a/bot/bot.py b/bot/bot.py index 03da1483..aff74b26 100644 --- a/bot/bot.py +++ b/bot/bot.py @@ -7,6 +7,7 @@ import discord import requests from discord import app_commands from dotenv import load_dotenv +from loguru import logger bot_url = "https://discord.com/api/oauth2/authorize?client_id=1051614245940375683&permissions=8&scope=bot" @@ -26,6 +27,9 @@ headers = {"X-API-Key": API_SERVER_KEY} # For testing only. TEST_GUILD = os.getenv("TEST_GUILD") +TEST_GUILD_LAION = os.getenv("TEST_GUILD_LAION") +# TEST_GUILD = False +guild_ids = [TEST_GUILD, TEST_GUILD_LAION] # Initiate the client and command tree to create slash commands. @@ -38,13 +42,18 @@ class OpenChatGPTClient(discord.Client): if TEST_GUILD: # When testing the bot it's handy to run in a single server (called a # Guide in the API). This is relatively fast. - guild = discord.Object(id=TEST_GUILD) - self.tree.copy_global_to(guild=guild) - await self.tree.sync(guild=guild) + for guild_id in guild_ids: + guild = discord.Object(id=guild_id) + self.tree.copy_global_to(guild=guild) + await self.tree.sync(guild=guild) + + # guild = discord.Object(id=TEST_GUILD) + # self.tree.copy_global_to(guild=guild) + # await self.tree.sync(guild=guild) else: # This can take up to an hour for the commands to be registered. await self.tree.sync() - print("Ready!") + logger.debug("Ready!") # List the set of intents needed for commands to operate properly. @@ -53,6 +62,46 @@ intents.message_content = True client = OpenChatGPTClient(intents=intents) +class LikeButton(discord.ui.Button): + def __init__(self, label, channel, username, prompt): + super().__init__(label=label, style=discord.ButtonStyle.green, emoji="👍") + self.channel = channel + self.username = username + self.prompt = prompt + + async def callback(self, interaction): + # interaction holds the interaction object + # await interaction.response.defer() + await interaction.response.send_message("Thanks for your feedback. You liked this 👍 ") + + +class NeutralButton(discord.ui.Button): + def __init__(self, label, channel, username, prompt): + super().__init__(label=label, style=discord.ButtonStyle.green, emoji="😐") + self.channel = channel + self.username = username + self.prompt = prompt + + async def callback(self, interaction): + # interaction holds the interaction object + # await interaction.response.defer() + await interaction.response.send_message("Thanks for your feedback. You thought this was neutral 😐 ") + + +class DislikeButton(discord.ui.Button): + def __init__(self, label, channel, username, prompt): + super().__init__(label=label, style=discord.ButtonStyle.green, emoji="👎") + self.channel = channel + self.username = username + self.prompt = prompt + + async def callback(self, interaction): + # interaction holds the interaction object + # await interaction.response.defer() + # send the feedback to the backend # + await interaction.response.send_message("Thanks for your feedback. You disliked this 👎 ") + + @client.tree.command() async def register(interaction: discord.Interaction): """Registers the user for submissions.""" @@ -65,7 +114,7 @@ async def register(interaction: discord.Interaction): if response.status_code == 200: await interaction.response.send_message(f"Added you {interaction.user.name}") else: - print(response) + logger.debug(response) await interaction.response.send_message("Failed to add you") @@ -80,6 +129,38 @@ async def list_participants(interaction: discord.Interaction): await interaction.response.send_message("Failed to fetch participants") +async def send_prompt_with_response_and_button(channel, username, prompt, response): + await channel.send(f"What do you think about the following interaction: \nprompt: {prompt} \nresponse: {response}") + # await channel.send(f'Please click on the button that best describes your reaction to the response:') + + # add buttons + view = discord.ui.View() + like = LikeButton(label="Like", channel=channel, username=username, prompt=prompt) + neutral = NeutralButton(label="Neutral", channel=channel, username=username, prompt=prompt) + dislike = DislikeButton(label="Dislike", channel=channel, username=username, prompt=prompt) + + view.add_item(item=like) + view.add_item(item=neutral) + view.add_item(item=dislike) + await channel.send(view=view) + + +@client.tree.command() +async def review_prompts(interaction: discord.Interaction, number_of_prompts: int): + # get the prompt from the db + url = f"{prompts_url}?begin_id=0&limit={number_of_prompts}" + response = requests.get(url, headers=headers) + if response.status_code == 200: + prompts = response.json() + logger.debug("the responses are:", prompts) + for prompt in prompts: + await send_prompt_with_response_and_button( + interaction.channel, interaction.user.name, prompt["prompt"], prompt["response"] + ) + else: + await interaction.response.send_message("Failed to get prompts for review") + + @client.tree.command() async def add_prompt(interaction: discord.Interaction, prompt: str, response: str, language: str = "en"): """Uploads a single prompt to the server.""" @@ -92,7 +173,11 @@ async def add_prompt(interaction: discord.Interaction, prompt: str, response: st } response = requests.post(prompts_url, headers=headers, json=prompt) if response.status_code == 200: - await interaction.response.send_message("Added your prompt") + await send_prompt_with_response_and_button( + interaction.channel, interaction.user.name, prompt["prompt"], prompt["response"] + ) + # send the prompt back with buttons for the user to click on + # await interaction.response.send_message("Added your prompt") else: await interaction.response.send_message("Failed to add the prompt")