diff --git a/backend/oasst_backend/api/v1/admin.py b/backend/oasst_backend/api/v1/admin.py index 59c5efda..584c5fb7 100644 --- a/backend/oasst_backend/api/v1/admin.py +++ b/backend/oasst_backend/api/v1/admin.py @@ -54,6 +54,7 @@ class PublicSettings(pydantic.BaseModel): PROJECT_NAME: str API_V1_STR: str + MESSAGE_SIZE_LIMIT: int DEBUG_USE_SEED_DATA: bool DEBUG_ALLOW_SELF_LABELING: bool DEBUG_SKIP_EMBEDDING_COMPUTATION: bool diff --git a/backend/oasst_backend/api/v1/messages.py b/backend/oasst_backend/api/v1/messages.py index 2dcca64b..1ef1e929 100644 --- a/backend/oasst_backend/api/v1/messages.py +++ b/backend/oasst_backend/api/v1/messages.py @@ -96,8 +96,15 @@ def get_messages_cursor( items = utils.prepare_message_list(messages) n, p = None, None if len(items) > 0: - p = str(items[0].id) + "$" + items[0].created_date.isoformat() - n = str(items[-1].id) + "$" + items[-1].created_date.isoformat() + if len(items) == max_count or gte_created_date: + p = str(items[0].id) + "$" + items[0].created_date.isoformat() + if len(items) == max_count or lte_created_date: + n = str(items[-1].id) + "$" + items[-1].created_date.isoformat() + else: + if gte_created_date: + p = gte_created_date.isoformat() + if lte_created_date: + n = lte_created_date.isoformat() order = "desc" if desc else "asc" return protocol.MessagePage(prev=p, next=n, sort_key="created_date", order=order, items=items) diff --git a/backend/oasst_backend/api/v1/text_labels.py b/backend/oasst_backend/api/v1/text_labels.py index c9afd88c..dc6cc889 100644 --- a/backend/oasst_backend/api/v1/text_labels.py +++ b/backend/oasst_backend/api/v1/text_labels.py @@ -4,8 +4,9 @@ from loguru import logger from oasst_backend.api import deps from oasst_backend.prompt_repository import PromptRepository from oasst_backend.schemas.text_labels import LabelOption, ValidLabelsResponse +from oasst_backend.utils.database_utils import CommitMode, managed_tx_function +from oasst_shared.exceptions import OasstError from oasst_shared.schemas import protocol as protocol_schema -from sqlmodel import Session from starlette.status import HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST router = APIRouter() @@ -14,20 +15,25 @@ router = APIRouter() @router.post("/", status_code=HTTP_204_NO_CONTENT) def label_text( *, - db: Session = Depends(deps.get_db), api_key: APIKey = Depends(deps.get_api_key), text_labels: protocol_schema.TextLabels, ) -> None: """ Label a piece of text. """ - api_client = deps.api_auth(api_key, db) + + @managed_tx_function(CommitMode.COMMIT) + def store_text_labels(session: deps.Session): + api_client = deps.api_auth(api_key, session) + pr = PromptRepository(session, api_client, client_user=text_labels.user) + pr.store_text_labels(text_labels) try: logger.info(f"Labeling text {text_labels=}.") - pr = PromptRepository(db, api_client, client_user=text_labels.user) - pr.store_text_labels(text_labels) + store_text_labels() + except OasstError: + raise except Exception: logger.exception("Failed to store label.") raise HTTPException( diff --git a/backend/oasst_backend/api/v1/users.py b/backend/oasst_backend/api/v1/users.py index 63a55691..c7ff9f9c 100644 --- a/backend/oasst_backend/api/v1/users.py +++ b/backend/oasst_backend/api/v1/users.py @@ -1,5 +1,5 @@ import datetime -from typing import Optional +from typing import Callable, Optional from uuid import UUID from fastapi import APIRouter, Depends, Query @@ -28,6 +28,7 @@ def get_users_ordered_by_username( search_text: Optional[str] = None, auth_method: Optional[str] = None, max_count: Optional[int] = Query(100, gt=0, le=10000), + desc: Optional[bool] = False, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db), ): @@ -41,6 +42,7 @@ def get_users_ordered_by_username( auth_method=auth_method, search_text=search_text, limit=max_count, + desc=desc, ) return [u.to_protocol_frontend_user() for u in users] @@ -55,6 +57,7 @@ def get_users_ordered_by_display_name( auth_method: Optional[str] = None, search_text: Optional[str] = None, max_count: Optional[int] = Query(100, gt=0, le=10000), + desc: Optional[bool] = False, api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db), ): @@ -68,6 +71,7 @@ def get_users_ordered_by_display_name( auth_method=auth_method, search_text=search_text, limit=max_count, + desc=desc, ) return [u.to_protocol_frontend_user() for u in users] @@ -84,6 +88,8 @@ def get_users_cursor( api_client: ApiClient = Depends(deps.get_api_client), db: Session = Depends(deps.get_db), ): + assert max_count is not None + def split_cursor(x: str | None) -> tuple[str, UUID]: if not x: return None, None @@ -93,6 +99,32 @@ def get_users_cursor( return x, None items: list[protocol.FrontEndUser] + qry_max_count = max_count + 1 if lt is None or gt is None else max_count + desc = lt and not gt + + def get_next_prev(num_rows: int, lt: str | None, gt: str | None, key_fn: Callable[[protocol.FrontEndUser], str]): + p, n = None, None + if len(items) > 0: + if (num_rows > max_count and lt) or gt: + p = str(items[0].user_id) + "$" + key_fn(items[0]) + if num_rows > max_count or lt: + n = str(items[-1].user_id) + "$" + key_fn(items[-1]) + else: + if gt: + p = gt + if lt: + n = lt + return p, n + + def remove_extra_item(items: list[protocol.FrontEndUser], lt: str | None, gt: str | None): + num_rows = len(items) + if qry_max_count > max_count and num_rows == qry_max_count: + assert not (lt and gt) + items = items[:-1] + if desc: + items.reverse() + return items, num_rows + n, p = None, None if sort_key == "username": lte_username, lt_id = split_cursor(lt) @@ -105,13 +137,14 @@ def get_users_cursor( lt_id=lt_id, auth_method=auth_method, search_text=search_text, - max_count=max_count, + max_count=qry_max_count, + desc=desc, api_client=api_client, db=db, ) - if len(items) > 0: - p = str(items[0].user_id) + "$" + items[0].id - n = str(items[-1].user_id) + "$" + items[-1].id + items, num_rows = remove_extra_item(items, lte_username, gte_username) + p, n = get_next_prev(num_rows, lte_username, gte_username, lambda x: x.id) + elif sort_key == "display_name": lte_display_name, lt_id = split_cursor(lt) gte_display_name, gt_id = split_cursor(gt) @@ -123,13 +156,14 @@ def get_users_cursor( lt_id=lt_id, auth_method=auth_method, search_text=search_text, - max_count=max_count, + max_count=qry_max_count, + desc=desc, api_client=api_client, db=db, ) - if len(items) > 0: - p = str(items[0].user_id) + "$" + items[0].display_name - n = str(items[-1].user_id) + "$" + items[-1].display_name + items, num_rows = remove_extra_item(items, lte_display_name, gte_display_name) + p, n = get_next_prev(num_rows, lte_display_name, gte_display_name, lambda x: x.display_name) + else: raise OasstError(f"Unsupported sort key: '{sort_key}'", OasstErrorCode.SORT_KEY_UNSUPPORTED) diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py index 99b10cb4..8ca2a413 100644 --- a/backend/oasst_backend/config.py +++ b/backend/oasst_backend/config.py @@ -72,6 +72,7 @@ class Settings(BaseSettings): DATABASE_MAX_TX_RETRY_COUNT: int = 3 RATE_LIMIT: bool = True + MESSAGE_SIZE_LIMIT: int = 2000 REDIS_HOST: str = "localhost" REDIS_PORT: str = "6379" diff --git a/backend/oasst_backend/prompt_repository.py b/backend/oasst_backend/prompt_repository.py index abf5b721..0a0fa61d 100644 --- a/backend/oasst_backend/prompt_repository.py +++ b/backend/oasst_backend/prompt_repository.py @@ -448,6 +448,11 @@ class PromptRepository: if task: message.review_count += 1 self.db.add(message) + # for the same User id with no task id associated with the message, then update existing record for repeated updates + existing_text_label = self.fetch_non_task_text_labels(message_id, self.user_id) + if existing_text_label is not None: + existing_text_label.labels = text_labels.labels + model = existing_text_label self.db.add(model) return model, task, message @@ -561,6 +566,16 @@ class PromptRepository: raise OasstError("Message not found", OasstErrorCode.MESSAGE_NOT_FOUND, HTTP_404_NOT_FOUND) return message + def fetch_non_task_text_labels(self, message_id: UUID, user_id: UUID) -> Optional[TextLabels]: + + query = ( + self.db.query(TextLabels) + .outerjoin(Task, Task.id == TextLabels.id) + .filter(Task.id.is_(None), TextLabels.message_id == message_id, TextLabels.user_id == user_id) + ) + text_label = query.one_or_none() + return text_label + @staticmethod def trace_conversation(messages: list[Message] | dict[UUID, Message], last_message: Message) -> list[Message]: """ @@ -693,7 +708,7 @@ class PromptRepository: if user_id: qry = qry.filter(Message.user_id == user_id) if username or auth_method: - if not username and auth_method: + if not (username and auth_method): raise OasstError("Auth method or username missing.", OasstErrorCode.AUTH_AND_USERNAME_REQUIRED) qry = qry.join(User) qry = qry.filter(User.username == username, User.auth_method == auth_method) diff --git a/backend/oasst_backend/tree_manager.py b/backend/oasst_backend/tree_manager.py index 026bb564..1224af4a 100644 --- a/backend/oasst_backend/tree_manager.py +++ b/backend/oasst_backend/tree_manager.py @@ -465,6 +465,13 @@ class TreeManager: f"Frontend reports text reply to {interaction.message_id=} with {interaction.text=} by {interaction.user=}." ) + # ensure message size is below the predefined limit + if len(interaction.text) > settings.MESSAGE_SIZE_LIMIT: + logger.error( + f"Message size {len(interaction.text)=} exceeds size limit of {settings.MESSAGE_SIZE_LIMIT=}." + ) + raise OasstError("Message size too long.", OasstErrorCode.TASK_MESSAGE_TOO_LONG) + # here we store the text reply in the database message = pr.store_text_reply( text=interaction.text, diff --git a/backend/oasst_backend/user_repository.py b/backend/oasst_backend/user_repository.py index 1e9ac78f..136d3d29 100644 --- a/backend/oasst_backend/user_repository.py +++ b/backend/oasst_backend/user_repository.py @@ -145,6 +145,7 @@ class UserRepository: auth_method: Optional[str] = None, search_text: Optional[str] = None, limit: Optional[int] = 100, + desc: bool = False, ) -> list[User]: if not self.api_client.trusted: if not api_client_id: @@ -153,7 +154,7 @@ class UserRepository: if api_client_id != self.api_client.id: raise OasstError("Forbidden", OasstErrorCode.API_CLIENT_NOT_AUTHORIZED, HTTP_403_FORBIDDEN) - qry = self.db.query(User).order_by(User.username, User.id) + qry = self.db.query(User) if gte_username is not None: if gt_id: @@ -184,6 +185,11 @@ class UserRepository: pattern = "%{}%".format(search_text.replace("\\", "\\\\").replace("_", "\\_").replace("%", "\\%")) qry = qry.filter(User.username.like(pattern)) + if desc: + qry = qry.order_by(User.username.desc(), User.id.desc()) + else: + qry = qry.order_by(User.username, User.id) + if limit is not None: qry = qry.limit(limit) @@ -199,7 +205,9 @@ class UserRepository: auth_method: Optional[str] = None, search_text: Optional[str] = None, limit: Optional[int] = 100, + desc: bool = False, ) -> list[User]: + if not self.api_client.trusted: if not api_client_id: # Let unprivileged api clients query their own users without api_client_id being set @@ -209,7 +217,7 @@ class UserRepository: # Unprivileged api client asks for foreign users raise OasstError("Forbidden", OasstErrorCode.API_CLIENT_NOT_AUTHORIZED, HTTP_403_FORBIDDEN) - qry = self.db.query(User).order_by(User.display_name, User.id) + qry = self.db.query(User) if gte_display_name is not None: if gt_id: @@ -249,6 +257,11 @@ class UserRepository: if auth_method: qry = qry.filter(User.auth_method == auth_method) + if desc: + qry = qry.order_by(User.display_name.desc(), User.id.desc()) + else: + qry = qry.order_by(User.display_name, User.id) + if limit is not None: qry = qry.limit(limit) diff --git a/inference/README.md b/inference/README.md new file mode 100644 index 00000000..3dee94f9 --- /dev/null +++ b/inference/README.md @@ -0,0 +1,35 @@ +# OpenAssitant Inference + +Preliminary implementation of the inference engine for OpenAssistant. + +## Development (you'll need multiple terminals) + +Run a redis container (or use the one of the general docker compose file): + +```bash +docker run --rm -it -p 6379:6379 redis +``` + +Run the inference server: + +```bash +cd server +pip install -r requirements.txt +uvicorn main:app --reload +``` + +Run one (or more) workers: + +```bash +cd worker +pip install -r requirements.txt +python __main__.py +``` + +Run the client: + +```bash +cd text-client +pip install -r requirements.txt +python __main__.py +``` diff --git a/inference/server/README.md b/inference/server/README.md new file mode 100644 index 00000000..a235a7e6 --- /dev/null +++ b/inference/server/README.md @@ -0,0 +1,10 @@ +# OpenAssistant Inference Server + +Workers communicate with the `/work` endpoint via Websocket. They provide their +configuration and if a task is available, the server returns it. The worker then +performs the task and returns the result in a streaming fashion to the server, +also via websocket. + +Clients first call `/chat` to make a new chat, then add to that via +`/chat//message`. The response is a SSE event source, which will send tokens +as they are available. diff --git a/inference/server/main.py b/inference/server/main.py new file mode 100644 index 00000000..f3ec02b1 --- /dev/null +++ b/inference/server/main.py @@ -0,0 +1,193 @@ +import asyncio +import enum +import uuid + +import fastapi +import pydantic +import redis.asyncio as redis +from fastapi.middleware.cors import CORSMiddleware +from loguru import logger +from oasst_shared.schemas import inference, protocol +from sse_starlette.sse import EventSourceResponse + +app = fastapi.FastAPI() + +# Allow CORS +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +class Settings(pydantic.BaseSettings): + redis_host: str = "localhost" + redis_port: int = 6379 + redis_db: int = 0 + + sse_retry_timeout: int = 15000 + + +settings = Settings() + +# create async redis client +redisClient = redis.Redis( + host=settings.redis_host, port=settings.redis_port, db=settings.redis_db, decode_responses=True +) + + +class CreateChatRequest(pydantic.BaseModel): + pass + + +class CreateChatResponse(pydantic.BaseModel): + id: str + + +class MessageRequest(pydantic.BaseModel): + message: str = pydantic.Field(..., repr=False) + model_name: str = "distilgpt2" + max_new_tokens: int = 100 + + def compatible_with(self, worker_config: inference.WorkerConfig) -> bool: + return self.model_name == worker_config.model_name + + +class TokenResponseEvent(pydantic.BaseModel): + token: str + + +class MessageRequestState(str, enum.Enum): + pending = "pending" + in_progress = "in_progress" + complete = "complete" + + +class DbChatEntry(pydantic.BaseModel): + id: str = pydantic.Field(default_factory=lambda: str(uuid.uuid4())) + conversation: protocol.Conversation = pydantic.Field(default_factory=protocol.Conversation) + pending_message_request: MessageRequest | None = None + message_request_state: MessageRequestState | None = None + + +# TODO: make real database +CHATS: dict[str, DbChatEntry] = {} + + +@app.post("/chat") +async def create_chat(request: CreateChatRequest) -> CreateChatResponse: + """Allows a client to create a new chat.""" + logger.info(f"Received {request}") + chat = DbChatEntry() + CHATS[chat.id] = chat + return CreateChatResponse(id=chat.id) + + +@app.get("/chat/{id}") +async def get_chat(id: str) -> protocol.Conversation: + """Allows a client to get the current state of a chat.""" + return CHATS[id].conversation + + +@app.post("/chat/{id}/message") +async def create_message(id: str, message_request: MessageRequest, fastapi_request: fastapi.Request): + """Allows the client to stream the results of a request.""" + + chat = CHATS[id] + if not chat.conversation.is_prompter_turn: + raise fastapi.HTTPException(status_code=400, detail="Not your turn") + if chat.pending_message_request is not None: + raise fastapi.HTTPException(status_code=400, detail="Already pending") + + chat.conversation.messages.append( + protocol.ConversationMessage( + text=message_request.message, + is_assistant=False, + ) + ) + + chat.pending_message_request = message_request + chat.message_request_state = MessageRequestState.pending + + async def event_generator(): + result_data = [] + + try: + while True: + if await fastapi_request.is_disconnected(): + logger.warning("Client disconnected") + break + + item = await redisClient.blpop(chat.id, 1) + if item is None: + continue + + _, response_packet_str = item + response_packet = inference.WorkResponsePacket.parse_raw(response_packet_str) + result_data.append(response_packet) + + if response_packet.is_end: + break + + yield { + "retry": settings.sse_retry_timeout, + "data": TokenResponseEvent(token=response_packet.token).json(), + } + logger.info(f"Finished streaming {chat.id} {len(result_data)=}") + except Exception: + logger.exception(f"Error streaming {chat.id}") + + chat.conversation.messages.append( + protocol.ConversationMessage( + text="".join([d.token for d in result_data[:-1]]), + is_assistant=True, + ) + ) + chat.pending_message_request = None + + return EventSourceResponse(event_generator()) + + +@app.websocket("/work") +async def work(websocket: fastapi.WebSocket): + await websocket.accept() + worker_config = inference.WorkerConfig.parse_raw(await websocket.receive_text()) + while True: + # find a pending task that matches the worker's config + # could also be implemented using task queues + # but general compatibility matching is tricky + for chat in CHATS.values(): + if (request := chat.pending_message_request) is not None: + if chat.message_request_state == MessageRequestState.pending: + if request.compatible_with(worker_config): + break + else: + logger.debug("No pending tasks") + await asyncio.sleep(1) + continue + + chat.message_request_state = MessageRequestState.in_progress + + work_request = inference.WorkRequest( + conversation=chat.conversation, + model_name=request.model_name, + max_new_tokens=request.max_new_tokens, + ) + + logger.info(f"Created {work_request}") + try: + await websocket.send_text(work_request.json()) + while True: + # maybe unnecessary to parse and re-serialize + # could just pass the raw string and mark end via empty string + response_packet = inference.WorkResponsePacket.parse_raw(await websocket.receive_text()) + await redisClient.rpush(chat.id, response_packet.json()) + if response_packet.is_end: + break + except fastapi.WebSocketException: + # TODO: handle this better + logger.exception(f"Websocket closed during handling of {chat.id}") + + chat.message_request_state = MessageRequestState.complete diff --git a/inference/server/requirements.txt b/inference/server/requirements.txt new file mode 100644 index 00000000..e0a00339 --- /dev/null +++ b/inference/server/requirements.txt @@ -0,0 +1,6 @@ +fastapi[all] +loguru +pydantic +redis +sse-starlette +websockets diff --git a/inference/text-client/__main__.py b/inference/text-client/__main__.py new file mode 100644 index 00000000..bf1f8b02 --- /dev/null +++ b/inference/text-client/__main__.py @@ -0,0 +1,40 @@ +"""Simple REPL frontend.""" + +import json + +import requests +import sseclient +import typer + +app = typer.Typer() + + +@app.command() +def main(backend_url: str = "http://127.0.0.1:8000"): + """Simple REPL client.""" + chat_id = requests.post(f"{backend_url}/chat", json={}).json()["id"] + while True: + message = typer.prompt("User").strip() + + # wait for stream to be ready + # could implement a queue position indicator + # could be implemented with long polling + # but server load needs to be considered + response = requests.post( + f"{backend_url}/chat/{chat_id}/message", + json={"message": message}, + stream=True, + headers={"Accept": "text/event-stream"}, + ) + response.raise_for_status() + + client = sseclient.SSEClient(response) + print("Assistant: ", end="", flush=True) + for event in client.events(): + data = json.loads(event.data) + print(data["token"], end="", flush=True) + print() + + +if __name__ == "__main__": + app() diff --git a/inference/text-client/requirements.txt b/inference/text-client/requirements.txt new file mode 100644 index 00000000..8d7bff7d --- /dev/null +++ b/inference/text-client/requirements.txt @@ -0,0 +1,3 @@ +requests +sseclient-py +typer diff --git a/inference/worker/__main__.py b/inference/worker/__main__.py new file mode 100644 index 00000000..ad5e5cef --- /dev/null +++ b/inference/worker/__main__.py @@ -0,0 +1,79 @@ +import re +import time + +import rel +import torch +import typer +import websocket +from loguru import logger +from oasst_shared.schemas import inference, protocol +from transformers import pipeline + +app = typer.Typer() + + +@app.command() +def main( + backend_url: str = "ws://localhost:8000", + model_name: str = "distilgpt2", +): + pipe = pipeline("text-generation", model=model_name) + + def on_open(ws: websocket.WebSocket): + worker_config = inference.WorkerConfig(model_name=model_name) + ws.send(worker_config.json()) + + def on_message(ws: websocket.WebSocket, message: str): + # TODO: what if this comes in, but one is already in progress? + # also need to think of enabling batching + work_request = inference.WorkRequest.parse_raw(message) + + def _prepare_message(message: protocol.ConversationMessage) -> str: + prefix = "Assistant: " if message.is_assistant else "User: " + return prefix + message.text + + # construct prompt + messages = [_prepare_message(message) for message in work_request.conversation.messages] + + prompt = "\n".join(messages) + "\nAssistant:" + + # TODO: replace this with incremental generation + torch.manual_seed(work_request.seed) + model_output = pipe(prompt, max_new_tokens=work_request.max_new_tokens, do_sample=True, return_full_text=False)[ + 0 + ]["generated_text"] + model_output = model_output.strip() + + # fake streaming + split_idcs = [m.start() for m in re.finditer(r"([\w:]+)", model_output)] + pieces = [model_output[a:b] for a, b in zip([0] + split_idcs, split_idcs + [None])] + for piece in pieces: + if not piece: + continue + if piece.strip() in ("User:", "Assistant:"): + break + ws.send(inference.WorkResponsePacket(token=piece).json()) + time.sleep(0.1) + ws.send(inference.WorkResponsePacket(is_end=True).json()) + + def on_error(ws: websocket.WebSocket, error: Exception): + logger.error(f"Connection error: {error}") + + def on_close(ws: websocket.WebSocket, close_status_code: int, close_msg: str): + logger.warning(f"Connection closed: {close_status_code=} {close_msg=}") + + ws = websocket.WebSocketApp( + f"{backend_url}/work", + on_message=on_message, + on_error=on_error, + on_close=on_close, + on_open=on_open, + ) + + ws.run_forever(dispatcher=rel, reconnect=5) + rel.signal(2, rel.abort) + rel.dispatch() + + +if __name__ == "__main__": + app() diff --git a/inference/worker/requirements.txt b/inference/worker/requirements.txt new file mode 100644 index 00000000..c248c652 --- /dev/null +++ b/inference/worker/requirements.txt @@ -0,0 +1,6 @@ +loguru +rel +torch +transformers +typer +websocket-client diff --git a/oasst-shared/oasst_shared/exceptions/oasst_api_error.py b/oasst-shared/oasst_shared/exceptions/oasst_api_error.py index bed6f942..0a548ebb 100644 --- a/oasst-shared/oasst_shared/exceptions/oasst_api_error.py +++ b/oasst-shared/oasst_shared/exceptions/oasst_api_error.py @@ -37,6 +37,7 @@ class OasstErrorCode(IntEnum): TASK_GENERATION_FAILED = 1005 TASK_REQUESTED_TYPE_NOT_AVAILABLE = 1006 TASK_AVAILABILITY_QUERY_FAILED = 1007 + TASK_MESSAGE_TOO_LONG = 1008 # 2000-3000: prompt_repository INVALID_FRONTEND_MESSAGE_ID = 2000 diff --git a/oasst-shared/oasst_shared/schemas/inference.py b/oasst-shared/oasst_shared/schemas/inference.py new file mode 100644 index 00000000..0acb5014 --- /dev/null +++ b/oasst-shared/oasst_shared/schemas/inference.py @@ -0,0 +1,21 @@ +import random + +import pydantic + +from . import protocol + + +class WorkerConfig(pydantic.BaseModel): + model_name: str = "distilgpt2" + + +class WorkRequest(pydantic.BaseModel): + conversation: protocol.Conversation = pydantic.Field(..., repr=False) + model_name: str = "distilgpt2" + max_new_tokens: int = 100 + seed: int = pydantic.Field(default_factory=lambda: random.randint(0, 2**32 - 1)) + + +class WorkResponsePacket(pydantic.BaseModel): + token: str | None = None + is_end: bool = False diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index f2164e8f..20bbdf9b 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -64,6 +64,18 @@ class Conversation(BaseModel): messages: list[ConversationMessage] = [] + def __len__(self): + return len(self.messages) + + @property + def is_prompter_turn(self) -> bool: + if len(self) == 0: + return True + last_message = self.messages[-1] + if last_message.is_assistant: + return True + return False + class Message(ConversationMessage): parent_id: Optional[UUID] = None diff --git a/openassistant/datasets/__init__.py b/openassistant/datasets/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openassistant/datasets/soda_synthetic_dialogue/README.md b/openassistant/datasets/soda_synthetic_dialogue/README.md new file mode 100644 index 00000000..c4866e16 --- /dev/null +++ b/openassistant/datasets/soda_synthetic_dialogue/README.md @@ -0,0 +1,108 @@ +--- +annotations_creators: + - no-annotation +language: + - en +language_creators: + - machine-generated +license: + - mit +multilinguality: + - monolingual +pretty_name: "SODA Synthetic Dialogue" +size_categories: + - 1M 1 and sys.argv[1] == "--print" + + +def main(output_dir: str = "data"): + """Download and prepare the dataset for use.""" + + random.seed(42) + dataset = load_dataset("allenai/soda") + os.makedirs(output_dir, exist_ok=True) + + for split in ["train", "test", "validation"]: + with open(f"{output_dir}/{split}.jsonl", "w", encoding="utf8") as output: + + for i in tqdm(range(len(dataset[split])), desc=split): + dat = dataset["train"][i] + title = dat["literal"] + story = dat["narrative"] + + if dat["relation"] == "xWant": + theme = "wanting " + dat["tail"] + elif dat["relation"] == "xNeed": + theme = "needing " + dat["tail"] + elif not dat["tail"].startswith("to ") and not dat["tail"].startswith("and "): + theme = "being " + dat["tail"] + elif dat["tail"].startswith("and "): + theme = "people are " + dat["tail"].replace("and PersonY ", "") + else: + theme = dat["tail"] + theme = theme.replace("PersonY", "another person") + theme = theme.replace("being is", "being") + + dialogue = [s2 + ": " + s1 for s1, s2 in zip(dat["dialogue"], dat["speakers"])] + + if random.randint(0, 6) == 0: + # print("##") + # print(f"User: Can you give me a short story description for this dialog?") + # print(" " + "\n ".join(dialog)) + # print(f"Assistant: Sure, a short story description for this dialog could be: \n {story}") + # print("User: And a title?") + # print(f"Assistant: Sure, a title for this dialog could be: \n {title}") + # if theme: + # print("User: What would be one theme of this story?") + # print(f'Assistant: One theme of this story could be: "{theme}"') + conversation = SUMMARY_TEMPLATE.format(dialogue="\n ".join(dialogue), story=story, title=title) + if theme: + conversation = conversation + THEME_TEMPLATE.format(theme=theme) + elif random.randint(0, 6) == 0: + # print("##") + # print(f"User: Can you write a short dialog based on this story:\n {story}") + # print(f"Assistant: Sure, a dialog for this story could be:") + # print(" " + "\n ".join(dialog)) + # print("User: And a title?") + # print(f"Assistant: Sure, a title for this dialog could be: \n {title}") + # if theme: + # print("User: What would be one theme of this story?") + # print(f'Assistant: One theme of this story could be: "{theme}"') + conversation = NEW_DIALOGUE_TEMPLATE.format( + story=story, dialogue="\n ".join(dialogue), title=title + ) + if theme: + conversation = conversation + THEME_TEMPLATE.format(theme=theme) + elif random.randint(0, 3) == 0: + # print("##") + # print(f"User: Can you write the next few lines of dialog for this scene:") + # if random.randint(0, 1) == 0: + # print(" " + "\n ".join(dialog[:-5])) + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-5:])) + # elif random.randint(0, 1) == 0: + # print(" " + "\n ".join(dialog[:-3])) + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-3:])) + # else: + # print(" " + "\n ".join(dialog[:-4])) + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-4:])) + # print("User: And a title?") + # print(f"Assistant: Sure, a title for this dialog could be: \n {title}") + # print("User: How about a short description?") + # print(f"Assistant: Sure, a short description for this dialog could be: \n {story}") + # if theme: + # print("User: What would be one theme of this story?") + # print(f'Assistant: One theme of this story could be: "{theme}"') + if random.randint(0, 1) == 0: + depth = -5 + elif random.randint(0, 1) == 0: + depth = -3 + else: + depth = -4 + conversation = NEXT_LINES_TEMPLATE.format( + scene="\n ".join(dialogue[:depth]), + dialogue="\n ".join(dialogue[depth:]), + title=title, + story=story, + ) + if theme: + conversation = conversation + THEME_TEMPLATE.format(theme=theme) + elif random.randint(0, 3) == 0: + # print("##") + # title1 = title.split(".")[0] + # title2 = title.split(".")[1] + # print(f"User: Can you write short story and dialog about: {title1}") + # print(f'Assistant: Sure, a short story and dialog about: "{title1}" could be:') + # print(f" {story}") + # if random.randint(0, 1) == 0: + # print(" " + "\n ".join(dialog)) + # elif random.randint(0, 1) == 0 and len(dialog) > 5: + # print(" " + "\n ".join(dialog[:-5])) + # print(f'User: Can you provide more dialog assuming "{title2}"?') + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-5:])) + # elif random.randint(0, 1) == 0: + # print(" " + "\n ".join(dialog[:-3])) + # print("User: more please.") + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-3:])) + # else: + # print(" " + "\n ".join(dialog[:-4])) + # print(f'User: Can you provide more dialog assuming "{title2}"?') + # print(f"Assistant: Sure, the next dialog for this scene could be:") + # print(" " + "\n ".join(dialog[-4:])) + # if theme: + # print("User: What would be one theme of this story?") + # print(f'Assistant: One theme of this story could be: "{theme}"') + title1 = title.split(".")[0] + title2 = title.split(".")[1] + conversation = NEW_STORY_AND_DIALOGUE_TEMPLATE.format(title1=title1, story=story) + if random.randint(0, 1) == 0: + conversation = FULL_DIALOGUE_TEMPLATE.format( + conversation=conversation, dialogue="\n ".join(dialogue) + ) + elif random.randint(0, 1) == 0 and len(dialogue) > 5: + conversation = MORE_DIALOGUE_TEMPLATE.format( + conversation=conversation, + dialogue1="\n ".join(dialogue[:-5]), + title2=title2, + dialogue2="\n ".join(dialogue[-5:]), + ) + elif random.randint(0, 1) == 0: + conversation = NEXT_DIALOGUE_TEMPLATE.format( + conversation=conversation, + dialogue1="\n ".join(dialogue[:-3]), + dialogue2="\n ".join(dialogue[-3:]), + ) + else: + conversation = MORE_DIALOGUE_TEMPLATE.format( + conversation=conversation, + dialogue1="\n ".join(dialogue[:-4]), + title2=title2, + dialogue2="\n ".join(dialogue[-4:]), + ) + if theme: + conversation = conversation + THEME_TEMPLATE.format(theme=theme) + else: + # print("##") + # print(f"User: Can you write short story and dialog based on the theme:\n {theme}") + # print(f'Assistant: Sure, a short story and dialog based on the theme "{theme}" could be:') + # print(f" {story}") + # print(" " + "\n ".join(dialog)) + # print("User: And a title?") + # print(f"Assistant: Sure, a title for this dialog could be: \n {title}") + conversation = NEW_STORY_AND_DIALOGUE_FROM_THEME_TEMPLATE.format( + theme=theme, story=story, dialogue="\n ".join(dialogue), title=title + ) + if PRINT: + print("##") + print(conversation) + + output.write(f"{json.dumps({'conversation': conversation})}\n") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py b/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py new file mode 100644 index 00000000..ddc7c883 --- /dev/null +++ b/openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py @@ -0,0 +1,108 @@ +# Copyright 2023 The OpenAssistant Authors and the current dataset script contributor. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This dataset is a set of dialogues synthesized from the SODA dataset. +In each dialogue, User and Assistant have a conversation about a story. + +The original collab notebook for this dataset can be found at: +https://colab.research.google.com/drive/1Sw3px5dP8whdqT7QMNoqwmqIasZkMbJi?usp=sharing +""" + +import json +from typing import Dict, List, Tuple + +import datasets + +from .hub import OpenAssistantConfig, features + +_CITATION = """\ +@article{ontocord2023sodasynth, + author = {ontocord and Jeffrey Quesnelle}, + title = {SODA Synthetic Dialogue}, + year = {2023} +} +""" +_DATASETNAME = "soda_synthetic_dialogue" +_DISPLAYNAME = "🥤SODA Synthetic Dialogue" +_DESCRIPTION = "A set of dialogues synthesized from the SODA dataset." +_HOMEPAGE = "" +_LICENSE = "mit" +_URLS = { + _DATASETNAME: {"train": "./data/train.jsonl", "test": "./data/test.jsonl", "validation": "./data/validation.jsonl"} +} +_SUPPORTED_TASKS = ["dialogue-modeling"] +_VERSION = "1.0.0" + + +class SODASyntheticDialogueDataset(datasets.GeneratorBasedBuilder): + """A set of dialogues synthesized from the SODA dataset.""" + + VERSION = datasets.Version(_VERSION) + + BUILDER_CONFIGS = [ + OpenAssistantConfig( + name=f"{_DATASETNAME}_dialogue_modeling", + version=VERSION, + description=f"OpenAssistant dataset config for {_DATASETNAME}", + schema="dialogue_modeling", + subset_id=_DATASETNAME, + ) + ] + + DEFAULT_CONFIG_NAME = f"{_DATASETNAME}_dialogue_modeling" + + def _info(self) -> datasets.DatasetInfo: + + return datasets.DatasetInfo( + description=_DESCRIPTION, + features=features, + homepage=_HOMEPAGE, + license=_LICENSE, + citation=_CITATION, + ) + + def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]: + """Returns SplitGenerators.""" + + urls = _URLS[_DATASETNAME] + data_dir = dl_manager.download_and_extract(urls) + + return [ + datasets.SplitGenerator( + name=datasets.Split.TRAIN, + gen_kwargs={"filepath": data_dir, "split": "train"}, + ), + datasets.SplitGenerator( + name=datasets.Split.TEST, + gen_kwargs={"filepath": data_dir, "split": "test"}, + ), + datasets.SplitGenerator( + name=datasets.Split.VALIDATION, + gen_kwargs={"filepath": data_dir, "split": "validation"}, + ), + ] + + def _generate_examples(self, filepath, split: str) -> Tuple[int, Dict]: + """Yields examples as (key, example) tuples.""" + + if self.config.schema == "dialogue_modeling": + key = 0 + with open(filepath[split], "r", encoding="utf8") as data: + while True: + line = data.readline() + if not line: + return + yield key, json.loads(line) + key += 1 diff --git a/scripts/data-collection/twitter/README.md b/scripts/data-collection/twitter/README.md new file mode 100644 index 00000000..b8c7bfc9 --- /dev/null +++ b/scripts/data-collection/twitter/README.md @@ -0,0 +1,80 @@ +# Twitter data collection for Open Assistant + +Conversations on Twitter can be an interesting and useful source of data for our +model to learn from. Certain twitter threads may contain helpful prompts and +replies, in a similar fashion to how we want our model to be able to respond to +prompts in a useful way. + +Thus, these scripts are intended to process twitter data from a variety of +sources, process them into cleaner and more useful formats, and then combine the +various outputs into a unified training set that can be fed to our model as a +conversation, or at least as a prompt with replies. + +**Note: Based on issue #126** + +## Possible Data Paths + +- Twitterstream archive: https://archive.org/details/twitterstream These are + large .tar files with compressed json files inside. However, some data points + such as reply counts seem to always be 0 due to limitations when scraping the + Twitter API. +- Alternative APIs such as snscrape, twint, etc. These alternative APIs often + are harder to use than the official Twitter API but can often bypass API + limits which can make it useful for larger scale data collection. The downside + is potentially slower speed, and less features. +- The official Twitter API + +## Currently Completed Items + +- Downloaded various archive files (both are .tar, but each have a different + format of json compression. One used .gz, and the other.bz2). Each json file + is roughly 2000 rows of tweets. There are thousands of these compressed json + files. Managing the IO of opening lots of small files is one of the + challenges, which is why future steps will consolidate data into larger easier + to process files. +- Wrote script that can loop through the compressed json files, cleans them up a + bit by removing truncated tweets or tweets that aren't replies. The script + then standardizes the columns, and exports the polars dataframes into parquet + files for future processing. Note: Using polars instead of pandas due to + performance reasons. +- Wrote scripts that process the large dump of tweets into conversation threads + using the tree and node architecture. This results in aroun 17K conversation + threads bassed on a dump of 90M tweets. +- Script can output the conversation threads into a jsonl file for further + filtering or use in models. + +## Main Issue + +- The issue is that we can easily scrape replies, but there is no guarantee the + original tweet is in the archive file. Furthermore, the archives are large so + they would need to be kept completely in-memory or in a db to reference. We + still need to decide if we want to try to mine the archive to piece together + the conversations, or we can take the list of replied tweets and loop through + those and use alternative apis to fetch the original tweet text, and then + match it with the confirmed replies already in our archive to generate the + prompt/replies data. Currently, my script can extract conversations based on + the dump, but it is a small percentage of the overall dump, and there is no + guarantee of the quality of the tweets. +- The tweet quality is the other major issue. We can get conversations through + the currently made scripts, but they most likely don't match a useful + instruction -> fulfilment. We are trying to filter the tweets through various + means such as matching useful hashtags, or by using cosine similarity against + known instructions. +- The modern Twitter API has conversation_id as a field which can be a way to + gather all tweets in a thread sort of automatically although there is + pagination limits. The main issue with this is it seems hard to search for it + using alternative APIs. + +## TODO + +- Write scripts to filter existing conversations into useful instructions -> + fulfilment with hashtags or cosine similarity. +- Train model to detect if text is a suitable instruction. This could then be + run through the conversations (or full tweet dump) to simplify the process. + Related to issue #143. +- Write script that matches the original tweets and their text with the archive + data to create the prompt/reply dataset. (Optional) +- Decide on final output format and storage options for the dataset. Currently + in JSONL with tree / node architecture as python dicts which is acceptable I + believe. +- Alternatively: Store processed tweets into DB or alternative option.(Optional) diff --git a/scripts/data-collection/twitter/requirements.txt b/scripts/data-collection/twitter/requirements.txt new file mode 100644 index 00000000..2b084674 --- /dev/null +++ b/scripts/data-collection/twitter/requirements.txt @@ -0,0 +1,3 @@ +numpy==1.21.5 +polars==0.15.14 +tqdm==4.64.0 diff --git a/scripts/data-collection/twitter/twitter_create_convs.py b/scripts/data-collection/twitter/twitter_create_convs.py new file mode 100644 index 00000000..b1fd709a --- /dev/null +++ b/scripts/data-collection/twitter/twitter_create_convs.py @@ -0,0 +1,141 @@ +import json +from pathlib import Path + +import polars as pl +from tqdm import tqdm + +# Sets up paths +# TODO: Source paths from env file +path_string = "PUT THE PATH HERE TO WHERE YOU STORED THE PARQUET FILES" +folder_path = Path(path_string) +processed_folder_path = folder_path / "processed" +output_path = folder_path / "twitter-conv-trees.jsonl" + +# Get parq files +parq_files = sorted(processed_folder_path.rglob("*.parquet")) + +wanted_cols = [ + "timestamp_ms", + "id", + "text", + "truncated", + "in_reply_to_status_id", + "in_reply_to_user_id", + "is_quote_status", + "quote_count", + "reply_count", + "retweet_count", + "favorite_count", + "filter_level", + "lang", + "possibly_sensitive", + "hashtags", + "user_id", + "user_verified", + "user_followers_count", + "user_statuses_count", +] + +# Load parqs into list. Using Polars for performance reasons. +df_list = [] +for p in parq_files: + df_list.append(pl.read_parquet(p, columns=wanted_cols)) + +# Create major dataframe. +# This can be done incrementally if RAM is constrained by modifying the above code. +p_df = pl.concat(df_list) + +# Clean up the reference just in case to help with memory if needed. +del df_list + +# Get tweets that are replies to other tweets +p_df_replies_only = p_df.filter(pl.col("in_reply_to_status_id").is_null().is_not()) + +# Group by replied to status id to see the most replied to statuses. This can take some time. +p_df_group_reply_to_status = p_df_replies_only.groupby("in_reply_to_status_id").count().sort("count", reverse=True) + +# Save output of grouping the top replied to statuses +group_reply_parq = folder_path / "group_reply_parq.parquet" +p_df_group_reply_to_status.write_parquet(group_reply_parq) + +# Join the main dataframe with the top replies to find tweets that have replies. +p_join = p_df.join(p_df_group_reply_to_status, left_on="id", right_on="in_reply_to_status_id", how="inner") + +# Save output of tweets that have replies +tweets_that_have_replies_path = folder_path / "tweets_that_have_replies.parquet" +p_join.write_parquet(tweets_that_have_replies_path) + +# Save output of tweets that are replies to other tweets +tweets_that_are_replies_path = folder_path / "tweets_that_are_replies.parquet" +p_df_replies_only.write_parquet(tweets_that_are_replies_path) + +# Filter the tweets that have replies to ones that aren't replies to others. +# Also filter for only english for now. +# This gives the root tweets that have replies but are the start of a conversation. +origin_tweets = p_join.filter((pl.col("in_reply_to_status_id").is_null()) & (pl.col("lang") == "en")) + + +# Helper functions and classes below for the next steps + + +def role_decide(user_id, prompt_user): + if user_id == prompt_user: + return "prompter" + else: + return "assistant" + + +class ConversationTreeNode: + def __init__(self, tweet_id, prompt_user, from_df, children_df, metadata=None): + + if metadata: + self.metadata = metadata + else: + self.metadata = from_df.filter(pl.col("id") == tweet_id).to_dicts()[0] + + self.metadata["prompt_user"] = prompt_user + self.role = role_decide(self.metadata["user_id"], prompt_user) + self.children = None + self.text = self.metadata["text"] + del self.metadata["text"] + self.get_children(tweet_id=tweet_id, children_df=children_df) + + def get_children(self, tweet_id, children_df): + children_dicts = children_df.filter(pl.col("in_reply_to_status_id") == tweet_id).to_dicts() + if len(children_dicts) > 0: + children = [ + ConversationTreeNode( + tweet_id=c["id"], + prompt_user=self.metadata["prompt_user"], + from_df=children_df, + children_df=children_df, + metadata=c, + ) + for c in children_dicts + ] + self.children = children + + +class ConversationTree: + def __init__(self, tweet_id, prompt_user, from_df, children_df, r_metadata=None): + + self.root = ConversationTreeNode( + tweet_id=tweet_id, prompt_user=prompt_user, from_df=from_df, children_df=children_df, metadata=r_metadata + ) + self.metadata = None + + +# Create conversation trees +conv_tree_list = [ + ConversationTree( + tweet_id=r["id"], prompt_user=r["user_id"], from_df=origin_tweets, children_df=p_df_replies_only, r_metadata=r + ) + for r in tqdm(origin_tweets.to_dicts()) +] + +# Write conversation trees to jsonl file. +# Might need to clean up the last newline. +with open(output_path, "w") as output: + for t in tqdm(conv_tree_list): + json.dump(obj=t, fp=output, default=lambda x: x.__dict__) + output.write("\n") diff --git a/scripts/data-collection/twitter/twitter_process_json.py b/scripts/data-collection/twitter/twitter_process_json.py new file mode 100644 index 00000000..a1e47d7d --- /dev/null +++ b/scripts/data-collection/twitter/twitter_process_json.py @@ -0,0 +1,233 @@ +# This file loops through compressed json tweet data, pre-processes them, +# and then extracts them into more unified parquet files that can be handed +# off for further processing. The main focus is on producing viable replies. + +# Initial data exploration seems that there is no guarantee that the original +# tweets are in the archive, so we might need to extract suitable replies +# then get the original tweets separately, and then combine them into a +# suitable thread format that can be used by our instruction model. + +# This assumes data downloaded from https://archive.org/details/twitterstream +# and that the internal .tar files are extracted locally. +# They are large files so using something like 7Zip or WinRar migth be easier +# than putting all of it in scripts, but it is a possibility. + +# I often work in notebooks. If you encounter any issue, please reach out to let me know. + +import bz2 +import gzip +import json +import pickle +from pathlib import Path + +import numpy as np +import polars as pl +from tqdm import tqdm + +# TODO: OPTIONAL - Put the Untar process in a script instead of doing that part externally. Twitterstream archives are .tar with folders and json.gz files inside. +# TODO: Set up list of important hashtags & keywords. This might have to be done after we get the original tweets in a separate file. +# TODO: Process data and filter based on hashtags & keywords + +# Sets up paths +# TODO: Source paths from env file +path_string = "PUT THE PATH HERE TO WHERE YOU DOWNLOADED AND EXTRACTED THE ARCHIVE .TAR" +folder_path = Path(path_string) +file_list_pkl = folder_path / "file_list.pkl" +processed_file_list_pkl = folder_path / "processed_file_list.pkl" + +# For the processed folder to save inside, we can create the directory if it doesn't exist +processed_folder_path = folder_path / "processed" +processed_folder_path.mkdir(parents=True, exist_ok=True) + +# Set max buffer to store temporary dataframes for processing +# Change this depending on the memory of your computer +processed_max_buffer = 5000 + +# Set up list of wanted column names. +# Note: User columns are prefixed with user_ +wanted_cols = [ + "timestamp_ms", + "id", + "text", + "truncated", + "in_reply_to_status_id", + "in_reply_to_user_id", + "is_quote_status", + "quote_count", + "reply_count", + "retweet_count", + "favorite_count", + "filter_level", + "lang", + "possibly_sensitive", + "hashtags", + "user_id", + "user_verified", + "user_followers_count", + "user_statuses_count", +] + + +def main(file_list_pkl, folder_path, processed_max_buffer): + """ + Runs the main processing script to get files, loop through them, and process them. + Outputs larger json.gz files made by concat the pre-filtered dataframes from + the original json.gz files. + """ + + file_list = get_file_paths(file_list_pkl, folder_path) + + process_json(file_list, processed_max_buffer) + + print("Done") + + +def get_file_paths(file_list_pkl, folder_path): + """ + Gets the file paths by recursively checking the folder structure. + # Based on code from stackoverflow https://stackoverflow.com/questions/26835477/pickle-load-variable-if-exists-or-create-and-save-it + """ + try: + allpaths = pickle.load(open(file_list_pkl, "rb")) + except (OSError, IOError) as e: + print(e) + allpaths = sorted(list(folder_path.rglob("*.[gz bz2]*"))) + pickle.dump(allpaths, open(file_list_pkl, "wb")) + print("Got file paths.") + return allpaths + + +def get_processed_list(processed_file_list_pkl): + # Gets processed file list if stored, if not, creates it. + try: + processed_list = pickle.load(open(processed_file_list_pkl, "rb")) + except (OSError, IOError) as e: + print(e) + processed_list = [] + pickle.dump(processed_list, open(processed_file_list_pkl, "wb")) + return processed_list + + +def modify_dict_cols(j_dict): + # Extracting some nested json + j_dict["user_id"] = np.int64(j_dict["user"]["id"]) + j_dict["user_followers_count"] = np.int64(j_dict["user"]["followers_count"]) + j_dict["user_statuses_count"] = np.int64(j_dict["user"]["statuses_count"]) + + # Get hashtags as a list of strings + j_dict["hashtags"] = [h["text"] for h in j_dict["entities"]["hashtags"]] + + j_dict["id"] = np.int64(j_dict["id"]) + + try: + j_dict["in_reply_to_status_id"] = np.int64(j_dict["in_reply_to_status_id"]) + except Exception as e: + print(e) + j_dict["in_reply_to_status_id"] = j_dict["in_reply_to_status_id"] + + try: + j_dict["in_reply_to_user_id"] = np.int64(j_dict["in_reply_to_user_id"]) + except Exception as e: + print(e) + j_dict["in_reply_to_user_id"] = j_dict["in_reply_to_user_id"] + + # Make sure relevant columns are available or none. + for key in wanted_cols: + if key not in j_dict: + j_dict[key] = None + + # Ordering keys and taking wanted columns + j_dict = {key: j_dict[key] for key in wanted_cols} + + return j_dict + + +def process_single_file(f, processed_list): + + j_dict_list = [] + if f not in processed_list: + # Check for compression type + if f.suffix == ".bz2": + with bz2.BZ2File(f) as file: + for line in file: + # Load JSON + j_dict = json.loads(line) + # Check if user key exists + if "delete" not in j_dict: + if j_dict["truncated"] is False: + + j_dict = modify_dict_cols(j_dict) + + j_dict_list.append(j_dict) + + else: + with gzip.open(f, "r") as file: + for line in file: + # Load JSON + j_dict = json.loads(line) + # Check if user key exists + if "delete" not in j_dict: + if j_dict["truncated"] is False: + + j_dict = modify_dict_cols(j_dict) + + j_dict_list.append(j_dict) + + return j_dict_list + + +def process_json(file_list, processed_max_buffer): + """ + Loops through file list and loads the compressed + json into a list of dicts after some pre-processing. + + Makes sure dicts are ordered in a specific + way to make sure polars can read them. + """ + + # Gets processed file list if stored, if not, creates it. + processed_list = get_processed_list(processed_file_list_pkl) + + j_list = [] + temp_processed_files = [] + + for i, f in enumerate(tqdm(file_list)): + + j_dict_list = process_single_file(f, processed_list) + + j_list.extend(j_dict_list) + + temp_processed_files.append(f) + + if len(temp_processed_files) == processed_max_buffer: + # If we reach our buffer, + # combine into polars dataframe + # and write to parquet as + # a checkpoint + processed_file_name = f"processed_json_{i}.parquet" + processed_file_path = processed_folder_path / processed_file_name + + pl.DataFrame(j_list, columns=wanted_cols).write_parquet(processed_file_path) + + # Make note of which files have been processed + processed_list.extend(temp_processed_files) + pickle.dump(processed_list, open(processed_file_list_pkl, "wb")) + + # Reset buffer lists + j_list = [] + temp_processed_files = [] + + # Process remaining files + processed_file_name = f"processed_json_{i}.parquet" + processed_file_path = processed_folder_path / processed_file_name + pl.from_dicts(j_dict_list).write_parquet(processed_file_path) + processed_list.extend(temp_processed_files) + pickle.dump(processed_list, open(processed_file_list_pkl, "wb")) + j_dict_list = [] + temp_processed_files = [] + + print("Processing completed") + + +if __name__ == "__main__": + main(file_list_pkl, folder_path, processed_max_buffer) diff --git a/website/next-i18next.config.js b/website/next-i18next.config.js index 7c87a7a4..40c4b14d 100644 --- a/website/next-i18next.config.js +++ b/website/next-i18next.config.js @@ -1,6 +1,6 @@ module.exports = { i18n: { defaultLocale: "en", - locales: ["en"], + locales: ["de", "en", "fr"], }, }; diff --git a/website/package-lock.json b/website/package-lock.json index 06f3c98d..0d38e98c 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -21,6 +21,8 @@ "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", "@tailwindcss/forms": "^0.5.3", + "@tanstack/react-table": "^8.7.6", + "accept-language-parser": "^1.5.0", "autoprefixer": "^10.4.13", "axios": "^1.2.1", "boolean": "^3.2.0", @@ -38,6 +40,7 @@ "npm": "^9.2.0", "postcss-focus-visible": "^7.1.0", "react": "18.2.0", + "react-cookies": "^0.1.1", "react-dom": "18.2.0", "react-feature-flags": "^1.0.0", "react-hook-form": "^7.42.1", @@ -12297,6 +12300,37 @@ "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1" } }, + "node_modules/@tanstack/react-table": { + "version": "8.7.6", + "resolved": "https://registry.npmjs.org/@tanstack/react-table/-/react-table-8.7.6.tgz", + "integrity": "sha512-/QijmMFeP7wDLBnr0MQ/5MlbXePbIL/1nOtkxBC9zvmBu4gDKJEDBqipUyM7Wc/iBpSd0IFyqBlvZvTPD9FYDA==", + "dependencies": { + "@tanstack/table-core": "8.7.6" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": ">=16", + "react-dom": ">=16" + } + }, + "node_modules/@tanstack/table-core": { + "version": "8.7.6", + "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.7.6.tgz", + "integrity": "sha512-sqiNTMzB6cpyL8DFH6/VqW48SwiflLqxQqYpo2wNock7rdVGvlm0BLNI8vZUJbr1+fmmWmHwBvi5OMgZw8n1DA==", + "engines": { + "node": ">=12" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, "node_modules/@testing-library/dom": { "version": "8.19.1", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-8.19.1.tgz", @@ -13616,6 +13650,11 @@ "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", "dev": true }, + "node_modules/accept-language-parser": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/accept-language-parser/-/accept-language-parser-1.5.0.tgz", + "integrity": "sha512-QhyTbMLYo0BBGg1aWbeMG4ekWtds/31BrEU+DONOg/7ax23vxpL03Pb7/zBmha2v7vdD3AyzZVWBVGEZxKOXWw==" + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -32466,6 +32505,23 @@ "react": "^15.3.0 || ^16.0.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/react-cookies": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/react-cookies/-/react-cookies-0.1.1.tgz", + "integrity": "sha512-PP75kJ4vtoHuuTdq0TAD3RmlAv7vuDQh9fkC4oDlhntgs9vX1DmREomO0Y1mcQKR9nMZ6/zxoflaMJ3MAmF5KQ==", + "dependencies": { + "cookie": "^0.3.1", + "object-assign": "^4.1.1" + } + }, + "node_modules/react-cookies/node_modules/cookie": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/react-docgen": { "version": "5.4.3", "resolved": "https://registry.npmjs.org/react-docgen/-/react-docgen-5.4.3.tgz", @@ -46682,6 +46738,19 @@ "mini-svg-data-uri": "^1.2.3" } }, + "@tanstack/react-table": { + "version": "8.7.6", + "resolved": "https://registry.npmjs.org/@tanstack/react-table/-/react-table-8.7.6.tgz", + "integrity": "sha512-/QijmMFeP7wDLBnr0MQ/5MlbXePbIL/1nOtkxBC9zvmBu4gDKJEDBqipUyM7Wc/iBpSd0IFyqBlvZvTPD9FYDA==", + "requires": { + "@tanstack/table-core": "8.7.6" + } + }, + "@tanstack/table-core": { + "version": "8.7.6", + "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.7.6.tgz", + "integrity": "sha512-sqiNTMzB6cpyL8DFH6/VqW48SwiflLqxQqYpo2wNock7rdVGvlm0BLNI8vZUJbr1+fmmWmHwBvi5OMgZw8n1DA==" + }, "@testing-library/dom": { "version": "8.19.1", "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-8.19.1.tgz", @@ -47817,6 +47886,11 @@ "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", "dev": true }, + "accept-language-parser": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/accept-language-parser/-/accept-language-parser-1.5.0.tgz", + "integrity": "sha512-QhyTbMLYo0BBGg1aWbeMG4ekWtds/31BrEU+DONOg/7ax23vxpL03Pb7/zBmha2v7vdD3AyzZVWBVGEZxKOXWw==" + }, "accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -61962,6 +62036,22 @@ "@babel/runtime": "^7.12.13" } }, + "react-cookies": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/react-cookies/-/react-cookies-0.1.1.tgz", + "integrity": "sha512-PP75kJ4vtoHuuTdq0TAD3RmlAv7vuDQh9fkC4oDlhntgs9vX1DmREomO0Y1mcQKR9nMZ6/zxoflaMJ3MAmF5KQ==", + "requires": { + "cookie": "^0.3.1", + "object-assign": "^4.1.1" + }, + "dependencies": { + "cookie": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", + "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw==" + } + } + }, "react-docgen": { "version": "5.4.3", "resolved": "https://registry.npmjs.org/react-docgen/-/react-docgen-5.4.3.tgz", diff --git a/website/package.json b/website/package.json index 4ae762e4..e4fb8319 100644 --- a/website/package.json +++ b/website/package.json @@ -38,6 +38,8 @@ "@next/font": "^13.1.0", "@prisma/client": "^4.7.1", "@tailwindcss/forms": "^0.5.3", + "@tanstack/react-table": "^8.7.6", + "accept-language-parser": "^1.5.0", "autoprefixer": "^10.4.13", "axios": "^1.2.1", "boolean": "^3.2.0", @@ -55,6 +57,7 @@ "npm": "^9.2.0", "postcss-focus-visible": "^7.1.0", "react": "18.2.0", + "react-cookies": "^0.1.1", "react-dom": "18.2.0", "react-feature-flags": "^1.0.0", "react-hook-form": "^7.42.1", diff --git a/website/public/locales/en/index.json b/website/public/locales/en/index.json index 3443e444..7d8c9152 100644 --- a/website/public/locales/en/index.json +++ b/website/public/locales/en/index.json @@ -1,16 +1,15 @@ { - "title": "Open Assistant", - "subtitle": "Conversational AI for everyone.", - "description": "Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world.", "blurb": "We believe we can create a revolution.", "blurb1": "In the same way that Stable Diffusion helped the world make art and images in new ways, we want to improve the world by providing amazing conversational AI.", - "join_us_title": "Join us", - "join_us_description": "All open source projects begin with people like you. Open source is the belief that if we collaborate we can together gift our knowledge and technology to the world for the benefit of humanity. Are you in? Find us here:", - "faq_title": "Frequently Asked Questions", + "description": "Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world.", "faq_items": { "q0": "How far along is this project?", "a0": "We are in the early stages of development, working from established research in applying RLHF to large language models.", "q1": "Who is behind Open Assistant?", "a1": "Open Assistant is a project organized by LAION and individuals around the world interested in bringing this technology to everyone." - } + }, + "faq_title": "Frequently Asked Questions", + "join_us_description": "All open source projects begin with people like you. Open source is the belief that if we collaborate we can together gift our knowledge and technology to the world for the benefit of humanity. Are you in? Find us here:", + "join_us_title": "Join us", + "subtitle": "Conversational AI for everyone." } diff --git a/website/public/locales/en/leaderboard.json b/website/public/locales/en/leaderboard.json new file mode 100644 index 00000000..c2dd0832 --- /dev/null +++ b/website/public/locales/en/leaderboard.json @@ -0,0 +1,11 @@ +{ + "daily": "Daily", + "last_updated_at": "Last updated at: {{val, datetime}}", + "leaderboard": "Leaderboard", + "monthly": "Monthly", + "overall": "Overall", + "rank": "Rank", + "score": "Score", + "user": "User", + "weekly": "Weekly" +} diff --git a/website/src/components/Dashboard/TaskOption.tsx b/website/src/components/Dashboard/TaskOption.tsx index e2bafac3..e73a06c8 100644 --- a/website/src/components/Dashboard/TaskOption.tsx +++ b/website/src/components/Dashboard/TaskOption.tsx @@ -1,51 +1,75 @@ import { Box, Flex, GridItem, Heading, SimpleGrid, Text, useColorModeValue } from "@chakra-ui/react"; import Link from "next/link"; +import { useMemo } from "react"; +import { TaskType } from "src/types/Task"; -import { TaskCategory, TaskCategoryLabels, TaskTypes } from "../Tasks/TaskTypes"; +import { TaskCategory, TaskCategoryLabels, TaskInfo, TaskInfos } from "../Tasks/TaskTypes"; -export const TaskOption = ({ displayTaskCategories }: { displayTaskCategories: TaskCategory[] }) => { +export interface TasksOptionProps { + content: Partial>; +} + +export const TaskOption = ({ content }: TasksOptionProps) => { const backgroundColor = useColorModeValue("white", "gray.700"); + const taskInfoMap = useMemo( + () => + Object.values(content) + .flat() + .reduce((obj, taskType) => { + obj[taskType] = TaskInfos.filter((t) => t.type === taskType).pop(); + return obj; + }, {} as Record), + [content] + ); + return ( - {displayTaskCategories.map((category) => ( + {Object.entries(content).map(([category, taskTypes]) => (
- {TaskCategoryLabels[category]} + + {TaskCategoryLabels[category]} + - {TaskTypes.filter((task) => task.category === category).map((item) => ( - - - - - - {item.label} - - - {item.desc} - - - - taskInfoMap[taskType]) + .map((item) => ( + + - + + {item.label} + {item.desc} + + Go -> - - - - ))} + + + ))}
))} ); }; + +export const allTaskOptions: TasksOptionProps["content"] = { + [TaskCategory.Random]: [TaskType.random], + [TaskCategory.Create]: [TaskType.initial_prompt, TaskType.prompter_reply, TaskType.assistant_reply], + [TaskCategory.Evaluate]: [ + TaskType.rank_initial_prompts, + TaskType.rank_prompter_replies, + TaskType.rank_assistant_replies, + ], + [TaskCategory.Label]: [TaskType.label_initial_prompt, TaskType.label_prompter_reply, TaskType.label_assistant_reply], +}; diff --git a/website/src/components/DataTable.tsx b/website/src/components/DataTable.tsx new file mode 100644 index 00000000..f9ef4e49 --- /dev/null +++ b/website/src/components/DataTable.tsx @@ -0,0 +1,166 @@ +import { + Box, + Button, + Card, + CardBody, + Flex, + FormControl, + FormLabel, + Input, + Popover, + PopoverArrow, + PopoverBody, + PopoverCloseButton, + PopoverContent, + PopoverTrigger, + Spacer, + Table, + TableCaption, + TableContainer, + Tbody, + Td, + Th, + Thead, + Tr, + useDisclosure, +} from "@chakra-ui/react"; +import { ColumnDef, flexRender, getCoreRowModel, useReactTable } from "@tanstack/react-table"; +import { ChangeEvent, ReactNode } from "react"; +import { FaFilter } from "react-icons/fa"; +import { useDebouncedCallback } from "use-debounce"; + +export type DataTableColumnDef = ColumnDef & { + filterable?: boolean; +}; + +// TODO: stricter type +export type FilterItem = { + id: string; + value: string; +}; + +export type DataTableProps = { + data: T[]; + columns: DataTableColumnDef[]; + caption?: string; + filterValues?: FilterItem[]; + onNextClick?: () => void; + onPreviousClick?: () => void; + onFilterChange?: (items: FilterItem[]) => void; + disableNext?: boolean; + disablePrevious?: boolean; +}; + +export const DataTable = ({ + data, + columns, + caption, + filterValues = [], + onNextClick, + onPreviousClick, + onFilterChange, + disableNext, + disablePrevious, +}: DataTableProps) => { + const { getHeaderGroups, getRowModel } = useReactTable({ + data, + columns, + getCoreRowModel: getCoreRowModel(), + }); + + const handleFilterChange = (value: FilterItem) => { + const idx = filterValues.findIndex((oldValue) => oldValue.id === value.id); + let newValues: FilterItem[] = []; + if (idx === -1) { + newValues = [...filterValues, value]; + } else { + newValues = filterValues.map((oldValue) => (oldValue.id === value.id ? value : oldValue)); + } + onFilterChange(newValues); + }; + return ( + + + + + + + + + + {caption} + + {getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + ))} + + ))} + + + {getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + ))} + + ))} + +
+ + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + {(header.column.columnDef as DataTableColumnDef).filterable && ( + value.id === header.id)?.value ?? ""} + onChange={(value) => handleFilterChange({ id: header.id, value })} + label={flexRender(header.column.columnDef.header, header.getContext())} + > + )} + +
{flexRender(cell.column.columnDef.cell, cell.getContext())}
+ + + + ); +}; + +const FilterModal = ({ + label, + onChange, + value, +}: { + label: ReactNode; + onChange: (val: string) => void; + value: string; +}) => { + const { isOpen, onOpen, onClose } = useDisclosure(); + + const handleInputChange = useDebouncedCallback((e: ChangeEvent) => { + onChange(e.target.value); + }, 500); + + return ( + + + + + + + + + + {label} + + + + + + ); +}; diff --git a/website/src/components/EmptyState.tsx b/website/src/components/EmptyState.tsx index 14715518..51e51a00 100644 --- a/website/src/components/EmptyState.tsx +++ b/website/src/components/EmptyState.tsx @@ -1,5 +1,5 @@ -import { Box, Link, Text, useColorModeValue } from "@chakra-ui/react"; -import { useRouter } from "next/router"; +import { Box, Text, useColorModeValue } from "@chakra-ui/react"; +import NextLink from "next/link"; import { FiAlertTriangle } from "react-icons/fi"; import { IconType } from "react-icons/lib"; @@ -10,16 +10,15 @@ type EmptyStateProps = { export const EmptyState = (props: EmptyStateProps) => { const backgroundColor = useColorModeValue("white", "gray.800"); - const router = useRouter(); return ( {props.text} - router.back()} color="blue.500" textUnderlineOffset="3px"> - Click here to go back - + + Go back to the dashboard + ); diff --git a/website/src/components/Header/Header.tsx b/website/src/components/Header/Header.tsx index a1b36123..64614578 100644 --- a/website/src/components/Header/Header.tsx +++ b/website/src/components/Header/Header.tsx @@ -5,6 +5,7 @@ import { useSession } from "next-auth/react"; import { useTranslation } from "next-i18next"; import { Flags } from "react-feature-flags"; import { FaUser } from "react-icons/fa"; +import { LanguageSelector } from "src/components/LanguageSelector"; import { UserMenu } from "./UserMenu"; @@ -45,6 +46,7 @@ export function Header() { FlagTest + diff --git a/website/src/components/Hero.tsx b/website/src/components/Hero.tsx index d401e47e..d9cab0c2 100644 --- a/website/src/components/Hero.tsx +++ b/website/src/components/Hero.tsx @@ -6,7 +6,7 @@ import { AnimatedCircles } from "./AnimatedCircles"; import { Container } from "./Container"; export function Hero() { - const { t } = useTranslation("index"); + const { t } = useTranslation(["index", "common"]); const { colorMode } = useColorMode(); const pTextColor = colorMode === "light" ? "text-gray-600" : "text-white"; const fancyTextGradientClasses = @@ -17,7 +17,7 @@ export function Hero() { - {t("title")} + {t("common:title")} { + const router = useRouter(); + const { i18n } = useTranslation(); + + const { language: currentLanguage } = i18n; + const languageNames = useMemo(() => { + return new Intl.DisplayNames([currentLanguage], { + type: "language", + }); + }, [currentLanguage]); + + const languageChanged = useCallback( + async (option) => { + const locale = option.target.value; + cookie.save("NEXT_LOCALE", locale, { path: "/" }); + const path = router.asPath; + return router.push(path, path, { locale }); + }, + [router] + ); + + const locales = router.locales; + return ( + + ); +}; + +export { LanguageSelector }; diff --git a/website/src/components/LanguageSelector/index.tsx b/website/src/components/LanguageSelector/index.tsx new file mode 100644 index 00000000..feb9f322 --- /dev/null +++ b/website/src/components/LanguageSelector/index.tsx @@ -0,0 +1 @@ +export * from "./LanguageSelector"; diff --git a/website/src/components/Layout.tsx b/website/src/components/Layout.tsx index 484d16ec..55085550 100644 --- a/website/src/components/Layout.tsx +++ b/website/src/components/Layout.tsx @@ -2,7 +2,7 @@ import { Box, Grid } from "@chakra-ui/react"; import type { NextPage } from "next"; -import { FiBarChart2, FiLayout, FiMessageSquare, FiUsers } from "react-icons/fi"; +import { FiBarChart2, FiLayout, FiMessageSquare, FiUsers, FiActivity } from "react-icons/fi"; import { Header } from "src/components/Header"; import { SlimFooter } from "./Dashboard/SlimFooter"; @@ -75,6 +75,12 @@ export const getAdminLayout = (page: React.ReactElement) => ( desc: "Users Dashboard", icon: FiUsers, }, + { + label: "Status", + pathname: "/admin/status", + desc: "Status Dashboard", + icon: FiActivity, + }, ]} > {page} diff --git a/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx b/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx index df18735d..7886784a 100644 --- a/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx +++ b/website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx @@ -1,23 +1,24 @@ -import { Table, TableContainer, Tbody, Td, Th, Thead, Tr, useColorModeValue } from "@chakra-ui/react"; -import React from "react"; +import { Table, TableContainer, Tbody, Td, Text, Th, Thead, Tr, useColorModeValue } from "@chakra-ui/react"; +import { useTranslation } from "next-i18next"; +import React, { useMemo } from "react"; import { useTable } from "react-table"; import { get } from "src/lib/api"; -import { LeaderboardEntity, LeaderboardTimeFrame } from "src/types/Leaderboard"; +import { LeaderboardReply, LeaderboardTimeFrame } from "src/types/Leaderboard"; import useSWRImmutable from "swr/immutable"; -const columns = [ +const getColumns = (t) => [ { - Header: "Rank", + Header: t("rank"), accessor: "rank", style: { width: "90px" }, }, { - Header: "Score", + Header: t("score"), accessor: "leader_score", style: { width: "90px" }, }, { - Header: "User", + Header: t("user"), accessor: "display_name", }, ]; @@ -26,13 +27,28 @@ const columns = [ * Presents a grid of leaderboard entries with more detailed information. */ const LeaderboardGridCell = ({ timeFrame }: { timeFrame: LeaderboardTimeFrame }) => { - const { data } = useSWRImmutable(`/api/leaderboard?time_frame=${timeFrame}`, get, { - fallbackData: [], + const { t } = useTranslation(["leaderboard", "common"]); + const { data: reply } = useSWRImmutable(`/api/leaderboard?time_frame=${timeFrame}`, get, { revalidateOnMount: true, }); + + const columns = useMemo(() => getColumns(t), [t]); + + const { getTableProps, getTableBodyProps, headerGroups, rows, prepareRow } = useTable({ + columns, + data: reply?.leaderboard ?? [], + }); + const backgroundColor = useColorModeValue("white", "gray.800"); - const { getTableProps, getTableBodyProps, headerGroups, rows, prepareRow } = useTable({ columns, data }); + const lastUpdated = useMemo(() => { + const val = new Date(reply?.last_updated); + return t("last_updated_at", { val, formatParams: { val: { dateStyle: "full", timeStyle: "short" } } }); + }, [t, reply?.last_updated]); + + if (!reply) { + return null; + } return ( @@ -66,6 +82,7 @@ const LeaderboardGridCell = ({ timeFrame }: { timeFrame: LeaderboardTimeFrame }) })} + {lastUpdated} ); }; diff --git a/website/src/components/Messages/MessageTableEntry.tsx b/website/src/components/Messages/MessageTableEntry.tsx index d18bd910..1205991e 100644 --- a/website/src/components/Messages/MessageTableEntry.tsx +++ b/website/src/components/Messages/MessageTableEntry.tsx @@ -50,6 +50,7 @@ export function MessageTableEntry(props: MessageTableEntryProps) { bg={item.is_assistant ? backgroundColor : backgroundColor2} onClick={props.enabled && goToMessage} _hover={props.enabled && { cursor: "pointer", opacity: 0.9 }} + whiteSpace="pre-wrap" > {inlineAvatar && avatar} {item.text} diff --git a/website/src/components/Tasks/Task/Task.tsx b/website/src/components/Tasks/Task/Task.tsx index 3d393575..45fb83d0 100644 --- a/website/src/components/Tasks/Task/Task.tsx +++ b/website/src/components/Tasks/Task/Task.tsx @@ -3,7 +3,7 @@ import { TaskControls } from "src/components/Survey/TaskControls"; import { CreateTask } from "src/components/Tasks/CreateTask"; import { EvaluateTask } from "src/components/Tasks/EvaluateTask"; import { LabelTask } from "src/components/Tasks/LabelTask"; -import { TaskCategory, TaskInfo, TaskTypes } from "src/components/Tasks/TaskTypes"; +import { TaskCategory, TaskInfo, TaskInfos } from "src/components/Tasks/TaskTypes"; import { UnchangedWarning } from "src/components/Tasks/UnchangedWarning"; import { post } from "src/lib/api"; import { TaskContent } from "src/types/Task"; @@ -29,7 +29,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => { const rootEl = useRef(null); - const taskType = TaskTypes.find((taskType) => taskType.type === task.type && taskType.mode === task.mode); + const taskType = TaskInfos.find((taskType) => taskType.type === task.type && taskType.mode === task.mode); const { trigger: sendRejection } = useSWRMutation("/api/reject_task", post, { onSuccess: async () => { diff --git a/website/src/components/Tasks/TaskTypes.tsx b/website/src/components/Tasks/TaskTypes.tsx index 4c6da92c..d10159d9 100644 --- a/website/src/components/Tasks/TaskTypes.tsx +++ b/website/src/components/Tasks/TaskTypes.tsx @@ -21,16 +21,16 @@ export interface TaskInfo { } export const TaskCategoryLabels: { [key in TaskCategory]: string } = { - [TaskCategory.Random]: "I'm feeling lucky", + [TaskCategory.Random]: "Grab a task!", [TaskCategory.Create]: "Create", [TaskCategory.Evaluate]: "Evaluate", [TaskCategory.Label]: "Label", }; -export const TaskTypes: TaskInfo[] = [ +export const TaskInfos: TaskInfo[] = [ // general/random { - label: "Start a Task", + label: "I'm feeling lucky", desc: "Help us improve Open Assistant by starting a random task.", category: TaskCategory.Random, pathname: "/tasks/random", @@ -104,7 +104,7 @@ export const TaskTypes: TaskInfo[] = [ category: TaskCategory.Evaluate, pathname: "/evaluate/rank_initial_prompts", help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting", - overview: "Given the following inital prompts, sort them from best to worst, best being first, worst being last.", + overview: "Given the following initial prompts, sort them from best to worst, best being first, worst being last.", type: "rank_initial_prompts", update_type: "message_ranking", unchanged_title: "Order Unchanged", diff --git a/website/src/components/UserTable.tsx b/website/src/components/UserTable.tsx new file mode 100644 index 00000000..df412bbc --- /dev/null +++ b/website/src/components/UserTable.tsx @@ -0,0 +1,108 @@ +import { IconButton } from "@chakra-ui/react"; +import { createColumnHelper } from "@tanstack/react-table"; +import Link from "next/link"; +import { memo, useState } from "react"; +import { FaPen } from "react-icons/fa"; +import { get } from "src/lib/api"; +import { FetchUsersResponse } from "src/lib/oasst_api_client"; +import type { User } from "src/types/Users"; +import useSWR from "swr"; + +import { DataTable, DataTableColumnDef, FilterItem } from "./DataTable"; + +interface Pagination { + /** + * The user's `display_name` used for pagination. + */ + cursor: string; + + /** + * The pagination direction. + */ + direction: "forward" | "back"; +} + +const columnHelper = createColumnHelper(); + +const columns: DataTableColumnDef[] = [ + columnHelper.accessor("user_id", { + header: "ID", + }), + columnHelper.accessor("id", { + header: "Auth ID", + }), + columnHelper.accessor("auth_method", { + header: "Auth Method", + }), + { + ...columnHelper.accessor("display_name", { + header: "Name", + }), + filterable: true, + }, + columnHelper.accessor("role", { + header: "Role", + }), + columnHelper.accessor((user) => user.user_id, { + cell: ({ getValue }) => ( + } + > + ), + header: "Update", + }), +]; + +export const UserTable = memo(function UserTable() { + const [pagination, setPagination] = useState({ cursor: "", direction: "forward" }); + const [filterValues, setFilterValues] = useState([]); + const handleFilterValuesChange = (values: FilterItem[]) => { + setFilterValues(values); + setPagination((old) => ({ ...old, cursor: "" })); + }; + // Fetch and save the users. + // This follows useSWR's recommendation for simple pagination: + // https://swr.vercel.app/docs/pagination#when-to-use-useswr + const display_name = filterValues.find((value) => value.id === "display_name")?.value ?? ""; + const { data, error } = useSWR>( + `/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}&searchDisplayName=${display_name}&sortKey=display_name`, + get, + { + keepPreviousData: true, + } + ); + + const toPreviousPage = () => { + setPagination({ + cursor: data.prev, + direction: "back", + }); + }; + + const toNextPage = () => { + setPagination({ + cursor: data.next, + direction: "forward", + }); + }; + + return ( + <> + + {error && "Unable to load users."} + + ); +}); diff --git a/website/src/components/UsersCell.tsx b/website/src/components/UsersCell.tsx deleted file mode 100644 index 99824090..00000000 --- a/website/src/components/UsersCell.tsx +++ /dev/null @@ -1,137 +0,0 @@ -import { - Button, - Flex, - Spacer, - Stack, - Table, - TableCaption, - TableContainer, - Tbody, - Td, - Th, - Thead, - Tr, - useToast, -} from "@chakra-ui/react"; -import Link from "next/link"; -import { useState } from "react"; -import { get } from "src/lib/api"; -import type { User } from "src/types/Users"; -import useSWR from "swr"; - -interface Pagination { - /** - * The user's `display_name` used for pagination. - */ - cursor: string; - - /** - * The pagination direction. - */ - direction: "forward" | "back"; -} - -/** - * Fetches users from the users api route and then presents them in a simple Chakra table. - */ -const UsersCell = () => { - const toast = useToast(); - const [pagination, setPagination] = useState({ cursor: "", direction: "forward" }); - const [users, setUsers] = useState([]); - - // Fetch and save the users. - // This follows useSWR's recommendation for simple pagination: - // https://swr.vercel.app/docs/pagination#when-to-use-useswr - useSWR(`/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}`, get, { - onSuccess: (data) => { - // When no more users can be found, trigger a toast to indicate why no - // changes have taken place. We have to maintain a non-empty set of - // users otherwise we can't paginate using a cursor (since we've lost the - // cursor). - if (data.length === 0) { - toast({ - title: "No more users", - status: "warning", - duration: 1000, - isClosable: true, - }); - return; - } - setUsers(data); - }, - }); - - const toPreviousPage = () => { - if (users.length >= 0) { - setPagination({ - cursor: users[0].display_name, - direction: "back", - }); - } else { - toast({ - title: "Can not paginate when no users are found", - status: "warning", - duration: 1000, - isClosable: true, - }); - } - }; - - const toNextPage = () => { - if (users.length >= 0) { - setPagination({ - cursor: users[users.length - 1].display_name, - direction: "forward", - }); - } else { - toast({ - title: "Can not paginate when no users are found", - status: "warning", - duration: 1000, - isClosable: true, - }); - } - }; - - // Present users in a naive table. - return ( - - - - - - - - - Users - - - - - - - - - - - - {users.map(({ id, user_id, auth_method, display_name, role }) => ( - - - - - - - - - ))} - -
IdAuth IdAuth MethodNameRoleUpdate
{user_id}{id}{auth_method}{display_name}{role} - Manage -
- - - ); -}; - -export default UsersCell; diff --git a/website/src/lib/oasst_api_client.ts b/website/src/lib/oasst_api_client.ts index b1639462..cd5e1abc 100644 --- a/website/src/lib/oasst_api_client.ts +++ b/website/src/lib/oasst_api_client.ts @@ -1,7 +1,7 @@ import type { Message } from "src/types/Conversation"; import { LeaderboardReply, LeaderboardTimeFrame } from "src/types/Leaderboard"; import type { AvailableTasks } from "src/types/Task"; -import type { BackendUser, BackendUserCore } from "src/types/Users"; +import type { BackendUser, BackendUserCore, User } from "src/types/Users"; export class OasstError { message: string; @@ -15,6 +15,22 @@ export class OasstError { } } +export type FetchUsersParams = { + limit: number; + cursor?: string; + direction: "forward" | "back"; + searchDisplayName?: string; + sortKey?: "username" | "display_name"; +}; + +export type FetchUsersResponse = { + items: T[]; + next?: string; + prev?: string; + sort_key: "username" | "display_name"; + order: "asc" | "desc"; +}; + export class OasstApiClient { oasstApiUrl: string; oasstApiKey: string; @@ -108,10 +124,11 @@ export class OasstApiClient { // TODO return a strongly typed Task? // This method is used to store a task in RegisteredTask.task. // This is a raw Json type, so we can't use it to strongly type the task. - async fetchTask(taskType: string, user: BackendUserCore): Promise { + async fetchTask(taskType: string, user: BackendUserCore, lang: string): Promise { return this.post("/api/v1/tasks/", { type: taskType, user, + lang, }); } @@ -136,7 +153,8 @@ export class OasstApiClient { messageId: string, userMessageId: string, content: object, - user: BackendUserCore + user: BackendUserCore, + lang: string ): Promise { return this.post("/api/v1/tasks/interaction", { type: updateType, @@ -144,15 +162,37 @@ export class OasstApiClient { task_id: taskId, message_id: messageId, user_message_id: userMessageId, + lang, ...content, }); } + /** + * Returns the tasks availability information for given `user`. + */ + async fetch_tasks_availability(user: object): Promise { + return this.post("/api/v1/tasks/availability", user); + } + + /** + * Returns the message stats from the backend. + */ + async fetch_stats(): Promise { + return this.get("/api/v1/stats/"); + } + + /** + * Returns the tree manager stats from the backend. + */ + async fetch_tree_manager(): Promise { + return this.get("/api/v1/stats/tree_manager"); + } + /** * Returns the `BackendUser` associated with `user_id` */ async fetch_user(user_id: string): Promise { - return this.get(`/api/v1/users/users/${user_id}`); + return this.get(`/api/v1/users/${user_id}`); } /** @@ -164,21 +204,40 @@ export class OasstApiClient { * forward. If false and `cursor` is not empty, pages backwards. * @returns {Promise} A Promise that returns an array of `BackendUser` objects. */ - async fetch_users(max_count: number, cursor: string, isForward: boolean): Promise { - const params = new URLSearchParams(); - params.append("max_count", max_count.toString()); + async fetch_users({ + direction, + limit, + cursor, + searchDisplayName, + sortKey = "display_name", + }: FetchUsersParams): Promise { + const params = new URLSearchParams({ + search_text: searchDisplayName, + sort_key: sortKey, + max_count: limit.toString(), + }); // The backend API uses different query parameters depending on the // pagination direction but they both take the same cursor value. // Depending on direction, pick the right query param. if (cursor !== "") { - params.append(isForward ? "gt" : "lt", cursor); + params.append(direction === "forward" ? "gt" : "lt", cursor); } - const BASE_URL = `/api/v1/frontend_users`; + const BASE_URL = `/api/v1/users/cursor`; const url = `${BASE_URL}/?${params.toString()}`; return this.get(url); } + // async fetch_user_by_display_name(name: string): Promise { + // const params = new URLSearchParams({ + // search_text: name, + // }); + + // const endpoint = `/api/v1/frontend_users/by_display_name`; + + // return this.get(`${endpoint}?${params.toString()}`); + // } + /** * Returns the `Message`s associated with `user_id` in the backend. */ diff --git a/website/src/lib/users.ts b/website/src/lib/users.ts index 2aa8c708..637e93a9 100644 --- a/website/src/lib/users.ts +++ b/website/src/lib/users.ts @@ -1,6 +1,32 @@ +import parser from "accept-language-parser"; +import type { NextApiRequest } from "next"; +import { i18n } from "src/../next-i18next.config"; import prisma from "src/lib/prismadb"; import type { BackendUserCore } from "src/types/Users"; +const LOCALE_SET = new Set(i18n.locales); + +/** + * Returns the most appropriate user language using the following priority: + * + * 1. The `NEXT_LOCALE` cookie which is set by the client side and will be in + * the set of supported locales. + * 2. The `accept-language` header if it contains a supported locale as set by + * the i18n module. + * 3. "en" as a final fallback. + */ +const getUserLanguage = (req: NextApiRequest) => { + const cookieLanguage = req.cookies["NEXT_LOCALE"]; + if (cookieLanguage) { + return cookieLanguage; + } + const headerLanguages = parser.parse(req.headers["accept-language"]); + if (headerLanguages.length > 0 && LOCALE_SET.has(headerLanguages[0].code)) { + return headerLanguages[0].code; + } + return "en"; +}; + /** * Returns a `BackendUserCore` that can be used for interacting with the Backend service. * @@ -35,4 +61,4 @@ const getBackendUserCore = async (id: string) => { } as BackendUserCore; }; -export { getBackendUserCore }; +export { getBackendUserCore, getUserLanguage }; diff --git a/website/src/pages/admin/index.tsx b/website/src/pages/admin/index.tsx index f8827049..ede9f59c 100644 --- a/website/src/pages/admin/index.tsx +++ b/website/src/pages/admin/index.tsx @@ -3,7 +3,7 @@ import { useRouter } from "next/router"; import { useSession } from "next-auth/react"; import { useEffect } from "react"; import { getAdminLayout } from "src/components/Layout"; -import UsersCell from "src/components/UsersCell"; +import { UserTable } from "src/components/UserTable"; export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props"; /** @@ -28,7 +28,6 @@ const AdminIndex = () => { } router.push("/"); }, [router, session, status]); - return ( <> @@ -38,7 +37,7 @@ const AdminIndex = () => { content="Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world." /> -
{status === "loading" ? "loading..." : }
+
{status === "loading" ? "loading..." : }
); }; diff --git a/website/src/pages/admin/status/index.tsx b/website/src/pages/admin/status/index.tsx new file mode 100644 index 00000000..12eb0785 --- /dev/null +++ b/website/src/pages/admin/status/index.tsx @@ -0,0 +1,174 @@ +import { + Box, + Card, + CardBody, + CircularProgress, + SimpleGrid, + Text, + Table, + TableCaption, + TableContainer, + Tbody, + Td, + Th, + Thead, + Tr, + useColorMode, +} from "@chakra-ui/react"; +import Head from "next/head"; +import { useRouter } from "next/router"; +import { useSession } from "next-auth/react"; +import { useEffect } from "react"; +import useSWRImmutable from "swr/immutable"; +import { getAdminLayout } from "src/components/Layout"; +import { get } from "src/lib/api"; + +/** + * Provides the admin status page that shows result of calls to several backend API endpoints, + * namely /api/v1/tasks/availability, /api/v1/stats/, /api/v1/stats/tree_manager + */ + +const StatusIndex = () => { + const router = useRouter(); + const { data: session, status } = useSession(); + + const { colorMode } = useColorMode(); + const dataBackgroundColor = colorMode === "light" ? "gray.100" : "gray.800"; + // Check when the user session is loaded and re-route if the user is not an + // admin. This follows the suggestion by NextJS for handling private pages: + // https://nextjs.org/docs/api-reference/next/router#usage + // + // All admin pages should use the same check and routing steps. + useEffect(() => { + if (status === "loading") { + return; + } + if (session?.user?.role === "admin") { + return; + } + router.push("/"); + }, [router, session, status]); + + const { + data: dataStatus, + error: errorStatus, + isLoading: isLoadingStatus, + } = useSWRImmutable("/api/admin/status", get); + + const { tasksAvailability, stats, treeManager } = dataStatus || {}; + + return ( + <> + + Status - Open Assistant + + + + + + + + /api/v1/tasks/availability + + + {tasksAvailability?.status === "fulfilled" ? ( + 
{JSON.stringify(tasksAvailability.value, null, 2)}
+ ) : tasksAvailability?.status === "rejected" ? ( + 
{JSON.stringify(tasksAvailability.reason, null, 2)}
+ ) : errorStatus ? ( + 
{JSON.stringify(errorStatus, null, 2)}
+ ) : ( + + )} + + + + + + + + /api/v1/stats/ + + + {stats?.status === "fulfilled" ? ( + 
{JSON.stringify(stats.value, null, 2)}
+ ) : stats?.status === "rejected" ? ( + 
{JSON.stringify(stats.reason, null, 2)}
+ ) : errorStatus ? ( + 
{JSON.stringify(errorStatus, null, 2)}
+ ) : ( + + )} + + + + +
+ + + + /api/v1/stats/tree_manager + + {treeManager?.status === "fulfilled" ? ( + + + state_counts + + + 
{JSON.stringify(treeManager.value.state_counts, null, 2)}
+ + +
+ + message_counts + + + Tree Manager + + + + + + + + + + + + + {treeManager.value.message_counts.map( + ({ message_tree_id, state, depth, oldest, youngest, count, goal_tree_size }) => ( + + + + + + + + + + ) + )} + +
Message Tree IDStateDepthOldestYoungestCountGoal Tree Size
{message_tree_id}{state}{depth}{oldest}{youngest}{count}{goal_tree_size}
+ + + ) : treeManager?.status === "rejected" ? ( + 
{JSON.stringify(treeManager.reason, null, 2)}
+ ) : errorStatus ? ( + 
{JSON.stringify(errorStatus, null, 2)}
+ ) : ( + + )} + + + + ); +}; + +StatusIndex.getLayout = getAdminLayout; + +export default StatusIndex; diff --git a/website/src/pages/api/admin/status.ts b/website/src/pages/api/admin/status.ts new file mode 100644 index 00000000..1da03da8 --- /dev/null +++ b/website/src/pages/api/admin/status.ts @@ -0,0 +1,30 @@ +import { getToken } from "next-auth/jwt"; +import { withRole } from "src/lib/auth"; +import { oasstApiClient } from "src/lib/oasst_api_client"; +import { getBackendUserCore } from "src/lib/users"; + +/** + * Returns tasks availability, stats, and tree manager stats. + */ +const handler = withRole("admin", async (req, res) => { + const dummyUser = { + id: "__dummy_user__", + display_name: "Dummy User", + auth_method: "local", + }; + const [tasksAvailabilityOutcome, statsOutcome, treeManagerOutcome] = await Promise.allSettled([ + oasstApiClient.fetch_tasks_availability(dummyUser), + oasstApiClient.fetch_stats(), + oasstApiClient.fetch_tree_manager(), + ]); + + const status = { + tasksAvailability: tasksAvailabilityOutcome, + stats: statsOutcome, + treeManager: treeManagerOutcome, + }; + + res.status(200).json(status); +}); + +export default handler; diff --git a/website/src/pages/api/admin/users.ts b/website/src/pages/api/admin/users.ts index e600650d..57944cff 100644 --- a/website/src/pages/api/admin/users.ts +++ b/website/src/pages/api/admin/users.ts @@ -1,5 +1,5 @@ import { withRole } from "src/lib/auth"; -import { oasstApiClient } from "src/lib/oasst_api_client"; +import { FetchUsersParams, oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; /** @@ -17,10 +17,16 @@ const PAGE_SIZE = 20; * direction. */ const handler = withRole("admin", async (req, res) => { - const { cursor, direction } = req.query; + const { cursor, direction, searchDisplayName = "", sortKey = "username" } = req.query; // First, get all the users according to the backend. - const all_users = await oasstApiClient.fetch_users(PAGE_SIZE, cursor as string, direction === "forward"); + const { items: all_users, ...rest } = await oasstApiClient.fetch_users({ + searchDisplayName: searchDisplayName as FetchUsersParams["searchDisplayName"], + direction: direction as FetchUsersParams["direction"], + limit: PAGE_SIZE, + cursor: cursor as FetchUsersParams["cursor"], + sortKey: sortKey === "username" || sortKey === "display_name" ? sortKey : undefined, + }); // Next, get all the users stored in the web's auth database to fetch their role. const local_user_ids = all_users.map(({ id }) => id); @@ -51,7 +57,10 @@ const handler = withRole("admin", async (req, res) => { }; }); - res.status(200).json(users); + res.status(200).json({ + items: users, + ...rest, + }); }); export default handler; diff --git a/website/src/pages/api/leaderboard.ts b/website/src/pages/api/leaderboard.ts index 592f3da5..1ddf947e 100644 --- a/website/src/pages/api/leaderboard.ts +++ b/website/src/pages/api/leaderboard.ts @@ -6,9 +6,9 @@ import { LeaderboardTimeFrame } from "src/types/Leaderboard"; * Returns the set of valid labels that can be applied to messages. */ const handler = withoutRole("banned", async (req, res) => { - const time_frame = (req.query.time_frame as LeaderboardTimeFrame) || LeaderboardTimeFrame.day; - const { leaderboard } = await oasstApiClient.fetch_leaderboard(time_frame); - res.status(200).json(leaderboard); + const time_frame = (req.query.time_frame as LeaderboardTimeFrame) ?? LeaderboardTimeFrame.day; + const info = await oasstApiClient.fetch_leaderboard(time_frame); + res.status(200).json(info); }); export default handler; diff --git a/website/src/pages/api/messages/user.ts b/website/src/pages/api/messages/user.ts index e5f361b8..6f39aad1 100644 --- a/website/src/pages/api/messages/user.ts +++ b/website/src/pages/api/messages/user.ts @@ -1,9 +1,12 @@ import { withoutRole } from "src/lib/auth"; +import { getBackendUserCore } from "src/lib/users"; const handler = withoutRole("banned", async (req, res, token) => { //TODO: add params if needed + const user = await getBackendUserCore(token.sub); const params = new URLSearchParams({ - username: token.sub, + username: user.id, + auth_method: user.auth_method, }); const messagesRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/messages?${params}`, { diff --git a/website/src/pages/api/new_task/[task_type].ts b/website/src/pages/api/new_task/[task_type].ts index c8255b18..360b8faa 100644 --- a/website/src/pages/api/new_task/[task_type].ts +++ b/website/src/pages/api/new_task/[task_type].ts @@ -1,7 +1,7 @@ import { withoutRole } from "src/lib/auth"; import { oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; -import { getBackendUserCore } from "src/lib/users"; +import { getBackendUserCore, getUserLanguage } from "src/lib/users"; /** * Returns a new task created from the Task Backend. We do a few things here: @@ -14,11 +14,12 @@ import { getBackendUserCore } from "src/lib/users"; const handler = withoutRole("banned", async (req, res, token) => { // Fetch the new task. const { task_type } = req.query; + const userLanguage = getUserLanguage(req); const user = await getBackendUserCore(token.sub); let task; try { - task = await oasstApiClient.fetchTask(task_type as string, user); + task = await oasstApiClient.fetchTask(task_type as string, user, userLanguage); } catch (err) { console.error(err); res.status(500).json(err); diff --git a/website/src/pages/api/update_task.ts b/website/src/pages/api/update_task.ts index c547503a..6f08d640 100644 --- a/website/src/pages/api/update_task.ts +++ b/website/src/pages/api/update_task.ts @@ -2,7 +2,7 @@ import { Prisma } from "@prisma/client"; import { withoutRole } from "src/lib/auth"; import { oasstApiClient } from "src/lib/oasst_api_client"; import prisma from "src/lib/prismadb"; -import { getBackendUserCore } from "src/lib/users"; +import { getBackendUserCore, getUserLanguage } from "src/lib/users"; /** * Stores the task interaction with the Task Backend and then returns the next task generated. @@ -41,9 +41,18 @@ const handler = withoutRole("banned", async (req, res, token) => { }); const user = await getBackendUserCore(token.sub); + const userLanguage = getUserLanguage(req); let newTask; try { - newTask = await oasstApiClient.interactTask(update_type, taskId, frontendId, interaction.id, content, user); + newTask = await oasstApiClient.interactTask( + update_type, + taskId, + frontendId, + interaction.id, + content, + user, + userLanguage + ); } catch (err) { console.error(JSON.stringify(err)); return res.status(500).json(err); diff --git a/website/src/pages/auth/verify.tsx b/website/src/pages/auth/verify.tsx index 876aa677..d7a64a63 100644 --- a/website/src/pages/auth/verify.tsx +++ b/website/src/pages/auth/verify.tsx @@ -1,6 +1,7 @@ import { useColorMode } from "@chakra-ui/react"; import Head from "next/head"; import { getCsrfToken, getProviders } from "next-auth/react"; +import { serverSideTranslations } from "next-i18next/serverSideTranslations"; export default function Verify() { const { colorMode } = useColorMode(); @@ -21,14 +22,14 @@ export default function Verify() { ); } -// eslint-disable-next-line @typescript-eslint/no-unused-vars -export async function getServerSideProps(context) { +export async function getServerSideProps({ locale }) { const csrfToken = await getCsrfToken(); const providers = await getProviders(); return { props: { csrfToken, providers, + ...(await serverSideTranslations(locale, ["common"])), }, }; } diff --git a/website/src/pages/dashboard.tsx b/website/src/pages/dashboard.tsx index e0b8bba4..4def6196 100644 --- a/website/src/pages/dashboard.tsx +++ b/website/src/pages/dashboard.tsx @@ -5,15 +5,17 @@ import { LeaderboardTable, TaskOption, WelcomeCard } from "src/components/Dashbo import { getDashboardLayout } from "src/components/Layout"; import { TaskCategory } from "src/components/Tasks/TaskTypes"; import { get } from "src/lib/api"; -import type { AvailableTasks, TaskType } from "src/types/Task"; +import { AvailableTasks, TaskType } from "src/types/Task"; export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props"; import useSWRImmutable from "swr/immutable"; const Dashboard = () => { const { data } = useSWRImmutable("/api/available_tasks", get); - // TODO: show only these tasks: - const availableTasks = useMemo(() => filterAvailableTasks(data ?? {}), [data]); + const availableTaskTypes = useMemo(() => { + const taskTypes = filterAvailableTasks(data ?? {}); + return { [TaskCategory.Random]: taskTypes }; + }, [data]); return ( <> @@ -23,7 +25,7 @@ const Dashboard = () => { - + diff --git a/website/src/pages/index.tsx b/website/src/pages/index.tsx index 8fe5d852..888c0d61 100644 --- a/website/src/pages/index.tsx +++ b/website/src/pages/index.tsx @@ -3,17 +3,17 @@ import Head from "next/head"; import { useRouter } from "next/router"; import { useSession } from "next-auth/react"; import { useTranslation } from "next-i18next"; -import { serverSideTranslations } from "next-i18next/serverSideTranslations"; import { useEffect } from "react"; import { CallToAction } from "src/components/CallToAction"; import { Faq } from "src/components/Faq"; import { Hero } from "src/components/Hero"; import { getTransparentHeaderLayout } from "src/components/Layout"; +export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props"; const Home = () => { const router = useRouter(); const { status } = useSession(); - const { t } = useTranslation("index"); + const { t } = useTranslation(); useEffect(() => { if (status === "authenticated") { router.push("/dashboard"); @@ -37,10 +37,4 @@ const Home = () => { Home.getLayout = getTransparentHeaderLayout; -export const getStaticProps = async ({ locale }) => ({ - props: { - ...(await serverSideTranslations(locale, ["index", "common"])), - }, -}); - export default Home; diff --git a/website/src/pages/leaderboard.tsx b/website/src/pages/leaderboard.tsx index f79dac52..e413366f 100644 --- a/website/src/pages/leaderboard.tsx +++ b/website/src/pages/leaderboard.tsx @@ -1,27 +1,29 @@ import { Box, Heading, Tab, TabList, TabPanel, TabPanels, Tabs } from "@chakra-ui/react"; import Head from "next/head"; +import { useTranslation } from "next-i18next"; import { getDashboardLayout } from "src/components/Layout"; import { LeaderboardGridCell } from "src/components/LeaderboardGridCell"; export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props"; import { LeaderboardTimeFrame } from "src/types/Leaderboard"; const Leaderboard = () => { + const { t } = useTranslation(["leaderboard", "common"]); return ( <> - Leaderboard - Open Assistant + {`${t("leaderboard")} - ${t("common:title")}`} - Leaderboard + {t("leaderboard")} - Daily - Weekly - Monthly - Overall + {t("daily")} + {t("weekly")} + {t("monthly")} + {t("overall")} diff --git a/website/src/pages/tasks/all.tsx b/website/src/pages/tasks/all.tsx index ed0659c8..01954c2f 100644 --- a/website/src/pages/tasks/all.tsx +++ b/website/src/pages/tasks/all.tsx @@ -1,7 +1,8 @@ import Head from "next/head"; import { TaskOption } from "src/components/Dashboard"; +import { allTaskOptions } from "src/components/Dashboard/TaskOption"; import { getDashboardLayout } from "src/components/Layout"; -import { TaskCategory } from "src/components/Tasks/TaskTypes"; +export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props"; const AllTasks = () => { return ( @@ -10,7 +11,7 @@ const AllTasks = () => { All Tasks - Open Assistant - + ); }; diff --git a/website/src/pages/tasks/random.tsx b/website/src/pages/tasks/random.tsx index be1809c3..f1c04d2c 100644 --- a/website/src/pages/tasks/random.tsx +++ b/website/src/pages/tasks/random.tsx @@ -4,6 +4,7 @@ import { getDashboardLayout } from "src/components/Layout"; import { LoadingScreen } from "src/components/Loading/LoadingScreen"; import { Task } from "src/components/Tasks/Task"; import { useGenericTaskAPI } from "src/hooks/tasks/useGenericTaskAPI"; +export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props"; import { TaskType } from "src/types/Task"; const RandomTask = () => { diff --git a/website/src/types/Leaderboard.ts b/website/src/types/Leaderboard.ts index 21c91766..5c0acfc3 100644 --- a/website/src/types/Leaderboard.ts +++ b/website/src/types/Leaderboard.ts @@ -12,6 +12,7 @@ export const enum LeaderboardTimeFrame { } export interface LeaderboardReply { time_frame: LeaderboardTimeFrame; + last_updated: string; // date time iso string leaderboard: LeaderboardEntity[]; }