Merge branch 'main' into sft-formatting

backend/oasst_backend/api/v1/admin.py

@@ -54,6 +54,7 @@ class PublicSettings(pydantic.BaseModel):
 
     PROJECT_NAME: str
     API_V1_STR: str
+    MESSAGE_SIZE_LIMIT: int
     DEBUG_USE_SEED_DATA: bool
     DEBUG_ALLOW_SELF_LABELING: bool
     DEBUG_SKIP_EMBEDDING_COMPUTATION: bool

backend/oasst_backend/api/v1/messages.py

@@ -96,8 +96,15 @@ def get_messages_cursor(
     items = utils.prepare_message_list(messages)
     n, p = None, None
     if len(items) > 0:
-        p = str(items[0].id) + "$" + items[0].created_date.isoformat()
-        n = str(items[-1].id) + "$" + items[-1].created_date.isoformat()
+        if len(items) == max_count or gte_created_date:
+            p = str(items[0].id) + "$" + items[0].created_date.isoformat()
+        if len(items) == max_count or lte_created_date:
+            n = str(items[-1].id) + "$" + items[-1].created_date.isoformat()
+    else:
+        if gte_created_date:
+            p = gte_created_date.isoformat()
+        if lte_created_date:
+            n = lte_created_date.isoformat()
 
     order = "desc" if desc else "asc"
     return protocol.MessagePage(prev=p, next=n, sort_key="created_date", order=order, items=items)

backend/oasst_backend/api/v1/text_labels.py

@@ -4,8 +4,9 @@ from loguru import logger
 from oasst_backend.api import deps
 from oasst_backend.prompt_repository import PromptRepository
 from oasst_backend.schemas.text_labels import LabelOption, ValidLabelsResponse
+from oasst_backend.utils.database_utils import CommitMode, managed_tx_function
+from oasst_shared.exceptions import OasstError
 from oasst_shared.schemas import protocol as protocol_schema
-from sqlmodel import Session
 from starlette.status import HTTP_204_NO_CONTENT, HTTP_400_BAD_REQUEST
 
 router = APIRouter()
@@ -14,20 +15,25 @@ router = APIRouter()
 @router.post("/", status_code=HTTP_204_NO_CONTENT)
 def label_text(
     *,
-    db: Session = Depends(deps.get_db),
     api_key: APIKey = Depends(deps.get_api_key),
     text_labels: protocol_schema.TextLabels,
 ) -> None:
     """
     Label a piece of text.
     """
-    api_client = deps.api_auth(api_key, db)
+
+    @managed_tx_function(CommitMode.COMMIT)
+    def store_text_labels(session: deps.Session):
+        api_client = deps.api_auth(api_key, session)
+        pr = PromptRepository(session, api_client, client_user=text_labels.user)
+        pr.store_text_labels(text_labels)
 
     try:
         logger.info(f"Labeling text {text_labels=}.")
-        pr = PromptRepository(db, api_client, client_user=text_labels.user)
-        pr.store_text_labels(text_labels)
+        store_text_labels()
 
+    except OasstError:
+        raise
     except Exception:
         logger.exception("Failed to store label.")
         raise HTTPException(

backend/oasst_backend/api/v1/users.py

@@ -1,5 +1,5 @@
 import datetime
-from typing import Optional
+from typing import Callable, Optional
 from uuid import UUID
 
 from fastapi import APIRouter, Depends, Query
@@ -28,6 +28,7 @@ def get_users_ordered_by_username(
     search_text: Optional[str] = None,
     auth_method: Optional[str] = None,
     max_count: Optional[int] = Query(100, gt=0, le=10000),
+    desc: Optional[bool] = False,
     api_client: ApiClient = Depends(deps.get_api_client),
     db: Session = Depends(deps.get_db),
 ):
@@ -41,6 +42,7 @@ def get_users_ordered_by_username(
         auth_method=auth_method,
         search_text=search_text,
         limit=max_count,
+        desc=desc,
     )
     return [u.to_protocol_frontend_user() for u in users]
 
@@ -55,6 +57,7 @@ def get_users_ordered_by_display_name(
     auth_method: Optional[str] = None,
     search_text: Optional[str] = None,
     max_count: Optional[int] = Query(100, gt=0, le=10000),
+    desc: Optional[bool] = False,
     api_client: ApiClient = Depends(deps.get_api_client),
     db: Session = Depends(deps.get_db),
 ):
@@ -68,6 +71,7 @@ def get_users_ordered_by_display_name(
         auth_method=auth_method,
         search_text=search_text,
         limit=max_count,
+        desc=desc,
     )
     return [u.to_protocol_frontend_user() for u in users]
 
@@ -84,6 +88,8 @@ def get_users_cursor(
     api_client: ApiClient = Depends(deps.get_api_client),
     db: Session = Depends(deps.get_db),
 ):
+    assert max_count is not None
+
     def split_cursor(x: str | None) -> tuple[str, UUID]:
         if not x:
             return None, None
@@ -93,6 +99,32 @@ def get_users_cursor(
         return x, None
 
     items: list[protocol.FrontEndUser]
+    qry_max_count = max_count + 1 if lt is None or gt is None else max_count
+    desc = lt and not gt
+
+    def get_next_prev(num_rows: int, lt: str | None, gt: str | None, key_fn: Callable[[protocol.FrontEndUser], str]):
+        p, n = None, None
+        if len(items) > 0:
+            if (num_rows > max_count and lt) or gt:
+                p = str(items[0].user_id) + "$" + key_fn(items[0])
+            if num_rows > max_count or lt:
+                n = str(items[-1].user_id) + "$" + key_fn(items[-1])
+        else:
+            if gt:
+                p = gt
+            if lt:
+                n = lt
+        return p, n
+
+    def remove_extra_item(items: list[protocol.FrontEndUser], lt: str | None, gt: str | None):
+        num_rows = len(items)
+        if qry_max_count > max_count and num_rows == qry_max_count:
+            assert not (lt and gt)
+            items = items[:-1]
+        if desc:
+            items.reverse()
+        return items, num_rows
+
     n, p = None, None
     if sort_key == "username":
         lte_username, lt_id = split_cursor(lt)
@@ -105,13 +137,14 @@ def get_users_cursor(
             lt_id=lt_id,
             auth_method=auth_method,
             search_text=search_text,
-            max_count=max_count,
+            max_count=qry_max_count,
+            desc=desc,
             api_client=api_client,
             db=db,
         )
-        if len(items) > 0:
-            p = str(items[0].user_id) + "$" + items[0].id
-            n = str(items[-1].user_id) + "$" + items[-1].id
+        items, num_rows = remove_extra_item(items, lte_username, gte_username)
+        p, n = get_next_prev(num_rows, lte_username, gte_username, lambda x: x.id)
+
     elif sort_key == "display_name":
         lte_display_name, lt_id = split_cursor(lt)
         gte_display_name, gt_id = split_cursor(gt)
@@ -123,13 +156,14 @@ def get_users_cursor(
             lt_id=lt_id,
             auth_method=auth_method,
             search_text=search_text,
-            max_count=max_count,
+            max_count=qry_max_count,
+            desc=desc,
             api_client=api_client,
             db=db,
         )
-        if len(items) > 0:
-            p = str(items[0].user_id) + "$" + items[0].display_name
-            n = str(items[-1].user_id) + "$" + items[-1].display_name
+        items, num_rows = remove_extra_item(items, lte_display_name, gte_display_name)
+        p, n = get_next_prev(num_rows, lte_display_name, gte_display_name, lambda x: x.display_name)
+
     else:
         raise OasstError(f"Unsupported sort key: '{sort_key}'", OasstErrorCode.SORT_KEY_UNSUPPORTED)
 

backend/oasst_backend/config.py

@@ -72,6 +72,7 @@ class Settings(BaseSettings):
     DATABASE_MAX_TX_RETRY_COUNT: int = 3
 
     RATE_LIMIT: bool = True
+    MESSAGE_SIZE_LIMIT: int = 2000
     REDIS_HOST: str = "localhost"
     REDIS_PORT: str = "6379"
 

backend/oasst_backend/prompt_repository.py

@@ -448,6 +448,11 @@ class PromptRepository:
             if task:
                 message.review_count += 1
                 self.db.add(message)
+            # for the same User id with no task id associated with the message, then update existing record for repeated updates
+            existing_text_label = self.fetch_non_task_text_labels(message_id, self.user_id)
+            if existing_text_label is not None:
+                existing_text_label.labels = text_labels.labels
+                model = existing_text_label
 
         self.db.add(model)
         return model, task, message
@@ -561,6 +566,16 @@ class PromptRepository:
             raise OasstError("Message not found", OasstErrorCode.MESSAGE_NOT_FOUND, HTTP_404_NOT_FOUND)
         return message
 
+    def fetch_non_task_text_labels(self, message_id: UUID, user_id: UUID) -> Optional[TextLabels]:
+
+        query = (
+            self.db.query(TextLabels)
+            .outerjoin(Task, Task.id == TextLabels.id)
+            .filter(Task.id.is_(None), TextLabels.message_id == message_id, TextLabels.user_id == user_id)
+        )
+        text_label = query.one_or_none()
+        return text_label
+
     @staticmethod
     def trace_conversation(messages: list[Message] | dict[UUID, Message], last_message: Message) -> list[Message]:
         """
@@ -693,7 +708,7 @@ class PromptRepository:
         if user_id:
             qry = qry.filter(Message.user_id == user_id)
         if username or auth_method:
-            if not username and auth_method:
+            if not (username and auth_method):
                 raise OasstError("Auth method or username missing.", OasstErrorCode.AUTH_AND_USERNAME_REQUIRED)
             qry = qry.join(User)
             qry = qry.filter(User.username == username, User.auth_method == auth_method)

backend/oasst_backend/tree_manager.py

@@ -465,6 +465,13 @@ class TreeManager:
                     f"Frontend reports text reply to {interaction.message_id=} with {interaction.text=} by {interaction.user=}."
                 )
 
+                # ensure message size is below the predefined limit
+                if len(interaction.text) > settings.MESSAGE_SIZE_LIMIT:
+                    logger.error(
+                        f"Message size {len(interaction.text)=} exceeds size limit of {settings.MESSAGE_SIZE_LIMIT=}."
+                    )
+                    raise OasstError("Message size too long.", OasstErrorCode.TASK_MESSAGE_TOO_LONG)
+
                 # here we store the text reply in the database
                 message = pr.store_text_reply(
                     text=interaction.text,

backend/oasst_backend/user_repository.py

@@ -145,6 +145,7 @@ class UserRepository:
         auth_method: Optional[str] = None,
         search_text: Optional[str] = None,
         limit: Optional[int] = 100,
+        desc: bool = False,
     ) -> list[User]:
         if not self.api_client.trusted:
             if not api_client_id:
@@ -153,7 +154,7 @@ class UserRepository:
             if api_client_id != self.api_client.id:
                 raise OasstError("Forbidden", OasstErrorCode.API_CLIENT_NOT_AUTHORIZED, HTTP_403_FORBIDDEN)
 
-        qry = self.db.query(User).order_by(User.username, User.id)
+        qry = self.db.query(User)
 
         if gte_username is not None:
             if gt_id:
@@ -184,6 +185,11 @@ class UserRepository:
             pattern = "%{}%".format(search_text.replace("\\", "\\\\").replace("_", "\\_").replace("%", "\\%"))
             qry = qry.filter(User.username.like(pattern))
 
+        if desc:
+            qry = qry.order_by(User.username.desc(), User.id.desc())
+        else:
+            qry = qry.order_by(User.username, User.id)
+
         if limit is not None:
             qry = qry.limit(limit)
 
@@ -199,7 +205,9 @@ class UserRepository:
         auth_method: Optional[str] = None,
         search_text: Optional[str] = None,
         limit: Optional[int] = 100,
+        desc: bool = False,
     ) -> list[User]:
+
         if not self.api_client.trusted:
             if not api_client_id:
                 # Let unprivileged api clients query their own users without api_client_id being set
@@ -209,7 +217,7 @@ class UserRepository:
                 # Unprivileged api client asks for foreign users
                 raise OasstError("Forbidden", OasstErrorCode.API_CLIENT_NOT_AUTHORIZED, HTTP_403_FORBIDDEN)
 
-        qry = self.db.query(User).order_by(User.display_name, User.id)
+        qry = self.db.query(User)
 
         if gte_display_name is not None:
             if gt_id:
@@ -249,6 +257,11 @@ class UserRepository:
         if auth_method:
             qry = qry.filter(User.auth_method == auth_method)
 
+        if desc:
+            qry = qry.order_by(User.display_name.desc(), User.id.desc())
+        else:
+            qry = qry.order_by(User.display_name, User.id)
+
         if limit is not None:
             qry = qry.limit(limit)
 

inference/README.md

@@ -0,0 +1,35 @@
+# OpenAssitant Inference
+
+Preliminary implementation of the inference engine for OpenAssistant.
+
+## Development (you'll need multiple terminals)
+
+Run a redis container (or use the one of the general docker compose file):
+
+```bash
+docker run --rm -it -p 6379:6379 redis
+```
+
+Run the inference server:
+
+```bash
+cd server
+pip install -r requirements.txt
+uvicorn main:app --reload
+```
+
+Run one (or more) workers:
+
+```bash
+cd worker
+pip install -r requirements.txt
+python __main__.py
+```
+
+Run the client:
+
+```bash
+cd text-client
+pip install -r requirements.txt
+python __main__.py
+```

inference/server/README.md

@@ -0,0 +1,10 @@
+# OpenAssistant Inference Server
+
+Workers communicate with the `/work` endpoint via Websocket. They provide their
+configuration and if a task is available, the server returns it. The worker then
+performs the task and returns the result in a streaming fashion to the server,
+also via websocket.
+
+Clients first call `/chat` to make a new chat, then add to that via
+`/chat/<id>/message`. The response is a SSE event source, which will send tokens
+as they are available.

inference/server/main.py

@@ -0,0 +1,193 @@
+import asyncio
+import enum
+import uuid
+
+import fastapi
+import pydantic
+import redis.asyncio as redis
+from fastapi.middleware.cors import CORSMiddleware
+from loguru import logger
+from oasst_shared.schemas import inference, protocol
+from sse_starlette.sse import EventSourceResponse
+
+app = fastapi.FastAPI()
+
+# Allow CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+class Settings(pydantic.BaseSettings):
+    redis_host: str = "localhost"
+    redis_port: int = 6379
+    redis_db: int = 0
+
+    sse_retry_timeout: int = 15000
+
+
+settings = Settings()
+
+# create async redis client
+redisClient = redis.Redis(
+    host=settings.redis_host, port=settings.redis_port, db=settings.redis_db, decode_responses=True
+)
+
+
+class CreateChatRequest(pydantic.BaseModel):
+    pass
+
+
+class CreateChatResponse(pydantic.BaseModel):
+    id: str
+
+
+class MessageRequest(pydantic.BaseModel):
+    message: str = pydantic.Field(..., repr=False)
+    model_name: str = "distilgpt2"
+    max_new_tokens: int = 100
+
+    def compatible_with(self, worker_config: inference.WorkerConfig) -> bool:
+        return self.model_name == worker_config.model_name
+
+
+class TokenResponseEvent(pydantic.BaseModel):
+    token: str
+
+
+class MessageRequestState(str, enum.Enum):
+    pending = "pending"
+    in_progress = "in_progress"
+    complete = "complete"
+
+
+class DbChatEntry(pydantic.BaseModel):
+    id: str = pydantic.Field(default_factory=lambda: str(uuid.uuid4()))
+    conversation: protocol.Conversation = pydantic.Field(default_factory=protocol.Conversation)
+    pending_message_request: MessageRequest | None = None
+    message_request_state: MessageRequestState | None = None
+
+
+# TODO: make real database
+CHATS: dict[str, DbChatEntry] = {}
+
+
+@app.post("/chat")
+async def create_chat(request: CreateChatRequest) -> CreateChatResponse:
+    """Allows a client to create a new chat."""
+    logger.info(f"Received {request}")
+    chat = DbChatEntry()
+    CHATS[chat.id] = chat
+    return CreateChatResponse(id=chat.id)
+
+
+@app.get("/chat/{id}")
+async def get_chat(id: str) -> protocol.Conversation:
+    """Allows a client to get the current state of a chat."""
+    return CHATS[id].conversation
+
+
+@app.post("/chat/{id}/message")
+async def create_message(id: str, message_request: MessageRequest, fastapi_request: fastapi.Request):
+    """Allows the client to stream the results of a request."""
+
+    chat = CHATS[id]
+    if not chat.conversation.is_prompter_turn:
+        raise fastapi.HTTPException(status_code=400, detail="Not your turn")
+    if chat.pending_message_request is not None:
+        raise fastapi.HTTPException(status_code=400, detail="Already pending")
+
+    chat.conversation.messages.append(
+        protocol.ConversationMessage(
+            text=message_request.message,
+            is_assistant=False,
+        )
+    )
+
+    chat.pending_message_request = message_request
+    chat.message_request_state = MessageRequestState.pending
+
+    async def event_generator():
+        result_data = []
+
+        try:
+            while True:
+                if await fastapi_request.is_disconnected():
+                    logger.warning("Client disconnected")
+                    break
+
+                item = await redisClient.blpop(chat.id, 1)
+                if item is None:
+                    continue
+
+                _, response_packet_str = item
+                response_packet = inference.WorkResponsePacket.parse_raw(response_packet_str)
+                result_data.append(response_packet)
+
+                if response_packet.is_end:
+                    break
+
+                yield {
+                    "retry": settings.sse_retry_timeout,
+                    "data": TokenResponseEvent(token=response_packet.token).json(),
+                }
+            logger.info(f"Finished streaming {chat.id} {len(result_data)=}")
+        except Exception:
+            logger.exception(f"Error streaming {chat.id}")
+
+        chat.conversation.messages.append(
+            protocol.ConversationMessage(
+                text="".join([d.token for d in result_data[:-1]]),
+                is_assistant=True,
+            )
+        )
+        chat.pending_message_request = None
+
+    return EventSourceResponse(event_generator())
+
+
+@app.websocket("/work")
+async def work(websocket: fastapi.WebSocket):
+    await websocket.accept()
+    worker_config = inference.WorkerConfig.parse_raw(await websocket.receive_text())
+    while True:
+        # find a pending task that matches the worker's config
+        # could also be implemented using task queues
+        # but general compatibility matching is tricky
+        for chat in CHATS.values():
+            if (request := chat.pending_message_request) is not None:
+                if chat.message_request_state == MessageRequestState.pending:
+                    if request.compatible_with(worker_config):
+                        break
+        else:
+            logger.debug("No pending tasks")
+            await asyncio.sleep(1)
+            continue
+
+        chat.message_request_state = MessageRequestState.in_progress
+
+        work_request = inference.WorkRequest(
+            conversation=chat.conversation,
+            model_name=request.model_name,
+            max_new_tokens=request.max_new_tokens,
+        )
+
+        logger.info(f"Created {work_request}")
+        try:
+            await websocket.send_text(work_request.json())
+            while True:
+                # maybe unnecessary to parse and re-serialize
+                # could just pass the raw string and mark end via empty string
+                response_packet = inference.WorkResponsePacket.parse_raw(await websocket.receive_text())
+                await redisClient.rpush(chat.id, response_packet.json())
+                if response_packet.is_end:
+                    break
+        except fastapi.WebSocketException:
+            # TODO: handle this better
+            logger.exception(f"Websocket closed during handling of {chat.id}")
+
+        chat.message_request_state = MessageRequestState.complete

inference/server/requirements.txt

@@ -0,0 +1,6 @@
+fastapi[all]
+loguru
+pydantic
+redis
+sse-starlette
+websockets

inference/text-client/__main__.py

@@ -0,0 +1,40 @@
+"""Simple REPL frontend."""
+
+import json
+
+import requests
+import sseclient
+import typer
+
+app = typer.Typer()
+
+
+@app.command()
+def main(backend_url: str = "http://127.0.0.1:8000"):
+    """Simple REPL client."""
+    chat_id = requests.post(f"{backend_url}/chat", json={}).json()["id"]
+    while True:
+        message = typer.prompt("User").strip()
+
+        # wait for stream to be ready
+        # could implement a queue position indicator
+        # could be implemented with long polling
+        # but server load needs to be considered
+        response = requests.post(
+            f"{backend_url}/chat/{chat_id}/message",
+            json={"message": message},
+            stream=True,
+            headers={"Accept": "text/event-stream"},
+        )
+        response.raise_for_status()
+
+        client = sseclient.SSEClient(response)
+        print("Assistant: ", end="", flush=True)
+        for event in client.events():
+            data = json.loads(event.data)
+            print(data["token"], end="", flush=True)
+        print()
+
+
+if __name__ == "__main__":
+    app()

inference/text-client/requirements.txt

@@ -0,0 +1,3 @@
+requests
+sseclient-py
+typer

inference/worker/__main__.py

@@ -0,0 +1,79 @@
+import re
+import time
+
+import rel
+import torch
+import typer
+import websocket
+from loguru import logger
+from oasst_shared.schemas import inference, protocol
+from transformers import pipeline
+
+app = typer.Typer()
+
+
+@app.command()
+def main(
+    backend_url: str = "ws://localhost:8000",
+    model_name: str = "distilgpt2",
+):
+    pipe = pipeline("text-generation", model=model_name)
+
+    def on_open(ws: websocket.WebSocket):
+        worker_config = inference.WorkerConfig(model_name=model_name)
+        ws.send(worker_config.json())
+
+    def on_message(ws: websocket.WebSocket, message: str):
+        # TODO: what if this comes in, but one is already in progress?
+        # also need to think of enabling batching
+        work_request = inference.WorkRequest.parse_raw(message)
+
+        def _prepare_message(message: protocol.ConversationMessage) -> str:
+            prefix = "Assistant: " if message.is_assistant else "User: "
+            return prefix + message.text
+
+        # construct prompt
+        messages = [_prepare_message(message) for message in work_request.conversation.messages]
+
+        prompt = "\n".join(messages) + "\nAssistant:"
+
+        # TODO: replace this with incremental generation
+        torch.manual_seed(work_request.seed)
+        model_output = pipe(prompt, max_new_tokens=work_request.max_new_tokens, do_sample=True, return_full_text=False)[
+            0
+        ]["generated_text"]
+        model_output = model_output.strip()
+
+        # fake streaming
+        split_idcs = [m.start() for m in re.finditer(r"([\w:]+)", model_output)]
+        pieces = [model_output[a:b] for a, b in zip([0] + split_idcs, split_idcs + [None])]
+        for piece in pieces:
+            if not piece:
+                continue
+            if piece.strip() in ("User:", "Assistant:"):
+                break
+            ws.send(inference.WorkResponsePacket(token=piece).json())
+            time.sleep(0.1)
+        ws.send(inference.WorkResponsePacket(is_end=True).json())
+
+    def on_error(ws: websocket.WebSocket, error: Exception):
+        logger.error(f"Connection error: {error}")
+
+    def on_close(ws: websocket.WebSocket, close_status_code: int, close_msg: str):
+        logger.warning(f"Connection closed: {close_status_code=} {close_msg=}")
+
+    ws = websocket.WebSocketApp(
+        f"{backend_url}/work",
+        on_message=on_message,
+        on_error=on_error,
+        on_close=on_close,
+        on_open=on_open,
+    )
+
+    ws.run_forever(dispatcher=rel, reconnect=5)
+    rel.signal(2, rel.abort)
+    rel.dispatch()
+
+
+if __name__ == "__main__":
+    app()

inference/worker/requirements.txt

@@ -0,0 +1,6 @@
+loguru
+rel
+torch
+transformers
+typer
+websocket-client

oasst-shared/oasst_shared/exceptions/oasst_api_error.py

@@ -37,6 +37,7 @@ class OasstErrorCode(IntEnum):
     TASK_GENERATION_FAILED = 1005
     TASK_REQUESTED_TYPE_NOT_AVAILABLE = 1006
     TASK_AVAILABILITY_QUERY_FAILED = 1007
+    TASK_MESSAGE_TOO_LONG = 1008
 
     # 2000-3000: prompt_repository
     INVALID_FRONTEND_MESSAGE_ID = 2000

oasst-shared/oasst_shared/schemas/inference.py

@@ -0,0 +1,21 @@
+import random
+
+import pydantic
+
+from . import protocol
+
+
+class WorkerConfig(pydantic.BaseModel):
+    model_name: str = "distilgpt2"
+
+
+class WorkRequest(pydantic.BaseModel):
+    conversation: protocol.Conversation = pydantic.Field(..., repr=False)
+    model_name: str = "distilgpt2"
+    max_new_tokens: int = 100
+    seed: int = pydantic.Field(default_factory=lambda: random.randint(0, 2**32 - 1))
+
+
+class WorkResponsePacket(pydantic.BaseModel):
+    token: str | None = None
+    is_end: bool = False

oasst-shared/oasst_shared/schemas/protocol.py

@@ -64,6 +64,18 @@ class Conversation(BaseModel):
 
     messages: list[ConversationMessage] = []
 
+    def __len__(self):
+        return len(self.messages)
+
+    @property
+    def is_prompter_turn(self) -> bool:
+        if len(self) == 0:
+            return True
+        last_message = self.messages[-1]
+        if last_message.is_assistant:
+            return True
+        return False
+
 
 class Message(ConversationMessage):
     parent_id: Optional[UUID] = None

openassistant/datasets/soda_synthetic_dialogue/README.md

@@ -0,0 +1,108 @@
+---
+annotations_creators:
+  - no-annotation
+language:
+  - en
+language_creators:
+  - machine-generated
+license:
+  - mit
+multilinguality:
+  - monolingual
+pretty_name: "SODA Synthetic Dialogue"
+size_categories:
+  - 1M<n<10M
+source_datasets:
+  - extended|allenai/soda
+tags:
+  - open-assistant
+  - conversations-summarization
+task_categories:
+  - conversational
+task_ids:
+  - dialogue-generation
+dataset_info:
+  features:
+    - name: conversation
+      dtype: string
+  config_name: soda_synthetic_dialogue_dialogue_modeling
+  splits:
+    - name: train
+      num_bytes: 1524941476
+      num_examples: 1191582
+    - name: test
+      num_bytes: 190565707
+      num_examples: 148968
+    - name: validation
+      num_bytes: 187161878
+      num_examples: 146346
+  download_size: 1952926579
+  dataset_size: 1902669061
+---
+
+# Dataset Card for 🥤SODA Synthetic Dialogue
+
+## Dataset Description
+
+- **Repository:**
+  [Code](https://github.com/LAION-AI/Open-Assistant/openassistant/datasets/soda_synthetic_dialogue/)
+
+## Dataset Summary
+
+🥤SODA Synthetic Dialogue is a set of synthetic dialogues between Assistant and
+User. In each conversation, User asks Assistant to perform summarization or
+story generation tasks based on a snippet of an existing dialogue, story, or
+from a title or theme.
+
+This data was created by synthesizing the dialogues in
+[🥤Soda](https://huggingface.co/datasets/allenai/soda) and applying a set of
+templates to generate the conversation. The original research paper can be found
+[here](https://arxiv.org/pdf/2212.10465v1.pdf).
+
+Example:
+
+```
+User: Can you write the next few lines of dialogue for this scene:
+  Cornell: I know what you're thinking, Dontrell. I can sense your feelings and know exactly what you're going to say or do next. You're thinking about how much you miss your family, and how you wish you could see them again. You're wondering if there's anything you can do to make that happen. And you're also feeling a little guilty, because you know that if you could see them again, it would only be for a short while before they died.
+  Dontrell: Cornell, how can you possibly know all that? It's like you can read my mind!
+Assistant: Sure, the next dialogue for this scene could be:
+  Cornell: I told you, I can sense your feelings. It's like I have a connection to your thoughts and emotions. I'm not sure why or how it works, but ever since we met, I've been able to understand what you're thinking and feeling pretty easily.
+  Dontrell: Wow...that's amazing! And a little bit scary too. But it does explain why we've always been so close - even though we've only known each other for a few months.
+  Cornell: Yeah, I guess our connection is pretty strong. But don't worry, Dontrell - I'll never use it to manipulate or control you in any way. I promise."
+  Dontrell: That's a relief! I was starting to feel a little bit like you could read my mind, and it was kind of creepy. But if you promise not to use your abilities to control me, then I'm okay with it.
+  Cornell: I swear, Dontrell. I'll never use my abilities to control or manipulate you in any way. You have my word.
+User: And a title?
+Assistant: Sure, a title for this dialogue could be:
+  Cornell is intuitive. Cornell reads Dontrell's mind.
+User: How about a short description?
+Assistant: Sure, a short description for this dialogue could be:
+  Cornell knows what Dontrell is thinking. He can sense her feelings and know exactly what she is going to say or do next.
+User: What would be one theme of this story?
+Assistant: One theme of this story could be:
+  being intuitive
+```
+
+## Usage
+
+The dataset contains one configuration, `dialogue_modeling`, which has a single
+text `conversation` feature.
+
+## Source data
+
+The script to synthesize this dataset was originally created by
+[ontocord](https://github.com/ontocord) in
+[this Colab notebook](https://colab.research.google.com/drive/1Sw3px5dP8whdqT7QMNoqwmqIasZkMbJi?usp=sharing)
+and prepared for Hugging Face by
+[Jeffrey Quesnelle](https://github.com/jquesnelle/).
+
+## Citation
+
+Please cite our work if you find the resources in this repository useful:
+
+```
+@article{ontocord2023sodasynth,
+  author    = {ontocord and Jeffrey Quesnelle},
+  title     = {SODA Synthetic Dialogue},
+  year      = {2023}
+}
+```

openassistant/datasets/soda_synthetic_dialogue/hub.py

@@ -0,0 +1,21 @@
+from dataclasses import dataclass
+
+import datasets
+
+
+@dataclass
+class OpenAssistantConfig(datasets.BuilderConfig):
+    """BuilderConfig for OpenAssistant datasets."""
+
+    name: str = None
+    version: datasets.Version = None
+    description: str = None
+    schema: str = None
+    subset_id: str = None
+
+
+features = datasets.Features(
+    {
+        "conversation": datasets.Value("string"),
+    }
+)

openassistant/datasets/soda_synthetic_dialogue/prepare.py

@@ -0,0 +1,246 @@
+"""Prepare the SODA Synthetic Dialogue Dataset"""
+
+import json
+import os
+import random
+import sys
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+# adapted from https://colab.research.google.com/drive/1Sw3px5dP8whdqT7QMNoqwmqIasZkMbJi?usp=sharing
+
+SUMMARY_TEMPLATE = """User: Can you give me a short story description for this dialogue?
+  {dialogue}
+Assistant: Sure, a short story description for this dialogue could be:
+  {story}
+User: And a title?
+Assistant: Sure, a title for this dialogue could be:
+  {title}"""
+
+THEME_TEMPLATE = """
+User: What would be one theme of this story?
+Assistant: One theme of this story could be:
+  {theme}"""
+
+NEW_DIALOGUE_TEMPLATE = """User: Can you write a short dialogue based on this story:
+  {story}
+Assistant: Sure, a dialogue for this story could be:
+  {dialogue}
+User: And a title?
+Assistant: Sure, a title for this dialogue could be:
+  {title}"""
+
+NEXT_LINES_TEMPLATE = """User: Can you write the next few lines of dialogue for this scene:
+  {scene}
+Assistant: Sure, the next dialogue for this scene could be:
+  {dialogue}
+User: And a title?
+Assistant: Sure, a title for this dialogue could be:
+  {title}
+User: How about a short description?
+Assistant: Sure, a short description for this dialogue could be:
+  {story}"""
+
+NEW_STORY_AND_DIALOGUE_TEMPLATE = """User: Can you write a short story and dialogue about:
+  {title1}
+Assistant: Sure, a short story and dialogue about: "{title1}" could be:
+  {story}"""
+
+FULL_DIALOGUE_TEMPLATE = """{conversation}
+  {dialogue}"""
+
+MORE_DIALOGUE_TEMPLATE = """{conversation}
+  {dialogue1}
+User: Can you provide more dialogue assuming "{title2}"?
+Assistant: Sure, the next dialogue for this scene could be:
+  {dialogue2}"""
+
+NEXT_DIALOGUE_TEMPLATE = """{conversation}
+  {dialogue1}
+User: More please.
+Assistant: Sure, the next dialogue for this scene could be:
+  {dialogue2}"""
+
+NEW_STORY_AND_DIALOGUE_FROM_THEME_TEMPLATE = """User: Can you write short story and dialogue based on the theme:
+  {theme}
+Assistant: Sure, a short story and dialogue based on the theme "{theme}" could be:
+  {story}
+  {dialogue}
+User: And a title?
+Assistant: Sure, a title for this dialogue could be:
+  {title}"""
+
+PRINT = len(sys.argv) > 1 and sys.argv[1] == "--print"
+
+
+def main(output_dir: str = "data"):
+    """Download and prepare the dataset for use."""
+
+    random.seed(42)
+    dataset = load_dataset("allenai/soda")
+    os.makedirs(output_dir, exist_ok=True)
+
+    for split in ["train", "test", "validation"]:
+        with open(f"{output_dir}/{split}.jsonl", "w", encoding="utf8") as output:
+
+            for i in tqdm(range(len(dataset[split])), desc=split):
+                dat = dataset["train"][i]
+                title = dat["literal"]
+                story = dat["narrative"]
+
+                if dat["relation"] == "xWant":
+                    theme = "wanting " + dat["tail"]
+                elif dat["relation"] == "xNeed":
+                    theme = "needing " + dat["tail"]
+                elif not dat["tail"].startswith("to ") and not dat["tail"].startswith("and "):
+                    theme = "being " + dat["tail"]
+                elif dat["tail"].startswith("and "):
+                    theme = "people are " + dat["tail"].replace("and PersonY ", "")
+                else:
+                    theme = dat["tail"]
+                theme = theme.replace("PersonY", "another person")
+                theme = theme.replace("being is", "being")
+
+                dialogue = [s2 + ": " + s1 for s1, s2 in zip(dat["dialogue"], dat["speakers"])]
+
+                if random.randint(0, 6) == 0:
+                    # print("##")
+                    # print(f"User: Can you give me a short story description for this dialog?")
+                    # print("  " + "\n  ".join(dialog))
+                    # print(f"Assistant: Sure, a short story description for this dialog could be: \n  {story}")
+                    # print("User: And a title?")
+                    # print(f"Assistant: Sure, a title for this dialog could be: \n  {title}")
+                    # if theme:
+                    #     print("User: What would be one theme of this story?")
+                    #     print(f'Assistant: One theme of this story could be: "{theme}"')
+                    conversation = SUMMARY_TEMPLATE.format(dialogue="\n  ".join(dialogue), story=story, title=title)
+                    if theme:
+                        conversation = conversation + THEME_TEMPLATE.format(theme=theme)
+                elif random.randint(0, 6) == 0:
+                    # print("##")
+                    # print(f"User: Can you write a short dialog based on this story:\n  {story}")
+                    # print(f"Assistant: Sure, a dialog for this story could be:")
+                    # print("  " + "\n  ".join(dialog))
+                    # print("User: And a title?")
+                    # print(f"Assistant: Sure, a title for this dialog could be: \n  {title}")
+                    # if theme:
+                    #     print("User: What would be one theme of this story?")
+                    #     print(f'Assistant: One theme of this story could be: "{theme}"')
+                    conversation = NEW_DIALOGUE_TEMPLATE.format(
+                        story=story, dialogue="\n  ".join(dialogue), title=title
+                    )
+                    if theme:
+                        conversation = conversation + THEME_TEMPLATE.format(theme=theme)
+                elif random.randint(0, 3) == 0:
+                    # print("##")
+                    # print(f"User: Can you write the next few lines of dialog for this scene:")
+                    # if random.randint(0, 1) == 0:
+                    #     print("  " + "\n  ".join(dialog[:-5]))
+                    #     print(f"Assistant: Sure, the next dialog for this scene could be:")
+                    #     print("  " + "\n  ".join(dialog[-5:]))
+                    # elif random.randint(0, 1) == 0:
+                    #     print("  " + "\n  ".join(dialog[:-3]))
+                    #     print(f"Assistant: Sure, the next dialog for this scene could be:")
+                    #     print("  " + "\n  ".join(dialog[-3:]))
+                    # else:
+                    #     print("  " + "\n  ".join(dialog[:-4]))
+                    #     print(f"Assistant: Sure, the next dialog for this scene could be:")
+                    #     print("  " + "\n  ".join(dialog[-4:]))
+                    # print("User: And a title?")
+                    # print(f"Assistant: Sure, a title for this dialog could be: \n  {title}")
+                    # print("User: How about a short description?")
+                    # print(f"Assistant: Sure, a short description for this dialog could be: \n  {story}")
+                    # if theme:
+                    #     print("User: What would be one theme of this story?")
+                    #     print(f'Assistant: One theme of this story could be: "{theme}"')
+                    if random.randint(0, 1) == 0:
+                        depth = -5
+                    elif random.randint(0, 1) == 0:
+                        depth = -3
+                    else:
+                        depth = -4
+                    conversation = NEXT_LINES_TEMPLATE.format(
+                        scene="\n  ".join(dialogue[:depth]),
+                        dialogue="\n  ".join(dialogue[depth:]),
+                        title=title,
+                        story=story,
+                    )
+                    if theme:
+                        conversation = conversation + THEME_TEMPLATE.format(theme=theme)
+                elif random.randint(0, 3) == 0:
+                    # print("##")
+                    # title1 = title.split(".")[0]
+                    # title2 = title.split(".")[1]
+                    # print(f"User: Can you write short story and dialog about: {title1}")
+                    # print(f'Assistant: Sure, a short story and dialog about: "{title1}" could be:')
+                    # print(f"  {story}")
+                    # if random.randint(0, 1) == 0:
+                    #     print("  " + "\n  ".join(dialog))
+                    # elif random.randint(0, 1) == 0 and len(dialog) > 5:
+                    #     print("  " + "\n  ".join(dialog[:-5]))
+                    #     print(f'User: Can you provide more dialog assuming "{title2}"?')
+                    #     print(f"Assistant: Sure, the next dialog for this scene could be:")
+                    #     print("  " + "\n  ".join(dialog[-5:]))
+                    # elif random.randint(0, 1) == 0:
+                    #     print("  " + "\n  ".join(dialog[:-3]))
+                    #     print("User: more please.")
+                    #     print(f"Assistant: Sure, the next dialog for this scene could be:")
+                    #     print("  " + "\n  ".join(dialog[-3:]))
+                    # else:
+                    #     print("  " + "\n  ".join(dialog[:-4]))
+                    #     print(f'User: Can you provide more dialog assuming "{title2}"?')
+                    #     print(f"Assistant: Sure, the next dialog for this scene could be:")
+                    #     print("  " + "\n  ".join(dialog[-4:]))
+                    # if theme:
+                    #     print("User: What would be one theme of this story?")
+                    #     print(f'Assistant: One theme of this story could be: "{theme}"')
+                    title1 = title.split(".")[0]
+                    title2 = title.split(".")[1]
+                    conversation = NEW_STORY_AND_DIALOGUE_TEMPLATE.format(title1=title1, story=story)
+                    if random.randint(0, 1) == 0:
+                        conversation = FULL_DIALOGUE_TEMPLATE.format(
+                            conversation=conversation, dialogue="\n  ".join(dialogue)
+                        )
+                    elif random.randint(0, 1) == 0 and len(dialogue) > 5:
+                        conversation = MORE_DIALOGUE_TEMPLATE.format(
+                            conversation=conversation,
+                            dialogue1="\n  ".join(dialogue[:-5]),
+                            title2=title2,
+                            dialogue2="\n  ".join(dialogue[-5:]),
+                        )
+                    elif random.randint(0, 1) == 0:
+                        conversation = NEXT_DIALOGUE_TEMPLATE.format(
+                            conversation=conversation,
+                            dialogue1="\n  ".join(dialogue[:-3]),
+                            dialogue2="\n  ".join(dialogue[-3:]),
+                        )
+                    else:
+                        conversation = MORE_DIALOGUE_TEMPLATE.format(
+                            conversation=conversation,
+                            dialogue1="\n  ".join(dialogue[:-4]),
+                            title2=title2,
+                            dialogue2="\n  ".join(dialogue[-4:]),
+                        )
+                    if theme:
+                        conversation = conversation + THEME_TEMPLATE.format(theme=theme)
+                else:
+                    # print("##")
+                    # print(f"User: Can you write short story and dialog based on the theme:\n  {theme}")
+                    # print(f'Assistant: Sure, a short story and dialog based on the theme "{theme}" could be:')
+                    # print(f"  {story}")
+                    # print("  " + "\n  ".join(dialog))
+                    # print("User: And a title?")
+                    # print(f"Assistant: Sure, a title for this dialog could be: \n  {title}")
+                    conversation = NEW_STORY_AND_DIALOGUE_FROM_THEME_TEMPLATE.format(
+                        theme=theme, story=story, dialogue="\n  ".join(dialogue), title=title
+                    )
+                if PRINT:
+                    print("##")
+                    print(conversation)
+
+                output.write(f"{json.dumps({'conversation': conversation})}\n")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

openassistant/datasets/soda_synthetic_dialogue/soda_synthetic_dialogue.py

@@ -0,0 +1,108 @@
+# Copyright 2023 The OpenAssistant Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This dataset is a set of dialogues synthesized from the SODA dataset.
+In each dialogue, User and Assistant have a conversation about a story.
+
+The original collab notebook for this dataset can be found at:
+https://colab.research.google.com/drive/1Sw3px5dP8whdqT7QMNoqwmqIasZkMbJi?usp=sharing
+"""
+
+import json
+from typing import Dict, List, Tuple
+
+import datasets
+
+from .hub import OpenAssistantConfig, features
+
+_CITATION = """\
+@article{ontocord2023sodasynth,
+  author    = {ontocord and Jeffrey Quesnelle},
+  title     = {SODA Synthetic Dialogue},
+  year      = {2023}
+}
+"""
+_DATASETNAME = "soda_synthetic_dialogue"
+_DISPLAYNAME = "🥤SODA Synthetic Dialogue"
+_DESCRIPTION = "A set of dialogues synthesized from the SODA dataset."
+_HOMEPAGE = ""
+_LICENSE = "mit"
+_URLS = {
+    _DATASETNAME: {"train": "./data/train.jsonl", "test": "./data/test.jsonl", "validation": "./data/validation.jsonl"}
+}
+_SUPPORTED_TASKS = ["dialogue-modeling"]
+_VERSION = "1.0.0"
+
+
+class SODASyntheticDialogueDataset(datasets.GeneratorBasedBuilder):
+    """A set of dialogues synthesized from the SODA dataset."""
+
+    VERSION = datasets.Version(_VERSION)
+
+    BUILDER_CONFIGS = [
+        OpenAssistantConfig(
+            name=f"{_DATASETNAME}_dialogue_modeling",
+            version=VERSION,
+            description=f"OpenAssistant dataset config for {_DATASETNAME}",
+            schema="dialogue_modeling",
+            subset_id=_DATASETNAME,
+        )
+    ]
+
+    DEFAULT_CONFIG_NAME = f"{_DATASETNAME}_dialogue_modeling"
+
+    def _info(self) -> datasets.DatasetInfo:
+
+        return datasets.DatasetInfo(
+            description=_DESCRIPTION,
+            features=features,
+            homepage=_HOMEPAGE,
+            license=_LICENSE,
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager) -> List[datasets.SplitGenerator]:
+        """Returns SplitGenerators."""
+
+        urls = _URLS[_DATASETNAME]
+        data_dir = dl_manager.download_and_extract(urls)
+
+        return [
+            datasets.SplitGenerator(
+                name=datasets.Split.TRAIN,
+                gen_kwargs={"filepath": data_dir, "split": "train"},
+            ),
+            datasets.SplitGenerator(
+                name=datasets.Split.TEST,
+                gen_kwargs={"filepath": data_dir, "split": "test"},
+            ),
+            datasets.SplitGenerator(
+                name=datasets.Split.VALIDATION,
+                gen_kwargs={"filepath": data_dir, "split": "validation"},
+            ),
+        ]
+
+    def _generate_examples(self, filepath, split: str) -> Tuple[int, Dict]:
+        """Yields examples as (key, example) tuples."""
+
+        if self.config.schema == "dialogue_modeling":
+            key = 0
+            with open(filepath[split], "r", encoding="utf8") as data:
+                while True:
+                    line = data.readline()
+                    if not line:
+                        return
+                    yield key, json.loads(line)
+                    key += 1

scripts/data-collection/twitter/README.md

@@ -0,0 +1,80 @@
+# Twitter data collection for Open Assistant
+
+Conversations on Twitter can be an interesting and useful source of data for our
+model to learn from. Certain twitter threads may contain helpful prompts and
+replies, in a similar fashion to how we want our model to be able to respond to
+prompts in a useful way.
+
+Thus, these scripts are intended to process twitter data from a variety of
+sources, process them into cleaner and more useful formats, and then combine the
+various outputs into a unified training set that can be fed to our model as a
+conversation, or at least as a prompt with replies.
+
+**Note: Based on issue #126**
+
+## Possible Data Paths
+
+- Twitterstream archive: https://archive.org/details/twitterstream These are
+  large .tar files with compressed json files inside. However, some data points
+  such as reply counts seem to always be 0 due to limitations when scraping the
+  Twitter API.
+- Alternative APIs such as snscrape, twint, etc. These alternative APIs often
+  are harder to use than the official Twitter API but can often bypass API
+  limits which can make it useful for larger scale data collection. The downside
+  is potentially slower speed, and less features.
+- The official Twitter API
+
+## Currently Completed Items
+
+- Downloaded various archive files (both are .tar, but each have a different
+  format of json compression. One used .gz, and the other.bz2). Each json file
+  is roughly 2000 rows of tweets. There are thousands of these compressed json
+  files. Managing the IO of opening lots of small files is one of the
+  challenges, which is why future steps will consolidate data into larger easier
+  to process files.
+- Wrote script that can loop through the compressed json files, cleans them up a
+  bit by removing truncated tweets or tweets that aren't replies. The script
+  then standardizes the columns, and exports the polars dataframes into parquet
+  files for future processing. Note: Using polars instead of pandas due to
+  performance reasons.
+- Wrote scripts that process the large dump of tweets into conversation threads
+  using the tree and node architecture. This results in aroun 17K conversation
+  threads bassed on a dump of 90M tweets.
+- Script can output the conversation threads into a jsonl file for further
+  filtering or use in models.
+
+## Main Issue
+
+- The issue is that we can easily scrape replies, but there is no guarantee the
+  original tweet is in the archive file. Furthermore, the archives are large so
+  they would need to be kept completely in-memory or in a db to reference. We
+  still need to decide if we want to try to mine the archive to piece together
+  the conversations, or we can take the list of replied tweets and loop through
+  those and use alternative apis to fetch the original tweet text, and then
+  match it with the confirmed replies already in our archive to generate the
+  prompt/replies data. Currently, my script can extract conversations based on
+  the dump, but it is a small percentage of the overall dump, and there is no
+  guarantee of the quality of the tweets.
+- The tweet quality is the other major issue. We can get conversations through
+  the currently made scripts, but they most likely don't match a useful
+  instruction -> fulfilment. We are trying to filter the tweets through various
+  means such as matching useful hashtags, or by using cosine similarity against
+  known instructions.
+- The modern Twitter API has conversation_id as a field which can be a way to
+  gather all tweets in a thread sort of automatically although there is
+  pagination limits. The main issue with this is it seems hard to search for it
+  using alternative APIs.
+
+## TODO
+
+- Write scripts to filter existing conversations into useful instructions ->
+  fulfilment with hashtags or cosine similarity.
+- Train model to detect if text is a suitable instruction. This could then be
+  run through the conversations (or full tweet dump) to simplify the process.
+  Related to issue #143.
+- Write script that matches the original tweets and their text with the archive
+  data to create the prompt/reply dataset. (Optional)
+- Decide on final output format and storage options for the dataset. Currently
+  in JSONL with tree / node architecture as python dicts which is acceptable I
+  believe.
+- Alternatively: Store processed tweets into DB or alternative option.(Optional)

scripts/data-collection/twitter/requirements.txt

@@ -0,0 +1,3 @@
+numpy==1.21.5
+polars==0.15.14
+tqdm==4.64.0

scripts/data-collection/twitter/twitter_create_convs.py

@@ -0,0 +1,141 @@
+import json
+from pathlib import Path
+
+import polars as pl
+from tqdm import tqdm
+
+# Sets up paths
+# TODO: Source paths from env file
+path_string = "PUT THE PATH HERE TO WHERE YOU STORED THE PARQUET FILES"
+folder_path = Path(path_string)
+processed_folder_path = folder_path / "processed"
+output_path = folder_path / "twitter-conv-trees.jsonl"
+
+# Get parq files
+parq_files = sorted(processed_folder_path.rglob("*.parquet"))
+
+wanted_cols = [
+    "timestamp_ms",
+    "id",
+    "text",
+    "truncated",
+    "in_reply_to_status_id",
+    "in_reply_to_user_id",
+    "is_quote_status",
+    "quote_count",
+    "reply_count",
+    "retweet_count",
+    "favorite_count",
+    "filter_level",
+    "lang",
+    "possibly_sensitive",
+    "hashtags",
+    "user_id",
+    "user_verified",
+    "user_followers_count",
+    "user_statuses_count",
+]
+
+# Load parqs into list. Using Polars for performance reasons.
+df_list = []
+for p in parq_files:
+    df_list.append(pl.read_parquet(p, columns=wanted_cols))
+
+# Create major dataframe.
+# This can be done incrementally if RAM is constrained by modifying the above code.
+p_df = pl.concat(df_list)
+
+# Clean up the reference just in case to help with memory if needed.
+del df_list
+
+# Get tweets that are replies to other tweets
+p_df_replies_only = p_df.filter(pl.col("in_reply_to_status_id").is_null().is_not())
+
+# Group by replied to status id to see the most replied to statuses. This can take some time.
+p_df_group_reply_to_status = p_df_replies_only.groupby("in_reply_to_status_id").count().sort("count", reverse=True)
+
+# Save output of grouping the top replied to statuses
+group_reply_parq = folder_path / "group_reply_parq.parquet"
+p_df_group_reply_to_status.write_parquet(group_reply_parq)
+
+# Join the main dataframe with the top replies to find tweets that have replies.
+p_join = p_df.join(p_df_group_reply_to_status, left_on="id", right_on="in_reply_to_status_id", how="inner")
+
+# Save output of tweets that have replies
+tweets_that_have_replies_path = folder_path / "tweets_that_have_replies.parquet"
+p_join.write_parquet(tweets_that_have_replies_path)
+
+# Save output of tweets that are replies to other tweets
+tweets_that_are_replies_path = folder_path / "tweets_that_are_replies.parquet"
+p_df_replies_only.write_parquet(tweets_that_are_replies_path)
+
+# Filter the tweets that have replies to ones that aren't replies to others.
+# Also filter for only english for now.
+# This gives the root tweets that have replies but are the start of a conversation.
+origin_tweets = p_join.filter((pl.col("in_reply_to_status_id").is_null()) & (pl.col("lang") == "en"))
+
+
+# Helper functions and classes below for the next steps
+
+
+def role_decide(user_id, prompt_user):
+    if user_id == prompt_user:
+        return "prompter"
+    else:
+        return "assistant"
+
+
+class ConversationTreeNode:
+    def __init__(self, tweet_id, prompt_user, from_df, children_df, metadata=None):
+
+        if metadata:
+            self.metadata = metadata
+        else:
+            self.metadata = from_df.filter(pl.col("id") == tweet_id).to_dicts()[0]
+
+        self.metadata["prompt_user"] = prompt_user
+        self.role = role_decide(self.metadata["user_id"], prompt_user)
+        self.children = None
+        self.text = self.metadata["text"]
+        del self.metadata["text"]
+        self.get_children(tweet_id=tweet_id, children_df=children_df)
+
+    def get_children(self, tweet_id, children_df):
+        children_dicts = children_df.filter(pl.col("in_reply_to_status_id") == tweet_id).to_dicts()
+        if len(children_dicts) > 0:
+            children = [
+                ConversationTreeNode(
+                    tweet_id=c["id"],
+                    prompt_user=self.metadata["prompt_user"],
+                    from_df=children_df,
+                    children_df=children_df,
+                    metadata=c,
+                )
+                for c in children_dicts
+            ]
+            self.children = children
+
+
+class ConversationTree:
+    def __init__(self, tweet_id, prompt_user, from_df, children_df, r_metadata=None):
+
+        self.root = ConversationTreeNode(
+            tweet_id=tweet_id, prompt_user=prompt_user, from_df=from_df, children_df=children_df, metadata=r_metadata
+        )
+        self.metadata = None
+
+
+# Create conversation trees
+conv_tree_list = [
+    ConversationTree(
+        tweet_id=r["id"], prompt_user=r["user_id"], from_df=origin_tweets, children_df=p_df_replies_only, r_metadata=r
+    )
+    for r in tqdm(origin_tweets.to_dicts())
+]
+
+# Write conversation trees to jsonl file.
+# Might need to clean up the last newline.
+with open(output_path, "w") as output:
+    for t in tqdm(conv_tree_list):
+        json.dump(obj=t, fp=output, default=lambda x: x.__dict__)
+        output.write("\n")

scripts/data-collection/twitter/twitter_process_json.py

@@ -0,0 +1,233 @@
+# This file loops through compressed json tweet data, pre-processes them,
+# and then extracts them into more unified parquet files that can be handed
+# off for further processing. The main focus is on producing viable replies.
+
+# Initial data exploration seems that there is no guarantee that the original
+# tweets are in the archive, so we might need to extract suitable replies
+# then get the original tweets separately, and then combine them into a
+# suitable thread format that can be used by our instruction model.
+
+# This assumes data downloaded from https://archive.org/details/twitterstream
+# and that the internal .tar files are extracted locally.
+# They are large files so using something like 7Zip or WinRar migth be easier
+# than putting all of it in scripts, but it is a possibility.
+
+# I often work in notebooks. If you encounter any issue, please reach out to let me know.
+
+import bz2
+import gzip
+import json
+import pickle
+from pathlib import Path
+
+import numpy as np
+import polars as pl
+from tqdm import tqdm
+
+# TODO: OPTIONAL - Put the Untar process in a script instead of doing that part externally. Twitterstream archives are .tar with folders and json.gz files inside.
+# TODO: Set up list of important hashtags & keywords. This might have to be done after we get the original tweets in a separate file.
+# TODO: Process data and filter based on hashtags & keywords
+
+# Sets up paths
+# TODO: Source paths from env file
+path_string = "PUT THE PATH HERE TO WHERE YOU DOWNLOADED AND EXTRACTED THE ARCHIVE .TAR"
+folder_path = Path(path_string)
+file_list_pkl = folder_path / "file_list.pkl"
+processed_file_list_pkl = folder_path / "processed_file_list.pkl"
+
+# For the processed folder to save inside, we can create the directory if it doesn't exist
+processed_folder_path = folder_path / "processed"
+processed_folder_path.mkdir(parents=True, exist_ok=True)
+
+# Set max buffer to store temporary dataframes for processing
+# Change this depending on the memory of your computer
+processed_max_buffer = 5000
+
+# Set up list of wanted column names.
+# Note: User columns are prefixed with user_
+wanted_cols = [
+    "timestamp_ms",
+    "id",
+    "text",
+    "truncated",
+    "in_reply_to_status_id",
+    "in_reply_to_user_id",
+    "is_quote_status",
+    "quote_count",
+    "reply_count",
+    "retweet_count",
+    "favorite_count",
+    "filter_level",
+    "lang",
+    "possibly_sensitive",
+    "hashtags",
+    "user_id",
+    "user_verified",
+    "user_followers_count",
+    "user_statuses_count",
+]
+
+
+def main(file_list_pkl, folder_path, processed_max_buffer):
+    """
+    Runs the main processing script to get files, loop through them, and process them.
+    Outputs larger json.gz files made by concat the pre-filtered dataframes from
+    the original json.gz files.
+    """
+
+    file_list = get_file_paths(file_list_pkl, folder_path)
+
+    process_json(file_list, processed_max_buffer)
+
+    print("Done")
+
+
+def get_file_paths(file_list_pkl, folder_path):
+    """
+    Gets the file paths by recursively checking the folder structure.
+    # Based on code from stackoverflow https://stackoverflow.com/questions/26835477/pickle-load-variable-if-exists-or-create-and-save-it
+    """
+    try:
+        allpaths = pickle.load(open(file_list_pkl, "rb"))
+    except (OSError, IOError) as e:
+        print(e)
+        allpaths = sorted(list(folder_path.rglob("*.[gz bz2]*")))
+        pickle.dump(allpaths, open(file_list_pkl, "wb"))
+    print("Got file paths.")
+    return allpaths
+
+
+def get_processed_list(processed_file_list_pkl):
+    # Gets processed file list if stored, if not, creates it.
+    try:
+        processed_list = pickle.load(open(processed_file_list_pkl, "rb"))
+    except (OSError, IOError) as e:
+        print(e)
+        processed_list = []
+        pickle.dump(processed_list, open(processed_file_list_pkl, "wb"))
+    return processed_list
+
+
+def modify_dict_cols(j_dict):
+    # Extracting some nested json
+    j_dict["user_id"] = np.int64(j_dict["user"]["id"])
+    j_dict["user_followers_count"] = np.int64(j_dict["user"]["followers_count"])
+    j_dict["user_statuses_count"] = np.int64(j_dict["user"]["statuses_count"])
+
+    # Get hashtags as a list of strings
+    j_dict["hashtags"] = [h["text"] for h in j_dict["entities"]["hashtags"]]
+
+    j_dict["id"] = np.int64(j_dict["id"])
+
+    try:
+        j_dict["in_reply_to_status_id"] = np.int64(j_dict["in_reply_to_status_id"])
+    except Exception as e:
+        print(e)
+        j_dict["in_reply_to_status_id"] = j_dict["in_reply_to_status_id"]
+
+    try:
+        j_dict["in_reply_to_user_id"] = np.int64(j_dict["in_reply_to_user_id"])
+    except Exception as e:
+        print(e)
+        j_dict["in_reply_to_user_id"] = j_dict["in_reply_to_user_id"]
+
+    # Make sure relevant columns are available or none.
+    for key in wanted_cols:
+        if key not in j_dict:
+            j_dict[key] = None
+
+    # Ordering keys and taking wanted columns
+    j_dict = {key: j_dict[key] for key in wanted_cols}
+
+    return j_dict
+
+
+def process_single_file(f, processed_list):
+
+    j_dict_list = []
+    if f not in processed_list:
+        # Check for compression type
+        if f.suffix == ".bz2":
+            with bz2.BZ2File(f) as file:
+                for line in file:
+                    # Load JSON
+                    j_dict = json.loads(line)
+                    # Check if user key exists
+                    if "delete" not in j_dict:
+                        if j_dict["truncated"] is False:
+
+                            j_dict = modify_dict_cols(j_dict)
+
+                            j_dict_list.append(j_dict)
+
+        else:
+            with gzip.open(f, "r") as file:
+                for line in file:
+                    # Load JSON
+                    j_dict = json.loads(line)
+                    # Check if user key exists
+                    if "delete" not in j_dict:
+                        if j_dict["truncated"] is False:
+
+                            j_dict = modify_dict_cols(j_dict)
+
+                            j_dict_list.append(j_dict)
+
+        return j_dict_list
+
+
+def process_json(file_list, processed_max_buffer):
+    """
+    Loops through file list and loads the compressed
+    json into a list of dicts after some pre-processing.
+
+    Makes sure dicts are ordered in a specific
+    way to make sure polars can read them.
+    """
+
+    # Gets processed file list if stored, if not, creates it.
+    processed_list = get_processed_list(processed_file_list_pkl)
+
+    j_list = []
+    temp_processed_files = []
+
+    for i, f in enumerate(tqdm(file_list)):
+
+        j_dict_list = process_single_file(f, processed_list)
+
+        j_list.extend(j_dict_list)
+
+        temp_processed_files.append(f)
+
+        if len(temp_processed_files) == processed_max_buffer:
+            # If we reach our buffer,
+            # combine into polars dataframe
+            # and write to parquet as
+            # a checkpoint
+            processed_file_name = f"processed_json_{i}.parquet"
+            processed_file_path = processed_folder_path / processed_file_name
+
+            pl.DataFrame(j_list, columns=wanted_cols).write_parquet(processed_file_path)
+
+            # Make note of which files have been processed
+            processed_list.extend(temp_processed_files)
+            pickle.dump(processed_list, open(processed_file_list_pkl, "wb"))
+
+            # Reset buffer lists
+            j_list = []
+            temp_processed_files = []
+
+    # Process remaining files
+    processed_file_name = f"processed_json_{i}.parquet"
+    processed_file_path = processed_folder_path / processed_file_name
+    pl.from_dicts(j_dict_list).write_parquet(processed_file_path)
+    processed_list.extend(temp_processed_files)
+    pickle.dump(processed_list, open(processed_file_list_pkl, "wb"))
+    j_dict_list = []
+    temp_processed_files = []
+
+    print("Processing completed")
+
+
+if __name__ == "__main__":
+    main(file_list_pkl, folder_path, processed_max_buffer)

website/next-i18next.config.js

@@ -1,6 +1,6 @@
 module.exports = {
   i18n: {
     defaultLocale: "en",
-    locales: ["en"],
+    locales: ["de", "en", "fr"],
   },
 };

website/package-lock.json

@@ -21,6 +21,8 @@
         "@next/font": "^13.1.0",
         "@prisma/client": "^4.7.1",
         "@tailwindcss/forms": "^0.5.3",
+        "@tanstack/react-table": "^8.7.6",
+        "accept-language-parser": "^1.5.0",
         "autoprefixer": "^10.4.13",
         "axios": "^1.2.1",
         "boolean": "^3.2.0",
@@ -38,6 +40,7 @@
         "npm": "^9.2.0",
         "postcss-focus-visible": "^7.1.0",
         "react": "18.2.0",
+        "react-cookies": "^0.1.1",
         "react-dom": "18.2.0",
         "react-feature-flags": "^1.0.0",
         "react-hook-form": "^7.42.1",
@@ -12297,6 +12300,37 @@
         "tailwindcss": ">=3.0.0 || >= 3.0.0-alpha.1"
       }
     },
+    "node_modules/@tanstack/react-table": {
+      "version": "8.7.6",
+      "resolved": "https://registry.npmjs.org/@tanstack/react-table/-/react-table-8.7.6.tgz",
+      "integrity": "sha512-/QijmMFeP7wDLBnr0MQ/5MlbXePbIL/1nOtkxBC9zvmBu4gDKJEDBqipUyM7Wc/iBpSd0IFyqBlvZvTPD9FYDA==",
+      "dependencies": {
+        "@tanstack/table-core": "8.7.6"
+      },
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      },
+      "peerDependencies": {
+        "react": ">=16",
+        "react-dom": ">=16"
+      }
+    },
+    "node_modules/@tanstack/table-core": {
+      "version": "8.7.6",
+      "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.7.6.tgz",
+      "integrity": "sha512-sqiNTMzB6cpyL8DFH6/VqW48SwiflLqxQqYpo2wNock7rdVGvlm0BLNI8vZUJbr1+fmmWmHwBvi5OMgZw8n1DA==",
+      "engines": {
+        "node": ">=12"
+      },
+      "funding": {
+        "type": "github",
+        "url": "https://github.com/sponsors/tannerlinsley"
+      }
+    },
     "node_modules/@testing-library/dom": {
       "version": "8.19.1",
       "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-8.19.1.tgz",
@@ -13616,6 +13650,11 @@
       "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==",
       "dev": true
     },
+    "node_modules/accept-language-parser": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/accept-language-parser/-/accept-language-parser-1.5.0.tgz",
+      "integrity": "sha512-QhyTbMLYo0BBGg1aWbeMG4ekWtds/31BrEU+DONOg/7ax23vxpL03Pb7/zBmha2v7vdD3AyzZVWBVGEZxKOXWw=="
+    },
     "node_modules/accepts": {
       "version": "1.3.8",
       "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@@ -32466,6 +32505,23 @@
         "react": "^15.3.0 || ^16.0.0 || ^17.0.0 || ^18.0.0"
       }
     },
+    "node_modules/react-cookies": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/react-cookies/-/react-cookies-0.1.1.tgz",
+      "integrity": "sha512-PP75kJ4vtoHuuTdq0TAD3RmlAv7vuDQh9fkC4oDlhntgs9vX1DmREomO0Y1mcQKR9nMZ6/zxoflaMJ3MAmF5KQ==",
+      "dependencies": {
+        "cookie": "^0.3.1",
+        "object-assign": "^4.1.1"
+      }
+    },
+    "node_modules/react-cookies/node_modules/cookie": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
+      "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw==",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/react-docgen": {
       "version": "5.4.3",
       "resolved": "https://registry.npmjs.org/react-docgen/-/react-docgen-5.4.3.tgz",
@@ -46682,6 +46738,19 @@
         "mini-svg-data-uri": "^1.2.3"
       }
     },
+    "@tanstack/react-table": {
+      "version": "8.7.6",
+      "resolved": "https://registry.npmjs.org/@tanstack/react-table/-/react-table-8.7.6.tgz",
+      "integrity": "sha512-/QijmMFeP7wDLBnr0MQ/5MlbXePbIL/1nOtkxBC9zvmBu4gDKJEDBqipUyM7Wc/iBpSd0IFyqBlvZvTPD9FYDA==",
+      "requires": {
+        "@tanstack/table-core": "8.7.6"
+      }
+    },
+    "@tanstack/table-core": {
+      "version": "8.7.6",
+      "resolved": "https://registry.npmjs.org/@tanstack/table-core/-/table-core-8.7.6.tgz",
+      "integrity": "sha512-sqiNTMzB6cpyL8DFH6/VqW48SwiflLqxQqYpo2wNock7rdVGvlm0BLNI8vZUJbr1+fmmWmHwBvi5OMgZw8n1DA=="
+    },
     "@testing-library/dom": {
       "version": "8.19.1",
       "resolved": "https://registry.npmjs.org/@testing-library/dom/-/dom-8.19.1.tgz",
@@ -47817,6 +47886,11 @@
       "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==",
       "dev": true
     },
+    "accept-language-parser": {
+      "version": "1.5.0",
+      "resolved": "https://registry.npmjs.org/accept-language-parser/-/accept-language-parser-1.5.0.tgz",
+      "integrity": "sha512-QhyTbMLYo0BBGg1aWbeMG4ekWtds/31BrEU+DONOg/7ax23vxpL03Pb7/zBmha2v7vdD3AyzZVWBVGEZxKOXWw=="
+    },
     "accepts": {
       "version": "1.3.8",
       "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz",
@@ -61962,6 +62036,22 @@
         "@babel/runtime": "^7.12.13"
       }
     },
+    "react-cookies": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/react-cookies/-/react-cookies-0.1.1.tgz",
+      "integrity": "sha512-PP75kJ4vtoHuuTdq0TAD3RmlAv7vuDQh9fkC4oDlhntgs9vX1DmREomO0Y1mcQKR9nMZ6/zxoflaMJ3MAmF5KQ==",
+      "requires": {
+        "cookie": "^0.3.1",
+        "object-assign": "^4.1.1"
+      },
+      "dependencies": {
+        "cookie": {
+          "version": "0.3.1",
+          "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
+          "integrity": "sha512-+IJOX0OqlHCszo2mBUq+SrEbCj6w7Kpffqx60zYbPTFaO4+yYgRjHwcZNpWvaTylDHaV7PPmBHzSecZiMhtPgw=="
+        }
+      }
+    },
     "react-docgen": {
       "version": "5.4.3",
       "resolved": "https://registry.npmjs.org/react-docgen/-/react-docgen-5.4.3.tgz",

website/package.json

@@ -38,6 +38,8 @@
     "@next/font": "^13.1.0",
     "@prisma/client": "^4.7.1",
     "@tailwindcss/forms": "^0.5.3",
+    "@tanstack/react-table": "^8.7.6",
+    "accept-language-parser": "^1.5.0",
     "autoprefixer": "^10.4.13",
     "axios": "^1.2.1",
     "boolean": "^3.2.0",
@@ -55,6 +57,7 @@
     "npm": "^9.2.0",
     "postcss-focus-visible": "^7.1.0",
     "react": "18.2.0",
+    "react-cookies": "^0.1.1",
     "react-dom": "18.2.0",
     "react-feature-flags": "^1.0.0",
     "react-hook-form": "^7.42.1",

website/public/locales/en/index.json

@@ -1,16 +1,15 @@
 {
-  "title": "Open Assistant",
-  "subtitle": "Conversational AI for everyone.",
-  "description": "Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world.",
   "blurb": "We believe we can create a revolution.",
   "blurb1": "In the same way that Stable Diffusion helped the world make art and images in new ways, we want to improve the world by providing amazing conversational AI.",
-  "join_us_title": "Join us",
-  "join_us_description": "All open source projects begin with people like you. Open source is the belief that if we collaborate we can together gift our knowledge and technology to the world for the benefit of humanity. Are you in? Find us here:",
-  "faq_title": "Frequently Asked Questions",
+  "description": "Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world.",
   "faq_items": {
     "q0": "How far along is this project?",
     "a0": "We are in the early stages of development, working from established research in applying RLHF to large language models.",
     "q1": "Who is behind Open Assistant?",
     "a1": "Open Assistant is a project organized by LAION and individuals around the world interested in bringing this technology to everyone."
-  }
+  },
+  "faq_title": "Frequently Asked Questions",
+  "join_us_description": "All open source projects begin with people like you. Open source is the belief that if we collaborate we can together gift our knowledge and technology to the world for the benefit of humanity. Are you in? Find us here:",
+  "join_us_title": "Join us",
+  "subtitle": "Conversational AI for everyone."
 }

website/public/locales/en/leaderboard.json

@@ -0,0 +1,11 @@
+{
+  "daily": "Daily",
+  "last_updated_at": "Last updated at: {{val, datetime}}",
+  "leaderboard": "Leaderboard",
+  "monthly": "Monthly",
+  "overall": "Overall",
+  "rank": "Rank",
+  "score": "Score",
+  "user": "User",
+  "weekly": "Weekly"
+}

website/src/components/Dashboard/TaskOption.tsx

@@ -1,51 +1,75 @@
 import { Box, Flex, GridItem, Heading, SimpleGrid, Text, useColorModeValue } from "@chakra-ui/react";
 import Link from "next/link";
+import { useMemo } from "react";
+import { TaskType } from "src/types/Task";
 
-import { TaskCategory, TaskCategoryLabels, TaskTypes } from "../Tasks/TaskTypes";
+import { TaskCategory, TaskCategoryLabels, TaskInfo, TaskInfos } from "../Tasks/TaskTypes";
 
-export const TaskOption = ({ displayTaskCategories }: { displayTaskCategories: TaskCategory[] }) => {
+export interface TasksOptionProps {
+  content: Partial<Record<TaskCategory, TaskType[]>>;
+}
+
+export const TaskOption = ({ content }: TasksOptionProps) => {
   const backgroundColor = useColorModeValue("white", "gray.700");
 
+  const taskInfoMap = useMemo(
+    () =>
+      Object.values(content)
+        .flat()
+        .reduce((obj, taskType) => {
+          obj[taskType] = TaskInfos.filter((t) => t.type === taskType).pop();
+          return obj;
+        }, {} as Record<TaskType, TaskInfo>),
+    [content]
+  );
+
   return (
     <Box className="flex flex-col gap-14">
-      {displayTaskCategories.map((category) => (
+      {Object.entries(content).map(([category, taskTypes]) => (
         <div key={category}>
-          <Text className="text-2xl font-bold pb-4">{TaskCategoryLabels[category]}</Text>
+          <Heading size="lg" className="pb-4">
+            {TaskCategoryLabels[category]}
+          </Heading>
           <SimpleGrid columns={[1, 1, 2, 2, 3, 4]} gap={4}>
-            {TaskTypes.filter((task) => task.category === category).map((item) => (
-              <Link key={category + item.label} href={item.pathname}>
-                <GridItem
-                  bg={backgroundColor}
-                  borderRadius="xl"
-                  boxShadow="base"
-                  className="flex flex-col justify-between h-full"
-                >
-                  <Box className="p-6 pb-10">
-                    <Flex flexDir="column" gap="3">
-                      <Heading size="md" fontFamily="inter">
-                        {item.label}
-                      </Heading>
-                      <Text size="sm" opacity="80%">
-                        {item.desc}
-                      </Text>
-                    </Flex>
-                  </Box>
-                  <Box
-                    bg="blue.500"
-                    borderBottomRadius="xl"
-                    className="px-6 py-2 transition-colors duration-300"
-                    _hover={{ backgroundColor: "blue.600" }}
+            {taskTypes
+              .map((taskType) => taskInfoMap[taskType])
+              .map((item) => (
+                <Link key={category + item.label} href={item.pathname}>
+                  <GridItem
+                    bg={backgroundColor}
+                    borderRadius="xl"
+                    boxShadow="base"
+                    className="flex flex-col justify-between h-full"
                   >
-                    <Text fontWeight="bold" color="white">
+                    <Flex className="p-6 pb-10" flexDir="column" gap="3">
+                      <Heading size="md">{item.label}</Heading>
+                      <Text size="sm">{item.desc}</Text>
+                    </Flex>
+                    <Text
+                      fontWeight="bold"
+                      color="white"
+                      borderBottomRadius="xl"
+                      className="px-6 py-2 transition-colors duration-300 bg-blue-500 hover:bg-blue-600"
+                    >
                       Go -&gt;
                     </Text>
-                  </Box>
-                </GridItem>
-              </Link>
-            ))}
+                  </GridItem>
+                </Link>
+              ))}
           </SimpleGrid>
         </div>
       ))}
     </Box>
   );
 };
+
+export const allTaskOptions: TasksOptionProps["content"] = {
+  [TaskCategory.Random]: [TaskType.random],
+  [TaskCategory.Create]: [TaskType.initial_prompt, TaskType.prompter_reply, TaskType.assistant_reply],
+  [TaskCategory.Evaluate]: [
+    TaskType.rank_initial_prompts,
+    TaskType.rank_prompter_replies,
+    TaskType.rank_assistant_replies,
+  ],
+  [TaskCategory.Label]: [TaskType.label_initial_prompt, TaskType.label_prompter_reply, TaskType.label_assistant_reply],
+};

website/src/components/DataTable.tsx

@@ -0,0 +1,166 @@
+import {
+  Box,
+  Button,
+  Card,
+  CardBody,
+  Flex,
+  FormControl,
+  FormLabel,
+  Input,
+  Popover,
+  PopoverArrow,
+  PopoverBody,
+  PopoverCloseButton,
+  PopoverContent,
+  PopoverTrigger,
+  Spacer,
+  Table,
+  TableCaption,
+  TableContainer,
+  Tbody,
+  Td,
+  Th,
+  Thead,
+  Tr,
+  useDisclosure,
+} from "@chakra-ui/react";
+import { ColumnDef, flexRender, getCoreRowModel, useReactTable } from "@tanstack/react-table";
+import { ChangeEvent, ReactNode } from "react";
+import { FaFilter } from "react-icons/fa";
+import { useDebouncedCallback } from "use-debounce";
+
+export type DataTableColumnDef<T> = ColumnDef<T> & {
+  filterable?: boolean;
+};
+
+// TODO: stricter type
+export type FilterItem = {
+  id: string;
+  value: string;
+};
+
+export type DataTableProps<T> = {
+  data: T[];
+  columns: DataTableColumnDef<T>[];
+  caption?: string;
+  filterValues?: FilterItem[];
+  onNextClick?: () => void;
+  onPreviousClick?: () => void;
+  onFilterChange?: (items: FilterItem[]) => void;
+  disableNext?: boolean;
+  disablePrevious?: boolean;
+};
+
+export const DataTable = <T,>({
+  data,
+  columns,
+  caption,
+  filterValues = [],
+  onNextClick,
+  onPreviousClick,
+  onFilterChange,
+  disableNext,
+  disablePrevious,
+}: DataTableProps<T>) => {
+  const { getHeaderGroups, getRowModel } = useReactTable<T>({
+    data,
+    columns,
+    getCoreRowModel: getCoreRowModel(),
+  });
+
+  const handleFilterChange = (value: FilterItem) => {
+    const idx = filterValues.findIndex((oldValue) => oldValue.id === value.id);
+    let newValues: FilterItem[] = [];
+    if (idx === -1) {
+      newValues = [...filterValues, value];
+    } else {
+      newValues = filterValues.map((oldValue) => (oldValue.id === value.id ? value : oldValue));
+    }
+    onFilterChange(newValues);
+  };
+  return (
+    <Card>
+      <CardBody>
+        <Flex mb="2">
+          <Button onClick={onPreviousClick} disabled={disablePrevious}>
+            Previous
+          </Button>
+          <Spacer />
+          <Button onClick={onNextClick} disabled={disableNext}>
+            Next
+          </Button>
+        </Flex>
+        <TableContainer>
+          <Table variant="simple">
+            <TableCaption>{caption}</TableCaption>
+            <Thead>
+              {getHeaderGroups().map((headerGroup) => (
+                <Tr key={headerGroup.id}>
+                  {headerGroup.headers.map((header) => (
+                    <Th key={header.id}>
+                      <Box display="flex" alignItems="center">
+                        {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())}
+                        {(header.column.columnDef as DataTableColumnDef<T>).filterable && (
+                          <FilterModal
+                            value={filterValues.find((value) => value.id === header.id)?.value ?? ""}
+                            onChange={(value) => handleFilterChange({ id: header.id, value })}
+                            label={flexRender(header.column.columnDef.header, header.getContext())}
+                          ></FilterModal>
+                        )}
+                      </Box>
+                    </Th>
+                  ))}
+                </Tr>
+              ))}
+            </Thead>
+            <Tbody>
+              {getRowModel().rows.map((row) => (
+                <Tr key={row.id}>
+                  {row.getVisibleCells().map((cell) => (
+                    <Td key={cell.id}>{flexRender(cell.column.columnDef.cell, cell.getContext())}</Td>
+                  ))}
+                </Tr>
+              ))}
+            </Tbody>
+          </Table>
+        </TableContainer>
+      </CardBody>
+    </Card>
+  );
+};
+
+const FilterModal = ({
+  label,
+  onChange,
+  value,
+}: {
+  label: ReactNode;
+  onChange: (val: string) => void;
+  value: string;
+}) => {
+  const { isOpen, onOpen, onClose } = useDisclosure();
+
+  const handleInputChange = useDebouncedCallback((e: ChangeEvent<HTMLInputElement>) => {
+    onChange(e.target.value);
+  }, 500);
+
+  return (
+    <Popover isOpen={isOpen} onOpen={onOpen} onClose={onClose}>
+      <PopoverTrigger>
+        <Button variant={"unstyled"} ml="2">
+          <FaFilter></FaFilter>
+        </Button>
+      </PopoverTrigger>
+      <PopoverContent w="fit-content">
+        <PopoverArrow />
+        <PopoverCloseButton />
+        <PopoverBody mt="4">
+          <FormControl>
+            <FormLabel>{label}</FormLabel>
+            <Input onChange={handleInputChange} defaultValue={value}></Input>
+          </FormControl>
+        </PopoverBody>
+      </PopoverContent>
+    </Popover>
+  );
+};

website/src/components/EmptyState.tsx

@@ -1,5 +1,5 @@
-import { Box, Link, Text, useColorModeValue } from "@chakra-ui/react";
-import { useRouter } from "next/router";
+import { Box, Text, useColorModeValue } from "@chakra-ui/react";
+import NextLink from "next/link";
 import { FiAlertTriangle } from "react-icons/fi";
 import { IconType } from "react-icons/lib";
 
@@ -10,16 +10,15 @@ type EmptyStateProps = {
 
 export const EmptyState = (props: EmptyStateProps) => {
   const backgroundColor = useColorModeValue("white", "gray.800");
-  const router = useRouter();
 
   return (
     <Box bg={backgroundColor} p="10" borderRadius="xl" shadow="base">
       <Box display="flex" flexDirection="column" alignItems="center" gap="8" fontSize="lg">
         <props.icon size="30" color="DarkOrange" />
         <Text>{props.text}</Text>
-        <Link onClick={() => router.back()} color="blue.500" textUnderlineOffset="3px">
-          <Text>Click here to go back</Text>
-        </Link>
+        <NextLink href="/dashboard">
+          <Text color="blue.500">Go back to the dashboard</Text>
+        </NextLink>
       </Box>
     </Box>
   );

website/src/components/Header/Header.tsx

@@ -5,6 +5,7 @@ import { useSession } from "next-auth/react";
 import { useTranslation } from "next-i18next";
 import { Flags } from "react-feature-flags";
 import { FaUser } from "react-icons/fa";
+import { LanguageSelector } from "src/components/LanguageSelector";
 
 import { UserMenu } from "./UserMenu";
 
@@ -45,6 +46,7 @@ export function Header() {
           <Flags authorizedFlags={["flagTest"]}>
             <Text>FlagTest</Text>
           </Flags>
+          <LanguageSelector />
           <AccountButton />
           <UserMenu />
         </Flex>

website/src/components/Hero.tsx

@@ -6,7 +6,7 @@ import { AnimatedCircles } from "./AnimatedCircles";
 import { Container } from "./Container";
 
 export function Hero() {
-  const { t } = useTranslation("index");
+  const { t } = useTranslation(["index", "common"]);
   const { colorMode } = useColorMode();
   const pTextColor = colorMode === "light" ? "text-gray-600" : "text-white";
   const fancyTextGradientClasses =
@@ -17,7 +17,7 @@ export function Hero() {
         <Box className="lg:grid lg:grid-cols-12 lg:gap-x-8 lg:gap-y-20">
           <Box className="relative mx-auto max-w-2xl lg:col-span-7 lg:max-w-none lg:pt-6 xl:col-span-6">
             <Text as="h1" className="text-5xl mb-6 font-bold tracking-tight">
-              {t("title")}
+              {t("common:title")}
             </Text>
             <Text
               as="h2"

website/src/components/LanguageSelector/LanguageSelector.tsx

@@ -0,0 +1,40 @@
+import { Select } from "@chakra-ui/react";
+import { useRouter } from "next/router";
+import { useTranslation } from "next-i18next";
+import { useCallback, useMemo } from "react";
+import cookie from "react-cookies";
+
+const LanguageSelector = () => {
+  const router = useRouter();
+  const { i18n } = useTranslation();
+
+  const { language: currentLanguage } = i18n;
+  const languageNames = useMemo(() => {
+    return new Intl.DisplayNames([currentLanguage], {
+      type: "language",
+    });
+  }, [currentLanguage]);
+
+  const languageChanged = useCallback(
+    async (option) => {
+      const locale = option.target.value;
+      cookie.save("NEXT_LOCALE", locale, { path: "/" });
+      const path = router.asPath;
+      return router.push(path, path, { locale });
+    },
+    [router]
+  );
+
+  const locales = router.locales;
+  return (
+    <Select onChange={languageChanged} defaultValue={currentLanguage}>
+      {locales.map((locale) => (
+        <option key={locale} value={locale}>
+          {languageNames.of(locale) ?? locale}
+        </option>
+      ))}
+    </Select>
+  );
+};
+
+export { LanguageSelector };

website/src/components/LanguageSelector/index.tsx

@@ -0,0 +1 @@
+export * from "./LanguageSelector";

website/src/components/Layout.tsx

@@ -2,7 +2,7 @@
 
 import { Box, Grid } from "@chakra-ui/react";
 import type { NextPage } from "next";
-import { FiBarChart2, FiLayout, FiMessageSquare, FiUsers } from "react-icons/fi";
+import { FiBarChart2, FiLayout, FiMessageSquare, FiUsers, FiActivity } from "react-icons/fi";
 import { Header } from "src/components/Header";
 
 import { SlimFooter } from "./Dashboard/SlimFooter";
@@ -75,6 +75,12 @@ export const getAdminLayout = (page: React.ReactElement) => (
           desc: "Users Dashboard",
           icon: FiUsers,
         },
+        {
+          label: "Status",
+          pathname: "/admin/status",
+          desc: "Status Dashboard",
+          icon: FiActivity,
+        },
       ]}
     >
       {page}

website/src/components/LeaderboardGridCell/LeaderboardGridCell.tsx

@@ -1,23 +1,24 @@
-import { Table, TableContainer, Tbody, Td, Th, Thead, Tr, useColorModeValue } from "@chakra-ui/react";
-import React from "react";
+import { Table, TableContainer, Tbody, Td, Text, Th, Thead, Tr, useColorModeValue } from "@chakra-ui/react";
+import { useTranslation } from "next-i18next";
+import React, { useMemo } from "react";
 import { useTable } from "react-table";
 import { get } from "src/lib/api";
-import { LeaderboardEntity, LeaderboardTimeFrame } from "src/types/Leaderboard";
+import { LeaderboardReply, LeaderboardTimeFrame } from "src/types/Leaderboard";
 import useSWRImmutable from "swr/immutable";
 
-const columns = [
+const getColumns = (t) => [
   {
-    Header: "Rank",
+    Header: t("rank"),
     accessor: "rank",
     style: { width: "90px" },
   },
   {
-    Header: "Score",
+    Header: t("score"),
     accessor: "leader_score",
     style: { width: "90px" },
   },
   {
-    Header: "User",
+    Header: t("user"),
     accessor: "display_name",
   },
 ];
@@ -26,13 +27,28 @@ const columns = [
  * Presents a grid of leaderboard entries with more detailed information.
  */
 const LeaderboardGridCell = ({ timeFrame }: { timeFrame: LeaderboardTimeFrame }) => {
-  const { data } = useSWRImmutable<LeaderboardEntity[]>(`/api/leaderboard?time_frame=${timeFrame}`, get, {
-    fallbackData: [],
+  const { t } = useTranslation(["leaderboard", "common"]);
+  const { data: reply } = useSWRImmutable<LeaderboardReply>(`/api/leaderboard?time_frame=${timeFrame}`, get, {
     revalidateOnMount: true,
   });
+
+  const columns = useMemo(() => getColumns(t), [t]);
+
+  const { getTableProps, getTableBodyProps, headerGroups, rows, prepareRow } = useTable({
+    columns,
+    data: reply?.leaderboard ?? [],
+  });
+
   const backgroundColor = useColorModeValue("white", "gray.800");
 
-  const { getTableProps, getTableBodyProps, headerGroups, rows, prepareRow } = useTable({ columns, data });
+  const lastUpdated = useMemo(() => {
+    const val = new Date(reply?.last_updated);
+    return t("last_updated_at", { val, formatParams: { val: { dateStyle: "full", timeStyle: "short" } } });
+  }, [t, reply?.last_updated]);
+
+  if (!reply) {
+    return null;
+  }
 
   return (
     <TableContainer>
@@ -66,6 +82,7 @@ const LeaderboardGridCell = ({ timeFrame }: { timeFrame: LeaderboardTimeFrame })
           })}
         </Tbody>
       </Table>
+      <Text p="2">{lastUpdated}</Text>
     </TableContainer>
   );
 };

website/src/components/Messages/MessageTableEntry.tsx

@@ -50,6 +50,7 @@ export function MessageTableEntry(props: MessageTableEntryProps) {
           bg={item.is_assistant ? backgroundColor : backgroundColor2}
           onClick={props.enabled && goToMessage}
           _hover={props.enabled && { cursor: "pointer", opacity: 0.9 }}
+          whiteSpace="pre-wrap"
         >
           {inlineAvatar && avatar}
           {item.text}

website/src/components/Tasks/Task/Task.tsx

@@ -3,7 +3,7 @@ import { TaskControls } from "src/components/Survey/TaskControls";
 import { CreateTask } from "src/components/Tasks/CreateTask";
 import { EvaluateTask } from "src/components/Tasks/EvaluateTask";
 import { LabelTask } from "src/components/Tasks/LabelTask";
-import { TaskCategory, TaskInfo, TaskTypes } from "src/components/Tasks/TaskTypes";
+import { TaskCategory, TaskInfo, TaskInfos } from "src/components/Tasks/TaskTypes";
 import { UnchangedWarning } from "src/components/Tasks/UnchangedWarning";
 import { post } from "src/lib/api";
 import { TaskContent } from "src/types/Task";
@@ -29,7 +29,7 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
 
   const rootEl = useRef<HTMLDivElement>(null);
 
-  const taskType = TaskTypes.find((taskType) => taskType.type === task.type && taskType.mode === task.mode);
+  const taskType = TaskInfos.find((taskType) => taskType.type === task.type && taskType.mode === task.mode);
 
   const { trigger: sendRejection } = useSWRMutation("/api/reject_task", post, {
     onSuccess: async () => {

website/src/components/Tasks/TaskTypes.tsx

@@ -21,16 +21,16 @@ export interface TaskInfo {
 }
 
 export const TaskCategoryLabels: { [key in TaskCategory]: string } = {
-  [TaskCategory.Random]: "I'm feeling lucky",
+  [TaskCategory.Random]: "Grab a task!",
   [TaskCategory.Create]: "Create",
   [TaskCategory.Evaluate]: "Evaluate",
   [TaskCategory.Label]: "Label",
 };
 
-export const TaskTypes: TaskInfo[] = [
+export const TaskInfos: TaskInfo[] = [
   // general/random
   {
-    label: "Start a Task",
+    label: "I'm feeling lucky",
     desc: "Help us improve Open Assistant by starting a random task.",
     category: TaskCategory.Random,
     pathname: "/tasks/random",
@@ -104,7 +104,7 @@ export const TaskTypes: TaskInfo[] = [
     category: TaskCategory.Evaluate,
     pathname: "/evaluate/rank_initial_prompts",
     help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
-    overview: "Given the following inital prompts, sort them from best to worst, best being first, worst being last.",
+    overview: "Given the following initial prompts, sort them from best to worst, best being first, worst being last.",
     type: "rank_initial_prompts",
     update_type: "message_ranking",
     unchanged_title: "Order Unchanged",

website/src/components/UserTable.tsx

@@ -0,0 +1,108 @@
+import { IconButton } from "@chakra-ui/react";
+import { createColumnHelper } from "@tanstack/react-table";
+import Link from "next/link";
+import { memo, useState } from "react";
+import { FaPen } from "react-icons/fa";
+import { get } from "src/lib/api";
+import { FetchUsersResponse } from "src/lib/oasst_api_client";
+import type { User } from "src/types/Users";
+import useSWR from "swr";
+
+import { DataTable, DataTableColumnDef, FilterItem } from "./DataTable";
+
+interface Pagination {
+  /**
+   * The user's `display_name` used for pagination.
+   */
+  cursor: string;
+
+  /**
+   * The pagination direction.
+   */
+  direction: "forward" | "back";
+}
+
+const columnHelper = createColumnHelper<User>();
+
+const columns: DataTableColumnDef<User>[] = [
+  columnHelper.accessor("user_id", {
+    header: "ID",
+  }),
+  columnHelper.accessor("id", {
+    header: "Auth ID",
+  }),
+  columnHelper.accessor("auth_method", {
+    header: "Auth Method",
+  }),
+  {
+    ...columnHelper.accessor("display_name", {
+      header: "Name",
+    }),
+    filterable: true,
+  },
+  columnHelper.accessor("role", {
+    header: "Role",
+  }),
+  columnHelper.accessor((user) => user.user_id, {
+    cell: ({ getValue }) => (
+      <IconButton
+        as={Link}
+        href={`/admin/manage_user/${getValue()}`}
+        aria-label="Manage"
+        icon={<FaPen></FaPen>}
+      ></IconButton>
+    ),
+    header: "Update",
+  }),
+];
+
+export const UserTable = memo(function UserTable() {
+  const [pagination, setPagination] = useState<Pagination>({ cursor: "", direction: "forward" });
+  const [filterValues, setFilterValues] = useState<FilterItem[]>([]);
+  const handleFilterValuesChange = (values: FilterItem[]) => {
+    setFilterValues(values);
+    setPagination((old) => ({ ...old, cursor: "" }));
+  };
+  // Fetch and save the users.
+  // This follows useSWR's recommendation for simple pagination:
+  //   https://swr.vercel.app/docs/pagination#when-to-use-useswr
+  const display_name = filterValues.find((value) => value.id === "display_name")?.value ?? "";
+  const { data, error } = useSWR<FetchUsersResponse<User>>(
+    `/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}&searchDisplayName=${display_name}&sortKey=display_name`,
+    get,
+    {
+      keepPreviousData: true,
+    }
+  );
+
+  const toPreviousPage = () => {
+    setPagination({
+      cursor: data.prev,
+      direction: "back",
+    });
+  };
+
+  const toNextPage = () => {
+    setPagination({
+      cursor: data.next,
+      direction: "forward",
+    });
+  };
+
+  return (
+    <>
+      <DataTable
+        data={data?.items || []}
+        columns={columns}
+        caption="Users"
+        onNextClick={toNextPage}
+        onPreviousClick={toPreviousPage}
+        disableNext={!data?.next}
+        disablePrevious={!data?.prev}
+        filterValues={filterValues}
+        onFilterChange={handleFilterValuesChange}
+      ></DataTable>
+      {error && "Unable to load users."}
+    </>
+  );
+});

website/src/components/UsersCell.tsx

@@ -1,137 +0,0 @@
-import {
-  Button,
-  Flex,
-  Spacer,
-  Stack,
-  Table,
-  TableCaption,
-  TableContainer,
-  Tbody,
-  Td,
-  Th,
-  Thead,
-  Tr,
-  useToast,
-} from "@chakra-ui/react";
-import Link from "next/link";
-import { useState } from "react";
-import { get } from "src/lib/api";
-import type { User } from "src/types/Users";
-import useSWR from "swr";
-
-interface Pagination {
-  /**
-   * The user's `display_name` used for pagination.
-   */
-  cursor: string;
-
-  /**
-   * The pagination direction.
-   */
-  direction: "forward" | "back";
-}
-
-/**
- * Fetches users from the users api route and then presents them in a simple Chakra table.
- */
-const UsersCell = () => {
-  const toast = useToast();
-  const [pagination, setPagination] = useState<Pagination>({ cursor: "", direction: "forward" });
-  const [users, setUsers] = useState<User[]>([]);
-
-  // Fetch and save the users.
-  // This follows useSWR's recommendation for simple pagination:
-  //   https://swr.vercel.app/docs/pagination#when-to-use-useswr
-  useSWR(`/api/admin/users?direction=${pagination.direction}&cursor=${pagination.cursor}`, get, {
-    onSuccess: (data) => {
-      // When no more users can be found, trigger a toast to indicate why no
-      // changes have taken place.  We have to maintain a non-empty set of
-      // users otherwise we can't paginate using a cursor (since we've lost the
-      // cursor).
-      if (data.length === 0) {
-        toast({
-          title: "No more users",
-          status: "warning",
-          duration: 1000,
-          isClosable: true,
-        });
-        return;
-      }
-      setUsers(data);
-    },
-  });
-
-  const toPreviousPage = () => {
-    if (users.length >= 0) {
-      setPagination({
-        cursor: users[0].display_name,
-        direction: "back",
-      });
-    } else {
-      toast({
-        title: "Can not paginate when no users are found",
-        status: "warning",
-        duration: 1000,
-        isClosable: true,
-      });
-    }
-  };
-
-  const toNextPage = () => {
-    if (users.length >= 0) {
-      setPagination({
-        cursor: users[users.length - 1].display_name,
-        direction: "forward",
-      });
-    } else {
-      toast({
-        title: "Can not paginate when no users are found",
-        status: "warning",
-        duration: 1000,
-        isClosable: true,
-      });
-    }
-  };
-
-  // Present users in a naive table.
-  return (
-    <Stack>
-      <Flex p="2">
-        <Button onClick={toPreviousPage}>Previous</Button>
-        <Spacer />
-        <Button onClick={toNextPage}>Next</Button>
-      </Flex>
-      <TableContainer>
-        <Table variant="simple">
-          <TableCaption>Users</TableCaption>
-          <Thead>
-            <Tr>
-              <Th>Id</Th>
-              <Th>Auth Id</Th>
-              <Th>Auth Method</Th>
-              <Th>Name</Th>
-              <Th>Role</Th>
-              <Th>Update</Th>
-            </Tr>
-          </Thead>
-          <Tbody>
-            {users.map(({ id, user_id, auth_method, display_name, role }) => (
-              <Tr key={user_id}>
-                <Td>{user_id}</Td>
-                <Td>{id}</Td>
-                <Td>{auth_method}</Td>
-                <Td>{display_name}</Td>
-                <Td>{role}</Td>
-                <Td>
-                  <Link href={`/admin/manage_user/${user_id}`}>Manage</Link>
-                </Td>
-              </Tr>
-            ))}
-          </Tbody>
-        </Table>
-      </TableContainer>
-    </Stack>
-  );
-};
-
-export default UsersCell;

website/src/lib/oasst_api_client.ts

@@ -1,7 +1,7 @@
 import type { Message } from "src/types/Conversation";
 import { LeaderboardReply, LeaderboardTimeFrame } from "src/types/Leaderboard";
 import type { AvailableTasks } from "src/types/Task";
-import type { BackendUser, BackendUserCore } from "src/types/Users";
+import type { BackendUser, BackendUserCore, User } from "src/types/Users";
 
 export class OasstError {
   message: string;
@@ -15,6 +15,22 @@ export class OasstError {
   }
 }
 
+export type FetchUsersParams = {
+  limit: number;
+  cursor?: string;
+  direction: "forward" | "back";
+  searchDisplayName?: string;
+  sortKey?: "username" | "display_name";
+};
+
+export type FetchUsersResponse<T extends User | BackendUser = BackendUser> = {
+  items: T[];
+  next?: string;
+  prev?: string;
+  sort_key: "username" | "display_name";
+  order: "asc" | "desc";
+};
+
 export class OasstApiClient {
   oasstApiUrl: string;
   oasstApiKey: string;
@@ -108,10 +124,11 @@ export class OasstApiClient {
   // TODO return a strongly typed Task?
   // This method is used to store a task in RegisteredTask.task.
   // This is a raw Json type, so we can't use it to strongly type the task.
-  async fetchTask(taskType: string, user: BackendUserCore): Promise<any> {
+  async fetchTask(taskType: string, user: BackendUserCore, lang: string): Promise<any> {
     return this.post("/api/v1/tasks/", {
       type: taskType,
       user,
+      lang,
     });
   }
 
@@ -136,7 +153,8 @@ export class OasstApiClient {
     messageId: string,
     userMessageId: string,
     content: object,
-    user: BackendUserCore
+    user: BackendUserCore,
+    lang: string
   ): Promise<any> {
     return this.post("/api/v1/tasks/interaction", {
       type: updateType,
@@ -144,15 +162,37 @@ export class OasstApiClient {
       task_id: taskId,
       message_id: messageId,
       user_message_id: userMessageId,
+      lang,
       ...content,
     });
   }
 
+  /**
+   * Returns the tasks availability information for given `user`.
+   */
+  async fetch_tasks_availability(user: object): Promise<any> {
+    return this.post("/api/v1/tasks/availability", user);
+  }
+
+  /**
+   * Returns the message stats from the backend.
+   */
+  async fetch_stats(): Promise<any> {
+    return this.get("/api/v1/stats/");
+  }
+
+  /**
+   * Returns the tree manager stats from the backend.
+   */
+  async fetch_tree_manager(): Promise<any> {
+    return this.get("/api/v1/stats/tree_manager");
+  }
+
   /**
    * Returns the `BackendUser` associated with `user_id`
    */
   async fetch_user(user_id: string): Promise<BackendUser> {
-    return this.get(`/api/v1/users/users/${user_id}`);
+    return this.get(`/api/v1/users/${user_id}`);
   }
 
   /**
@@ -164,21 +204,40 @@ export class OasstApiClient {
    *        forward.  If false and `cursor` is not empty, pages backwards.
    * @returns {Promise<BackendUser[]>} A Promise that returns an array of `BackendUser` objects.
    */
-  async fetch_users(max_count: number, cursor: string, isForward: boolean): Promise<BackendUser[]> {
-    const params = new URLSearchParams();
-    params.append("max_count", max_count.toString());
+  async fetch_users({
+    direction,
+    limit,
+    cursor,
+    searchDisplayName,
+    sortKey = "display_name",
+  }: FetchUsersParams): Promise<FetchUsersResponse> {
+    const params = new URLSearchParams({
+      search_text: searchDisplayName,
+      sort_key: sortKey,
+      max_count: limit.toString(),
+    });
 
     // The backend API uses different query parameters depending on the
     // pagination direction but they both take the same cursor value.
     // Depending on direction, pick the right query param.
     if (cursor !== "") {
-      params.append(isForward ? "gt" : "lt", cursor);
+      params.append(direction === "forward" ? "gt" : "lt", cursor);
     }
-    const BASE_URL = `/api/v1/frontend_users`;
+    const BASE_URL = `/api/v1/users/cursor`;
     const url = `${BASE_URL}/?${params.toString()}`;
     return this.get(url);
   }
 
+  // async fetch_user_by_display_name(name: string): Promise<BackendUser[]> {
+  //   const params = new URLSearchParams({
+  //     search_text: name,
+  //   });
+
+  //   const endpoint = `/api/v1/frontend_users/by_display_name`;
+
+  //   return this.get(`${endpoint}?${params.toString()}`);
+  // }
+
   /**
    * Returns the `Message`s associated with `user_id` in the backend.
    */

website/src/lib/users.ts

@@ -1,6 +1,32 @@
+import parser from "accept-language-parser";
+import type { NextApiRequest } from "next";
+import { i18n } from "src/../next-i18next.config";
 import prisma from "src/lib/prismadb";
 import type { BackendUserCore } from "src/types/Users";
 
+const LOCALE_SET = new Set(i18n.locales);
+
+/**
+ * Returns the most appropriate user language using the following priority:
+ *
+ * 1. The `NEXT_LOCALE` cookie which is set by the client side and will be in
+ *    the set of supported locales.
+ * 2. The `accept-language` header if it contains a supported locale as set by
+ *    the i18n module.
+ * 3. "en" as a final fallback.
+ */
+const getUserLanguage = (req: NextApiRequest) => {
+  const cookieLanguage = req.cookies["NEXT_LOCALE"];
+  if (cookieLanguage) {
+    return cookieLanguage;
+  }
+  const headerLanguages = parser.parse(req.headers["accept-language"]);
+  if (headerLanguages.length > 0 && LOCALE_SET.has(headerLanguages[0].code)) {
+    return headerLanguages[0].code;
+  }
+  return "en";
+};
+
 /**
  * Returns a `BackendUserCore` that can be used for interacting with the Backend service.
  *
@@ -35,4 +61,4 @@ const getBackendUserCore = async (id: string) => {
   } as BackendUserCore;
 };
 
-export { getBackendUserCore };
+export { getBackendUserCore, getUserLanguage };

website/src/pages/admin/index.tsx

@@ -3,7 +3,7 @@ import { useRouter } from "next/router";
 import { useSession } from "next-auth/react";
 import { useEffect } from "react";
 import { getAdminLayout } from "src/components/Layout";
-import UsersCell from "src/components/UsersCell";
+import { UserTable } from "src/components/UserTable";
 export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props";
 
 /**
@@ -28,7 +28,6 @@ const AdminIndex = () => {
     }
     router.push("/");
   }, [router, session, status]);
-
   return (
     <>
       <Head>
@@ -38,7 +37,7 @@ const AdminIndex = () => {
           content="Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world."
         />
       </Head>
-      <main className="oa-basic-theme">{status === "loading" ? "loading..." : <UsersCell />}</main>
+      <main>{status === "loading" ? "loading..." : <UserTable />}</main>
     </>
   );
 };

website/src/pages/admin/status/index.tsx

@@ -0,0 +1,174 @@
+import {
+  Box,
+  Card,
+  CardBody,
+  CircularProgress,
+  SimpleGrid,
+  Text,
+  Table,
+  TableCaption,
+  TableContainer,
+  Tbody,
+  Td,
+  Th,
+  Thead,
+  Tr,
+  useColorMode,
+} from "@chakra-ui/react";
+import Head from "next/head";
+import { useRouter } from "next/router";
+import { useSession } from "next-auth/react";
+import { useEffect } from "react";
+import useSWRImmutable from "swr/immutable";
+import { getAdminLayout } from "src/components/Layout";
+import { get } from "src/lib/api";
+
+/**
+ * Provides the admin status page that shows result of calls to several backend API endpoints,
+ * namely /api/v1/tasks/availability, /api/v1/stats/, /api/v1/stats/tree_manager
+ */
+
+const StatusIndex = () => {
+  const router = useRouter();
+  const { data: session, status } = useSession();
+
+  const { colorMode } = useColorMode();
+  const dataBackgroundColor = colorMode === "light" ? "gray.100" : "gray.800";
+  // Check when the user session is loaded and re-route if the user is not an
+  // admin.  This follows the suggestion by NextJS for handling private pages:
+  //   https://nextjs.org/docs/api-reference/next/router#usage
+  //
+  // All admin pages should use the same check and routing steps.
+  useEffect(() => {
+    if (status === "loading") {
+      return;
+    }
+    if (session?.user?.role === "admin") {
+      return;
+    }
+    router.push("/");
+  }, [router, session, status]);
+
+  const {
+    data: dataStatus,
+    error: errorStatus,
+    isLoading: isLoadingStatus,
+  } = useSWRImmutable("/api/admin/status", get);
+
+  const { tasksAvailability, stats, treeManager } = dataStatus || {};
+
+  return (
+    <>
+      <Head>
+        <title>Status - Open Assistant</title>
+        <meta
+          name="description"
+          content="Conversational AI for everyone. An open source project to create a chat enabled GPT LLM run by LAION and contributors around the world."
+        />
+      </Head>
+
+      <SimpleGrid columns={[1, 1, 1, 1, 1, 2]} gap={4}>
+        <Card>
+          <CardBody>
+            <Text as="h1" fontSize="3xl" textAlign="center">
+              /api/v1/tasks/availability
+            </Text>
+            <Box bg={dataBackgroundColor} borderRadius="xl" p="6" pt="4" pr="12">
+              {tasksAvailability?.status === "fulfilled" ? (
+                <pre>{JSON.stringify(tasksAvailability.value, null, 2)}</pre>
+              ) : tasksAvailability?.status === "rejected" ? (
+                <pre>{JSON.stringify(tasksAvailability.reason, null, 2)}</pre>
+              ) : errorStatus ? (
+                <pre>{JSON.stringify(errorStatus, null, 2)}</pre>
+              ) : (
+                <CircularProgress isIndeterminate />
+              )}
+            </Box>
+          </CardBody>
+        </Card>
+
+        <Card>
+          <CardBody>
+            <Text as="h1" fontSize="3xl" textAlign="center">
+              /api/v1/stats/
+            </Text>
+            <Box bg={dataBackgroundColor} borderRadius="xl" p="6" pt="4" pr="12">
+              {stats?.status === "fulfilled" ? (
+                <pre>{JSON.stringify(stats.value, null, 2)}</pre>
+              ) : stats?.status === "rejected" ? (
+                <pre>{JSON.stringify(stats.reason, null, 2)}</pre>
+              ) : errorStatus ? (
+                <pre>{JSON.stringify(errorStatus, null, 2)}</pre>
+              ) : (
+                <CircularProgress isIndeterminate />
+              )}
+            </Box>
+          </CardBody>
+        </Card>
+      </SimpleGrid>
+      <br />
+      <Card>
+        <CardBody>
+          <Text as="h1" fontSize="3xl" textAlign="center">
+            /api/v1/stats/tree_manager
+          </Text>
+          {treeManager?.status === "fulfilled" ? (
+            <Box>
+              <Text as="h2" fontSize="2xl">
+                state_counts
+              </Text>
+              <Box bg={dataBackgroundColor} borderRadius="xl" p="6" pt="4" pr="12">
+                <pre>{JSON.stringify(treeManager.value.state_counts, null, 2)}</pre>
+              </Box>
+              <TableContainer>
+                <br />
+                <Text as="h2" fontSize="2xl">
+                  message_counts
+                </Text>
+                <Table variant="simple">
+                  <TableCaption>Tree Manager</TableCaption>
+                  <Thead>
+                    <Tr>
+                      <Th>Message Tree ID</Th>
+                      <Th>State</Th>
+                      <Th>Depth</Th>
+                      <Th>Oldest</Th>
+                      <Th>Youngest</Th>
+                      <Th>Count</Th>
+                      <Th>Goal Tree Size</Th>
+                    </Tr>
+                  </Thead>
+                  <Tbody>
+                    {treeManager.value.message_counts.map(
+                      ({ message_tree_id, state, depth, oldest, youngest, count, goal_tree_size }) => (
+                        <Tr key={message_tree_id}>
+                          <Td>{message_tree_id}</Td>
+                          <Td>{state}</Td>
+                          <Td>{depth}</Td>
+                          <Td>{oldest}</Td>
+                          <Td>{youngest}</Td>
+                          <Td>{count}</Td>
+                          <Td>{goal_tree_size}</Td>
+                        </Tr>
+                      )
+                    )}
+                  </Tbody>
+                </Table>
+              </TableContainer>
+            </Box>
+          ) : treeManager?.status === "rejected" ? (
+            <pre>{JSON.stringify(treeManager.reason, null, 2)}</pre>
+          ) : errorStatus ? (
+            <pre>{JSON.stringify(errorStatus, null, 2)}</pre>
+          ) : (
+            <CircularProgress isIndeterminate />
+          )}
+        </CardBody>
+      </Card>
+    </>
+  );
+};
+
+StatusIndex.getLayout = getAdminLayout;
+
+export default StatusIndex;

website/src/pages/api/admin/status.ts

@@ -0,0 +1,30 @@
+import { getToken } from "next-auth/jwt";
+import { withRole } from "src/lib/auth";
+import { oasstApiClient } from "src/lib/oasst_api_client";
+import { getBackendUserCore } from "src/lib/users";
+
+/**
+ * Returns tasks availability, stats, and tree manager stats.
+ */
+const handler = withRole("admin", async (req, res) => {
+  const dummyUser = {
+    id: "__dummy_user__",
+    display_name: "Dummy User",
+    auth_method: "local",
+  };
+  const [tasksAvailabilityOutcome, statsOutcome, treeManagerOutcome] = await Promise.allSettled([
+    oasstApiClient.fetch_tasks_availability(dummyUser),
+    oasstApiClient.fetch_stats(),
+    oasstApiClient.fetch_tree_manager(),
+  ]);
+
+  const status = {
+    tasksAvailability: tasksAvailabilityOutcome,
+    stats: statsOutcome,
+    treeManager: treeManagerOutcome,
+  };
+
+  res.status(200).json(status);
+});
+
+export default handler;

website/src/pages/api/admin/users.ts

@@ -1,5 +1,5 @@
 import { withRole } from "src/lib/auth";
-import { oasstApiClient } from "src/lib/oasst_api_client";
+import { FetchUsersParams, oasstApiClient } from "src/lib/oasst_api_client";
 import prisma from "src/lib/prismadb";
 
 /**
@@ -17,10 +17,16 @@ const PAGE_SIZE = 20;
  *   direction.
  */
 const handler = withRole("admin", async (req, res) => {
-  const { cursor, direction } = req.query;
+  const { cursor, direction, searchDisplayName = "", sortKey = "username" } = req.query;
 
   // First, get all the users according to the backend.
-  const all_users = await oasstApiClient.fetch_users(PAGE_SIZE, cursor as string, direction === "forward");
+  const { items: all_users, ...rest } = await oasstApiClient.fetch_users({
+    searchDisplayName: searchDisplayName as FetchUsersParams["searchDisplayName"],
+    direction: direction as FetchUsersParams["direction"],
+    limit: PAGE_SIZE,
+    cursor: cursor as FetchUsersParams["cursor"],
+    sortKey: sortKey === "username" || sortKey === "display_name" ? sortKey : undefined,
+  });
 
   // Next, get all the users stored in the web's auth database to fetch their role.
   const local_user_ids = all_users.map(({ id }) => id);
@@ -51,7 +57,10 @@ const handler = withRole("admin", async (req, res) => {
     };
   });
 
-  res.status(200).json(users);
+  res.status(200).json({
+    items: users,
+    ...rest,
+  });
 });
 
 export default handler;

website/src/pages/api/leaderboard.ts

@@ -6,9 +6,9 @@ import { LeaderboardTimeFrame } from "src/types/Leaderboard";
  * Returns the set of valid labels that can be applied to messages.
  */
 const handler = withoutRole("banned", async (req, res) => {
-  const time_frame = (req.query.time_frame as LeaderboardTimeFrame) || LeaderboardTimeFrame.day;
-  const { leaderboard } = await oasstApiClient.fetch_leaderboard(time_frame);
-  res.status(200).json(leaderboard);
+  const time_frame = (req.query.time_frame as LeaderboardTimeFrame) ?? LeaderboardTimeFrame.day;
+  const info = await oasstApiClient.fetch_leaderboard(time_frame);
+  res.status(200).json(info);
 });
 
 export default handler;

website/src/pages/api/messages/user.ts

@@ -1,9 +1,12 @@
 import { withoutRole } from "src/lib/auth";
+import { getBackendUserCore } from "src/lib/users";
 
 const handler = withoutRole("banned", async (req, res, token) => {
   //TODO: add params if needed
+  const user = await getBackendUserCore(token.sub);
   const params = new URLSearchParams({
-    username: token.sub,
+    username: user.id,
+    auth_method: user.auth_method,
   });
 
   const messagesRes = await fetch(`${process.env.FASTAPI_URL}/api/v1/messages?${params}`, {

website/src/pages/api/new_task/[task_type].ts

@@ -1,7 +1,7 @@
 import { withoutRole } from "src/lib/auth";
 import { oasstApiClient } from "src/lib/oasst_api_client";
 import prisma from "src/lib/prismadb";
-import { getBackendUserCore } from "src/lib/users";
+import { getBackendUserCore, getUserLanguage } from "src/lib/users";
 
 /**
  * Returns a new task created from the Task Backend.  We do a few things here:
@@ -14,11 +14,12 @@ import { getBackendUserCore } from "src/lib/users";
 const handler = withoutRole("banned", async (req, res, token) => {
   // Fetch the new task.
   const { task_type } = req.query;
+  const userLanguage = getUserLanguage(req);
 
   const user = await getBackendUserCore(token.sub);
   let task;
   try {
-    task = await oasstApiClient.fetchTask(task_type as string, user);
+    task = await oasstApiClient.fetchTask(task_type as string, user, userLanguage);
   } catch (err) {
     console.error(err);
     res.status(500).json(err);

website/src/pages/api/update_task.ts

@@ -2,7 +2,7 @@ import { Prisma } from "@prisma/client";
 import { withoutRole } from "src/lib/auth";
 import { oasstApiClient } from "src/lib/oasst_api_client";
 import prisma from "src/lib/prismadb";
-import { getBackendUserCore } from "src/lib/users";
+import { getBackendUserCore, getUserLanguage } from "src/lib/users";
 
 /**
  * Stores the task interaction with the Task Backend and then returns the next task generated.
@@ -41,9 +41,18 @@ const handler = withoutRole("banned", async (req, res, token) => {
   });
 
   const user = await getBackendUserCore(token.sub);
+  const userLanguage = getUserLanguage(req);
   let newTask;
   try {
-    newTask = await oasstApiClient.interactTask(update_type, taskId, frontendId, interaction.id, content, user);
+    newTask = await oasstApiClient.interactTask(
+      update_type,
+      taskId,
+      frontendId,
+      interaction.id,
+      content,
+      user,
+      userLanguage
+    );
   } catch (err) {
     console.error(JSON.stringify(err));
     return res.status(500).json(err);

website/src/pages/auth/verify.tsx

@@ -1,6 +1,7 @@
 import { useColorMode } from "@chakra-ui/react";
 import Head from "next/head";
 import { getCsrfToken, getProviders } from "next-auth/react";
+import { serverSideTranslations } from "next-i18next/serverSideTranslations";
 
 export default function Verify() {
   const { colorMode } = useColorMode();
@@ -21,14 +22,14 @@ export default function Verify() {
   );
 }
 
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
-export async function getServerSideProps(context) {
+export async function getServerSideProps({ locale }) {
   const csrfToken = await getCsrfToken();
   const providers = await getProviders();
   return {
     props: {
       csrfToken,
       providers,
+      ...(await serverSideTranslations(locale, ["common"])),
     },
   };
 }

website/src/pages/dashboard.tsx

@@ -5,15 +5,17 @@ import { LeaderboardTable, TaskOption, WelcomeCard } from "src/components/Dashbo
 import { getDashboardLayout } from "src/components/Layout";
 import { TaskCategory } from "src/components/Tasks/TaskTypes";
 import { get } from "src/lib/api";
-import type { AvailableTasks, TaskType } from "src/types/Task";
+import { AvailableTasks, TaskType } from "src/types/Task";
 export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props";
 import useSWRImmutable from "swr/immutable";
 
 const Dashboard = () => {
   const { data } = useSWRImmutable<AvailableTasks>("/api/available_tasks", get);
 
-  // TODO: show only these tasks:
-  const availableTasks = useMemo(() => filterAvailableTasks(data ?? {}), [data]);
+  const availableTaskTypes = useMemo(() => {
+    const taskTypes = filterAvailableTasks(data ?? {});
+    return { [TaskCategory.Random]: taskTypes };
+  }, [data]);
 
   return (
     <>
@@ -23,7 +25,7 @@ const Dashboard = () => {
       </Head>
       <Flex direction="column" gap="10">
         <WelcomeCard />
-        <TaskOption displayTaskCategories={[TaskCategory.Random]} />
+        <TaskOption content={availableTaskTypes} />
         <LeaderboardTable />
       </Flex>
     </>

website/src/pages/index.tsx

@@ -3,17 +3,17 @@ import Head from "next/head";
 import { useRouter } from "next/router";
 import { useSession } from "next-auth/react";
 import { useTranslation } from "next-i18next";
-import { serverSideTranslations } from "next-i18next/serverSideTranslations";
 import { useEffect } from "react";
 import { CallToAction } from "src/components/CallToAction";
 import { Faq } from "src/components/Faq";
 import { Hero } from "src/components/Hero";
 import { getTransparentHeaderLayout } from "src/components/Layout";
+export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props";
 
 const Home = () => {
   const router = useRouter();
   const { status } = useSession();
-  const { t } = useTranslation("index");
+  const { t } = useTranslation();
   useEffect(() => {
     if (status === "authenticated") {
       router.push("/dashboard");
@@ -37,10 +37,4 @@ const Home = () => {
 
 Home.getLayout = getTransparentHeaderLayout;
 
-export const getStaticProps = async ({ locale }) => ({
-  props: {
-    ...(await serverSideTranslations(locale, ["index", "common"])),
-  },
-});
-
 export default Home;

website/src/pages/leaderboard.tsx

@@ -1,27 +1,29 @@
 import { Box, Heading, Tab, TabList, TabPanel, TabPanels, Tabs } from "@chakra-ui/react";
 import Head from "next/head";
+import { useTranslation } from "next-i18next";
 import { getDashboardLayout } from "src/components/Layout";
 import { LeaderboardGridCell } from "src/components/LeaderboardGridCell";
 export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props";
 import { LeaderboardTimeFrame } from "src/types/Leaderboard";
 
 const Leaderboard = () => {
+  const { t } = useTranslation(["leaderboard", "common"]);
   return (
     <>
       <Head>
-        <title>Leaderboard - Open Assistant</title>
+        <title>{`${t("leaderboard")} - ${t("common:title")}`}</title>
         <meta name="description" content="Leaderboard Rankings" charSet="UTF-8" />
       </Head>
       <Box display="flex" flexDirection="column">
         <Heading fontSize="2xl" fontWeight="bold" pb="4">
-          Leaderboard
+          {t("leaderboard")}
         </Heading>
         <Tabs isFitted isLazy>
           <TabList>
-            <Tab>Daily</Tab>
-            <Tab>Weekly</Tab>
-            <Tab>Monthly</Tab>
-            <Tab>Overall</Tab>
+            <Tab>{t("daily")}</Tab>
+            <Tab>{t("weekly")}</Tab>
+            <Tab>{t("monthly")}</Tab>
+            <Tab>{t("overall")}</Tab>
           </TabList>
 
           <TabPanels>

website/src/pages/tasks/all.tsx

@@ -1,7 +1,8 @@
 import Head from "next/head";
 import { TaskOption } from "src/components/Dashboard";
+import { allTaskOptions } from "src/components/Dashboard/TaskOption";
 import { getDashboardLayout } from "src/components/Layout";
-import { TaskCategory } from "src/components/Tasks/TaskTypes";
+export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props";
 
 const AllTasks = () => {
   return (
@@ -10,7 +11,7 @@ const AllTasks = () => {
         <title>All Tasks - Open Assistant</title>
         <meta name="description" content="All tasks for Open Assistant." />
       </Head>
-      <TaskOption displayTaskCategories={[TaskCategory.Create, TaskCategory.Evaluate, TaskCategory.Label]} />
+      <TaskOption content={allTaskOptions} />
     </>
   );
 };

website/src/pages/tasks/random.tsx

@@ -4,6 +4,7 @@ import { getDashboardLayout } from "src/components/Layout";
 import { LoadingScreen } from "src/components/Loading/LoadingScreen";
 import { Task } from "src/components/Tasks/Task";
 import { useGenericTaskAPI } from "src/hooks/tasks/useGenericTaskAPI";
+export { getDefaultStaticProps as getStaticProps } from "src/lib/default_static_props";
 import { TaskType } from "src/types/Task";
 
 const RandomTask = () => {

website/src/types/Leaderboard.ts

@@ -12,6 +12,7 @@ export const enum LeaderboardTimeFrame {
 }
 export interface LeaderboardReply {
   time_frame: LeaderboardTimeFrame;
+  last_updated: string; // date time iso string
   leaderboard: LeaderboardEntity[];
 }