Merge branch 'main' into refactor_oasst_api_client

This commit is contained in:
notmd
2023-01-25 16:36:32 +07:00
26 changed files with 552 additions and 134 deletions
+1
View File
@@ -77,6 +77,7 @@ repos:
hooks:
- id: prettier
args: [--prose-wrap=always, --write]
exclude: website/tailwind.config.js|website/.storybook/main.js|website/.eslintrc.json
- repo: local
hooks:
@@ -0,0 +1,44 @@
"""add message_emoji
Revision ID: 40ed93df0ed5
Revises: 8ba17b5f467a
Create Date: 2023-01-24 22:56:28.229408
"""
import sqlalchemy as sa
import sqlmodel
from alembic import op
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision = "40ed93df0ed5"
down_revision = "8ba17b5f467a"
branch_labels = None
depends_on = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"message_emoji",
sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False),
sa.Column(
"created_date", sa.DateTime(timezone=True), server_default=sa.text("CURRENT_TIMESTAMP"), nullable=False
),
sa.Column("emoji", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False),
sa.ForeignKeyConstraint(["message_id"], ["message.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["user_id"], ["user.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("message_id", "user_id", "emoji"),
)
op.create_index("ix_message_emoji__user_id__message_id", "message_emoji", ["user_id", "message_id"], unique=False)
op.add_column("message", sa.Column("emojis", postgresql.JSONB(astext_type=sa.Text()), nullable=True))
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column("message", "emojis")
op.drop_index("ix_message_emoji__user_id__message_id", table_name="message_emoji")
op.drop_table("message_emoji")
# ### end Alembic commands ###
+28
View File
@@ -273,6 +273,24 @@ def get_openapi_schema():
return json.dumps(app.openapi())
def export_ready_trees(file: Optional[str] = None, use_compression: bool = False):
try:
with Session(engine) as db:
api_client = api_auth(settings.OFFICIAL_WEB_API_KEY, db=db)
dummy_user = protocol_schema.User(id="__dummy_user__", display_name="Dummy User", auth_method="local")
ur = UserRepository(db=db, api_client=api_client)
tr = TaskRepository(db=db, api_client=api_client, client_user=dummy_user, user_repository=ur)
pr = PromptRepository(
db=db, api_client=api_client, client_user=dummy_user, user_repository=ur, task_repository=tr
)
tm = TreeManager(db, pr)
tm.export_all_ready_trees(file, use_compression=use_compression)
except Exception:
logger.exception("Error exporting trees.")
def main():
# Importing here so we don't import packages unnecessarily if we're
# importing main as a module.
@@ -289,11 +307,21 @@ def main():
)
parser.add_argument("--host", help="The host to run the server", default="0.0.0.0")
parser.add_argument("--port", help="The port to run the server", default=8080)
parser.add_argument(
"--export", help="Export all trees which are ready for exporting.", action=argparse.BooleanOptionalAction
)
parser.add_argument(
"--export-file",
help="Name of file to export trees to. If not provided when exporting, output will be send to STDOUT",
)
args = parser.parse_args()
if args.print_openapi_schema:
print(get_openapi_schema())
elif args.export:
use_compression: bool = ".gz" in args.export_file
export_ready_trees(file=args.export_file, use_compression=use_compression)
else:
uvicorn.run(app, host=args.host, port=args.port)
+20
View File
@@ -7,6 +7,7 @@ from oasst_backend.api import deps
from oasst_backend.api.v1 import utils
from oasst_backend.models import ApiClient
from oasst_backend.prompt_repository import PromptRepository
from oasst_backend.utils.database_utils import CommitMode, managed_tx_function
from oasst_shared.exceptions.oasst_api_error import OasstError, OasstErrorCode
from oasst_shared.schemas import protocol
from sqlmodel import Session
@@ -229,3 +230,22 @@ def mark_message_deleted(
):
pr = PromptRepository(db, api_client)
pr.mark_messages_deleted(message_id)
@router.post("/{message_id}/emoji", response_model=protocol.Message)
def post_message_emoji(
*,
message_id: UUID,
request: protocol.MessageEmojiRequest,
api_client: ApiClient = Depends(deps.get_api_client),
) -> protocol.Message:
"""
Toggle, add or remove message emoji.
"""
@managed_tx_function(CommitMode.COMMIT)
def emoji_tx(session: deps.Session):
pr = PromptRepository(session, api_client, client_user=request.user)
return pr.handle_message_emoji(message_id, request.op, request.emoji)
return utils.prepare_message(emoji_tx())
+1
View File
@@ -14,6 +14,7 @@ def prepare_message(m: Message) -> protocol.Message:
lang=m.lang,
is_assistant=(m.role == "assistant"),
created_date=m.created_date,
emojis=m.emojis,
)
+2
View File
@@ -2,6 +2,7 @@ from .api_client import ApiClient
from .journal import Journal, JournalIntegration
from .message import Message
from .message_embedding import MessageEmbedding
from .message_emoji import MessageEmoji
from .message_reaction import MessageReaction
from .message_toxicity import MessageToxicity
from .message_tree_state import MessageTreeState
@@ -24,4 +25,5 @@ __all__ = [
"TextLabels",
"Journal",
"JournalIntegration",
"MessageEmoji",
]
+2
View File
@@ -49,6 +49,8 @@ class Message(SQLModel, table=True):
rank: Optional[int] = Field(nullable=True)
emojis: dict[str, int] = Field(default={}, sa_column=sa.Column(pg.JSONB), nullable=False)
def ensure_is_message(self) -> None:
if not self.payload or not isinstance(self.payload.payload, MessagePayload):
raise OasstError("Invalid message", OasstErrorCode.INVALID_MESSAGE, HTTPStatus.INTERNAL_SERVER_ERROR)
@@ -0,0 +1,27 @@
from datetime import datetime
from typing import Optional
from uuid import UUID
import sqlalchemy as sa
import sqlalchemy.dialects.postgresql as pg
from sqlmodel import Field, Index, SQLModel
class MessageEmoji(SQLModel, table=True):
__tablename__ = "message_emoji"
__table_args__ = (Index("ix_message_emoji__user_id__message_id", "user_id", "message_id", unique=False),)
message_id: Optional[UUID] = Field(
sa_column=sa.Column(
pg.UUID(as_uuid=True), sa.ForeignKey("message.id", ondelete="CASCADE"), nullable=False, primary_key=True
)
)
user_id: UUID = Field(
sa_column=sa.Column(
pg.UUID(as_uuid=True), sa.ForeignKey("user.id", ondelete="CASCADE"), nullable=False, primary_key=True
)
)
emoji: str = Field(nullable=False, max_length=128, primary_key=True)
created_date: Optional[datetime] = Field(
sa_column=sa.Column(sa.DateTime(timezone=True), nullable=False, server_default=sa.func.current_timestamp())
)
+82 -5
View File
@@ -2,7 +2,7 @@ import random
from collections import defaultdict
from datetime import datetime
from http import HTTPStatus
from typing import List, Optional, Tuple
from typing import Optional
from uuid import UUID, uuid4
import oasst_backend.models.db_payload as db_payload
@@ -13,6 +13,7 @@ from oasst_backend.models import (
ApiClient,
Message,
MessageEmbedding,
MessageEmoji,
MessageReaction,
MessageToxicity,
MessageTreeState,
@@ -29,6 +30,7 @@ from oasst_shared.exceptions import OasstError, OasstErrorCode
from oasst_shared.schemas import protocol as protocol_schema
from oasst_shared.schemas.protocol import SystemStats
from oasst_shared.utils import unaware_to_utc
from sqlalchemy.orm.attributes import flag_modified
from sqlmodel import Session, and_, func, not_, or_, text, update
from starlette.status import HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND
@@ -255,7 +257,7 @@ class PromptRepository:
return reaction
@managed_tx_method(CommitMode.COMMIT)
def store_ranking(self, ranking: protocol_schema.MessageRanking) -> Tuple[MessageReaction, Task]:
def store_ranking(self, ranking: protocol_schema.MessageRanking) -> tuple[MessageReaction, Task]:
# fetch task
task = self.task_repository.fetch_task_by_frontend_message_id(ranking.message_id)
self._validate_task(task, frontend_message_id=ranking.message_id)
@@ -345,13 +347,13 @@ class PromptRepository:
return message_toxicity
@managed_tx_method(CommitMode.FLUSH)
def insert_message_embedding(self, message_id: UUID, model: str, embedding: List[float]) -> MessageEmbedding:
def insert_message_embedding(self, message_id: UUID, model: str, embedding: list[float]) -> MessageEmbedding:
"""Insert the embedding of a new message in the database.
Args:
message_id (UUID): the identifier of the message we want to save its embedding
model (str): the model used for creating the embedding
embedding (List[float]): the values obtained from the message & model
embedding (list[float]): the values obtained from the message & model
Raises:
OasstError: if misses some of the before params
@@ -383,7 +385,7 @@ class PromptRepository:
return reaction
@managed_tx_method(CommitMode.FLUSH)
def store_text_labels(self, text_labels: protocol_schema.TextLabels) -> Tuple[TextLabels, Task, Message]:
def store_text_labels(self, text_labels: protocol_schema.TextLabels) -> tuple[TextLabels, Task, Message]:
valid_labels: Optional[list[str]] = None
mandatory_labels: Optional[list[str]] = None
@@ -529,6 +531,22 @@ class PromptRepository:
qry = qry.filter(not_(Message.deleted))
return qry.all()
def fetch_user_message_trees(
self, user_id: Message.user_id, reviewed: bool = True, include_deleted: bool = False
) -> list[Message]:
qry = self.db.query(Message).filter(Message.user_id == user_id)
if reviewed:
qry = qry.filter(Message.review_result)
if not include_deleted:
qry = qry.filter(not_(Message.deleted))
return qry.all()
def fetch_message_trees_ready_for_export(self) -> list[MessageTreeState]:
qry = self.db.query(MessageTreeState).filter(
MessageTreeState.state == message_tree_state.State.READY_FOR_EXPORT
)
return qry.all()
def fetch_multiple_random_replies(self, max_size: int = 5, message_role: str = None):
"""
Fetch a conversation with multiple possible replies to it.
@@ -827,3 +845,62 @@ WHERE message.id = cc.id;
deleted=result.get(True, 0),
message_trees=result.get(None, 0),
)
def handle_message_emoji(self, message_id: UUID, op: protocol_schema.EmojiOp, emoji: protocol_schema) -> Message:
self.ensure_user_is_enabled()
message = self.fetch_message(message_id)
# check if emoji exists
existing_emoji = (
self.db.query(MessageEmoji)
.filter(
MessageEmoji.message_id == message_id, MessageEmoji.user_id == self.user_id, MessageEmoji.emoji == emoji
)
.one_or_none()
)
if existing_emoji:
if op == protocol_schema.EmojiOp.add:
logger.info(f"Emoji record already exists {message_id=}, {emoji=}, {self.user_id=}")
return message
elif op == protocol_schema.EmojiOp.togggle:
op = protocol_schema.EmojiOp.remove
if existing_emoji is None:
if op == protocol_schema.EmojiOp.remove:
logger.info(f"Emoji record not found {message_id=}, {emoji=}, {self.user_id=}")
return message
elif op == protocol_schema.EmojiOp.togggle:
op = protocol_schema.EmojiOp.add
if op == protocol_schema.EmojiOp.add:
# insert emoji record & increment count
message_emoji = MessageEmoji(message_id=message.id, user_id=self.user_id, emoji=emoji)
self.db.add(message_emoji)
emoji_counts = message.emojis
if not emoji_counts:
message.emojis = {emoji.value: 1}
else:
count = emoji_counts.get(emoji.value) or 0
emoji_counts[emoji.value] = count + 1
elif op == protocol_schema.EmojiOp.remove:
# remove emoji record and & decrement count
message = self.fetch_message(message_id)
self.db.delete(existing_emoji)
emoji_counts = message.emojis
count = emoji_counts.get(emoji.value)
if count is not None:
if count == 1:
del emoji_counts[emoji.value]
else:
emoji_counts[emoji.value] = count - 1
flag_modified(message, "emojis")
self.db.add(message)
else:
raise OasstError("Emoji op not supported", OasstErrorCode.EMOJI_OP_UNSUPPORTED)
flag_modified(message, "emojis")
self.db.add(message)
self.db.flush()
return message
+50 -8
View File
@@ -1,4 +1,6 @@
import json
import random
import sys
from datetime import datetime
from enum import Enum
from http import HTTPStatus
@@ -7,11 +9,13 @@ from uuid import UUID
import numpy as np
import pydantic
from fastapi.encoders import jsonable_encoder
from loguru import logger
from oasst_backend.api.v1.utils import prepare_conversation, prepare_conversation_message_list
from oasst_backend.config import TreeManagerConfiguration, settings
from oasst_backend.models import Message, MessageReaction, MessageTreeState, Task, TextLabels, User, message_tree_state
from oasst_backend.prompt_repository import PromptRepository
from oasst_backend.utils import tree_export
from oasst_backend.utils.database_utils import CommitMode, async_managed_tx_method, managed_tx_method
from oasst_backend.utils.hugging_face import HfClassificationModel, HfEmbeddingModel, HfUrl, HuggingFaceAPI
from oasst_backend.utils.ranking import ranked_pairs
@@ -1184,14 +1188,55 @@ DELETE FROM user_stats WHERE user_id = :user_id;
if ban:
self.db.execute(update(User).filter(User.id == user_id).values(deleted=True, enabled=False))
def export_trees_to_file(
self,
message_tree_ids: list[str],
file=None,
reviewed: bool = True,
include_deleted: bool = False,
use_compression: bool = False,
) -> None:
trees_to_export: List[tree_export.ExportMessageTree] = []
for message_tree_id in message_tree_ids:
messages: List[Message] = self.pr.fetch_message_tree(message_tree_id, reviewed, include_deleted)
trees_to_export.append(tree_export.build_export_tree(message_tree_id, messages))
if file:
tree_export.write_trees_to_file(file, trees_to_export, use_compression)
else:
sys.stdout.write(json.dumps(jsonable_encoder(trees_to_export), indent=2))
def export_all_ready_trees(
self, file: str, reviewed: bool = True, include_deleted: bool = False, use_compression: bool = False
) -> None:
message_tree_states: MessageTreeState = self.pr.fetch_message_trees_ready_for_export()
message_tree_ids = [ms.message_tree_id for ms in message_tree_states]
self.export_trees_to_file(message_tree_ids, file, reviewed, include_deleted, use_compression)
def export_all_user_trees(
self,
user_id: str,
file: str,
reviewed: bool = True,
include_deleted: bool = False,
use_compression: bool = False,
) -> None:
messages = self.pr.fetch_user_message_trees(UUID(user_id))
message_tree_ids = [ms.message_tree_id for ms in messages]
self.export_trees_to_file(message_tree_ids, file, reviewed, include_deleted, use_compression)
if __name__ == "__main__":
from oasst_backend.api.deps import api_auth
# from oasst_backend.api.deps import create_api_client
from oasst_backend.database import engine
from oasst_backend.prompt_repository import PromptRepository
with Session(engine) as db:
api_client = api_auth(settings.OFFICIAL_WEB_API_KEY, db=db)
# api_client = create_api_client(session=db, description="test", frontend_type="bot")
dummy_user = protocol_schema.User(id="__dummy_user__", display_name="Dummy User", auth_method="local")
pr = PromptRepository(db=db, api_client=api_client, client_user=dummy_user)
@@ -1200,25 +1245,22 @@ if __name__ == "__main__":
tm = TreeManager(db, pr, cfg)
tm.ensure_tree_states()
tm.purge_user_messages(user_id=UUID("2ef9ad21-0dc5-442d-8750-6f7f1790723f"), purge_initial_prompts=False)
# tm.purge_user_messages(user_id=UUID("2ef9ad21-0dc5-442d-8750-6f7f1790723f"), purge_initial_prompts=False)
# tm.purge_user(user_id=UUID("2ef9ad21-0dc5-442d-8750-6f7f1790723f"))
# db.commit()
# print("query_num_active_trees", tm.query_num_active_trees())
# print("query_incomplete_rankings", tm.query_incomplete_rankings())
# print("query_replies_need_review", tm.query_replies_need_review())
# print("query_incomplete_reply_reviews", tm.query_replies_need_review())
# print("query_incomplete_initial_prompt_reviews", tm.query_prompts_need_review())
# print("query_extendible_trees", tm.query_extendible_trees())
# print("query_extendible_parents", tm.query_extendible_parents())
# print("query_tree_size", tm.query_tree_size(message_tree_id=UUID("bdf434cf-4df5-4b74-949c-a5a157bc3292")))
# print(
# "query_reviews_for_message",
# tm.query_reviews_for_message(message_id=UUID("6a444493-0d48-4316-a9f1-7e263f5a2473")),
# )
# print("next_task:", tm.next_task())
# print(
# "query_tree_ranking_results", tm.query_tree_ranking_results(UUID("6036f58f-41b5-48c4-bdd9-b16f34ab1312"))
# ".query_tree_ranking_results", tm.query_tree_ranking_results(UUID("2ac20d38-6650-43aa-8bb3-f61080c0d921"))
# )
print(tm.export_trees_to_file(message_tree_ids=["7e75fb38-e664-4e2b-817c-b9a0b01b0074"], file="lol.jsonl"))
@@ -107,6 +107,7 @@ def managed_tx_function(
auto_commit: CommitMode = CommitMode.COMMIT,
num_retries=settings.DATABASE_MAX_TX_RETRY_COUNT,
session_factory: Callable[..., Session] = default_session_factor,
refresh_result: bool = True,
):
"""Passes Session object as first argument to wrapped function."""
@@ -124,6 +125,8 @@ def managed_tx_function(
session.flush()
elif auto_commit == CommitMode.ROLLBACK:
session.rollback()
if refresh_result and isinstance(result, SQLModel):
session.refresh(result)
return result
except OperationalError:
logger.info(f"Retry {i+1}/{num_retries} after possible DB concurrent update conflict.")
@@ -0,0 +1,71 @@
from __future__ import annotations
import gzip
import json
from collections import defaultdict
from typing import Optional, TextIO
from fastapi.encoders import jsonable_encoder
from oasst_backend.models import Message
from pydantic import BaseModel
class ExportMessageNode(BaseModel):
message_id: str
parent_id: Optional[str]
text: Optional[str]
role: str
review_count: Optional[int]
rank: Optional[int]
replies: Optional[list[ExportMessageNode]]
@classmethod
def prep_message_export(cls, message: Message) -> ExportMessageNode:
return cls(
message_id=str(message.id),
parent_id=str(message.parent_id) if message.parent_id else None,
text=str(message.payload.payload.text),
role=message.role,
review_count=message.review_count,
rank=message.rank,
)
class ExportMessageTree(BaseModel):
message_tree_id: str
replies: Optional[ExportMessageNode]
def build_export_tree(message_tree_id: str, messages: list[Message]) -> ExportMessageTree:
export_tree = ExportMessageTree(message_tree_id=str(message_tree_id))
export_tree_data = [ExportMessageNode.prep_message_export(m) for m in messages]
message_parents = defaultdict(list)
for message in export_tree_data:
message_parents[message.parent_id].append(message)
def build_tree(tree: dict, parent: Optional[str], messages: list[Message]):
children = message_parents[parent]
tree.replies = children
for idx, child in enumerate(tree.replies):
build_tree(tree.replies[idx], child.message_id, messages)
build_tree(export_tree, None, export_tree_data)
return export_tree
def write_trees_to_file(file, trees: list[ExportMessageTree], use_compression: bool = True) -> None:
out_buff: TextIO
if use_compression:
out_buff = gzip.open(file, "wt", encoding="UTF-8")
else:
out_buff = open(file, "wt", encoding="UTF-8")
with out_buff as f:
for tree in trees:
file_data = jsonable_encoder(tree)
json.dump(file_data, f)
f.write("\n")
@@ -76,6 +76,8 @@ class OasstErrorCode(IntEnum):
USER_DISABLED = 4001
USER_NOT_FOUND = 4002
EMOJI_OP_UNSUPPORTED = 5000
class OasstError(Exception):
"""Base class for Open-Assistant exceptions."""
@@ -80,6 +80,7 @@ class Conversation(BaseModel):
class Message(ConversationMessage):
parent_id: Optional[UUID] = None
created_date: Optional[datetime] = None
emojis: Optional[dict] = None
class MessagePage(PageResult):
@@ -432,3 +433,27 @@ class OasstErrorResponse(BaseModel):
error_code: OasstErrorCode
message: str
class EmojiCode(str, enum.Enum):
thumbs_up = "+1" # 👍
thumbs_down = "-1" # 👎
red_flag = "red_flag" # 🚩
hundred = "100" # 💯
rofl = "rofl" # 🤣"
heart_eyes = "heart_eyes" # 😍
disappointed = "disappointed" # 😞
poop = "poop" # 💩
skull = "skull" # 💀
class EmojiOp(str, enum.Enum):
togggle = "toggle"
add = "add"
remove = "remove"
class MessageEmojiRequest(BaseModel):
user: User
op: EmojiOp = EmojiOp.togggle
emoji: EmojiCode
+2 -1
View File
@@ -14,5 +14,6 @@
"sign_in": "Sign In",
"sign_out": "Sign Out",
"terms_of_service": "Terms of Service",
"title": "Open Assistant"
"title": "Open Assistant",
"more_information": "More Information"
}
+8
View File
@@ -0,0 +1,8 @@
{
"grab_a_task": "Grab a task!",
"create": "Create",
"evaluate": "Evaluate",
"label": "Label",
"dashboard": "Dashboard",
"go": "Go"
}
+79
View File
@@ -0,0 +1,79 @@
{
"write_initial_prompt": "Write your prompt here...",
"default": {
"unchanged_title": "No changes",
"unchanged_message": "Are you sure you would like to continue?"
},
"random": {
"label": "I'm feeling lucky",
"desc": "Help us improve Open Assistant by starting a random task."
},
"create_initial_prompt": {
"label": "Create Initial Prompts",
"desc": "Write initial prompts to help Open Assistant to try replying to diverse messages.",
"overview": "Create an initial message to send to the assistant",
"instruction": "Provide the initial prompts"
},
"reply_as_user": {
"label": "Reply as User",
"desc": "Chat with Open Assistant and help improve it's responses as you interact with it.",
"overview": "Given the following conversation, provide an adequate reply",
"instruction": "Provide the user's reply"
},
"reply_as_assistant": {
"label": "Reply as Assistant",
"desc": "Help Open Assistant improve its responses to conversations with other users.",
"overview": "Given the following conversation, provide an adequate reply"
},
"rank_user_replies": {
"label": "Rank User Replies",
"desc": "Help Open Assistant improve its responses to conversations with other users.",
"overview": "Given the following User replies, sort them from best to worst, best being first, worst being last.",
"unchanged_title": "Order Unchanged",
"unchanged_message": "You have not changed the order of the prompts. Are you sure you would like to continue?"
},
"rank_assistant_replies": {
"label": "Rank Assistant Replies",
"desc": "Score prompts given by Open Assistant based on their accuracy and readability.",
"overview": "Given the following Assistant replies, sort them from best to worst, best being first, worst being last.",
"unchanged_title": "Order Unchanged",
"unchanged_message": "You have not changed the order of the prompts. Are you sure you would like to continue?"
},
"rank_initial_prompts": {
"label": "Rank Initial Prompts",
"desc": "Score prompts given by Open Assistant based on their accuracy and readability.",
"overview": "Given the following initial prompts, sort them from best to worst, best being first, worst being last.",
"unchanged_title": "Order Unchanged",
"unchanged_message": "You have not changed the order of the prompts. Are you sure you would like to continue?"
},
"label_initial_prompt": {
"label": "Label Initial Prompt",
"desc": "Provide labels for a prompt.",
"overview": "Provide labels for the following prompt"
},
"label_prompter_reply": {
"label": "Label Prompter Reply",
"desc": "Provide labels for a prompt.",
"overview": "Given the following discussion, provide labels for the final prompt."
},
"label_assistant_reply": {
"label": "Label Assistant Reply",
"desc": "Provide labels for a prompt.",
"overview": "Given the following discussion, provide labels for the final prompt."
},
"classify_initial_prompt": {
"label": "Classify Initial Prompt",
"desc": "Provide labels for a prompt.",
"overview": "Read the following prompt and then answer the question about it."
},
"classify_prompter_reply": {
"label": "Classify Prompter Reply",
"desc": "Provide labels for a prompt.",
"overview": "Read the following conversation and then answer the question about the last reply in the discussion."
},
"classify_assistant_reply": {
"label": "Classify Assistant Reply",
"desc": "Provide labels for a prompt.",
"overview": "Read the following conversation and then answer the question about the last reply in the discussion."
}
}
@@ -12,7 +12,9 @@ import {
} from "@chakra-ui/react";
import { HelpCircle } from "lucide-react";
import Link from "next/link";
import { useTranslation } from "next-i18next";
import { useMemo } from "react";
import { getTypeSafei18nKey } from "src/lib/i18n";
import { TaskType } from "src/types/Task";
import { TaskCategory, TaskCategoryLabels, TaskInfo, TaskInfos } from "../Tasks/TaskTypes";
@@ -22,6 +24,7 @@ export interface TasksOptionProps {
}
export const TaskOption = ({ content }: TasksOptionProps) => {
const { t } = useTranslation(["dashboard", "tasks"]);
const backgroundColor = useColorModeValue("white", "gray.700");
const taskInfoMap = useMemo(
@@ -41,7 +44,7 @@ export const TaskOption = ({ content }: TasksOptionProps) => {
<div key={category}>
<Flex>
<Heading size="lg" className="pb-4">
{TaskCategoryLabels[category]}
{t(TaskCategoryLabels[category])}
</Heading>
<Spacer />
<ExternalLink href="https://projects.laion.ai/Open-Assistant/" isExternal>
@@ -52,7 +55,7 @@ export const TaskOption = ({ content }: TasksOptionProps) => {
{taskTypes
.map((taskType) => taskInfoMap[taskType])
.map((item) => (
<Link key={category + item.label} href={item.pathname}>
<Link key={category + item.id} href={item.pathname}>
<GridItem
bg={backgroundColor}
borderRadius="xl"
@@ -60,8 +63,8 @@ export const TaskOption = ({ content }: TasksOptionProps) => {
className="flex flex-col justify-between h-full"
>
<Flex className="p-6 pb-10" flexDir="column" gap="3">
<Heading size="md">{item.label}</Heading>
<Text size="sm">{item.desc}</Text>
<Heading size="md">{t(getTypeSafei18nKey(`tasks:${item.id}.label`))}</Heading>
<Text size="sm">{t(getTypeSafei18nKey(`tasks:${item.id}.desc`))}</Text>
</Flex>
<Text
fontWeight="bold"
@@ -69,7 +72,7 @@ export const TaskOption = ({ content }: TasksOptionProps) => {
borderBottomRadius="xl"
className="px-6 py-2 transition-colors duration-300 bg-blue-500 hover:bg-blue-600"
>
Go -&gt;
{t("go")} -&gt;
</Text>
</GridItem>
</Link>
+13 -8
View File
@@ -1,10 +1,12 @@
import { Box, Stack, Text, useColorModeValue } from "@chakra-ui/react";
import { useTranslation } from "next-i18next";
import { useState } from "react";
import { MessageTable } from "src/components/Messages/MessageTable";
import { TrackedTextarea } from "src/components/Survey/TrackedTextarea";
import { TwoColumnsWithCards } from "src/components/Survey/TwoColumnsWithCards";
import { TaskSurveyProps } from "src/components/Tasks/Task";
import { TaskHeader } from "src/components/Tasks/TaskHeader";
import { getTypeSafei18nKey } from "src/lib/i18n";
export const CreateTask = ({
task,
@@ -14,14 +16,15 @@ export const CreateTask = ({
onReplyChanged,
onValidityChanged,
}: TaskSurveyProps<{ text: string }>) => {
const { t, i18n } = useTranslation(["tasks", "common"]);
const cardColor = useColorModeValue("gray.50", "gray.800");
const titleColor = useColorModeValue("gray.800", "gray.300");
const [inputText, setInputText] = useState("");
const textChangeHandler = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
const text = event.target.value;
const isTextBlank = !text || /^\s*$/.test(text) ? true : false;
onReplyChanged({ text });
const isTextBlank = !text || /^\s*$/.test(text);
if (!isTextBlank) {
onValidityChanged("VALID");
setInputText(text);
@@ -36,22 +39,24 @@ export const CreateTask = ({
<TwoColumnsWithCards>
<>
<TaskHeader taskType={taskType} />
{task.conversation ? (
{!!task.conversation && (
<Box mt="4" borderRadius="lg" bg={cardColor} className="p-3 sm:p-6">
<MessageTable messages={task.conversation.messages} highlightLastMessage />
</Box>
) : null}
)}
</>
<>
<Stack spacing="4">
<Text fontSize="xl" fontWeight="bold" color={titleColor}>
{taskType.instruction}
</Text>
{!!i18n.exists(`task.${taskType.id}.instruction`) && (
<Text fontSize="xl" fontWeight="bold" color={titleColor}>
{t(getTypeSafei18nKey(`${taskType.id}.instruction`))}
</Text>
)}
<TrackedTextarea
text={inputText}
onTextChange={textChangeHandler}
thresholds={{ low: 20, medium: 40, goal: 50 }}
textareaProps={{ placeholder: "Write your prompt here...", isDisabled, isReadOnly: !isEditable }}
textareaProps={{ placeholder: t("tasks:write_initial_prompt"), isDisabled, isReadOnly: !isEditable }}
/>
</Stack>
</>
+5 -5
View File
@@ -1,3 +1,4 @@
import { useTranslation } from "next-i18next";
import { useRef, useState } from "react";
import { TaskControls } from "src/components/Survey/TaskControls";
import { CreateTask } from "src/components/Tasks/CreateTask";
@@ -6,6 +7,7 @@ import { LabelTask } from "src/components/Tasks/LabelTask";
import { TaskCategory, TaskInfo, TaskInfos } from "src/components/Tasks/TaskTypes";
import { UnchangedWarning } from "src/components/Tasks/UnchangedWarning";
import { post } from "src/lib/api";
import { getTypeSafei18nKey } from "src/lib/i18n";
import { TaskContent, TaskReplyValidity } from "src/types/Task";
import useSWRMutation from "swr/mutation";
@@ -23,6 +25,7 @@ export interface TaskSurveyProps<T> {
}
export const Task = ({ frontendId, task, trigger, mutate }) => {
const { t } = useTranslation("tasks");
const [taskStatus, setTaskStatus] = useState<TaskStatus>("NOT_SUBMITTABLE");
const replyContent = useRef<TaskContent>(null);
const [showUnchangedWarning, setShowUnchangedWarning] = useState(false);
@@ -111,7 +114,6 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
case TaskCategory.Create:
return (
<CreateTask
key={task.id}
task={task}
taskType={taskType}
isEditable={edit_mode}
@@ -123,7 +125,6 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
case TaskCategory.Evaluate:
return (
<EvaluateTask
key={task.id}
task={task}
taskType={taskType}
isEditable={edit_mode}
@@ -135,7 +136,6 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
case TaskCategory.Label:
return (
<LabelTask
key={task.id}
task={task}
taskType={taskType}
isEditable={edit_mode}
@@ -160,8 +160,8 @@ export const Task = ({ frontendId, task, trigger, mutate }) => {
/>
<UnchangedWarning
show={showUnchangedWarning}
title={taskType.unchanged_title || "No changes"}
message={taskType.unchanged_message || "Are you sure you would like to continue?"}
title={t(getTypeSafei18nKey(`${taskType.id}.unchanged_title`)) || t("default.unchanged_title")}
message={t(getTypeSafei18nKey(`${taskType.id}.unchanged_message`)) || t("default.unchanged_message")}
continueButtonText={"Continue anyway"}
onClose={() => setShowUnchangedWarning(false)}
onContinueAnyway={() => {
@@ -1,6 +1,8 @@
import { HStack, IconButton, Link, Stack, Text, useColorModeValue } from "@chakra-ui/react";
import { HelpCircle } from "lucide-react";
import { useTranslation } from "next-i18next";
import type { TaskInfo } from "src/components/Tasks/TaskTypes";
import { getTypeSafei18nKey } from "src/lib/i18n";
interface TaskHeaderProps {
/**
@@ -13,20 +15,21 @@ interface TaskHeaderProps {
* Presents the Task label, instructions, and help link
*/
const TaskHeader = ({ taskType }: TaskHeaderProps) => {
const { t } = useTranslation(["tasks", "common"]);
const labelColor = useColorModeValue("gray.600", "gray.400");
const titleColor = useColorModeValue("gray.800", "gray.300");
return (
<Stack spacing="1">
<HStack>
<Text fontSize="xl" fontWeight="bold" color={titleColor}>
{taskType.label}
{t(getTypeSafei18nKey(`${taskType.id}.label`))}
</Text>
<Link href={taskType.help_link} isExternal>
<IconButton variant="ghost" aria-label="More Information" icon={<HelpCircle size="1em" />} />
</Link>
</HStack>
<Text fontSize="md" color={labelColor}>
{taskType.overview}
{t(getTypeSafei18nKey(`${taskType.id}.overview`))}
</Text>
</Stack>
);
+56 -87
View File
@@ -1,181 +1,150 @@
import { TaskType } from "src/types/Task";
export enum TaskCategory {
Random = "Random",
Create = "Create",
Evaluate = "Evaluate",
Label = "Label",
Random = "Random",
}
export enum TaskUpdateType {
MessageRanking = "message_ranking",
Random = "random",
TextLabels = "text_labels",
TextReplyToMessage = "text_reply_to_message",
}
export interface TaskInfo {
label: string;
desc: string;
category: TaskCategory;
help_link: string;
id: string;
mode?: string;
pathname: string;
type: string;
help_link: string;
mode?: string;
overview?: string;
instruction?: string;
update_type: string;
unchanged_title?: string;
unchanged_message?: string;
}
export const TaskCategoryLabels: { [key in TaskCategory]: string } = {
[TaskCategory.Random]: "Grab a task!",
[TaskCategory.Create]: "Create",
[TaskCategory.Evaluate]: "Evaluate",
[TaskCategory.Label]: "Label",
[TaskCategory.Random]: "grab_a_task",
[TaskCategory.Create]: "create",
[TaskCategory.Evaluate]: "evaluate",
[TaskCategory.Label]: "label",
};
export const TaskInfos: TaskInfo[] = [
// general/random
{
label: "I'm feeling lucky",
desc: "Help us improve Open Assistant by starting a random task.",
id: "random",
category: TaskCategory.Random,
pathname: "/tasks/random",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
type: "random",
update_type: "random",
type: TaskType.random,
update_type: TaskUpdateType.Random,
},
// create
{
label: "Create Initial Prompts",
desc: "Write initial prompts to help Open Assistant to try replying to diverse messages.",
id: "create_initial_prompt",
category: TaskCategory.Create,
pathname: "/create/initial_prompt",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
type: "initial_prompt",
overview: "Create an initial message to send to the assistant",
instruction: "Provide the initial prompt",
update_type: "text_reply_to_message",
type: TaskType.initial_prompt,
update_type: TaskUpdateType.TextReplyToMessage,
},
{
label: "Reply as User",
desc: "Chat with Open Assistant and help improve its responses as you interact with it.",
id: "reply_as_user",
category: TaskCategory.Create,
pathname: "/create/user_reply",
help_link: "https://projects.laion.ai/Open-Assistant/docs/tasks/reply_as_user",
type: "prompter_reply",
overview: "Given the following conversation, provide an adequate reply",
instruction: "Provide the user's reply",
update_type: "text_reply_to_message",
type: TaskType.prompter_reply,
update_type: TaskUpdateType.TextReplyToMessage,
},
{
label: "Reply as Assistant",
desc: "Help Open Assistant improve its responses to conversations with other users.",
id: "reply_as_assistant",
category: TaskCategory.Create,
pathname: "/create/assistant_reply",
help_link: "https://projects.laion.ai/Open-Assistant/docs/tasks/reply_as_assistant",
type: "assistant_reply",
overview: "Given the following conversation, provide an adequate reply",
instruction: "Provide the assistant's reply",
update_type: "text_reply_to_message",
type: TaskType.assistant_reply,
update_type: TaskUpdateType.TextReplyToMessage,
},
// evaluate
{
label: "Rank User Replies",
id: "rank_user_replies",
category: TaskCategory.Evaluate,
desc: "Help Open Assistant improve its responses to conversations with other users.",
pathname: "/evaluate/rank_user_replies",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
overview: "Given the following User replies, sort them from best to worst, best being first, worst being last.",
type: "rank_prompter_replies",
update_type: "message_ranking",
unchanged_title: "Order Unchanged",
unchanged_message: "You have not changed the order of the prompts. Are you sure you would like to continue?",
type: TaskType.rank_prompter_replies,
update_type: TaskUpdateType.MessageRanking,
},
{
label: "Rank Assistant Replies",
desc: "Score prompts given by Open Assistant based on their accuracy and readability.",
id: "rank_assistant_replies",
category: TaskCategory.Evaluate,
pathname: "/evaluate/rank_assistant_replies",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
overview:
"Given the following Assistant replies, sort them from best to worst, best being first, worst being last.",
type: "rank_assistant_replies",
update_type: "message_ranking",
unchanged_title: "Order Unchanged",
unchanged_message: "You have not changed the order of the prompts. Are you sure you would like to continue?",
type: TaskType.rank_assistant_replies,
update_type: TaskUpdateType.MessageRanking,
},
{
label: "Rank Initial Prompts",
desc: "Score prompts given by Open Assistant based on their accuracy and readability.",
id: "rank_initial_prompts",
category: TaskCategory.Evaluate,
pathname: "/evaluate/rank_initial_prompts",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
overview: "Given the following initial prompts, sort them from best to worst, best being first, worst being last.",
type: "rank_initial_prompts",
update_type: "message_ranking",
unchanged_title: "Order Unchanged",
unchanged_message: "You have not changed the order of the prompts. Are you sure you would like to continue?",
type: TaskType.rank_initial_prompts,
update_type: TaskUpdateType.MessageRanking,
},
// label (full)
{
label: "Label Initial Prompt",
desc: "Provide labels for a prompt.",
id: "label_initial_prompt",
category: TaskCategory.Label,
pathname: "/label/label_initial_prompt",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
overview: "Provide labels for the following prompt",
type: "label_initial_prompt",
type: TaskType.label_initial_prompt,
mode: "full",
update_type: "text_labels",
update_type: TaskUpdateType.TextLabels,
},
{
label: "Label Prompter Reply",
desc: "Provide labels for a prompt.",
id: "label_prompter_reply",
category: TaskCategory.Label,
pathname: "/label/label_prompter_reply",
help_link: "https://projects.laion.ai/Open-Assistant/docs/tasks/label_prompter_reply",
overview: "Given the following discussion, provide labels for the final prompt.",
type: "label_prompter_reply",
type: TaskType.label_prompter_reply,
mode: "full",
update_type: "text_labels",
update_type: TaskUpdateType.TextLabels,
},
{
label: "Label Assistant Reply",
desc: "Provide labels for a prompt.",
id: "label_assistant_reply",
category: TaskCategory.Label,
pathname: "/label/label_assistant_reply",
help_link: "https://projects.laion.ai/Open-Assistant/docs/tasks/label_assistant_reply",
overview: "Given the following discussion, provide labels for the final prompt.",
type: "label_assistant_reply",
type: TaskType.label_assistant_reply,
mode: "full",
update_type: "text_labels",
update_type: TaskUpdateType.TextLabels,
},
// label (simple)
{
label: "Classify Initial Prompt",
desc: "Provide labels for a prompt.",
id: "classify_initial_prompt",
category: TaskCategory.Label,
pathname: "/label/label_initial_prompt",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
overview: "Read the following prompt and then answer the question about it.",
type: "label_initial_prompt",
type: TaskType.label_initial_prompt,
mode: "simple",
update_type: "text_labels",
update_type: TaskUpdateType.TextLabels,
},
{
label: "Classify Prompter Reply",
desc: "Provide labels for a prompt.",
id: "classify_prompter_reply",
category: TaskCategory.Label,
pathname: "/label/label_prompter_reply",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
overview: "Read the following conversation and then answer the question about the last reply in the discussion.",
type: "label_prompter_reply",
type: TaskType.label_prompter_reply,
mode: "simple",
update_type: "text_labels",
update_type: TaskUpdateType.TextLabels,
},
{
label: "Classify Assistant Reply",
desc: "Provide labels for a prompt.",
id: "classify_assistant_reply",
category: TaskCategory.Label,
pathname: "/label/label_assistant_reply",
help_link: "https://projects.laion.ai/Open-Assistant/docs/guides/prompting",
overview: "Read the following conversation and then answer the question about the last reply in the discussion.",
type: "label_assistant_reply",
type: TaskType.label_assistant_reply,
mode: "simple",
update_type: "text_labels",
update_type: TaskUpdateType.TextLabels,
},
];
+1
View File
@@ -0,0 +1 @@
export const getTypeSafei18nKey = (key: string) => key as unknown as TemplateStringsArray;
+4 -3
View File
@@ -12,8 +12,9 @@ import useSWR from "swr";
const Dashboard = () => {
const {
t,
i18n: { language },
} = useTranslation();
} = useTranslation(["dashboard", "common", "tasks"]);
const [activeLang, setLang] = useState<string>(null);
const { data, mutate: fetchTasks } = useSWR<AvailableTasks>("/api/available_tasks", get, {
refreshInterval: 2 * 60 * 1000, //2 minutes
@@ -36,7 +37,7 @@ const Dashboard = () => {
return (
<>
<Head>
<title>Dashboard - Open Assistant</title>
<title>{`${t("dashboard")} - ${t("common:title")}`}</title>
<meta name="description" content="Chat with Open Assistant and provide feedback." />
</Head>
<Flex direction="column" gap="10">
@@ -54,6 +55,6 @@ export default Dashboard;
const filterAvailableTasks = (availableTasks: Partial<AvailableTasks>) =>
Object.entries(availableTasks)
.filter(([_, count]) => count > 0)
.filter(([, count]) => count > 0)
.sort((a, b) => b[1] - a[1])
.map(([taskType]) => taskType) as TaskType[];
+6 -7
View File
@@ -11,23 +11,22 @@ const Leaderboard = () => {
return (
<>
<Head>
<title>{`${t("leaderboard:leaderboard")} - ${t("common:title")}`}</title>
<title>{`${t("leaderboard")} - ${t("common:title")}`}</title>
<meta name="description" content="Leaderboard Rankings" charSet="UTF-8" />
</Head>
<Box display="flex" flexDirection="column">
<Heading fontSize="2xl" fontWeight="bold" pb="4">
{t("leaderboard:leaderboard")}
{t("leaderboard")}
</Heading>
<Card>
<CardBody>
<Tabs isFitted isLazy>
<TabList>
<Tab>{t("leaderboard:daily")}</Tab>
<Tab>{t("leaderboard:weekly")}</Tab>
<Tab>{t("leaderboard:monthly")}</Tab>
<Tab>{t("leaderboard:overall")}</Tab>
<Tab>{t("daily")}</Tab>
<Tab>{t("weekly")}</Tab>
<Tab>{t("monthly")}</Tab>
<Tab>{t("overall")}</Tab>
</TabList>
<TabPanels>
<TabPanel p="0">
<LeaderboardTable timeFrame={LeaderboardTimeFrame.day} limit={20} />
+7 -3
View File
@@ -1,15 +1,19 @@
import "i18next";
import type common from "../public/locales/en/common.json";
import type index from "../public/locales/en/index.json";
import type leaderboard from "../public/locales/en/leaderboard.json";
import type common from "public/locales/en/common.json";
import type dashboard from "public/locales/en/dashboard.json";
import type index from "public/locales/en/index.json";
import type leaderboard from "public/locales/en/leaderboard.json";
import type tasks from "public/locales/en/tasks.json";
declare module "i18next" {
interface CustomTypeOptions {
resources: {
common: typeof common;
dashboard: typeof dashboard;
index: typeof index;
leaderboard: typeof leaderboard;
tasks: typeof tasks;
};
}
}