From af0ae71bbe5f4da5a53c4406f69bce55efe8c177 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Tue, 31 Jan 2023 15:42:04 +0100 Subject: [PATCH] Add message to prompt-labeling task (#1035) * add full message to prompt-labeling & always require full prompt labeling * always sending conversation, also for initial prompt labeling * update type annotation --- backend/oasst_backend/config.py | 2 +- backend/oasst_backend/models/db_payload.py | 8 +++++--- backend/oasst_backend/task_repository.py | 4 ---- backend/oasst_backend/tree_manager.py | 10 +++------- oasst-shared/oasst_shared/schemas/protocol.py | 7 +++---- 5 files changed, 12 insertions(+), 19 deletions(-) diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py index 638dca30..449543d5 100644 --- a/backend/oasst_backend/config.py +++ b/backend/oasst_backend/config.py @@ -31,7 +31,7 @@ class TreeManagerConfiguration(BaseModel): num_reviews_reply: int = 3 """Number of peer review checks to collect per reply (other than initial_prompt).""" - p_full_labeling_review_prompt: float = 0.1 + p_full_labeling_review_prompt: float = 1.0 """Probability of full text-labeling (instead of mandatory only) for initial prompts.""" p_full_labeling_review_reply_assistant: float = 0.1 diff --git a/backend/oasst_backend/models/db_payload.py b/backend/oasst_backend/models/db_payload.py index b0eb38df..2b7d051a 100644 --- a/backend/oasst_backend/models/db_payload.py +++ b/backend/oasst_backend/models/db_payload.py @@ -3,7 +3,7 @@ from uuid import UUID from oasst_backend.models.payload_column_type import payload_type from oasst_shared.schemas import protocol as protocol_schema -from pydantic import BaseModel +from pydantic import BaseModel, Field @payload_type @@ -117,8 +117,10 @@ class LabelConversationReplyPayload(TaskPayload): message_id: UUID conversation: protocol_schema.Conversation - reply: str # deprecated - reply_message: Optional[protocol_schema.ConversationMessage] + reply: Optional[str] = Field(None, deprecated=True, description="deprecated") + reply_message: Optional[protocol_schema.ConversationMessage] = Field( + None, deprecated=True, description="deprecated" + ) valid_labels: list[str] mandatory_labels: Optional[list[str]] mode: Optional[protocol_schema.LabelTaskMode] diff --git a/backend/oasst_backend/task_repository.py b/backend/oasst_backend/task_repository.py index fdf6ebb7..db4ba576 100644 --- a/backend/oasst_backend/task_repository.py +++ b/backend/oasst_backend/task_repository.py @@ -100,8 +100,6 @@ class TaskRepository: type=task.type, message_id=task.message_id, conversation=task.conversation, - reply=task.reply, - reply_message=task.reply_message, valid_labels=task.valid_labels, mandatory_labels=task.mandatory_labels, mode=task.mode, @@ -112,8 +110,6 @@ class TaskRepository: type=task.type, message_id=task.message_id, conversation=task.conversation, - reply=task.reply, - reply_message=task.reply_message, valid_labels=task.valid_labels, mandatory_labels=task.mandatory_labels, mode=task.mode, diff --git a/backend/oasst_backend/tree_manager.py b/backend/oasst_backend/tree_manager.py index f1cec103..b1d50d35 100644 --- a/backend/oasst_backend/tree_manager.py +++ b/backend/oasst_backend/tree_manager.py @@ -11,11 +11,7 @@ import numpy as np import pydantic from fastapi.encoders import jsonable_encoder from loguru import logger -from oasst_backend.api.v1.utils import ( - prepare_conversation, - prepare_conversation_message, - prepare_conversation_message_list, -) +from oasst_backend.api.v1.utils import prepare_conversation, prepare_conversation_message_list from oasst_backend.config import TreeManagerConfiguration, settings from oasst_backend.models import Message, MessageReaction, MessageTreeState, Task, TextLabels, User, message_tree_state from oasst_backend.prompt_repository import PromptRepository @@ -386,7 +382,6 @@ class TreeManager: message_id=message.id, conversation=conversation, reply=message.text, - reply_message=prepare_conversation_message(message), valid_labels=list(map(lambda x: x.value, valid_labels)), mandatory_labels=list(map(lambda x: x.value, self.cfg.mandatory_labels_assistant_reply)), mode=label_mode, @@ -412,7 +407,6 @@ class TreeManager: message_id=message.id, conversation=conversation, reply=message.text, - reply_message=prepare_conversation_message(message), valid_labels=list(map(lambda x: x.value, valid_labels)), mandatory_labels=list(map(lambda x: x.value, self.cfg.mandatory_labels_prompter_reply)), mode=label_mode, @@ -479,6 +473,7 @@ class TreeManager: case TaskType.LABEL_PROMPT: assert len(prompts_need_review) > 0 message = random.choice(prompts_need_review) + message = self.pr.fetch_message(message.id) # re-fetch message including emojis label_mode = protocol_schema.LabelTaskMode.full label_disposition = protocol_schema.LabelTaskDisposition.quality @@ -495,6 +490,7 @@ class TreeManager: task = protocol_schema.LabelInitialPromptTask( message_id=message.id, prompt=message.text, + conversation=prepare_conversation([message]), valid_labels=list(map(lambda x: x.value, valid_labels)), mandatory_labels=list(map(lambda x: x.value, self.cfg.mandatory_labels_initial_prompt)), mode=label_mode, diff --git a/oasst-shared/oasst_shared/schemas/protocol.py b/oasst-shared/oasst_shared/schemas/protocol.py index f772eb81..8929251c 100644 --- a/oasst-shared/oasst_shared/schemas/protocol.py +++ b/oasst-shared/oasst_shared/schemas/protocol.py @@ -252,22 +252,21 @@ class AbstractLabelTask(Task): mode: Optional[LabelTaskMode] disposition: Optional[LabelTaskDisposition] labels: Optional[list[LabelDescription]] + conversation: Conversation # the conversation so far (labeling -> last message) class LabelInitialPromptTask(AbstractLabelTask): """A task to label an initial prompt.""" type: Literal["label_initial_prompt"] = "label_initial_prompt" - prompt: str + prompt: str | None = Field(None, deprecated=True, description="deprecated, use `prompt_message`") class LabelConversationReplyTask(AbstractLabelTask): """A task to label a reply to a conversation.""" type: Literal["label_conversation_reply"] = "label_conversation_reply" - conversation: Conversation # the conversation so far (new: including the reply message) - reply_message: Optional[ConversationMessage] - reply: str + reply: str | None = Field(None, deprecated=True, description="deprecated, use last message of `conversation`") class LabelPrompterReplyTask(LabelConversationReplyTask):