Add message to prompt-labeling task (#1035)

* add full message to prompt-labeling & always require full prompt labeling

* always sending conversation, also for initial prompt labeling

* update type annotation

backend/oasst_backend/config.py

@@ -31,7 +31,7 @@ class TreeManagerConfiguration(BaseModel):
     num_reviews_reply: int = 3
     """Number of peer review checks to collect per reply (other than initial_prompt)."""
 
-    p_full_labeling_review_prompt: float = 0.1
+    p_full_labeling_review_prompt: float = 1.0
     """Probability of full text-labeling (instead of mandatory only) for initial prompts."""
 
     p_full_labeling_review_reply_assistant: float = 0.1

backend/oasst_backend/models/db_payload.py

@@ -3,7 +3,7 @@ from uuid import UUID
 
 from oasst_backend.models.payload_column_type import payload_type
 from oasst_shared.schemas import protocol as protocol_schema
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
 @payload_type
@@ -117,8 +117,10 @@ class LabelConversationReplyPayload(TaskPayload):
 
     message_id: UUID
     conversation: protocol_schema.Conversation
-    reply: str  # deprecated
-    reply_message: Optional[protocol_schema.ConversationMessage]
+    reply: Optional[str] = Field(None, deprecated=True, description="deprecated")
+    reply_message: Optional[protocol_schema.ConversationMessage] = Field(
+        None, deprecated=True, description="deprecated"
+    )
     valid_labels: list[str]
     mandatory_labels: Optional[list[str]]
     mode: Optional[protocol_schema.LabelTaskMode]

backend/oasst_backend/task_repository.py

@@ -100,8 +100,6 @@ class TaskRepository:
                     type=task.type,
                     message_id=task.message_id,
                     conversation=task.conversation,
-                    reply=task.reply,
-                    reply_message=task.reply_message,
                     valid_labels=task.valid_labels,
                     mandatory_labels=task.mandatory_labels,
                     mode=task.mode,
@@ -112,8 +110,6 @@ class TaskRepository:
                     type=task.type,
                     message_id=task.message_id,
                     conversation=task.conversation,
-                    reply=task.reply,
-                    reply_message=task.reply_message,
                     valid_labels=task.valid_labels,
                     mandatory_labels=task.mandatory_labels,
                     mode=task.mode,

backend/oasst_backend/tree_manager.py

@@ -11,11 +11,7 @@ import numpy as np
 import pydantic
 from fastapi.encoders import jsonable_encoder
 from loguru import logger
-from oasst_backend.api.v1.utils import (
-    prepare_conversation,
-    prepare_conversation_message,
-    prepare_conversation_message_list,
-)
+from oasst_backend.api.v1.utils import prepare_conversation, prepare_conversation_message_list
 from oasst_backend.config import TreeManagerConfiguration, settings
 from oasst_backend.models import Message, MessageReaction, MessageTreeState, Task, TextLabels, User, message_tree_state
 from oasst_backend.prompt_repository import PromptRepository
@@ -386,7 +382,6 @@ class TreeManager:
                             message_id=message.id,
                             conversation=conversation,
                             reply=message.text,
-                            reply_message=prepare_conversation_message(message),
                             valid_labels=list(map(lambda x: x.value, valid_labels)),
                             mandatory_labels=list(map(lambda x: x.value, self.cfg.mandatory_labels_assistant_reply)),
                             mode=label_mode,
@@ -412,7 +407,6 @@ class TreeManager:
                             message_id=message.id,
                             conversation=conversation,
                             reply=message.text,
-                            reply_message=prepare_conversation_message(message),
                             valid_labels=list(map(lambda x: x.value, valid_labels)),
                             mandatory_labels=list(map(lambda x: x.value, self.cfg.mandatory_labels_prompter_reply)),
                             mode=label_mode,
@@ -479,6 +473,7 @@ class TreeManager:
             case TaskType.LABEL_PROMPT:
                 assert len(prompts_need_review) > 0
                 message = random.choice(prompts_need_review)
+                message = self.pr.fetch_message(message.id)  # re-fetch message including emojis
 
                 label_mode = protocol_schema.LabelTaskMode.full
                 label_disposition = protocol_schema.LabelTaskDisposition.quality
@@ -495,6 +490,7 @@ class TreeManager:
                 task = protocol_schema.LabelInitialPromptTask(
                     message_id=message.id,
                     prompt=message.text,
+                    conversation=prepare_conversation([message]),
                     valid_labels=list(map(lambda x: x.value, valid_labels)),
                     mandatory_labels=list(map(lambda x: x.value, self.cfg.mandatory_labels_initial_prompt)),
                     mode=label_mode,

oasst-shared/oasst_shared/schemas/protocol.py

@@ -252,22 +252,21 @@ class AbstractLabelTask(Task):
     mode: Optional[LabelTaskMode]
     disposition: Optional[LabelTaskDisposition]
     labels: Optional[list[LabelDescription]]
+    conversation: Conversation  # the conversation so far (labeling -> last message)
 
 
 class LabelInitialPromptTask(AbstractLabelTask):
     """A task to label an initial prompt."""
 
     type: Literal["label_initial_prompt"] = "label_initial_prompt"
-    prompt: str
+    prompt: str | None = Field(None, deprecated=True, description="deprecated, use `prompt_message`")
 
 
 class LabelConversationReplyTask(AbstractLabelTask):
     """A task to label a reply to a conversation."""
 
     type: Literal["label_conversation_reply"] = "label_conversation_reply"
-    conversation: Conversation  # the conversation so far (new: including the reply message)
-    reply_message: Optional[ConversationMessage]
-    reply: str
+    reply: str | None = Field(None, deprecated=True, description="deprecated, use last message of `conversation`")
 
 
 class LabelPrompterReplyTask(LabelConversationReplyTask):