Add lang-tag based task selection (lang-separation) (#863)

* lang based task selection

* use BCP 47 instead of ISO 639-1

* add Field(None, nullable=True)

* update migration script down_revision

backend/oasst_backend/api/v1/tasks.py

@@ -39,7 +39,7 @@ def request_task(
         pr.ensure_user_is_enabled()
 
         tm = TreeManager(db, pr)
-        task, message_tree_id, parent_message_id = tm.next_task(request.type)
+        task, message_tree_id, parent_message_id = tm.next_task(desired_task_type=request.type, lang=request.lang)
         pr.task_repository.store_task(task, message_tree_id, parent_message_id, request.collective)
 
     except OasstError:
@@ -54,6 +54,7 @@ def request_task(
 def tasks_availability(
     *,
     user: Optional[protocol_schema.User] = None,
+    lang: Optional[str] = "en",
     db: Session = Depends(deps.get_db),
     api_key: APIKey = Depends(deps.get_api_key),
 ):
@@ -62,7 +63,7 @@ def tasks_availability(
     try:
         pr = PromptRepository(db, api_client, client_user=user)
         tm = TreeManager(db, pr)
-        return tm.determine_task_availability()
+        return tm.determine_task_availability(lang)
 
     except OasstError:
         raise

backend/oasst_backend/api/v1/utils.py

@@ -10,6 +10,7 @@ def prepare_message(m: Message) -> protocol.Message:
         frontend_message_id=m.frontend_message_id,
         parent_id=m.parent_id,
         text=m.text,
+        lang=m.lang,
         is_assistant=(m.role == "assistant"),
         created_date=m.created_date,
     )
@@ -22,10 +23,11 @@ def prepare_message_list(messages: list[Message]) -> list[protocol.Message]:
 def prepare_conversation_message_list(messages: list[Message]) -> list[protocol.ConversationMessage]:
     return [
         protocol.ConversationMessage(
-            text=message.text,
-            is_assistant=(message.role == "assistant"),
             id=message.id,
             frontend_message_id=message.frontend_message_id,
+            text=message.text,
+            lang=message.lang,
+            is_assistant=(message.role == "assistant"),
         )
         for message in messages
     ]