diff --git a/backend/oasst_backend/api/v1/tasks.py b/backend/oasst_backend/api/v1/tasks.py index e46c2358..7db42662 100644 --- a/backend/oasst_backend/api/v1/tasks.py +++ b/backend/oasst_backend/api/v1/tasks.py @@ -5,6 +5,7 @@ from fastapi import APIRouter, Depends from fastapi.security.api_key import APIKey from loguru import logger from oasst_backend.api import deps +from oasst_backend.config import settings from oasst_backend.prompt_repository import PromptRepository, TaskRepository from oasst_backend.tree_manager import TreeManager from oasst_backend.user_repository import UserRepository @@ -21,8 +22,18 @@ router = APIRouter() "/", response_model=protocol_schema.AnyTask, dependencies=[ - Depends(deps.UserRateLimiter(times=100, minutes=5)), - Depends(deps.APIClientRateLimiter(times=10_000, minutes=1)), + Depends( + deps.UserRateLimiter( + times=settings.RATE_LIMIT_TASK_USER_TIMES, + minutes=settings.RATE_LIMIT_TASK_USER_MINUTES, + ) + ), + Depends( + deps.APIClientRateLimiter( + times=settings.RATE_LIMIT_TASK_API_TIMES, + minutes=settings.RATE_LIMIT_TASK_API_MINUTES, + ) + ), ], ) # work with Union once more types are added def request_task( diff --git a/backend/oasst_backend/config.py b/backend/oasst_backend/config.py index 3056d96c..638dca30 100644 --- a/backend/oasst_backend/config.py +++ b/backend/oasst_backend/config.py @@ -19,6 +19,9 @@ class TreeManagerConfiguration(BaseModel): max_children_count: int = 3 """Maximum number of reply messages per tree node.""" + num_prompter_replies: int = 1 + """Number of prompter replies to collect per assistant reply.""" + goal_tree_size: int = 15 """Total number of messages to gather per tree.""" @@ -109,11 +112,11 @@ class TreeManagerConfiguration(BaseModel): rank_prompter_replies: bool = False - lonely_children_count: int = 2 + lonely_children_count: int = 3 """Number of children below which parents are preferred during sampling for reply tasks.""" p_lonely_child_extension: float = 0.8 - """Probability to select a parent with less than lonely_children_count children.""" + """Probability to select a prompter message parent with less than lonely_children_count children.""" recent_tasks_span_sec: int = 3 * 60 # 3 min """Time in seconds of recent tasks to consider for exclusion during task selection.""" @@ -192,19 +195,24 @@ class Settings(BaseSettings): USER_STATS_INTERVAL_WEEK: int = 15 # minutes USER_STATS_INTERVAL_MONTH: int = 60 # minutes USER_STATS_INTERVAL_TOTAL: int = 240 # minutes + USER_STREAK_UPDATE_INTERVAL: int = 4 # Hours @validator( "USER_STATS_INTERVAL_DAY", "USER_STATS_INTERVAL_WEEK", "USER_STATS_INTERVAL_MONTH", "USER_STATS_INTERVAL_TOTAL", + "USER_STREAK_UPDATE_INTERVAL", ) def validate_user_stats_intervals(cls, v: int): if v < 1: raise ValueError(v) return v - USER_STREAK_UPDATE_INTERVAL: int = 4 # Hours + RATE_LIMIT_TASK_USER_TIMES: int = 60 + RATE_LIMIT_TASK_USER_MINUTES: int = 5 + RATE_LIMIT_TASK_API_TIMES: int = 10_000 + RATE_LIMIT_TASK_API_MINUTES: int = 1 class Config: env_file = ".env" diff --git a/backend/oasst_backend/tree_manager.py b/backend/oasst_backend/tree_manager.py index e86b73c7..f1cec103 100644 --- a/backend/oasst_backend/tree_manager.py +++ b/backend/oasst_backend/tree_manager.py @@ -439,12 +439,13 @@ class TreeManager: if len(extendible_parents) > 0: random_parent: ExtendibleParentRow = None if self.cfg.p_lonely_child_extension > 0 and self.cfg.lonely_children_count > 1: - # check if we have extendible parents with a small number of replies + # check if we have extendible prompter parents with a small number of replies lonely_children_parents = [ p for p in extendible_parents if 0 < p.active_children_count < self.cfg.lonely_children_count + and p.parent_role == "prompter" and p.parent_id not in recent_reply_task_parents ] if len(lonely_children_parents) > 0 and random.random() < self.cfg.p_lonely_child_extension: @@ -933,6 +934,7 @@ WHERE mts.active -- only consider active trees AND (c.review_result OR coalesce(c.review_count, 0) < :num_reviews_reply) -- don't count children with negative review but count elements under review GROUP BY m.id, m.role, m.depth, m.message_tree_id, mts.max_children_count HAVING COUNT(c.id) < mts.max_children_count -- below maximum number of children + AND (COUNT(c.id) < :num_prompter_replies OR m.role = 'prompter') -- limit replies to assistant messages AND COUNT(c.id) FILTER (WHERE c.user_id = :user_id) = 0 -- without reply by user """ @@ -945,6 +947,7 @@ HAVING COUNT(c.id) < mts.max_children_count -- below maximum number of children { "growing_state": message_tree_state.State.GROWING, "num_reviews_reply": self.cfg.num_reviews_reply, + "num_prompter_replies": self.cfg.num_prompter_replies, "lang": lang, "user_id": user_id, }, @@ -982,6 +985,7 @@ HAVING COUNT(m.id) < mts.goal_tree_size { "growing_state": message_tree_state.State.GROWING, "num_reviews_reply": self.cfg.num_reviews_reply, + "num_prompter_replies": self.cfg.num_prompter_replies, "lang": lang, "user_id": user_id, }, diff --git a/text-frontend/auto_main.py b/text-frontend/auto_main.py index c4ae19d3..485ee1cb 100644 --- a/text-frontend/auto_main.py +++ b/text-frontend/auto_main.py @@ -2,6 +2,7 @@ import http import random +from uuid import uuid4 import requests import typer @@ -14,7 +15,7 @@ USER = {"id": "1234", "display_name": "John Doe", "auth_method": "local"} def _random_message_id(): - return str(random.randint(1000, 9999)) + return str(uuid4()) def _render_message(message: dict) -> str: @@ -88,22 +89,21 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "1234"): _post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id}) valid_labels = task["valid_labels"] + mandatory_labels = task["mandatory_labels"] labels_dict = None if task["mode"] == "simple" and len(valid_labels) == 1: answer = random.choice([True, False]) labels_dict = {valid_labels[0]: 1 if answer else 0} else: - while labels_dict is None: - labels = random.sample(valid_labels, random.randint(1, len(valid_labels))) - if all([label in valid_labels for label in labels]): - labels_dict = { - label: 1 if label != "lang_mismatch" and label in labels else 0 - for label in valid_labels - } - else: - invalid_labels = [label for label in labels if label not in valid_labels] - typer.echo(f"Invalid labels: {', '.join(invalid_labels)}. Valid: {', '.join(valid_labels)}") + labels = random.sample(valid_labels, random.randint(1, len(valid_labels))) + for l in mandatory_labels: + if l not in labels: + labels.append(l) + labels_dict = {label: random.random() for label in valid_labels} + if random.random() < 0.9: + labels_dict["spam"] = 0 + labels_dict["lang_mismatch"] = 0 # send labels new_task = _post( @@ -206,22 +206,22 @@ def main(backend_url: str = "http://127.0.0.1:8080", api_key: str = "1234"): user_message_id = _random_message_id() _post(f"/api/v1/tasks/{task['id']}/ack", {"message_id": message_id}) valid_labels = task["valid_labels"] + mandatory_labels = task["mandatory_labels"] labels_dict = None if task["mode"] == "simple" and len(valid_labels) == 1: answer = random.choice([True, False]) labels_dict = {valid_labels[0]: 1 if answer else 0} else: - while labels_dict is None: - labels = random.sample(valid_labels, random.randint(1, len(valid_labels))) - if all([label in valid_labels for label in labels]): - labels_dict = { - label: 1 if label != "lang_mismatch" and label in labels else 0 - for label in valid_labels - } - else: - invalid_labels = [label for label in labels if label not in valid_labels] - typer.echo(f"Invalid labels: {', '.join(invalid_labels)}. Valid: {', '.join(valid_labels)}") + labels = random.sample(valid_labels, random.randint(1, len(valid_labels))) + for l in mandatory_labels: + if l not in labels: + labels.append(l) + labels_dict = {label: random.random() for label in valid_labels} + if random.random() < 0.9: + labels_dict["spam"] = 0 + labels_dict["lang_mismatch"] = 0 + # send interaction new_task = _post( "/api/v1/tasks/interaction",