From 59e325219011eacb9d598b7f5d73d12d9157772f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20K=C3=B6pf?= Date: Sun, 5 Feb 2023 00:18:05 +0100 Subject: [PATCH] limit number of trees activated during _prompt_lottery() calls --- backend/oasst_backend/tree_manager.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/oasst_backend/tree_manager.py b/backend/oasst_backend/tree_manager.py index a6c803be..cdf61d3f 100644 --- a/backend/oasst_backend/tree_manager.py +++ b/backend/oasst_backend/tree_manager.py @@ -220,7 +220,17 @@ class TreeManager: def _prompt_lottery(self, lang: str) -> int: MAX_RETRIES = 5 + + # Under high load the DB runs into deadlocks when many trees are released + # simultaneously (happens whens the max_active_trees setting is increased). + # To reduce the chance of write conflicts during updates of rows in the + # message_tree_state table we limit the number of trees that are activated + # per _prompt_lottery() call to MAX_ACTIVATE. + MAX_ACTIVATE = 2 + retry = 0 + activated = 0 + while True: stats = self.tree_counts_by_state_stats(lang=lang, only_active=True) @@ -229,8 +239,8 @@ class TreeManager: num_missing_growing = max(0, self.cfg.max_active_trees - stats.growing) logger.debug(f"_prompt_lottery {remaining_prompt_review=}, {num_missing_growing=}") - if num_missing_growing == 0: - return remaining_prompt_review + if num_missing_growing == 0 or activated >= MAX_ACTIVATE: + return num_missing_growing + remaining_prompt_review # select among distinct users authors_qry = ( @@ -288,6 +298,7 @@ class TreeManager: mts: MessageTreeState = winner_prompt.MessageTreeState self._enter_state(mts, message_tree_state.State.GROWING) self.db.flush() + activated += 1 def _auto_moderation(self, lang: str) -> None: if not self.cfg.auto_mod_enabled: