[tune] Support true pooling and batched concurrency (#10352)

python/ray/tune/tests/test_actor_reuse.py

@@ -29,7 +29,11 @@ def create_resettable_class():
             print("PRINT_STDERR: {}".format(self.msg), file=sys.stderr)
             logger.info("LOG_STDERR: {}".format(self.msg))
 
-            return {"num_resets": self.num_resets, "done": self.iter > 1}
+            return {
+                "num_resets": self.num_resets,
+                "done": self.iter > 1,
+                "iter": self.iter
+            }
 
         def save_checkpoint(self, chkpt_dir):
             return {"iter": self.iter}
@@ -64,7 +68,9 @@ class ActorReuseTest(unittest.TestCase):
                 }
             },
             reuse_actors=False,
-            scheduler=FrequentPausesScheduler())
+            scheduler=FrequentPausesScheduler(),
+            verbose=0)
+        self.assertEqual([t.last_result["iter"] for t in trials], [2, 2, 2, 2])
         self.assertEqual([t.last_result["num_resets"] for t in trials],
                          [0, 0, 0, 0])
 
@@ -78,11 +84,13 @@ class ActorReuseTest(unittest.TestCase):
                 }
             },
             reuse_actors=True,
-            scheduler=FrequentPausesScheduler())
+            scheduler=FrequentPausesScheduler(),
+            verbose=0)
+        self.assertEqual([t.last_result["iter"] for t in trials], [2, 2, 2, 2])
         self.assertEqual([t.last_result["num_resets"] for t in trials],
                          [1, 2, 3, 4])
 
-    def testTrialReuseEnabledError(self):
+    def testReuseEnabledError(self):
         def run():
             run_experiments(
                 {

python/ray/tune/tests/test_progress_reporter.py

@@ -70,7 +70,7 @@ tune.run_experiments({
             "c": tune.grid_search(list(range(10))),
         },
     },
-}, reuse_actors=True, verbose=1)"""
+}, verbose=1)"""
 
 EXPECTED_END_TO_END_START = """Number of trials: 30 (29 PENDING, 1 RUNNING)
 +---------------+----------+-------+-----+-----+

python/ray/tune/tests/test_trial_runner_3.py

@@ -764,6 +764,33 @@ class SearchAlgorithmTest(unittest.TestCase):
         limiter2.on_trial_complete("test_2", {"result": 3})
         assert limiter2.suggest("test_3")["score"] == 3
 
+    def testBatchLimiter(self):
+        ray.init(num_cpus=4)
+
+        class TestSuggestion(Searcher):
+            def __init__(self, index):
+                self.index = index
+                self.returned_result = []
+                super().__init__(metric="result", mode="max")
+
+            def suggest(self, trial_id):
+                self.index += 1
+                return {"score": self.index}
+
+            def on_trial_complete(self, trial_id, result=None, **kwargs):
+                self.returned_result.append(result)
+
+        searcher = TestSuggestion(0)
+        limiter = ConcurrencyLimiter(searcher, max_concurrent=2, batch=True)
+        assert limiter.suggest("test_1")["score"] == 1
+        assert limiter.suggest("test_2")["score"] == 2
+        assert limiter.suggest("test_3") is None
+
+        limiter.on_trial_complete("test_1", {"result": 3})
+        assert limiter.suggest("test_3") is None
+        limiter.on_trial_complete("test_2", {"result": 3})
+        assert limiter.suggest("test_3") is not None
+
 
 class ResourcesTest(unittest.TestCase):
     def testSubtraction(self):