[tune] lazy trials (#10802)

* Lazily fill trial queue

* Update interface

* Update end to end reporter test

* Removed `next_trials()` method

* Lint

* Print total number of samples to be generated in progress reporter. Allow infinite samples.

* Nit check

python/ray/tune/tests/test_automl_searcher.py

@@ -5,6 +5,16 @@ from ray.tune import register_trainable
 from ray.tune.automl import SearchSpace, DiscreteSpace, GridSearch
 
 
+def next_trials(searcher):
+    trials = []
+    while not searcher.is_finished():
+        trial = searcher.next_trial()
+        if not trial:
+            break
+        trials.append(trial)
+    return trials
+
+
 class AutoMLSearcherTest(unittest.TestCase):
     def setUp(self):
         def dummy_train(config, reporter):
@@ -20,7 +30,7 @@ class AutoMLSearcherTest(unittest.TestCase):
         ])
         searcher = GridSearch(space, "reward")
         searcher.add_configurations(exp)
-        trials = searcher.next_trials()
+        trials = next_trials(searcher)
 
         self.assertEqual(len(trials), 4)
         self.assertTrue(trials[0].config["a"]["b"]["c"] in [1, 2])
@@ -34,9 +44,9 @@ class AutoMLSearcherTest(unittest.TestCase):
         ])
         searcher = GridSearch(space, "reward")
         searcher.add_configurations(exp)
-        trials = searcher.next_trials()
+        trials = next_trials(searcher)
 
-        self.assertEqual(len(searcher.next_trials()), 0)
+        self.assertEqual(searcher.next_trial(), None)
         for trial in trials[1:]:
             searcher.on_trial_complete(trial.trial_id)
         searcher.on_trial_complete(trials[0].trial_id, error=True)
@@ -51,10 +61,11 @@ class AutoMLSearcherTest(unittest.TestCase):
         ])
         searcher = GridSearch(space, "reward")
         searcher.add_configurations(exp)
-        trials = searcher.next_trials()
+        trials = next_trials(searcher)
 
-        self.assertEqual(len(searcher.next_trials()), 0)
+        self.assertEqual(searcher.next_trial(), None)
         for i, trial in enumerate(trials):
+            print("TRIAL {}".format(trial))
             rewards = list(range(i, i + 10))
             random.shuffle(rewards)
             for reward in rewards:

python/ray/tune/tests/test_progress_reporter.py

@@ -73,34 +73,14 @@ tune.run_experiments({
     },
 }, verbose=1)"""
 
-EXPECTED_END_TO_END_START = """Number of trials: 30 (29 PENDING, 1 RUNNING)
-+---------------+----------+-------+-----+-----+
-| Trial name    | status   | loc   |   a |   b |
-|---------------+----------+-------+-----+-----|
-| f_xxxxx_00001 | PENDING  |       |   1 |     |
-| f_xxxxx_00002 | PENDING  |       |   2 |     |
-| f_xxxxx_00003 | PENDING  |       |   3 |     |
-| f_xxxxx_00004 | PENDING  |       |   4 |     |
-| f_xxxxx_00005 | PENDING  |       |   5 |     |
-| f_xxxxx_00006 | PENDING  |       |   6 |     |
-| f_xxxxx_00007 | PENDING  |       |   7 |     |
-| f_xxxxx_00008 | PENDING  |       |   8 |     |
-| f_xxxxx_00009 | PENDING  |       |   9 |     |
-| f_xxxxx_00010 | PENDING  |       |     |   0 |
-| f_xxxxx_00011 | PENDING  |       |     |   1 |
-| f_xxxxx_00012 | PENDING  |       |     |   2 |
-| f_xxxxx_00013 | PENDING  |       |     |   3 |
-| f_xxxxx_00014 | PENDING  |       |     |   4 |
-| f_xxxxx_00015 | PENDING  |       |     |   5 |
-| f_xxxxx_00016 | PENDING  |       |     |   6 |
-| f_xxxxx_00017 | PENDING  |       |     |   7 |
-| f_xxxxx_00018 | PENDING  |       |     |   8 |
-| f_xxxxx_00019 | PENDING  |       |     |   9 |
-| f_xxxxx_00000 | RUNNING  |       |   0 |     |
-+---------------+----------+-------+-----+-----+
-... 10 more trials not shown (10 PENDING)"""
+EXPECTED_END_TO_END_START = """Number of trials: 1/30 (1 RUNNING)
++---------------+----------+-------+-----+
+| Trial name    | status   | loc   |   a |
+|---------------+----------+-------+-----|
+| f_xxxxx_00000 | RUNNING  |       |   0 |
++---------------+----------+-------+-----+"""
 
-EXPECTED_END_TO_END_END = """Number of trials: 30 (30 TERMINATED)
+EXPECTED_END_TO_END_END = """Number of trials: 30/30 (30 TERMINATED)
 +---------------+------------+-------+-----+-----+-----+
 | Trial name    | status     | loc   |   a |   b |   c |
 |---------------+------------+-------+-----+-----+-----|
@@ -136,7 +116,7 @@ EXPECTED_END_TO_END_END = """Number of trials: 30 (30 TERMINATED)
 | f_xxxxx_00029 | TERMINATED |       |     |     |   9 |
 +---------------+------------+-------+-----+-----+-----+"""
 
-EXPECTED_END_TO_END_AC = """Number of trials: 30 (30 TERMINATED)
+EXPECTED_END_TO_END_AC = """Number of trials: 30/30 (30 TERMINATED)
 +---------------+------------+-------+-----+-----+-----+
 | Trial name    | status     | loc   |   a |   b |   c |
 |---------------+------------+-------+-----+-----+-----|

python/ray/tune/tests/test_ray_trial_executor.py

@@ -170,7 +170,14 @@ class RayTrialExecutorTest(unittest.TestCase):
     def generate_trials(spec, name):
         suggester = BasicVariantGenerator()
         suggester.add_configurations({name: spec})
-        return suggester.next_trials()
+        trials = []
+        while not suggester.is_finished():
+            trial = suggester.next_trial()
+            if trial:
+                trials.append(trial)
+            else:
+                break
+        return trials
 
     def process_trial_save(self, trial):
         """Simulates trial runner save."""

python/ray/tune/tests/test_trial_runner.py

@@ -57,7 +57,10 @@ class TrialRunnerTest(unittest.TestCase):
         for name, spec in experiments.items():
             trial_generator = BasicVariantGenerator()
             trial_generator.add_configurations({name: spec})
-            for trial in trial_generator.next_trials():
+            while not trial_generator.is_finished():
+                trial = trial_generator.next_trial()
+                if not trial:
+                    break
                 trial_executor.start_trial(trial)
                 self.assertLessEqual(len(os.path.basename(trial.logdir)), 200)
                 trial_executor.stop_trial(trial)

python/ray/tune/tests/test_trial_runner_3.py

@@ -228,19 +228,18 @@ class TrialRunnerTest3(unittest.TestCase):
         class FinishFastAlg(_MockSuggestionAlgorithm):
             _index = 0
 
-            def next_trials(self):
+            def next_trial(self):
                 spec = self._experiment.spec
-                trials = []
+                trial = None
                 if self._index < spec["num_samples"]:
                     trial = Trial(
                         spec.get("run"), stopping_criterion=spec.get("stop"))
-                    trials.append(trial)
                 self._index += 1
 
                 if self._index > 4:
                     self.set_finished()
 
-                return trials
+                return trial
 
             def suggest(self, trial_id):
                 return {}
@@ -625,7 +624,7 @@ class SearchAlgorithmTest(unittest.TestCase):
         searcher = TestSuggestion()
         alg = SearchGenerator(searcher)
         alg.add_configurations({"test": {"run": "__fake"}})
-        trial = alg.next_trials()[0]
+        trial = alg.next_trial()
         self.assertTrue("e=5" in trial.experiment_tag)
         self.assertTrue("d=4" in trial.experiment_tag)
 

python/ray/tune/tests/test_var.py

@@ -24,7 +24,14 @@ class VariantGeneratorTest(unittest.TestCase):
     def generate_trials(self, spec, name):
         suggester = BasicVariantGenerator()
         suggester.add_configurations({name: spec})
-        return suggester.next_trials()
+        trials = []
+        while not suggester.is_finished():
+            trial = suggester.next_trial()
+            if trial:
+                trials.append(trial)
+            else:
+                break
+        return trials
 
     def testParseToTrials(self):
         trials = self.generate_trials({