[tune] better error when metric or mode unset in search algorithms (#11646)

python/ray/tune/suggest/ax.py

@@ -3,7 +3,8 @@ from typing import Dict, List, Optional, Union
 from ax.service.ax_client import AxClient
 from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
     Quantized, Uniform
-from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
+from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
+    UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils import flatten_dict
 from ray.tune.utils.util import unflatten_dict
@@ -203,10 +204,15 @@ class AxSearch(Searcher):
     def suggest(self, trial_id: str) -> Optional[Dict]:
         if not self._ax:
             raise RuntimeError(
-                "Trying to sample a configuration from {}, but no search "
-                "space has been defined. Either pass the `{}` argument when "
-                "instantiating the search algorithm, or pass a `config` to "
-                "`tune.run()`.".format(self.__class__.__name__, "space"))
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"))
+
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__,
+                    metric=self._metric,
+                    mode=self._mode))
 
         if self.max_concurrent:
             if len(self._live_trial_mapping) >= self.max_concurrent:

python/ray/tune/suggest/bayesopt.py

@@ -6,7 +6,8 @@ from typing import Dict, Optional, Tuple
 
 from ray.tune import ExperimentAnalysis
 from ray.tune.sample import Domain, Float, Quantized
-from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
+from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
+    UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils.util import unflatten_dict
 
@@ -242,10 +243,15 @@ class BayesOptSearch(Searcher):
         """
         if not self.optimizer:
             raise RuntimeError(
-                "Trying to sample a configuration from {}, but no search "
-                "space has been defined. Either pass the `{}` argument when "
-                "instantiating the search algorithm, or pass a `config` to "
-                "`tune.run()`.".format(self.__class__.__name__, "space"))
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"))
+
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__,
+                    metric=self._metric,
+                    mode=self._mode))
 
         # If we have more active trials than the allowed maximum
         total_live_trials = len(self._live_trial_mapping)

python/ray/tune/suggest/bohb.py

@@ -11,7 +11,8 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
     Quantized, \
     Uniform
 from ray.tune.suggest import Searcher
-from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
+from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
+    UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils import flatten_dict
 from ray.tune.utils.util import unflatten_dict
@@ -156,10 +157,15 @@ class TuneBOHB(Searcher):
     def suggest(self, trial_id: str) -> Optional[Dict]:
         if not self._space:
             raise RuntimeError(
-                "Trying to sample a configuration from {}, but no search "
-                "space has been defined. Either pass the `{}` argument when "
-                "instantiating the search algorithm, or pass a `config` to "
-                "`tune.run()`.".format(self.__class__.__name__, "space"))
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"))
+
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__,
+                    metric=self._metric,
+                    mode=self._mode))
 
         if len(self.running) < self._max_concurrent:
             # This parameter is not used in hpbandster implementation.

python/ray/tune/suggest/dragonfly.py

@@ -8,7 +8,8 @@ import pickle
 from typing import Dict, List, Optional, Union
 
 from ray.tune.sample import Domain, Float, Quantized
-from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
+from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
+    UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils.util import flatten_dict
 
@@ -272,10 +273,15 @@ class DragonflySearch(Searcher):
     def suggest(self, trial_id: str) -> Optional[Dict]:
         if not self._opt:
             raise RuntimeError(
-                "Trying to sample a configuration from {}, but no search "
-                "space has been defined. Either pass the `{}` argument when "
-                "instantiating the search algorithm, or pass a `config` to "
-                "`tune.run()`.".format(self.__class__.__name__, "space"))
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"))
+
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__,
+                    metric=self._metric,
+                    mode=self._mode))
 
         if self._initial_points:
             suggested_config = self._initial_points[0]

python/ray/tune/suggest/hyperopt.py

@@ -10,7 +10,8 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
     Normal, \
     Quantized, \
     Uniform
-from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
+from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
+    UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import assign_value, parse_spec_vars
 
 try:
@@ -200,10 +201,15 @@ class HyperOptSearch(Searcher):
     def suggest(self, trial_id: str) -> Optional[Dict]:
         if not self.domain:
             raise RuntimeError(
-                "Trying to sample a configuration from {}, but no search "
-                "space has been defined. Either pass the `{}` argument when "
-                "instantiating the search algorithm, or pass a `config` to "
-                "`tune.run()`.".format(self.__class__.__name__, "space"))
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"))
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__,
+                    metric=self._metric,
+                    mode=self._mode))
+
         if self.max_concurrent:
             if len(self._live_trial_mapping) >= self.max_concurrent:
                 return None

python/ray/tune/suggest/nevergrad.py

@@ -4,7 +4,8 @@ from typing import Dict, Optional, Union
 
 from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
     Quantized
-from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
+from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
+    UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils import flatten_dict
 from ray.tune.utils.util import unflatten_dict
@@ -189,10 +190,14 @@ class NevergradSearch(Searcher):
     def suggest(self, trial_id: str) -> Optional[Dict]:
         if not self._nevergrad_opt:
             raise RuntimeError(
-                "Trying to sample a configuration from {}, but no search "
-                "space has been defined. Either pass the `{}` argument when "
-                "instantiating the search algorithm, or pass a `config` to "
-                "`tune.run()`.".format(self.__class__.__name__, "space"))
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"))
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__,
+                    metric=self._metric,
+                    mode=self._mode))
 
         if self.max_concurrent:
             if len(self._live_trial_mapping) >= self.max_concurrent:

python/ray/tune/suggest/optuna.py

@@ -5,7 +5,8 @@ from typing import Dict, List, Optional, Tuple, Union
 from ray.tune.result import TRAINING_ITERATION
 from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
     Quantized, Uniform
-from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
+from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
+    UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils import flatten_dict
 from ray.tune.utils.util import unflatten_dict
@@ -165,10 +166,14 @@ class OptunaSearch(Searcher):
     def suggest(self, trial_id: str) -> Optional[Dict]:
         if not self._space:
             raise RuntimeError(
-                "Trying to sample a configuration from {}, but no search "
-                "space has been defined. Either pass the `{}` argument when "
-                "instantiating the search algorithm, or pass a `config` to "
-                "`tune.run()`.".format(self.__class__.__name__, "space"))
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"))
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__,
+                    metric=self._metric,
+                    mode=self._mode))
 
         if trial_id not in self._ot_trials:
             ot_trial_id = self._storage.create_new_trial(

python/ray/tune/suggest/skopt.py

@@ -3,7 +3,8 @@ import pickle
 from typing import Dict, List, Optional, Tuple, Union
 
 from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized
-from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
+from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
+    UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils import flatten_dict
 from ray.tune.utils.util import unflatten_dict
@@ -230,10 +231,14 @@ class SkOptSearch(Searcher):
     def suggest(self, trial_id: str) -> Optional[Dict]:
         if not self._skopt_opt:
             raise RuntimeError(
-                "Trying to sample a configuration from {}, but no search "
-                "space has been defined. Either pass the `{}` argument when "
-                "instantiating the search algorithm, or pass a `config` to "
-                "`tune.run()`.".format(self.__class__.__name__, "space"))
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="space"))
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__,
+                    metric=self._metric,
+                    mode=self._mode))
 
         if self.max_concurrent:
             if len(self._live_trial_mapping) >= self.max_concurrent:

python/ray/tune/suggest/suggestion.py

@@ -15,6 +15,18 @@ UNRESOLVED_SEARCH_SPACE = str(
     "conversion, pass the space definition as part of the `config` argument "
     "to `tune.run()` instead.")
 
+UNDEFINED_SEARCH_SPACE = str(
+    "Trying to sample a configuration from {cls}, but no search "
+    "space has been defined. Either pass the `{space}` argument when "
+    "instantiating the search algorithm, or pass a `config` to "
+    "`tune.run()`.")
+
+UNDEFINED_METRIC_MODE = str(
+    "Trying to sample a configuration from {cls}, but the `metric` "
+    "({metric}) or `mode` ({mode}) parameters have not been set. "
+    "Either pass these arguments when instantiating the search algorithm, "
+    "or pass them to `tune.run()`.")
+
 
 class Searcher:
     """Abstract class for wrapping suggesting algorithms.

python/ray/tune/suggest/zoopt.py

@@ -6,7 +6,8 @@ import ray
 import ray.cloudpickle as pickle
 from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized, \
     Uniform
-from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
+from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
+    UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils.util import unflatten_dict
 from zoopt import ValueType
@@ -208,10 +209,14 @@ class ZOOptSearch(Searcher):
     def suggest(self, trial_id: str) -> Optional[Dict]:
         if not self._dim_dict or not self.optimizer:
             raise RuntimeError(
-                "Trying to sample a configuration from {}, but no search "
-                "space has been defined. Either pass the `{}` argument when "
-                "instantiating the search algorithm, or pass a `config` to "
-                "`tune.run()`.".format(self.__class__.__name__, "space"))
+                UNDEFINED_SEARCH_SPACE.format(
+                    cls=self.__class__.__name__, space="dim_dict"))
+        if not self._metric or not self._mode:
+            raise RuntimeError(
+                UNDEFINED_METRIC_MODE.format(
+                    cls=self.__class__.__name__,
+                    metric=self._metric,
+                    mode=self._mode))
 
         _solution = self.optimizer.suggest()
 

python/ray/tune/tests/test_sample.py

@@ -194,11 +194,11 @@ class SearchSpaceTest(unittest.TestCase):
 
         client1 = AxClient(random_seed=1234)
         client1.create_experiment(parameters=converted_config)
-        searcher1 = AxSearch(ax_client=client1)
+        searcher1 = AxSearch(ax_client=client1, metric="a", mode="max")
 
         client2 = AxClient(random_seed=1234)
         client2.create_experiment(parameters=ax_config)
-        searcher2 = AxSearch(ax_client=client2)
+        searcher2 = AxSearch(ax_client=client2, metric="a", mode="max")
 
         config1 = searcher1.suggest("0")
         config2 = searcher2.suggest("0")
@@ -240,8 +240,10 @@ class SearchSpaceTest(unittest.TestCase):
         bayesopt_config = {"b/z": (1e-4, 1e-2)}
         converted_config = BayesOptSearch.convert_search_space(config)
 
-        searcher1 = BayesOptSearch(space=converted_config, metric="none")
-        searcher2 = BayesOptSearch(space=bayesopt_config, metric="none")
+        searcher1 = BayesOptSearch(
+            space=converted_config, metric="none", mode="max")
+        searcher2 = BayesOptSearch(
+            space=bayesopt_config, metric="none", mode="max")
 
         config1 = searcher1.suggest("0")
         config2 = searcher2.suggest("0")
@@ -295,8 +297,8 @@ class SearchSpaceTest(unittest.TestCase):
         converted_config.seed(1234)
         bohb_config.seed(1234)
 
-        searcher1 = TuneBOHB(space=converted_config)
-        searcher2 = TuneBOHB(space=bohb_config)
+        searcher1 = TuneBOHB(space=converted_config, metric="a", mode="max")
+        searcher2 = TuneBOHB(space=bohb_config, metric="a", mode="max")
 
         config1 = searcher1.suggest("0")
         config2 = searcher2.suggest("0")
@@ -356,7 +358,8 @@ class SearchSpaceTest(unittest.TestCase):
             optimizer="bandit",
             domain="euclidean",
             space=converted_config,
-            metric="none")
+            metric="none",
+            mode="max")
 
         config1 = searcher1.suggest("0")
 
@@ -365,7 +368,8 @@ class SearchSpaceTest(unittest.TestCase):
             optimizer="bandit",
             domain="euclidean",
             space=dragonfly_config,
-            metric="none")
+            metric="none",
+            mode="max")
         config2 = searcher2.suggest("0")
 
         self.assertEqual(config1, config2)
@@ -424,9 +428,15 @@ class SearchSpaceTest(unittest.TestCase):
         }
 
         searcher1 = HyperOptSearch(
-            space=converted_config, random_state_seed=1234)
+            space=converted_config,
+            random_state_seed=1234,
+            metric="a",
+            mode="max")
         searcher2 = HyperOptSearch(
-            space=hyperopt_config, random_state_seed=1234)
+            space=hyperopt_config,
+            random_state_seed=1234,
+            metric="a",
+            mode="max")
 
         config1 = searcher1.suggest("0")
         config2 = searcher2.suggest("0")
@@ -516,9 +526,15 @@ class SearchSpaceTest(unittest.TestCase):
                 z=ng.p.Log(lower=1e-4, upper=1e-2)))
 
         searcher1 = NevergradSearch(
-            optimizer=ng.optimizers.OnePlusOne, space=converted_config)
+            optimizer=ng.optimizers.OnePlusOne,
+            space=converted_config,
+            metric="a",
+            mode="max")
         searcher2 = NevergradSearch(
-            optimizer=ng.optimizers.OnePlusOne, space=nevergrad_config)
+            optimizer=ng.optimizers.OnePlusOne,
+            space=nevergrad_config,
+            metric="a",
+            mode="max")
 
         np.random.seed(1234)
         config1 = searcher1.suggest("0")
@@ -571,10 +587,12 @@ class SearchSpaceTest(unittest.TestCase):
         ]
 
         sampler1 = RandomSampler(seed=1234)
-        searcher1 = OptunaSearch(space=converted_config, sampler=sampler1)
+        searcher1 = OptunaSearch(
+            space=converted_config, sampler=sampler1, metric="a", mode="max")
 
         sampler2 = RandomSampler(seed=1234)
-        searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2)
+        searcher2 = OptunaSearch(
+            space=optuna_config, sampler=sampler2, metric="a", mode="max")
 
         config1 = searcher1.suggest("0")
         config2 = searcher2.suggest("0")
@@ -614,8 +632,8 @@ class SearchSpaceTest(unittest.TestCase):
         converted_config = SkOptSearch.convert_search_space(config)
         skopt_config = {"a": [2, 3, 4], "b/x": (0, 5), "b/z": (1e-4, 1e-2)}
 
-        searcher1 = SkOptSearch(space=converted_config)
-        searcher2 = SkOptSearch(space=skopt_config)
+        searcher1 = SkOptSearch(space=converted_config, metric="a", mode="max")
+        searcher2 = SkOptSearch(space=skopt_config, metric="a", mode="max")
 
         np.random.seed(1234)
         config1 = searcher1.suggest("0")
@@ -675,9 +693,17 @@ class SearchSpaceTest(unittest.TestCase):
         zoopt_search_config = {"parallel_num": 4}
 
         searcher1 = ZOOptSearch(
-            dim_dict=converted_config, budget=5, **zoopt_search_config)
+            dim_dict=converted_config,
+            budget=5,
+            metric="a",
+            mode="max",
+            **zoopt_search_config)
         searcher2 = ZOOptSearch(
-            dim_dict=zoopt_config, budget=5, **zoopt_search_config)
+            dim_dict=zoopt_config,
+            budget=5,
+            metric="a",
+            mode="max",
+            **zoopt_search_config)
 
         np.random.seed(1234)
         config1 = searcher1.suggest("0")