diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py index 0379b6fae..6e5cd3cdd 100644 --- a/python/ray/tune/suggest/ax.py +++ b/python/ray/tune/suggest/ax.py @@ -3,7 +3,8 @@ from typing import Dict, List, Optional, Union from ax.service.ax_client import AxClient from ray.tune.sample import Categorical, Float, Integer, LogUniform, \ Quantized, Uniform -from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE +from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ + UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars from ray.tune.utils import flatten_dict from ray.tune.utils.util import unflatten_dict @@ -203,10 +204,15 @@ class AxSearch(Searcher): def suggest(self, trial_id: str) -> Optional[Dict]: if not self._ax: raise RuntimeError( - "Trying to sample a configuration from {}, but no search " - "space has been defined. Either pass the `{}` argument when " - "instantiating the search algorithm, or pass a `config` to " - "`tune.run()`.".format(self.__class__.__name__, "space")) + UNDEFINED_SEARCH_SPACE.format( + cls=self.__class__.__name__, space="space")) + + if not self._metric or not self._mode: + raise RuntimeError( + UNDEFINED_METRIC_MODE.format( + cls=self.__class__.__name__, + metric=self._metric, + mode=self._mode)) if self.max_concurrent: if len(self._live_trial_mapping) >= self.max_concurrent: diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py index d6610bad5..d285ad00a 100644 --- a/python/ray/tune/suggest/bayesopt.py +++ b/python/ray/tune/suggest/bayesopt.py @@ -6,7 +6,8 @@ from typing import Dict, Optional, Tuple from ray.tune import ExperimentAnalysis from ray.tune.sample import Domain, Float, Quantized -from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE +from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ + UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars from ray.tune.utils.util import unflatten_dict @@ -242,10 +243,15 @@ class BayesOptSearch(Searcher): """ if not self.optimizer: raise RuntimeError( - "Trying to sample a configuration from {}, but no search " - "space has been defined. Either pass the `{}` argument when " - "instantiating the search algorithm, or pass a `config` to " - "`tune.run()`.".format(self.__class__.__name__, "space")) + UNDEFINED_SEARCH_SPACE.format( + cls=self.__class__.__name__, space="space")) + + if not self._metric or not self._mode: + raise RuntimeError( + UNDEFINED_METRIC_MODE.format( + cls=self.__class__.__name__, + metric=self._metric, + mode=self._mode)) # If we have more active trials than the allowed maximum total_live_trials = len(self._live_trial_mapping) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index be11d3ade..745122d31 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -11,7 +11,8 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \ Quantized, \ Uniform from ray.tune.suggest import Searcher -from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE +from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ + UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars from ray.tune.utils import flatten_dict from ray.tune.utils.util import unflatten_dict @@ -156,10 +157,15 @@ class TuneBOHB(Searcher): def suggest(self, trial_id: str) -> Optional[Dict]: if not self._space: raise RuntimeError( - "Trying to sample a configuration from {}, but no search " - "space has been defined. Either pass the `{}` argument when " - "instantiating the search algorithm, or pass a `config` to " - "`tune.run()`.".format(self.__class__.__name__, "space")) + UNDEFINED_SEARCH_SPACE.format( + cls=self.__class__.__name__, space="space")) + + if not self._metric or not self._mode: + raise RuntimeError( + UNDEFINED_METRIC_MODE.format( + cls=self.__class__.__name__, + metric=self._metric, + mode=self._mode)) if len(self.running) < self._max_concurrent: # This parameter is not used in hpbandster implementation. diff --git a/python/ray/tune/suggest/dragonfly.py b/python/ray/tune/suggest/dragonfly.py index c3436246b..10a26add7 100644 --- a/python/ray/tune/suggest/dragonfly.py +++ b/python/ray/tune/suggest/dragonfly.py @@ -8,7 +8,8 @@ import pickle from typing import Dict, List, Optional, Union from ray.tune.sample import Domain, Float, Quantized -from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE +from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ + UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars from ray.tune.utils.util import flatten_dict @@ -272,10 +273,15 @@ class DragonflySearch(Searcher): def suggest(self, trial_id: str) -> Optional[Dict]: if not self._opt: raise RuntimeError( - "Trying to sample a configuration from {}, but no search " - "space has been defined. Either pass the `{}` argument when " - "instantiating the search algorithm, or pass a `config` to " - "`tune.run()`.".format(self.__class__.__name__, "space")) + UNDEFINED_SEARCH_SPACE.format( + cls=self.__class__.__name__, space="space")) + + if not self._metric or not self._mode: + raise RuntimeError( + UNDEFINED_METRIC_MODE.format( + cls=self.__class__.__name__, + metric=self._metric, + mode=self._mode)) if self._initial_points: suggested_config = self._initial_points[0] diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py index 06a4e65b6..953fb4bb1 100644 --- a/python/ray/tune/suggest/hyperopt.py +++ b/python/ray/tune/suggest/hyperopt.py @@ -10,7 +10,8 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \ Normal, \ Quantized, \ Uniform -from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE +from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ + UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import assign_value, parse_spec_vars try: @@ -200,10 +201,15 @@ class HyperOptSearch(Searcher): def suggest(self, trial_id: str) -> Optional[Dict]: if not self.domain: raise RuntimeError( - "Trying to sample a configuration from {}, but no search " - "space has been defined. Either pass the `{}` argument when " - "instantiating the search algorithm, or pass a `config` to " - "`tune.run()`.".format(self.__class__.__name__, "space")) + UNDEFINED_SEARCH_SPACE.format( + cls=self.__class__.__name__, space="space")) + if not self._metric or not self._mode: + raise RuntimeError( + UNDEFINED_METRIC_MODE.format( + cls=self.__class__.__name__, + metric=self._metric, + mode=self._mode)) + if self.max_concurrent: if len(self._live_trial_mapping) >= self.max_concurrent: return None diff --git a/python/ray/tune/suggest/nevergrad.py b/python/ray/tune/suggest/nevergrad.py index af39cfc21..75a399a6c 100644 --- a/python/ray/tune/suggest/nevergrad.py +++ b/python/ray/tune/suggest/nevergrad.py @@ -4,7 +4,8 @@ from typing import Dict, Optional, Union from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \ Quantized -from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE +from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ + UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars from ray.tune.utils import flatten_dict from ray.tune.utils.util import unflatten_dict @@ -189,10 +190,14 @@ class NevergradSearch(Searcher): def suggest(self, trial_id: str) -> Optional[Dict]: if not self._nevergrad_opt: raise RuntimeError( - "Trying to sample a configuration from {}, but no search " - "space has been defined. Either pass the `{}` argument when " - "instantiating the search algorithm, or pass a `config` to " - "`tune.run()`.".format(self.__class__.__name__, "space")) + UNDEFINED_SEARCH_SPACE.format( + cls=self.__class__.__name__, space="space")) + if not self._metric or not self._mode: + raise RuntimeError( + UNDEFINED_METRIC_MODE.format( + cls=self.__class__.__name__, + metric=self._metric, + mode=self._mode)) if self.max_concurrent: if len(self._live_trial_mapping) >= self.max_concurrent: diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py index 26d98964c..9c5470135 100644 --- a/python/ray/tune/suggest/optuna.py +++ b/python/ray/tune/suggest/optuna.py @@ -5,7 +5,8 @@ from typing import Dict, List, Optional, Tuple, Union from ray.tune.result import TRAINING_ITERATION from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \ Quantized, Uniform -from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE +from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ + UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars from ray.tune.utils import flatten_dict from ray.tune.utils.util import unflatten_dict @@ -165,10 +166,14 @@ class OptunaSearch(Searcher): def suggest(self, trial_id: str) -> Optional[Dict]: if not self._space: raise RuntimeError( - "Trying to sample a configuration from {}, but no search " - "space has been defined. Either pass the `{}` argument when " - "instantiating the search algorithm, or pass a `config` to " - "`tune.run()`.".format(self.__class__.__name__, "space")) + UNDEFINED_SEARCH_SPACE.format( + cls=self.__class__.__name__, space="space")) + if not self._metric or not self._mode: + raise RuntimeError( + UNDEFINED_METRIC_MODE.format( + cls=self.__class__.__name__, + metric=self._metric, + mode=self._mode)) if trial_id not in self._ot_trials: ot_trial_id = self._storage.create_new_trial( diff --git a/python/ray/tune/suggest/skopt.py b/python/ray/tune/suggest/skopt.py index 89492b360..fd8c6fc3e 100644 --- a/python/ray/tune/suggest/skopt.py +++ b/python/ray/tune/suggest/skopt.py @@ -3,7 +3,8 @@ import pickle from typing import Dict, List, Optional, Tuple, Union from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized -from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE +from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ + UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars from ray.tune.utils import flatten_dict from ray.tune.utils.util import unflatten_dict @@ -230,10 +231,14 @@ class SkOptSearch(Searcher): def suggest(self, trial_id: str) -> Optional[Dict]: if not self._skopt_opt: raise RuntimeError( - "Trying to sample a configuration from {}, but no search " - "space has been defined. Either pass the `{}` argument when " - "instantiating the search algorithm, or pass a `config` to " - "`tune.run()`.".format(self.__class__.__name__, "space")) + UNDEFINED_SEARCH_SPACE.format( + cls=self.__class__.__name__, space="space")) + if not self._metric or not self._mode: + raise RuntimeError( + UNDEFINED_METRIC_MODE.format( + cls=self.__class__.__name__, + metric=self._metric, + mode=self._mode)) if self.max_concurrent: if len(self._live_trial_mapping) >= self.max_concurrent: diff --git a/python/ray/tune/suggest/suggestion.py b/python/ray/tune/suggest/suggestion.py index fdb5b1f5a..99a6001d1 100644 --- a/python/ray/tune/suggest/suggestion.py +++ b/python/ray/tune/suggest/suggestion.py @@ -15,6 +15,18 @@ UNRESOLVED_SEARCH_SPACE = str( "conversion, pass the space definition as part of the `config` argument " "to `tune.run()` instead.") +UNDEFINED_SEARCH_SPACE = str( + "Trying to sample a configuration from {cls}, but no search " + "space has been defined. Either pass the `{space}` argument when " + "instantiating the search algorithm, or pass a `config` to " + "`tune.run()`.") + +UNDEFINED_METRIC_MODE = str( + "Trying to sample a configuration from {cls}, but the `metric` " + "({metric}) or `mode` ({mode}) parameters have not been set. " + "Either pass these arguments when instantiating the search algorithm, " + "or pass them to `tune.run()`.") + class Searcher: """Abstract class for wrapping suggesting algorithms. diff --git a/python/ray/tune/suggest/zoopt.py b/python/ray/tune/suggest/zoopt.py index 64a51525b..23177ddaf 100644 --- a/python/ray/tune/suggest/zoopt.py +++ b/python/ray/tune/suggest/zoopt.py @@ -6,7 +6,8 @@ import ray import ray.cloudpickle as pickle from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized, \ Uniform -from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE +from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ + UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars from ray.tune.utils.util import unflatten_dict from zoopt import ValueType @@ -208,10 +209,14 @@ class ZOOptSearch(Searcher): def suggest(self, trial_id: str) -> Optional[Dict]: if not self._dim_dict or not self.optimizer: raise RuntimeError( - "Trying to sample a configuration from {}, but no search " - "space has been defined. Either pass the `{}` argument when " - "instantiating the search algorithm, or pass a `config` to " - "`tune.run()`.".format(self.__class__.__name__, "space")) + UNDEFINED_SEARCH_SPACE.format( + cls=self.__class__.__name__, space="dim_dict")) + if not self._metric or not self._mode: + raise RuntimeError( + UNDEFINED_METRIC_MODE.format( + cls=self.__class__.__name__, + metric=self._metric, + mode=self._mode)) _solution = self.optimizer.suggest() diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py index a95d154c0..f8951aa13 100644 --- a/python/ray/tune/tests/test_sample.py +++ b/python/ray/tune/tests/test_sample.py @@ -194,11 +194,11 @@ class SearchSpaceTest(unittest.TestCase): client1 = AxClient(random_seed=1234) client1.create_experiment(parameters=converted_config) - searcher1 = AxSearch(ax_client=client1) + searcher1 = AxSearch(ax_client=client1, metric="a", mode="max") client2 = AxClient(random_seed=1234) client2.create_experiment(parameters=ax_config) - searcher2 = AxSearch(ax_client=client2) + searcher2 = AxSearch(ax_client=client2, metric="a", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") @@ -240,8 +240,10 @@ class SearchSpaceTest(unittest.TestCase): bayesopt_config = {"b/z": (1e-4, 1e-2)} converted_config = BayesOptSearch.convert_search_space(config) - searcher1 = BayesOptSearch(space=converted_config, metric="none") - searcher2 = BayesOptSearch(space=bayesopt_config, metric="none") + searcher1 = BayesOptSearch( + space=converted_config, metric="none", mode="max") + searcher2 = BayesOptSearch( + space=bayesopt_config, metric="none", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") @@ -295,8 +297,8 @@ class SearchSpaceTest(unittest.TestCase): converted_config.seed(1234) bohb_config.seed(1234) - searcher1 = TuneBOHB(space=converted_config) - searcher2 = TuneBOHB(space=bohb_config) + searcher1 = TuneBOHB(space=converted_config, metric="a", mode="max") + searcher2 = TuneBOHB(space=bohb_config, metric="a", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") @@ -356,7 +358,8 @@ class SearchSpaceTest(unittest.TestCase): optimizer="bandit", domain="euclidean", space=converted_config, - metric="none") + metric="none", + mode="max") config1 = searcher1.suggest("0") @@ -365,7 +368,8 @@ class SearchSpaceTest(unittest.TestCase): optimizer="bandit", domain="euclidean", space=dragonfly_config, - metric="none") + metric="none", + mode="max") config2 = searcher2.suggest("0") self.assertEqual(config1, config2) @@ -424,9 +428,15 @@ class SearchSpaceTest(unittest.TestCase): } searcher1 = HyperOptSearch( - space=converted_config, random_state_seed=1234) + space=converted_config, + random_state_seed=1234, + metric="a", + mode="max") searcher2 = HyperOptSearch( - space=hyperopt_config, random_state_seed=1234) + space=hyperopt_config, + random_state_seed=1234, + metric="a", + mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") @@ -516,9 +526,15 @@ class SearchSpaceTest(unittest.TestCase): z=ng.p.Log(lower=1e-4, upper=1e-2))) searcher1 = NevergradSearch( - optimizer=ng.optimizers.OnePlusOne, space=converted_config) + optimizer=ng.optimizers.OnePlusOne, + space=converted_config, + metric="a", + mode="max") searcher2 = NevergradSearch( - optimizer=ng.optimizers.OnePlusOne, space=nevergrad_config) + optimizer=ng.optimizers.OnePlusOne, + space=nevergrad_config, + metric="a", + mode="max") np.random.seed(1234) config1 = searcher1.suggest("0") @@ -571,10 +587,12 @@ class SearchSpaceTest(unittest.TestCase): ] sampler1 = RandomSampler(seed=1234) - searcher1 = OptunaSearch(space=converted_config, sampler=sampler1) + searcher1 = OptunaSearch( + space=converted_config, sampler=sampler1, metric="a", mode="max") sampler2 = RandomSampler(seed=1234) - searcher2 = OptunaSearch(space=optuna_config, sampler=sampler2) + searcher2 = OptunaSearch( + space=optuna_config, sampler=sampler2, metric="a", mode="max") config1 = searcher1.suggest("0") config2 = searcher2.suggest("0") @@ -614,8 +632,8 @@ class SearchSpaceTest(unittest.TestCase): converted_config = SkOptSearch.convert_search_space(config) skopt_config = {"a": [2, 3, 4], "b/x": (0, 5), "b/z": (1e-4, 1e-2)} - searcher1 = SkOptSearch(space=converted_config) - searcher2 = SkOptSearch(space=skopt_config) + searcher1 = SkOptSearch(space=converted_config, metric="a", mode="max") + searcher2 = SkOptSearch(space=skopt_config, metric="a", mode="max") np.random.seed(1234) config1 = searcher1.suggest("0") @@ -675,9 +693,17 @@ class SearchSpaceTest(unittest.TestCase): zoopt_search_config = {"parallel_num": 4} searcher1 = ZOOptSearch( - dim_dict=converted_config, budget=5, **zoopt_search_config) + dim_dict=converted_config, + budget=5, + metric="a", + mode="max", + **zoopt_search_config) searcher2 = ZOOptSearch( - dim_dict=zoopt_config, budget=5, **zoopt_search_config) + dim_dict=zoopt_config, + budget=5, + metric="a", + mode="max", + **zoopt_search_config) np.random.seed(1234) config1 = searcher1.suggest("0")