wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-19 11:27:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tune] better error when metric or mode unset in search algorithms (#11646)

Browse Source
This commit is contained in:
Kai Fricke
2020-10-28 13:17:59 -07:00
committed by GitHub
parent 58891551d3
commit ba63ded311
11 changed files with 151 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/ray/tune/suggest/ax.py
+11 -5
View File
@@ -3,7 +3,8 @@ from typing import Dict, List, Optional, Union
from ax.service.ax_client import AxClient
from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
Quantized, Uniform
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
from ray.tune.suggest.variant_generator import parse_spec_vars
from ray.tune.utils import flatten_dict
from ray.tune.utils.util import unflatten_dict
@@ -203,10 +204,15 @@ class AxSearch(Searcher):
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._ax:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self.max_concurrent:
if len(self._live_trial_mapping) >= self.max_concurrent:
python/ray/tune/suggest/bayesopt.py
+11 -5
View File
@@ -6,7 +6,8 @@ from typing import Dict, Optional, Tuple
from ray.tune import ExperimentAnalysis
from ray.tune.sample import Domain, Float, Quantized
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
from ray.tune.suggest.variant_generator import parse_spec_vars
from ray.tune.utils.util import unflatten_dict
@@ -242,10 +243,15 @@ class BayesOptSearch(Searcher):
"""
if not self.optimizer:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
# If we have more active trials than the allowed maximum
total_live_trials = len(self._live_trial_mapping)
python/ray/tune/suggest/bohb.py
+11 -5
View File
@@ -11,7 +11,8 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
Quantized, \
Uniform
from ray.tune.suggest import Searcher
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
from ray.tune.suggest.variant_generator import parse_spec_vars
from ray.tune.utils import flatten_dict
from ray.tune.utils.util import unflatten_dict
@@ -156,10 +157,15 @@ class TuneBOHB(Searcher):
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._space:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if len(self.running) < self._max_concurrent:
# This parameter is not used in hpbandster implementation.
python/ray/tune/suggest/dragonfly.py
+11 -5
View File
@@ -8,7 +8,8 @@ import pickle
from typing import Dict, List, Optional, Union
from ray.tune.sample import Domain, Float, Quantized
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
from ray.tune.suggest.variant_generator import parse_spec_vars
from ray.tune.utils.util import flatten_dict
@@ -272,10 +273,15 @@ class DragonflySearch(Searcher):
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._opt:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self._initial_points:
suggested_config = self._initial_points[0]
python/ray/tune/suggest/hyperopt.py
+11 -5
View File
@@ -10,7 +10,8 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
Normal, \
Quantized, \
Uniform
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
from ray.tune.suggest.variant_generator import assign_value, parse_spec_vars
try:
@@ -200,10 +201,15 @@ class HyperOptSearch(Searcher):
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self.domain:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self.max_concurrent:
if len(self._live_trial_mapping) >= self.max_concurrent:
return None
python/ray/tune/suggest/nevergrad.py
+10 -5
View File
@@ -4,7 +4,8 @@ from typing import Dict, Optional, Union
from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
Quantized
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
from ray.tune.suggest.variant_generator import parse_spec_vars
from ray.tune.utils import flatten_dict
from ray.tune.utils.util import unflatten_dict
@@ -189,10 +190,14 @@ class NevergradSearch(Searcher):
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._nevergrad_opt:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self.max_concurrent:
if len(self._live_trial_mapping) >= self.max_concurrent:
python/ray/tune/suggest/optuna.py
+10 -5
View File
@@ -5,7 +5,8 @@ from typing import Dict, List, Optional, Tuple, Union
from ray.tune.result import TRAINING_ITERATION
from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
Quantized, Uniform
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
from ray.tune.suggest.variant_generator import parse_spec_vars
from ray.tune.utils import flatten_dict
from ray.tune.utils.util import unflatten_dict
@@ -165,10 +166,14 @@ class OptunaSearch(Searcher):
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._space:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if trial_id not in self._ot_trials:
ot_trial_id = self._storage.create_new_trial(
python/ray/tune/suggest/skopt.py
+10 -5
View File
@@ -3,7 +3,8 @@ import pickle
from typing import Dict, List, Optional, Tuple, Union
from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
from ray.tune.suggest.variant_generator import parse_spec_vars
from ray.tune.utils import flatten_dict
from ray.tune.utils.util import unflatten_dict
@@ -230,10 +231,14 @@ class SkOptSearch(Searcher):
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._skopt_opt:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
if self.max_concurrent:
if len(self._live_trial_mapping) >= self.max_concurrent:
python/ray/tune/suggest/suggestion.py
+12
View File
@@ -15,6 +15,18 @@ UNRESOLVED_SEARCH_SPACE = str(
"conversion, pass the space definition as part of the `config` argument "
"to `tune.run()` instead.")
UNDEFINED_SEARCH_SPACE = str(
"Trying to sample a configuration from {cls}, but no search "
"space has been defined. Either pass the `{space}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.")
UNDEFINED_METRIC_MODE = str(
"Trying to sample a configuration from {cls}, but the `metric` "
"({metric}) or `mode` ({mode}) parameters have not been set. "
"Either pass these arguments when instantiating the search algorithm, "
"or pass them to `tune.run()`.")
class Searcher:
"""Abstract class for wrapping suggesting algorithms.
python/ray/tune/suggest/zoopt.py
+10 -5
View File
@@ -6,7 +6,8 @@ import ray
import ray.cloudpickle as pickle
from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized, \
Uniform
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE
from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
from ray.tune.suggest.variant_generator import parse_spec_vars
from ray.tune.utils.util import unflatten_dict
from zoopt import ValueType
@@ -208,10 +209,14 @@ class ZOOptSearch(Searcher):
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._dim_dict or not self.optimizer:
raise RuntimeError(
"Trying to sample a configuration from {}, but no search "
"space has been defined. Either pass the `{}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`.".format(self.__class__.__name__, "space"))
UNDEFINED_SEARCH_SPACE.format(
cls=self.__class__.__name__, space="dim_dict"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(
cls=self.__class__.__name__,
metric=self._metric,
mode=self._mode))
_solution = self.optimizer.suggest()