From d4b0a9fadf1120fbc2e1cf6557c2d37d5844c4d2 Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Sat, 9 Jan 2021 18:21:49 +0100 Subject: [PATCH] [tune] convert search spaces: parse spec before flattening (#12785) * Parse spec before flattening * flatten after parse * Test for ValueError if grid search is passed to search algorithms --- python/ray/tune/suggest/ax.py | 8 +++-- python/ray/tune/suggest/bayesopt.py | 5 ++- python/ray/tune/suggest/bohb.py | 8 +++-- python/ray/tune/suggest/dragonfly.py | 5 ++- python/ray/tune/suggest/nevergrad.py | 8 +++-- python/ray/tune/suggest/optuna.py | 8 +++-- python/ray/tune/suggest/skopt.py | 5 ++- python/ray/tune/tests/test_sample.py | 50 ++++++++++++++++++++++++++++ 8 files changed, 82 insertions(+), 15 deletions(-) diff --git a/python/ray/tune/suggest/ax.py b/python/ray/tune/suggest/ax.py index 5f15611bf..7cccf74a7 100644 --- a/python/ray/tune/suggest/ax.py +++ b/python/ray/tune/suggest/ax.py @@ -8,8 +8,7 @@ from ray.tune.sample import Categorical, Float, Integer, LogUniform, \ from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars -from ray.tune.utils import flatten_dict -from ray.tune.utils.util import unflatten_dict +from ray.tune.utils.util import flatten_dict, unflatten_dict try: import ax @@ -269,7 +268,6 @@ class AxSearch(Searcher): @staticmethod def convert_search_space(spec: Dict): - spec = flatten_dict(spec, prevent_delimiter=True) resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) if grid_vars: @@ -277,6 +275,10 @@ class AxSearch(Searcher): "Grid search parameters cannot be automatically converted " "to an Ax search space.") + # Flatten and resolve again after checking for grid search. + spec = flatten_dict(spec, prevent_delimiter=True) + resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) + def resolve_value(par, domain): sampler = domain.get_sampler() if isinstance(sampler, Quantized): diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py index 489e92709..7856c1152 100644 --- a/python/ray/tune/suggest/bayesopt.py +++ b/python/ray/tune/suggest/bayesopt.py @@ -366,7 +366,6 @@ class BayesOptSearch(Searcher): @staticmethod def convert_search_space(spec: Dict, join: bool = False) -> Dict: - spec = flatten_dict(spec, prevent_delimiter=True) resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) if grid_vars: @@ -374,6 +373,10 @@ class BayesOptSearch(Searcher): "Grid search parameters cannot be automatically converted " "to a BayesOpt search space.") + # Flatten and resolve again after checking for grid search. + spec = flatten_dict(spec, prevent_delimiter=True) + resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) + def resolve_value(domain: Domain) -> Tuple[float, float]: sampler = domain.get_sampler() if isinstance(sampler, Quantized): diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 21de8fe14..23c0811fc 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -15,8 +15,7 @@ from ray.tune.suggest import Searcher from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars -from ray.tune.utils import flatten_dict -from ray.tune.utils.util import unflatten_dict +from ray.tune.utils.util import flatten_dict, unflatten_dict logger = logging.getLogger(__name__) @@ -238,7 +237,6 @@ class TuneBOHB(Searcher): @staticmethod def convert_search_space(spec: Dict) -> ConfigSpace.ConfigurationSpace: - spec = flatten_dict(spec, prevent_delimiter=True) resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) if grid_vars: @@ -246,6 +244,10 @@ class TuneBOHB(Searcher): "Grid search parameters cannot be automatically converted " "to a TuneBOHB search space.") + # Flatten and resolve again after checking for grid search. + spec = flatten_dict(spec, prevent_delimiter=True) + resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) + def resolve_value(par: str, domain: Domain ) -> ConfigSpace.hyperparameters.Hyperparameter: quantize = None diff --git a/python/ray/tune/suggest/dragonfly.py b/python/ray/tune/suggest/dragonfly.py index 3037fbbbf..a46cea143 100644 --- a/python/ray/tune/suggest/dragonfly.py +++ b/python/ray/tune/suggest/dragonfly.py @@ -344,7 +344,6 @@ class DragonflySearch(Searcher): @staticmethod def convert_search_space(spec: Dict) -> List[Dict]: - spec = flatten_dict(spec, prevent_delimiter=True) resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) if grid_vars: @@ -352,6 +351,10 @@ class DragonflySearch(Searcher): "Grid search parameters cannot be automatically converted " "to a Dragonfly search space.") + # Flatten and resolve again after checking for grid search. + spec = flatten_dict(spec, prevent_delimiter=True) + resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) + def resolve_value(par: str, domain: Domain) -> Dict: sampler = domain.get_sampler() if isinstance(sampler, Quantized): diff --git a/python/ray/tune/suggest/nevergrad.py b/python/ray/tune/suggest/nevergrad.py index 8df7269ae..93ecbd5a7 100644 --- a/python/ray/tune/suggest/nevergrad.py +++ b/python/ray/tune/suggest/nevergrad.py @@ -8,8 +8,7 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \ from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars -from ray.tune.utils import flatten_dict -from ray.tune.utils.util import unflatten_dict +from ray.tune.utils.util import flatten_dict, unflatten_dict try: import nevergrad as ng @@ -287,7 +286,6 @@ class NevergradSearch(Searcher): @staticmethod def convert_search_space(spec: Dict) -> Parameter: - spec = flatten_dict(spec, prevent_delimiter=True) resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) if grid_vars: @@ -295,6 +293,10 @@ class NevergradSearch(Searcher): "Grid search parameters cannot be automatically converted " "to a Nevergrad search space.") + # Flatten and resolve again after checking for grid search. + spec = flatten_dict(spec, prevent_delimiter=True) + resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) + def resolve_value(domain: Domain) -> Parameter: sampler = domain.get_sampler() if isinstance(sampler, Quantized): diff --git a/python/ray/tune/suggest/optuna.py b/python/ray/tune/suggest/optuna.py index f2947e895..a6468b861 100644 --- a/python/ray/tune/suggest/optuna.py +++ b/python/ray/tune/suggest/optuna.py @@ -8,8 +8,7 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \ from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \ UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE from ray.tune.suggest.variant_generator import parse_spec_vars -from ray.tune.utils import flatten_dict -from ray.tune.utils.util import unflatten_dict +from ray.tune.utils.util import flatten_dict, unflatten_dict try: import optuna as ot @@ -240,7 +239,6 @@ class OptunaSearch(Searcher): @staticmethod def convert_search_space(spec: Dict) -> List[Tuple]: - spec = flatten_dict(spec, prevent_delimiter=True) resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) if not domain_vars and not grid_vars: @@ -251,6 +249,10 @@ class OptunaSearch(Searcher): "Grid search parameters cannot be automatically converted " "to an Optuna search space.") + # Flatten and resolve again after checking for grid search. + spec = flatten_dict(spec, prevent_delimiter=True) + resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) + def resolve_value(par: str, domain: Domain) -> Tuple: quantize = None diff --git a/python/ray/tune/suggest/skopt.py b/python/ray/tune/suggest/skopt.py index 7c4f337af..23f24e437 100644 --- a/python/ray/tune/suggest/skopt.py +++ b/python/ray/tune/suggest/skopt.py @@ -319,7 +319,6 @@ class SkOptSearch(Searcher): @staticmethod def convert_search_space(spec: Dict, join: bool = False) -> Dict: - spec = flatten_dict(spec, prevent_delimiter=True) resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) if grid_vars: @@ -327,6 +326,10 @@ class SkOptSearch(Searcher): "Grid search parameters cannot be automatically converted " "to a SkOpt search space.") + # Flatten and resolve again after checking for grid search. + spec = flatten_dict(spec, prevent_delimiter=True) + resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec) + def resolve_value(domain: Domain) -> Union[Tuple, List]: sampler = domain.get_sampler() if isinstance(sampler, Quantized): diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py index 1b6821439..378a2c1ef 100644 --- a/python/ray/tune/tests/test_sample.py +++ b/python/ray/tune/tests/test_sample.py @@ -197,6 +197,10 @@ class SearchSpaceTest(unittest.TestCase): from ray.tune.suggest.ax import AxSearch from ax.service.ax_client import AxClient + # Grid search not supported, should raise ValueError + with self.assertRaises(ValueError): + AxSearch.convert_search_space({"grid": tune.grid_search([0, 1])}) + config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { @@ -265,6 +269,12 @@ class SearchSpaceTest(unittest.TestCase): def testConvertBayesOpt(self): from ray.tune.suggest.bayesopt import BayesOptSearch + # Grid search not supported, should raise ValueError + with self.assertRaises(ValueError): + BayesOptSearch.convert_search_space({ + "grid": tune.grid_search([0, 1]) + }) + config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { @@ -320,6 +330,10 @@ class SearchSpaceTest(unittest.TestCase): from ray.tune.suggest.bohb import TuneBOHB import ConfigSpace + # Grid search not supported, should raise ValueError + with self.assertRaises(ValueError): + TuneBOHB.convert_search_space({"grid": tune.grid_search([0, 1])}) + config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { @@ -372,6 +386,12 @@ class SearchSpaceTest(unittest.TestCase): def testConvertDragonfly(self): from ray.tune.suggest.dragonfly import DragonflySearch + # Grid search not supported, should raise ValueError + with self.assertRaises(ValueError): + DragonflySearch.convert_search_space({ + "grid": tune.grid_search([0, 1]) + }) + config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { @@ -453,6 +473,12 @@ class SearchSpaceTest(unittest.TestCase): from ray.tune.suggest.hyperopt import HyperOptSearch from hyperopt import hp + # Grid search not supported, should raise ValueError + with self.assertRaises(ValueError): + HyperOptSearch.convert_search_space({ + "grid": tune.grid_search([0, 1]) + }) + config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { @@ -554,6 +580,12 @@ class SearchSpaceTest(unittest.TestCase): from ray.tune.suggest.nevergrad import NevergradSearch import nevergrad as ng + # Grid search not supported, should raise ValueError + with self.assertRaises(ValueError): + NevergradSearch.convert_search_space({ + "grid": tune.grid_search([0, 1]) + }) + config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { @@ -615,6 +647,12 @@ class SearchSpaceTest(unittest.TestCase): from ray.tune.suggest.optuna import OptunaSearch, param from optuna.samplers import RandomSampler + # Grid search not supported, should raise ValueError + with self.assertRaises(ValueError): + OptunaSearch.convert_search_space({ + "grid": tune.grid_search([0, 1]) + }) + config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { @@ -666,6 +704,12 @@ class SearchSpaceTest(unittest.TestCase): from ray.tune.suggest.skopt import SkOptSearch from skopt.space import Real, Integer + # Grid search not supported, should raise ValueError + with self.assertRaises(ValueError): + SkOptSearch.convert_search_space({ + "grid": tune.grid_search([0, 1]) + }) + config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { @@ -712,6 +756,12 @@ class SearchSpaceTest(unittest.TestCase): from ray.tune.suggest.zoopt import ZOOptSearch from zoopt import ValueType + # Grid search not supported, should raise ValueError + with self.assertRaises(ValueError): + ZOOptSearch.convert_search_space({ + "grid": tune.grid_search([0, 1]) + }) + config = { "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": {