[tune] convert search spaces: parse spec before flattening (#12785)

* Parse spec before flattening

* flatten after parse

* Test for ValueError if grid search is passed to search algorithms

python/ray/tune/suggest/ax.py

@@ -8,8 +8,7 @@ from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
-from ray.tune.utils import flatten_dict
-from ray.tune.utils.util import unflatten_dict
+from ray.tune.utils.util import flatten_dict, unflatten_dict
 
 try:
     import ax
@@ -269,7 +268,6 @@ class AxSearch(Searcher):
 
     @staticmethod
     def convert_search_space(spec: Dict):
-        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if grid_vars:
@@ -277,6 +275,10 @@ class AxSearch(Searcher):
                 "Grid search parameters cannot be automatically converted "
                 "to an Ax search space.")
 
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
         def resolve_value(par, domain):
             sampler = domain.get_sampler()
             if isinstance(sampler, Quantized):

python/ray/tune/suggest/bayesopt.py

@@ -366,7 +366,6 @@ class BayesOptSearch(Searcher):
 
     @staticmethod
     def convert_search_space(spec: Dict, join: bool = False) -> Dict:
-        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if grid_vars:
@@ -374,6 +373,10 @@ class BayesOptSearch(Searcher):
                 "Grid search parameters cannot be automatically converted "
                 "to a BayesOpt search space.")
 
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
         def resolve_value(domain: Domain) -> Tuple[float, float]:
             sampler = domain.get_sampler()
             if isinstance(sampler, Quantized):

python/ray/tune/suggest/bohb.py

@@ -15,8 +15,7 @@ from ray.tune.suggest import Searcher
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
-from ray.tune.utils import flatten_dict
-from ray.tune.utils.util import unflatten_dict
+from ray.tune.utils.util import flatten_dict, unflatten_dict
 
 logger = logging.getLogger(__name__)
 
@@ -238,7 +237,6 @@ class TuneBOHB(Searcher):
 
     @staticmethod
     def convert_search_space(spec: Dict) -> ConfigSpace.ConfigurationSpace:
-        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if grid_vars:
@@ -246,6 +244,10 @@ class TuneBOHB(Searcher):
                 "Grid search parameters cannot be automatically converted "
                 "to a TuneBOHB search space.")
 
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
         def resolve_value(par: str, domain: Domain
                           ) -> ConfigSpace.hyperparameters.Hyperparameter:
             quantize = None

python/ray/tune/suggest/dragonfly.py

@@ -344,7 +344,6 @@ class DragonflySearch(Searcher):
 
     @staticmethod
     def convert_search_space(spec: Dict) -> List[Dict]:
-        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if grid_vars:
@@ -352,6 +351,10 @@ class DragonflySearch(Searcher):
                 "Grid search parameters cannot be automatically converted "
                 "to a Dragonfly search space.")
 
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
         def resolve_value(par: str, domain: Domain) -> Dict:
             sampler = domain.get_sampler()
             if isinstance(sampler, Quantized):

python/ray/tune/suggest/nevergrad.py

@@ -8,8 +8,7 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
-from ray.tune.utils import flatten_dict
-from ray.tune.utils.util import unflatten_dict
+from ray.tune.utils.util import flatten_dict, unflatten_dict
 
 try:
     import nevergrad as ng
@@ -287,7 +286,6 @@ class NevergradSearch(Searcher):
 
     @staticmethod
     def convert_search_space(spec: Dict) -> Parameter:
-        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if grid_vars:
@@ -295,6 +293,10 @@ class NevergradSearch(Searcher):
                 "Grid search parameters cannot be automatically converted "
                 "to a Nevergrad search space.")
 
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
         def resolve_value(domain: Domain) -> Parameter:
             sampler = domain.get_sampler()
             if isinstance(sampler, Quantized):

python/ray/tune/suggest/optuna.py

@@ -8,8 +8,7 @@ from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
-from ray.tune.utils import flatten_dict
-from ray.tune.utils.util import unflatten_dict
+from ray.tune.utils.util import flatten_dict, unflatten_dict
 
 try:
     import optuna as ot
@@ -240,7 +239,6 @@ class OptunaSearch(Searcher):
 
     @staticmethod
     def convert_search_space(spec: Dict) -> List[Tuple]:
-        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if not domain_vars and not grid_vars:
@@ -251,6 +249,10 @@ class OptunaSearch(Searcher):
                 "Grid search parameters cannot be automatically converted "
                 "to an Optuna search space.")
 
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
         def resolve_value(par: str, domain: Domain) -> Tuple:
             quantize = None
 

python/ray/tune/suggest/skopt.py

@@ -319,7 +319,6 @@ class SkOptSearch(Searcher):
 
     @staticmethod
     def convert_search_space(spec: Dict, join: bool = False) -> Dict:
-        spec = flatten_dict(spec, prevent_delimiter=True)
         resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
 
         if grid_vars:
@@ -327,6 +326,10 @@ class SkOptSearch(Searcher):
                 "Grid search parameters cannot be automatically converted "
                 "to a SkOpt search space.")
 
+        # Flatten and resolve again after checking for grid search.
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
         def resolve_value(domain: Domain) -> Union[Tuple, List]:
             sampler = domain.get_sampler()
             if isinstance(sampler, Quantized):

python/ray/tune/tests/test_sample.py

@@ -197,6 +197,10 @@ class SearchSpaceTest(unittest.TestCase):
         from ray.tune.suggest.ax import AxSearch
         from ax.service.ax_client import AxClient
 
+        # Grid search not supported, should raise ValueError
+        with self.assertRaises(ValueError):
+            AxSearch.convert_search_space({"grid": tune.grid_search([0, 1])})
+
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
@@ -265,6 +269,12 @@ class SearchSpaceTest(unittest.TestCase):
     def testConvertBayesOpt(self):
         from ray.tune.suggest.bayesopt import BayesOptSearch
 
+        # Grid search not supported, should raise ValueError
+        with self.assertRaises(ValueError):
+            BayesOptSearch.convert_search_space({
+                "grid": tune.grid_search([0, 1])
+            })
+
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
@@ -320,6 +330,10 @@ class SearchSpaceTest(unittest.TestCase):
         from ray.tune.suggest.bohb import TuneBOHB
         import ConfigSpace
 
+        # Grid search not supported, should raise ValueError
+        with self.assertRaises(ValueError):
+            TuneBOHB.convert_search_space({"grid": tune.grid_search([0, 1])})
+
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
@@ -372,6 +386,12 @@ class SearchSpaceTest(unittest.TestCase):
     def testConvertDragonfly(self):
         from ray.tune.suggest.dragonfly import DragonflySearch
 
+        # Grid search not supported, should raise ValueError
+        with self.assertRaises(ValueError):
+            DragonflySearch.convert_search_space({
+                "grid": tune.grid_search([0, 1])
+            })
+
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
@@ -453,6 +473,12 @@ class SearchSpaceTest(unittest.TestCase):
         from ray.tune.suggest.hyperopt import HyperOptSearch
         from hyperopt import hp
 
+        # Grid search not supported, should raise ValueError
+        with self.assertRaises(ValueError):
+            HyperOptSearch.convert_search_space({
+                "grid": tune.grid_search([0, 1])
+            })
+
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
@@ -554,6 +580,12 @@ class SearchSpaceTest(unittest.TestCase):
         from ray.tune.suggest.nevergrad import NevergradSearch
         import nevergrad as ng
 
+        # Grid search not supported, should raise ValueError
+        with self.assertRaises(ValueError):
+            NevergradSearch.convert_search_space({
+                "grid": tune.grid_search([0, 1])
+            })
+
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
@@ -615,6 +647,12 @@ class SearchSpaceTest(unittest.TestCase):
         from ray.tune.suggest.optuna import OptunaSearch, param
         from optuna.samplers import RandomSampler
 
+        # Grid search not supported, should raise ValueError
+        with self.assertRaises(ValueError):
+            OptunaSearch.convert_search_space({
+                "grid": tune.grid_search([0, 1])
+            })
+
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
@@ -666,6 +704,12 @@ class SearchSpaceTest(unittest.TestCase):
         from ray.tune.suggest.skopt import SkOptSearch
         from skopt.space import Real, Integer
 
+        # Grid search not supported, should raise ValueError
+        with self.assertRaises(ValueError):
+            SkOptSearch.convert_search_space({
+                "grid": tune.grid_search([0, 1])
+            })
+
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {
@@ -712,6 +756,12 @@ class SearchSpaceTest(unittest.TestCase):
         from ray.tune.suggest.zoopt import ZOOptSearch
         from zoopt import ValueType
 
+        # Grid search not supported, should raise ValueError
+        with self.assertRaises(ValueError):
+            ZOOptSearch.convert_search_space({
+                "grid": tune.grid_search([0, 1])
+            })
+
         config = {
             "a": tune.sample.Categorical([2, 3, 4]).uniform(),
             "b": {