[tune] Add algorithms for search space conversion (#10621)

python/ray/tune/examples/bohb_example.py

@@ -6,7 +6,8 @@ import os
 import numpy as np
 
 import ray
-from ray.tune import Trainable, run
+from ray import tune
+from ray.tune import Trainable
 from ray.tune.schedulers.hb_bohb import HyperBandForBOHB
 from ray.tune.suggest.bohb import TuneBOHB
 
@@ -42,27 +43,43 @@ class MyTrainableClass(Trainable):
 
 
 if __name__ == "__main__":
-    import ConfigSpace as CS
+    import ConfigSpace as CS  # noqa: F401
     ray.init(num_cpus=8)
 
-    # BOHB uses ConfigSpace for their hyperparameter search space
-    config_space = CS.ConfigurationSpace()
-    config_space.add_hyperparameter(
-        CS.UniformFloatHyperparameter("height", lower=10, upper=100))
-    config_space.add_hyperparameter(
-        CS.UniformFloatHyperparameter("width", lower=0, upper=100))
+    config = {
+        "iterations": 100,
+        "width": tune.uniform(0, 20),
+        "height": tune.uniform(-100, 100),
+        "activation": tune.choice(["relu", "tanh"])
+    }
+
+    # Optional: Pass the parameter space yourself
+    # config_space = CS.ConfigurationSpace()
+    # config_space.add_hyperparameter(
+    #     CS.UniformFloatHyperparameter("width", lower=0, upper=20))
+    # config_space.add_hyperparameter(
+    #     CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
+    # config_space.add_hyperparameter(
+    #     CS.CategoricalHyperparameter(
+    #         "activation", choices=["relu", "tanh"]))
 
     experiment_metrics = dict(metric="episode_reward_mean", mode="max")
+
     bohb_hyperband = HyperBandForBOHB(
         time_attr="training_iteration",
         max_t=100,
         reduction_factor=4,
         **experiment_metrics)
-    bohb_search = TuneBOHB(
-        config_space, max_concurrent=4, **experiment_metrics)
 
-    run(MyTrainableClass,
+    bohb_search = TuneBOHB(
+        # space=config_space,  # If you want to set the space manually
+        max_concurrent=4,
+        **experiment_metrics)
+
+    tune.run(
+        MyTrainableClass,
         name="bohb_test",
+        config=config,
         scheduler=bohb_hyperband,
         search_alg=bohb_search,
         num_samples=10,

python/ray/tune/examples/dragonfly_example.py

@@ -31,9 +31,6 @@ def objective(config):
 
 if __name__ == "__main__":
     import argparse
-    from dragonfly.opt.gp_bandit import EuclideanGPBandit
-    from dragonfly.exd.experiment_caller import EuclideanFunctionCaller
-    from dragonfly import load_config
 
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -41,40 +38,45 @@ if __name__ == "__main__":
     args, _ = parser.parse_known_args()
     ray.init()
 
-    config = {
+    tune_kwargs = {
         "num_samples": 10 if args.smoke_test else 50,
         "config": {
             "iterations": 100,
+            "LiNO3_vol": tune.uniform(0, 7),
+            "Li2SO4_vol": tune.uniform(0, 7),
+            "NaClO4_vol": tune.uniform(0, 7)
         },
     }
 
-    domain_vars = [{
-        "name": "LiNO3_vol",
-        "type": "float",
-        "min": 0,
-        "max": 7
-    }, {
-        "name": "Li2SO4_vol",
-        "type": "float",
-        "min": 0,
-        "max": 7
-    }, {
-        "name": "NaClO4_vol",
-        "type": "float",
-        "min": 0,
-        "max": 7
-    }]
+    # Optional: Pass the parameter space yourself
+    # space = [{
+    #     "name": "LiNO3_vol",
+    #     "type": "float",
+    #     "min": 0,
+    #     "max": 7
+    # }, {
+    #     "name": "Li2SO4_vol",
+    #     "type": "float",
+    #     "min": 0,
+    #     "max": 7
+    # }, {
+    #     "name": "NaClO4_vol",
+    #     "type": "float",
+    #     "min": 0,
+    #     "max": 7
+    # }]
 
-    domain_config = load_config({"domain": domain_vars})
+    df_search = DragonflySearch(
+        optimizer="bandit",
+        domain="euclidean",
+        # space=space,  # If you want to set the space manually
+        metric="objective",
+        mode="max")
 
-    func_caller = EuclideanFunctionCaller(
-        None, domain_config.domain.list_of_domains[0])
-    optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True)
-    algo = DragonflySearch(optimizer, metric="objective", mode="max")
     scheduler = AsyncHyperBandScheduler(metric="objective", mode="max")
     tune.run(
         objective,
         name="dragonfly_search",
-        search_alg=algo,
+        search_alg=df_search,
         scheduler=scheduler,
-        **config)
+        **tune_kwargs)

python/ray/tune/examples/nevergrad_example.py

@@ -28,7 +28,7 @@ def easy_objective(config):
 
 if __name__ == "__main__":
     import argparse
-    from nevergrad.optimization import optimizerlib
+    import nevergrad as ng
 
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -36,27 +36,35 @@ if __name__ == "__main__":
     args, _ = parser.parse_known_args()
     ray.init()
 
-    config = {
+    # The config will be automatically converted to Nevergrad's search space
+    tune_kwargs = {
         "num_samples": 10 if args.smoke_test else 50,
         "config": {
             "steps": 100,
+            "width": tune.uniform(0, 20),
+            "height": tune.uniform(-100, 100),
+            "activation": tune.choice(["relu", "tanh"])
         }
     }
-    instrumentation = 2
-    parameter_names = ["height", "width"]
-    # With nevergrad v0.2.0+ the following is also possible:
-    # from nevergrad import instrumentation as inst
-    # instrumentation = inst.Instrumentation(
-    #     height=inst.var.Array(1).bounded(0, 200).asfloat(),
-    #     width=inst.var.OrderedDiscrete([0, 10, 20, 30, 40, 50]))
-    # parameter_names = None  # names are provided by the instrumentation
-    optimizer = optimizerlib.OnePlusOne(instrumentation)
+
+    # Optional: Pass the parameter space yourself
+    # space = ng.p.Dict(
+    #     width=ng.p.Scalar(lower=0, upper=20),
+    #     height=ng.p.Scalar(lower=-100, upper=100),
+    #     activation=ng.p.Choice(choices=["relu", "tanh"])
+    # )
+
     algo = NevergradSearch(
-        optimizer, parameter_names, metric="mean_loss", mode="min")
+        optimizer=ng.optimizers.OnePlusOne,
+        # space=space,  # If you want to set the space manually
+        metric="mean_loss",
+        mode="min")
+
     scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
+
     tune.run(
         easy_objective,
         name="nevergrad",
         search_alg=algo,
         scheduler=scheduler,
-        **config)
+        **tune_kwargs)

python/ray/tune/examples/skopt_example.py

@@ -28,7 +28,6 @@ def easy_objective(config):
 
 if __name__ == "__main__":
     import argparse
-    from skopt import Optimizer
 
     parser = argparse.ArgumentParser()
     parser.add_argument(
@@ -36,40 +35,40 @@ if __name__ == "__main__":
     args, _ = parser.parse_known_args()
     ray.init()
 
-    config = {
+    # The config will be automatically converted to SkOpt's search space
+    tune_kwargs = {
         "num_samples": 10 if args.smoke_test else 50,
         "config": {
             "steps": 100,
+            "width": tune.uniform(0, 20),
+            "height": tune.uniform(-100, 100),
+            "activation": tune.choice(["relu", "tanh"])
         }
     }
-    optimizer = Optimizer([(0, 20), (-100, 100)])
-    previously_run_params = [[10, 0], [15, -20]]
+
+    # Optional: Pass the parameter space yourself
+    # space = {
+    #     "width": (0, 20),
+    #     "height": (-100, 100),
+    #     "activation": ["relu", "tanh"]
+    # }
+
+    previously_run_params = [[10, 0, "relu"], [15, -20, "tanh"]]
     known_rewards = [-189, -1144]
+
     algo = SkOptSearch(
-        optimizer, ["width", "height"],
+        # parameter_names=space.keys(),  # If you want to set the space
+        # parameter_ranges=space.values(), # If you want to set the space
         metric="mean_loss",
         mode="min",
         points_to_evaluate=previously_run_params,
         evaluated_rewards=known_rewards)
+
     scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
+
     tune.run(
         easy_objective,
         name="skopt_exp_with_warmstart",
         search_alg=algo,
         scheduler=scheduler,
-        **config)
-
-    # Now run the experiment without known rewards
-
-    algo = SkOptSearch(
-        optimizer, ["width", "height"],
-        metric="mean_loss",
-        mode="min",
-        points_to_evaluate=previously_run_params)
-    scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min")
-    tune.run(
-        easy_objective,
-        name="skopt_exp",
-        search_alg=algo,
-        scheduler=scheduler,
-        **config)
+        **tune_kwargs)

python/ray/tune/examples/zoopt_example.py

@@ -8,7 +8,7 @@ import ray
 from ray import tune
 from ray.tune.suggest.zoopt import ZOOptSearch
 from ray.tune.schedulers import AsyncHyperBandScheduler
-from zoopt import ValueType
+from zoopt import ValueType  # noqa: F401
 
 
 def evaluation_fn(step, width, height):
@@ -36,26 +36,27 @@ if __name__ == "__main__":
     args, _ = parser.parse_known_args()
     ray.init()
 
-    # This dict could mix continuous dimensions and discrete dimensions,
-    # for example:
-    dim_dict = {
-        # for continuous dimensions: (continuous, search_range, precision)
-        "height": (ValueType.CONTINUOUS, [-10, 10], 1e-2),
-        # for discrete dimensions: (discrete, search_range, has_order)
-        "width": (ValueType.DISCRETE, [0, 10], False)
-    }
-
-    config = {
+    tune_kwargs = {
         "num_samples": 10 if args.smoke_test else 1000,
         "config": {
-            "steps": 10,  # evaluation times
+            "steps": 10,
+            "height": tune.quniform(-10, 10, 1e-2),
+            "width": tune.randint(0, 10)
         }
     }
 
+    # Optional: Pass the parameter space yourself
+    # space = {
+    #     # for continuous dimensions: (continuous, search_range, precision)
+    #     "height": (ValueType.CONTINUOUS, [-10, 10], 1e-2),
+    #     # for discrete dimensions: (discrete, search_range, has_order)
+    #     "width": (ValueType.DISCRETE, [0, 10], True)
+    # }
+
     zoopt_search = ZOOptSearch(
         algo="Asracos",  # only support ASRacos currently
-        budget=config["num_samples"],
-        dim_dict=dim_dict,
+        budget=tune_kwargs["num_samples"],
+        # dim_dict=space,  # If you want to set the space yourself
         metric="mean_loss",
         mode="min")
 
@@ -66,4 +67,4 @@ if __name__ == "__main__":
         search_alg=zoopt_search,
         name="zoopt_search",
         scheduler=scheduler,
-        **config)
+        **tune_kwargs)

python/ray/tune/sample.py

@@ -3,7 +3,6 @@ import random
 from copy import copy
 from inspect import signature
 from math import isclose
-from numbers import Number
 from typing import Any, Callable, Dict, List, Optional, Sequence, Union
 
 import numpy as np
@@ -223,7 +222,7 @@ class Integer(Domain):
     def cast(self, value):
         return int(value)
 
-    def quantized(self, q: Number):
+    def quantized(self, q: int):
         new = copy(self)
         new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
         return new
@@ -298,7 +297,7 @@ class Function(Domain):
 
 
 class Quantized(Sampler):
-    def __init__(self, sampler: Sampler, q: Number):
+    def __init__(self, sampler: Sampler, q: Union[float, int]):
         self.sampler = sampler
         self.q = q
 

python/ray/tune/suggest/bohb.py

@@ -2,8 +2,17 @@
 
 import copy
 import logging
+import math
+from typing import Dict
 
+import ConfigSpace
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, Normal, \
+    Quantized, \
+    Uniform
 from ray.tune.suggest import Searcher
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils import flatten_dict
+from ray.tune.utils.util import unflatten_dict
 
 logger = logging.getLogger(__name__)
 
@@ -37,7 +46,26 @@ class TuneBOHB(Searcher):
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
 
-    Example:
+    Tune automatically converts search spaces to TuneBOHB's format:
+
+    .. code-block:: python
+
+        config = {
+            "width": tune.uniform(0, 20),
+            "height": tune.uniform(-100, 100),
+            "activation": tune.choice(["relu", "tanh"])
+        }
+
+        algo = TuneBOHB(max_concurrent=4, metric="mean_loss", mode="min")
+        bohb = HyperBandForBOHB(
+            time_attr="training_iteration",
+            metric="mean_loss",
+            mode="min",
+            max_t=100)
+        run(my_trainable, config=config, scheduler=bohb, search_alg=algo)
+
+    If you would like to pass the search space manually, the code would
+    look like this:
 
     .. code-block:: python
 
@@ -45,53 +73,86 @@ class TuneBOHB(Searcher):
 
         config_space = CS.ConfigurationSpace()
         config_space.add_hyperparameter(
-            CS.UniformFloatHyperparameter('width', lower=0, upper=20))
+            CS.UniformFloatHyperparameter("width", lower=0, upper=20))
         config_space.add_hyperparameter(
-            CS.UniformFloatHyperparameter('height', lower=-100, upper=100))
+            CS.UniformFloatHyperparameter("height", lower=-100, upper=100))
         config_space.add_hyperparameter(
             CS.CategoricalHyperparameter(
-                name='activation', choices=['relu', 'tanh']))
+                name="activation", choices=["relu", "tanh"]))
 
         algo = TuneBOHB(
-            config_space, max_concurrent=4, metric='mean_loss', mode='min')
+            config_space, max_concurrent=4, metric="mean_loss", mode="min")
         bohb = HyperBandForBOHB(
-            time_attr='training_iteration',
-            metric='mean_loss',
-            mode='min',
+            time_attr="training_iteration",
+            metric="mean_loss",
+            mode="min",
             max_t=100)
-        run(MyTrainableClass, scheduler=bohb, search_alg=algo)
+        run(my_trainable, scheduler=bohb, search_alg=algo)
 
     """
 
     def __init__(self,
-                 space,
+                 space=None,
                  bohb_config=None,
                  max_concurrent=10,
                  metric="neg_mean_loss",
                  mode="max"):
         from hpbandster.optimizers.config_generators.bohb import BOHB
         assert BOHB is not None, "HpBandSter must be installed!"
-        assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!"
+        assert mode in ["min", "max"], "`mode` must be in [min, max]!"
         self._max_concurrent = max_concurrent
         self.trial_to_params = {}
         self.running = set()
         self.paused = set()
         self._metric = metric
-        if mode == "max":
-            self._metric_op = -1.
-        elif mode == "min":
-            self._metric_op = 1.
-        bohb_config = bohb_config or {}
-        self.bohber = BOHB(space, **bohb_config)
+
+        self._bohb_config = bohb_config
+        self._space = space
+
         super(TuneBOHB, self).__init__(metric=self._metric, mode=mode)
 
+        if self._space:
+            self.setup_bohb()
+
+    def setup_bohb(self):
+        from hpbandster.optimizers.config_generators.bohb import BOHB
+
+        if self._mode == "max":
+            self._metric_op = -1.
+        elif self._mode == "min":
+            self._metric_op = 1.
+
+        bohb_config = self._bohb_config or {}
+        self.bohber = BOHB(self._space, **bohb_config)
+
+    def set_search_properties(self, metric, mode, config):
+        if self._space:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self.setup_bohb()
+        return True
+
     def suggest(self, trial_id):
+        if not self._space:
+            raise RuntimeError(
+                "Trying to sample a configuration from {}, but no search "
+                "space has been defined. Either pass the `{}` argument when "
+                "instantiating the search algorithm, or pass a `config` to "
+                "`tune.run()`.".format(self.__class__.__name__, "space"))
+
         if len(self.running) < self._max_concurrent:
             # This parameter is not used in hpbandster implementation.
             config, info = self.bohber.get_config(None)
             self.trial_to_params[trial_id] = copy.deepcopy(config)
             self.running.add(trial_id)
-            return config
+            return unflatten_dict(config)
         return None
 
     def on_trial_result(self, trial_id, result):
@@ -123,3 +184,74 @@ class TuneBOHB(Searcher):
     def on_unpause(self, trial_id):
         self.paused.remove(trial_id)
         self.running.add(trial_id)
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a TuneBOHB search space.")
+
+        def resolve_value(par, domain):
+            quantize = None
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                quantize = sampler.q
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    lower = domain.lower
+                    upper = domain.upper
+                    if quantize:
+                        lower = math.ceil(domain.lower / quantize) * quantize
+                        upper = math.floor(domain.upper / quantize) * quantize
+                    return ConfigSpace.UniformFloatHyperparameter(
+                        par, lower=lower, upper=upper, q=quantize, log=True)
+                elif isinstance(sampler, Uniform):
+                    lower = domain.lower
+                    upper = domain.upper
+                    if quantize:
+                        lower = math.ceil(domain.lower / quantize) * quantize
+                        upper = math.floor(domain.upper / quantize) * quantize
+                    return ConfigSpace.UniformFloatHyperparameter(
+                        par, lower=lower, upper=upper, q=quantize, log=False)
+                elif isinstance(sampler, Normal):
+                    return ConfigSpace.NormalFloatHyperparameter(
+                        par,
+                        mu=sampler.mean,
+                        sigma=sampler.sd,
+                        q=quantize,
+                        log=False)
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, Uniform):
+                    lower = domain.lower
+                    upper = domain.upper
+                    if quantize:
+                        lower = math.ceil(domain.lower / quantize) * quantize
+                        upper = math.floor(domain.upper / quantize) * quantize
+                    return ConfigSpace.UniformIntegerHyperparameter(
+                        par, lower=lower, upper=upper, q=quantize, log=False)
+
+            elif isinstance(domain, Categorical):
+                if isinstance(sampler, Uniform):
+                    return ConfigSpace.CategoricalHyperparameter(
+                        par, choices=domain.categories)
+
+            raise ValueError("TuneBOHB does not support parameters of type "
+                             "`{}` with samplers of type `{}`".format(
+                                 type(domain).__name__,
+                                 type(domain.sampler).__name__))
+
+        cs = ConfigSpace.ConfigurationSpace()
+        for path, domain in domain_vars:
+            par = "/".join(path)
+            value = resolve_value(par, domain)
+            cs.add_hyperparameter(value)
+
+        return cs

python/ray/tune/suggest/dragonfly.py

@@ -2,8 +2,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import inspect
 import logging
 import pickle
+from typing import Dict
+
+from ray.tune.sample import Float, Quantized
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils.util import flatten_dict
 
 try:  # Python 3 only -- needed for lint test.
     import dragonfly
@@ -37,14 +43,62 @@ class DragonflySearch(Searcher):
     This interface requires using FunctionCallers and optimizers provided by
     Dragonfly.
 
+    Parameters:
+        optimizer (dragonfly.opt.BlackboxOptimiser|str): Optimizer provided
+            from dragonfly. Choose an optimiser that extends BlackboxOptimiser.
+            If this is a string, `domain` must be set and `optimizer` must be
+            one of [random, bandit, genetic].
+        domain (str): Optional domain. Should only be set if you don't pass
+            an optimizer as the `optimizer` argument.
+            Has to be one of [cartesian, euclidean].
+        space (list): Search space. Should only be set if you don't pass
+            an optimizer as the `optimizer` argument. Defines the search space
+            and requires a `domain` to be set. Can be automatically converted
+            from the `config` dict passed to `tune.run()`.
+        metric (str): The training result objective value attribute.
+        mode (str): One of {min, max}. Determines whether objective is
+            minimizing or maximizing the metric attribute.
+        points_to_evaluate (list of lists): A list of points you'd like to run
+            first before sampling from the optimiser, e.g. these could be
+            parameter configurations you already know work well to help
+            the optimiser select good values. Each point is a list of the
+            parameters using the order definition given by parameter_names.
+        evaluated_rewards (list): If you have previously evaluated the
+            parameters passed in as points_to_evaluate you can avoid
+            re-running those trials by passing in the reward attributes
+            as a list so the optimiser can be told the results without
+            needing to re-compute the trial. Must be the same length as
+            points_to_evaluate.
+
+    Tune automatically converts search spaces to Dragonfly's format:
+
+
     .. code-block:: python
 
         from ray import tune
-        from dragonfly.opt.gp_bandit import EuclideanGPBandit
-        from dragonfly.exd.experiment_caller import EuclideanFunctionCaller
-        from dragonfly import load_config
 
-        domain_vars = [{
+        config = {
+            "LiNO3_vol": tune.uniform(0, 7),
+            "Li2SO4_vol": tune.uniform(0, 7),
+            "NaClO4_vol": tune.uniform(0, 7)
+        }
+
+        df_search = DragonflySearch(
+            optimizer="bandit",
+            domain="euclidean",
+            metric="objective",
+            mode="max")
+
+        tune.run(my_func, config=config, search_alg=df_search)
+
+    If you would like to pass the search space/optimizer manually,
+    the code would look like this:
+
+    .. code-block:: python
+
+        from ray import tune
+
+        space = [{
             "name": "LiNO3_vol",
             "type": "float",
             "min": 0,
@@ -61,37 +115,21 @@ class DragonflySearch(Searcher):
             "max": 7
         }]
 
-        domain_config = load_config({"domain": domain_vars})
-        func_caller = EuclideanFunctionCaller(None,
-            domain_config.domain.list_of_domains[0])
-        optimizer = EuclideanGPBandit(func_caller, ask_tell_mode=True)
+        df_search = DragonflySearch(
+            optimizer="bandit",
+            domain="euclidean",
+            space=space,
+            metric="objective",
+            mode="max")
 
-        algo = DragonflySearch(optimizer, metric="objective", mode="max")
-
-        tune.run(my_func, search_alg=algo)
-
-    Parameters:
-        optimizer (dragonfly.opt.BlackboxOptimiser): Optimizer provided
-            from dragonfly. Choose an optimiser that extends BlackboxOptimiser.
-        metric (str): The training result objective value attribute.
-        mode (str): One of {min, max}. Determines whether objective is
-            minimizing or maximizing the metric attribute.
-        points_to_evaluate (list of lists): A list of points you'd like to run
-            first before sampling from the optimiser, e.g. these could be
-            parameter configurations you already know work well to help
-            the optimiser select good values. Each point is a list of the
-            parameters using the order definition given by parameter_names.
-        evaluated_rewards (list): If you have previously evaluated the
-            parameters passed in as points_to_evaluate you can avoid
-            re-running those trials by passing in the reward attributes
-            as a list so the optimiser can be told the results without
-            needing to re-compute the trial. Must be the same length as
-            points_to_evaluate.
+        tune.run(my_func, search_alg=df_search)
 
     """
 
     def __init__(self,
-                 optimizer,
+                 optimizer=None,
+                 domain=None,
+                 space=None,
                  metric="episode_reward_mean",
                  mode="max",
                  points_to_evaluate=None,
@@ -102,23 +140,131 @@ class DragonflySearch(Searcher):
             `pip install dragonfly-opt`."""
         assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!"
 
-        self._initial_points = []
-        self._opt = optimizer
-        self._opt.initialise()
-        if points_to_evaluate and evaluated_rewards:
-            self._opt.tell([(points_to_evaluate, evaluated_rewards)])
-        elif points_to_evaluate:
-            self._initial_points = points_to_evaluate
-        # Dragonfly internally maximizes, so "min" => -1
-        if mode == "min":
-            self._metric_op = -1.
-        elif mode == "max":
-            self._metric_op = 1.
-        self._live_trial_mapping = {}
         super(DragonflySearch, self).__init__(
             metric=metric, mode=mode, **kwargs)
 
+        from dragonfly.opt.blackbox_optimiser import BlackboxOptimiser
+
+        self._opt_arg = optimizer
+        self._domain = domain
+        self._space = space
+        self._points_to_evaluate = points_to_evaluate
+        self._evaluated_rewards = evaluated_rewards
+        self._initial_points = []
+        self._live_trial_mapping = {}
+
+        self._opt = None
+        if isinstance(optimizer, BlackboxOptimiser):
+            if domain or space:
+                raise ValueError(
+                    "If you pass an optimizer instance to dragonfly, do not "
+                    "pass a `domain` or `space`.")
+            self._opt = optimizer
+            self.init_dragonfly()
+        elif self._space:
+            self.setup_dragonfly()
+
+    def setup_dragonfly(self):
+        """Setup dragonfly when no optimizer has been passed."""
+        assert not self._opt, "Optimizer already set."
+
+        from dragonfly import load_config
+        from dragonfly.exd.experiment_caller import CPFunctionCaller, \
+            EuclideanFunctionCaller
+        from dragonfly.opt.blackbox_optimiser import BlackboxOptimiser
+        from dragonfly.opt.random_optimiser import CPRandomOptimiser, \
+            EuclideanRandomOptimiser
+        from dragonfly.opt.cp_ga_optimiser import CPGAOptimiser
+        from dragonfly.opt.gp_bandit import CPGPBandit, EuclideanGPBandit
+
+        if not self._space:
+            raise ValueError(
+                "You have to pass a `space` when initializing dragonfly, or "
+                "pass a search space definition to the `config` parameter "
+                "of `tune.run()`.")
+
+        if not self._domain:
+            raise ValueError(
+                "You have to set a `domain` when initializing dragonfly. "
+                "Choose one of [Cartesian, Euclidean].")
+
+        if self._domain.lower().startswith("cartesian"):
+            function_caller_cls = CPFunctionCaller
+        elif self._domain.lower().startswith("euclidean"):
+            function_caller_cls = EuclideanFunctionCaller
+        else:
+            raise ValueError("Dragonfly's `domain` argument must be one of "
+                             "[Cartesian, Euclidean].")
+
+        optimizer_cls = None
+        if inspect.isclass(self._opt_arg) and issubclass(
+                self._opt_arg, BlackboxOptimiser):
+            optimizer_cls = self._opt_arg
+        elif isinstance(self._opt_arg, str):
+            if self._opt_arg.lower().startswith("random"):
+                if function_caller_cls == CPFunctionCaller:
+                    optimizer_cls = CPRandomOptimiser
+                else:
+                    optimizer_cls = EuclideanRandomOptimiser
+            elif self._opt_arg.lower().startswith("bandit"):
+                if function_caller_cls == CPFunctionCaller:
+                    optimizer_cls = CPGPBandit
+                else:
+                    optimizer_cls = EuclideanGPBandit
+            elif self._opt_arg.lower().startswith("genetic"):
+                if function_caller_cls == CPFunctionCaller:
+                    optimizer_cls = CPGAOptimiser
+                else:
+                    raise ValueError(
+                        "Currently only the `cartesian` domain works with "
+                        "the `genetic` optimizer.")
+            else:
+                raise ValueError(
+                    "Invalid optimizer specification. Either pass a full "
+                    "dragonfly optimizer, or a string "
+                    "in [random, bandit, genetic].")
+
+        assert optimizer_cls, "No optimizer could be determined."
+        domain_config = load_config({"domain": self._space})
+        function_caller = function_caller_cls(
+            None, domain_config.domain.list_of_domains[0])
+        self._opt = optimizer_cls(function_caller, ask_tell_mode=True)
+        self.init_dragonfly()
+
+    def init_dragonfly(self):
+        self._opt.initialise()
+        if self._points_to_evaluate and self._evaluated_rewards:
+            self._opt.tell([(self._points_to_evaluate,
+                             self._evaluated_rewards)])
+        elif self._points_to_evaluate:
+            self._initial_points = self._points_to_evaluate
+        # Dragonfly internally maximizes, so "min" => -1
+        if self._mode == "min":
+            self._metric_op = -1.
+        elif self._mode == "max":
+            self._metric_op = 1.
+
+    def set_search_properties(self, metric, mode, config):
+        if self._opt:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self.setup_dragonfly()
+        return True
+
     def suggest(self, trial_id):
+        if not self._opt:
+            raise RuntimeError(
+                "Trying to sample a configuration from {}, but no search "
+                "space has been defined. Either pass the `{}` argument when "
+                "instantiating the search algorithm, or pass a `config` to "
+                "`tune.run()`.".format(self.__class__.__name__, "space"))
+
         if self._initial_points:
             suggested_config = self._initial_points[0]
             del self._initial_points[0]
@@ -151,3 +297,44 @@ class DragonflySearch(Searcher):
             trials_object = pickle.load(inputFile)
         self._initial_points = trials_object[0]
         self._opt = trials_object[1]
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a Dragonfly search space.")
+
+        def resolve_value(par, domain):
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning(
+                    "Dragonfly search does not support quantization. "
+                    "Dropped quantization.")
+                sampler = sampler.get_sampler()
+
+            if isinstance(domain, Float):
+                if domain.sampler is not None:
+                    logger.warning(
+                        "Dragonfly does not support specific sampling methods."
+                        " The {} sampler will be dropped.".format(sampler))
+                return {
+                    "name": par,
+                    "type": "float",
+                    "min": domain.lower,
+                    "max": domain.upper
+                }
+
+            raise ValueError("Dragonfly does not support parameters of type "
+                             "`{}`".format(type(domain).__name__))
+
+        # Parameter name is e.g. "a/b/c" for nested dicts
+        space = [
+            resolve_value("/".join(path), domain)
+            for path, domain in domain_vars
+        ]
+
+        return space

python/ray/tune/suggest/nevergrad.py

@@ -1,5 +1,12 @@
 import logging
 import pickle
+from typing import Dict
+
+from ray.tune.sample import Categorical, Float, Integer, LogUniform, Quantized
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils import flatten_dict
+from ray.tune.utils.util import unflatten_dict
+
 try:
     import nevergrad as ng
 except ImportError:
@@ -23,50 +30,63 @@ class NevergradSearch(Searcher):
 
         $ pip install nevergrad
 
-    This algorithm requires using an optimizer provided by Nevergrad, of
-    which there are many options. A good rundown can be found on
-    the `Nevergrad README's Optimization section`_.
-
-    .. code-block:: python
-
-        from nevergrad.optimization import optimizerlib
-
-        instrumentation = 1
-        optimizer = optimizerlib.OnePlusOne(instrumentation, budget=100)
-        algo = NevergradSearch(
-            optimizer, ["lr"], metric="mean_loss", mode="min")
-
     Parameters:
-        optimizer (nevergrad.optimization.Optimizer): Optimizer provided
-            from Nevergrad.
-        parameter_names (list): List of parameter names. Should match
-            the dimension of the optimizer output. Alternatively, set to None
-            if the optimizer is already instrumented with kwargs
-            (see nevergrad v0.2.0+).
+        optimizer (nevergrad.optimization.Optimizer|class): Optimizer provided
+            from Nevergrad. Alter
+        space (list|nevergrad.parameter.Parameter): Nevergrad parametrization
+            to be passed to optimizer on instantiation, or list of parameter
+            names if you passed an optimizer object.
         metric (str): The training result objective value attribute.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
         use_early_stopped_trials: Deprecated.
         max_concurrent: Deprecated.
 
-    Note:
-        In nevergrad v0.2.0+, optimizers can be instrumented.
-        For instance, the following will specifies searching
-        for "lr" from 1 to 2.
+    Tune automatically converts search spaces to Nevergrad's format:
 
-        >>> from nevergrad.optimization import optimizerlib
-        >>> from nevergrad import instrumentation as inst
-        >>> lr = inst.var.Array(1).bounded(1, 2).asfloat()
-        >>> instrumentation = inst.Instrumentation(lr=lr)
-        >>> optimizer = optimizerlib.OnePlusOne(instrumentation, budget=100)
-        >>> algo = NevergradSearch(
-                optimizer, None, metric="mean_loss", mode="min")
+    .. code-block:: python
+
+        import nevergrad as ng
+
+        config = {
+            "width": tune.uniform(0, 20),
+            "height": tune.uniform(-100, 100),
+            "activation": tune.choice(["relu", "tanh"])
+        }
+
+        ng_search = NevergradSearch(
+            optimizer=ng.optimizers.OnePlusOne,
+            metric="mean_loss",
+            mode="min")
+
+        run(my_trainable, config=config, search_alg=ng_search)
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        import nevergrad as ng
+
+        space = ng.p.Dict(
+            width=ng.p.Scalar(lower=0, upper=20),
+            height=ng.p.Scalar(lower=-100, upper=100),
+            activation=ng.p.Choice(choices=["relu", "tanh"])
+        )
+
+        ng_search = NevergradSearch(
+            optimizer=ng.optimizers.OnePlusOne,
+            space=space,
+            metric="mean_loss",
+            mode="min")
+
+        run(my_trainable, search_alg=ng_search)
 
     """
 
     def __init__(self,
-                 optimizer,
-                 parameter_names,
+                 optimizer=None,
+                 space=None,
                  metric="episode_reward_mean",
                  mode="max",
                  max_concurrent=None,
@@ -74,39 +94,88 @@ class NevergradSearch(Searcher):
         assert ng is not None, "Nevergrad must be installed!"
         assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!"
 
-        self._parameters = parameter_names
-        # nevergrad.tell internally minimizes, so "max" => -1
-        if mode == "max":
-            self._metric_op = -1.
-        elif mode == "min":
-            self._metric_op = 1.
-        self._nevergrad_opt = optimizer
-        self._live_trial_mapping = {}
-        self.max_concurrent = max_concurrent
         super(NevergradSearch, self).__init__(
             metric=metric, mode=mode, max_concurrent=max_concurrent, **kwargs)
-        # validate parameters
-        if hasattr(optimizer, "instrumentation"):  # added in v0.2.0
-            if optimizer.instrumentation.kwargs:
-                if optimizer.instrumentation.args:
+
+        self._space = None
+        self._opt_factory = None
+        self._nevergrad_opt = None
+
+        if isinstance(optimizer, ng.optimization.Optimizer):
+            if space is not None or isinstance(space, list):
+                raise ValueError(
+                    "If you pass a configured optimizer to Nevergrad, either "
+                    "pass a list of parameter names or None as the `space` "
+                    "parameter.")
+            self._parameters = space
+            self._nevergrad_opt = optimizer
+        elif isinstance(optimizer, ng.optimization.base.ConfiguredOptimizer):
+            self._opt_factory = optimizer
+            self._parameters = None
+            self._space = space
+        else:
+            raise ValueError(
+                "The `optimizer` argument passed to NevergradSearch must be "
+                "either an `Optimizer` or a `ConfiguredOptimizer`.")
+
+        self._live_trial_mapping = {}
+        self.max_concurrent = max_concurrent
+
+        if self._nevergrad_opt or self._space:
+            self.setup_nevergrad()
+
+    def setup_nevergrad(self):
+        if self._opt_factory:
+            self._nevergrad_opt = self._opt_factory(self._space)
+
+        # nevergrad.tell internally minimizes, so "max" => -1
+        if self._mode == "max":
+            self._metric_op = -1.
+        elif self._mode == "min":
+            self._metric_op = 1.
+
+        if hasattr(self._nevergrad_opt, "instrumentation"):  # added in v0.2.0
+            if self._nevergrad_opt.instrumentation.kwargs:
+                if self._nevergrad_opt.instrumentation.args:
                     raise ValueError(
                         "Instrumented optimizers should use kwargs only")
-                if parameter_names is not None:
+                if self._parameters is not None:
                     raise ValueError("Instrumented optimizers should provide "
                                      "None as parameter_names")
             else:
-                if parameter_names is None:
+                if self._parameters is None:
                     raise ValueError("Non-instrumented optimizers should have "
                                      "a list of parameter_names")
-                if len(optimizer.instrumentation.args) != 1:
+                if len(self._nevergrad_opt.instrumentation.args) != 1:
                     raise ValueError(
                         "Instrumented optimizers should use kwargs only")
-        if parameter_names is not None and optimizer.dimension != len(
-                parameter_names):
+        if self._parameters is not None and \
+           self._nevergrad_opt.dimension != len(self._parameters):
             raise ValueError("len(parameters_names) must match optimizer "
                              "dimension for non-instrumented optimizers")
 
+    def set_search_properties(self, metric, mode, config):
+        if self._nevergrad_opt or self._space:
+            return False
+        space = self.convert_search_space(config)
+        self._space = space
+
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self.setup_nevergrad()
+        return True
+
     def suggest(self, trial_id):
+        if not self._nevergrad_opt:
+            raise RuntimeError(
+                "Trying to sample a configuration from {}, but no search "
+                "space has been defined. Either pass the `{}` argument when "
+                "instantiating the search algorithm, or pass a `config` to "
+                "`tune.run()`.".format(self.__class__.__name__, "space"))
+
         if self.max_concurrent:
             if len(self._live_trial_mapping) >= self.max_concurrent:
                 return None
@@ -115,9 +184,12 @@ class NevergradSearch(Searcher):
         # in v0.2.0+, output of ask() is a Candidate,
         # with fields args and kwargs
         if not suggested_config.kwargs:
-            return dict(zip(self._parameters, suggested_config.args[0]))
+            if self._parameters:
+                return unflatten_dict(
+                    dict(zip(self._parameters, suggested_config.args[0])))
+            return unflatten_dict(suggested_config.value)
         else:
-            return suggested_config.kwargs
+            return unflatten_dict(suggested_config.kwargs)
 
     def on_trial_complete(self, trial_id, result=None, error=False):
         """Notification for the completion of trial.
@@ -146,3 +218,47 @@ class NevergradSearch(Searcher):
             trials_object = pickle.load(inputFile)
         self._nevergrad_opt = trials_object[0]
         self._parameters = trials_object[1]
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a Nevergrad search space.")
+
+        def resolve_value(domain):
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning("Nevergrad does not support quantization. "
+                               "Dropped quantization.")
+                sampler = sampler.get_sampler()
+
+            if isinstance(domain, Float):
+                if isinstance(sampler, LogUniform):
+                    return ng.p.Log(
+                        lower=domain.lower,
+                        upper=domain.upper,
+                        exponent=sampler.base)
+                return ng.p.Scalar(lower=domain.lower, upper=domain.upper)
+
+            if isinstance(domain, Integer):
+                return ng.p.Scalar(
+                    lower=domain.lower,
+                    upper=domain.upper).set_integer_casting()
+
+            if isinstance(domain, Categorical):
+                return ng.p.Choice(choices=domain.categories)
+
+            raise ValueError("SkOpt does not support parameters of type "
+                             "`{}`".format(type(domain).__name__))
+
+        # Parameter name is e.g. "a/b/c" for nested dicts
+        space = {
+            "/".join(path): resolve_value(domain)
+            for path, domain in domain_vars
+        }
+
+        return ng.p.Dict(**space)

python/ray/tune/suggest/skopt.py

@@ -1,5 +1,12 @@
 import logging
 import pickle
+from typing import Dict
+
+from ray.tune.sample import Categorical, Float, Integer, Quantized
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils import flatten_dict
+from ray.tune.utils.util import unflatten_dict
+
 try:
     import skopt as sko
 except ImportError:
@@ -52,14 +59,16 @@ class SkOptSearch(Searcher):
 
         pip install scikit-optimize
 
-
     This Search Algorithm requires you to pass in a `skopt Optimizer object`_.
 
     Parameters:
         optimizer (skopt.optimizer.Optimizer): Optimizer provided
             from skopt.
-        parameter_names (list): List of parameter names. Should match
-            the dimension of the optimizer output.
+        space (dict|list): A dict mapping parameter names to valid parameters,
+            i.e. tuples for numerical parameters and lists for categorical
+            parameters. If you passed an optimizer instance as the
+            `optimizer` argument, this should be a list of parameter names
+            instead.
         metric (str): The training result objective value attribute.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
@@ -77,24 +86,47 @@ class SkOptSearch(Searcher):
         max_concurrent: Deprecated.
         use_early_stopped_trials: Deprecated.
 
-    Example:
+    Tune automatically converts search spaces to SkOpt's format:
 
     .. code-block:: python
 
-        from skopt import Optimizer
-        optimizer = Optimizer([(0,20),(-100,100)])
+        config = {
+            "width": tune.uniform(0, 20),
+            "height": tune.uniform(-100, 100)
+        }
+
         current_best_params = [[10, 0], [15, -20]]
 
-        algo = SkOptSearch(optimizer,
-            ["width", "height"],
+        skopt_search = SkOptSearch(
             metric="mean_loss",
             mode="min",
             points_to_evaluate=current_best_params)
+
+        tune.run(my_trainable, config=config, search_alg=skopt_search)
+
+    If you would like to pass the search space/optimizer manually,
+    the code would look like this:
+
+    .. code-block:: python
+
+        parameter_names = ["width", "height"]
+        parameter_ranges = [(0,20),(-100,100)]
+        current_best_params = [[10, 0], [15, -20]]
+
+        skopt_search = SkOptSearch(
+            parameter_names=parameter_names,
+            parameter_ranges=parameter_ranges,
+            metric="mean_loss",
+            mode="min",
+            points_to_evaluate=current_best_params)
+
+        tune.run(my_trainable, search_alg=skopt_search)
+
     """
 
     def __init__(self,
-                 optimizer,
-                 parameter_names,
+                 optimizer=None,
+                 space=None,
                  metric="episode_reward_mean",
                  mode="max",
                  points_to_evaluate=None,
@@ -104,8 +136,7 @@ class SkOptSearch(Searcher):
         assert sko is not None, """skopt must be installed!
             You can install Skopt with the command:
             `pip install scikit-optimize`."""
-        _validate_warmstart(parameter_names, points_to_evaluate,
-                            evaluated_rewards)
+
         assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!"
         self.max_concurrent = max_concurrent
         super(SkOptSearch, self).__init__(
@@ -115,20 +146,83 @@ class SkOptSearch(Searcher):
             use_early_stopped_trials=use_early_stopped_trials)
 
         self._initial_points = []
-        if points_to_evaluate and evaluated_rewards:
-            optimizer.tell(points_to_evaluate, evaluated_rewards)
-        elif points_to_evaluate:
-            self._initial_points = points_to_evaluate
-        self._parameters = parameter_names
-        # Skopt internally minimizes, so "max" => -1
-        if mode == "max":
-            self._metric_op = -1.
-        elif mode == "min":
-            self._metric_op = 1.
+        self._parameters = None
+        self._parameter_names = None
+        self._parameter_ranges = None
+
+        self._space = space
+
+        if self._space:
+            if isinstance(optimizer, sko.Optimizer):
+                if not isinstance(space, list):
+                    raise ValueError(
+                        "You passed an optimizer instance to SkOpt. Your "
+                        "`space` parameter should be a list of parameter"
+                        "names.")
+                self._parameter_names = space
+            else:
+                self._parameter_names = space.keys()
+                self._parameter_ranges = space.values()
+
+        self._points_to_evaluate = points_to_evaluate
+        self._evaluated_rewards = evaluated_rewards
+
         self._skopt_opt = optimizer
+        if self._skopt_opt or self._space:
+            self.setup_skopt()
+
         self._live_trial_mapping = {}
 
+    def setup_skopt(self):
+        _validate_warmstart(self._parameter_names, self._points_to_evaluate,
+                            self._evaluated_rewards)
+
+        if not self._skopt_opt:
+            if not self._space:
+                raise ValueError(
+                    "If you don't pass an optimizer instance to SkOptSearch, "
+                    "pass a valid `space` parameter.")
+
+            self._skopt_opt = sko.Optimizer(self._parameter_ranges)
+
+        if self._points_to_evaluate and self._evaluated_rewards:
+            self._skopt_opt.tell(self._points_to_evaluate,
+                                 self._evaluated_rewards)
+        elif self._points_to_evaluate:
+            self._initial_points = self._points_to_evaluate
+        self._parameters = self._parameter_names
+
+        # Skopt internally minimizes, so "max" => -1
+        if self._mode == "max":
+            self._metric_op = -1.
+        elif self._mode == "min":
+            self._metric_op = 1.
+
+    def set_search_properties(self, metric, mode, config):
+        if self._skopt_opt:
+            return False
+        space = self.convert_search_space(config)
+
+        self._space = space
+        self._parameter_names = space.keys()
+        self._parameter_ranges = space.values()
+
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        self.setup_skopt()
+        return True
+
     def suggest(self, trial_id):
+        if not self._skopt_opt:
+            raise RuntimeError(
+                "Trying to sample a configuration from {}, but no search "
+                "space has been defined. Either pass the `{}` argument when "
+                "instantiating the search algorithm, or pass a `config` to "
+                "`tune.run()`.".format(self.__class__.__name__, "space"))
+
         if self.max_concurrent:
             if len(self._live_trial_mapping) >= self.max_concurrent:
                 return None
@@ -138,7 +232,7 @@ class SkOptSearch(Searcher):
         else:
             suggested_config = self._skopt_opt.ask()
         self._live_trial_mapping[trial_id] = suggested_config
-        return dict(zip(self._parameters, suggested_config))
+        return unflatten_dict(dict(zip(self._parameters, suggested_config)))
 
     def on_trial_complete(self, trial_id, result=None, error=False):
         """Notification for the completion of trial.
@@ -167,3 +261,48 @@ class SkOptSearch(Searcher):
             trials_object = pickle.load(inputFile)
         self._initial_points = trials_object[0]
         self._skopt_opt = trials_object[1]
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = flatten_dict(spec, prevent_delimiter=True)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a SkOpt search space.")
+
+        def resolve_value(domain):
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                logger.warning("SkOpt search does not support quantization. "
+                               "Dropped quantization.")
+                sampler = sampler.get_sampler()
+
+            if isinstance(domain, Float):
+                if domain.sampler is not None:
+                    logger.warning(
+                        "SkOpt does not support specific sampling methods."
+                        " The {} sampler will be dropped.".format(sampler))
+                return domain.lower, domain.upper
+
+            if isinstance(domain, Integer):
+                if domain.sampler is not None:
+                    logger.warning(
+                        "SkOpt does not support specific sampling methods."
+                        " The {} sampler will be dropped.".format(sampler))
+                return domain.lower, domain.upper
+
+            if isinstance(domain, Categorical):
+                return domain.categories
+
+            raise ValueError("SkOpt does not support parameters of type "
+                             "`{}`".format(type(domain).__name__))
+
+        # Parameter name is e.g. "a/b/c" for nested dicts
+        space = {
+            "/".join(path): resolve_value(domain)
+            for path, domain in domain_vars
+        }
+
+        return space

python/ray/tune/suggest/zoopt.py

@@ -1,6 +1,12 @@
 import copy
 import logging
+from typing import Dict
+
 import ray.cloudpickle as pickle
+from ray.tune.sample import Categorical, Float, Integer, Quantized, Uniform
+from ray.tune.suggest.variant_generator import parse_spec_vars
+from ray.tune.utils.util import unflatten_dict
+from zoopt import ValueType
 
 try:
     import zoopt
@@ -22,9 +28,39 @@ class ZOOptSearch(Searcher):
 
     To use ZOOptSearch, install zoopt (>=0.4.0): ``pip install -U zoopt``.
 
+    Tune automatically converts search spaces to ZOOpt"s format:
+
     .. code-block:: python
 
-        from ray.tune import run
+        from ray import tune
+        from ray.tune.suggest.zoopt import ZOOptSearch
+
+        "config": {
+            "iterations": 10,  # evaluation times
+            "width": tune.uniform(-10, 10),
+            "height": tune.uniform(-10, 10)
+        }
+
+        zoopt_search = ZOOptSearch(
+            algo="Asracos",  # only support Asracos currently
+            budget=20,  # must match `num_samples` in `tune.run()`.
+            dim_dict=dim_dict,
+            metric="mean_loss",
+            mode="min")
+
+        tune.run(my_objective,
+            config=config,
+            search_alg=zoopt_search,
+            name="zoopt_search",
+            num_samples=20,
+            stop={"timesteps_total": 10})
+
+    If you would like to pass the search space manually, the code would
+    look like this:
+
+    .. code-block:: python
+
+        from ray import tune
         from ray.tune.suggest.zoopt import ZOOptSearch
         from zoopt import ValueType
 
@@ -33,27 +69,23 @@ class ZOOptSearch(Searcher):
             "width": (ValueType.DISCRETE, [-10, 10], False)
         }
 
-        config = {
-            "num_samples": 200,
-            "config": {
-                "iterations": 10,  # evaluation times
-            },
-            "stop": {
-                "timesteps_total": 10  # cumstom stop rules
-            }
+        "config": {
+            "iterations": 10,  # evaluation times
         }
 
         zoopt_search = ZOOptSearch(
             algo="Asracos",  # only support Asracos currently
-            budget=config["num_samples"],
+            budget=20,  # must match `num_samples` in `tune.run()`.
             dim_dict=dim_dict,
             metric="mean_loss",
             mode="min")
 
-        run(my_objective,
+        tune.run(my_objective,
+            config=config,
             search_alg=zoopt_search,
             name="zoopt_search",
-            **config)
+            num_samples=20,
+            stop={"timesteps_total": 10})
 
     Parameters:
         algo (str): To specify an algorithm in zoopt you want to use.
@@ -82,12 +114,15 @@ class ZOOptSearch(Searcher):
                  **kwargs):
         assert zoopt is not None, "Zoopt not found - please install zoopt."
         assert budget is not None, "`budget` should not be None!"
-        assert dim_dict is not None, "`dim_list` should not be None!"
         assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!"
         _algo = algo.lower()
         assert _algo in ["asracos", "sracos"
                          ], "`algo` must be in ['asracos', 'sracos'] currently"
 
+        self._algo = _algo
+        self._dim_dict = dim_dict
+        self._budget = budget
+
         self._metric = metric
         if mode == "max":
             self._metric_op = -1.
@@ -96,31 +131,62 @@ class ZOOptSearch(Searcher):
         self._live_trial_mapping = {}
 
         self._dim_keys = []
-        _dim_list = []
-        for k in dim_dict:
-            self._dim_keys.append(k)
-            _dim_list.append(dim_dict[k])
-
-        dim = zoopt.Dimension2(_dim_list)
-        par = zoopt.Parameter(budget=budget)
-        if _algo == "sracos" or _algo == "asracos":
-            from zoopt.algos.opt_algorithms.racos.sracos import SRacosTune
-            self.optimizer = SRacosTune(dimension=dim, parameter=par)
-
         self.solution_dict = {}
         self.best_solution_list = []
+        self.optimizer = None
 
         super(ZOOptSearch, self).__init__(
             metric=self._metric, mode=mode, **kwargs)
 
+        if self._dim_dict:
+            self.setup_zoopt()
+
+    def setup_zoopt(self):
+        _dim_list = []
+        for k in self._dim_dict:
+            self._dim_keys.append(k)
+            _dim_list.append(self._dim_dict[k])
+
+        dim = zoopt.Dimension2(_dim_list)
+        par = zoopt.Parameter(budget=self._budget)
+        if self._algo == "sracos" or self._algo == "asracos":
+            from zoopt.algos.opt_algorithms.racos.sracos import SRacosTune
+            self.optimizer = SRacosTune(dimension=dim, parameter=par)
+
+    def set_search_properties(self, metric, mode, config):
+        if self._dim_dict:
+            return False
+        space = self.convert_search_space(config)
+        self._dim_dict = space
+
+        if metric:
+            self._metric = metric
+        if mode:
+            self._mode = mode
+
+        if self._mode == "max":
+            self._metric_op = -1.
+        elif self._mode == "min":
+            self._metric_op = 1.
+
+        self.setup_zoopt()
+        return True
+
     def suggest(self, trial_id):
+        if not self._dim_dict or not self.optimizer:
+            raise RuntimeError(
+                "Trying to sample a configuration from {}, but no search "
+                "space has been defined. Either pass the `{}` argument when "
+                "instantiating the search algorithm, or pass a `config` to "
+                "`tune.run()`.".format(self.__class__.__name__, "space"))
+
         _solution = self.optimizer.suggest()
         if _solution:
             self.solution_dict[str(trial_id)] = _solution
             _x = _solution.get_x()
             new_trial = dict(zip(self._dim_keys, _x))
             self._live_trial_mapping[trial_id] = new_trial
-            return copy.deepcopy(new_trial)
+            return unflatten_dict(new_trial)
 
     def on_trial_complete(self, trial_id, result=None, error=False):
         """Notification for the completion of trial."""
@@ -142,3 +208,57 @@ class ZOOptSearch(Searcher):
         with open(checkpoint_path, "rb") as input:
             trials_object = pickle.load(input)
         self.optimizer = trials_object
+
+    @staticmethod
+    def convert_search_space(spec: Dict):
+        spec = copy.deepcopy(spec)
+        resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
+
+        if not domain_vars and not grid_vars:
+            return []
+
+        if grid_vars:
+            raise ValueError(
+                "Grid search parameters cannot be automatically converted "
+                "to a ZOOpt search space.")
+
+        def resolve_value(domain):
+            quantize = None
+
+            sampler = domain.get_sampler()
+            if isinstance(sampler, Quantized):
+                quantize = sampler.q
+                sampler = sampler.sampler
+
+            if isinstance(domain, Float):
+                precision = quantize or 1e-12
+                if isinstance(sampler, Uniform):
+                    return (ValueType.CONTINUOUS, [domain.lower, domain.upper],
+                            precision)
+
+            elif isinstance(domain, Integer):
+                if isinstance(sampler, Uniform):
+                    return (ValueType.DISCRETE, [domain.lower, domain.upper],
+                            True)
+
+            elif isinstance(domain, Categorical):
+                # Categorical variables would use ValjeType.DISCRETE with
+                # has_partial_order=False, however, currently we do not
+                # keep track of category values and cannot automatically
+                # translate back and forth between them.
+                raise ValueError(
+                    "ZOOpt does not support automatic conversion for "
+                    "categorical variables. Please instantiate ZOOpt with "
+                    "a manually defined search space.")
+
+            raise ValueError("ZOOpt does not support parameters of type "
+                             "`{}` with samplers of type `{}`".format(
+                                 type(domain).__name__,
+                                 type(domain.sampler).__name__))
+
+        spec = {
+            "/".join(path): resolve_value(domain)
+            for path, domain in domain_vars
+        }
+
+        return spec

python/ray/tune/tests/test_sample.py

@@ -258,6 +258,113 @@ class SearchSpaceTest(unittest.TestCase):
         trial = analysis.trials[0]
         self.assertLess(trial.config["b"]["z"], 1e-2)
 
+    def testConvertBOHB(self):
+        from ray.tune.suggest.bohb import TuneBOHB
+        import ConfigSpace
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        converted_config = TuneBOHB.convert_search_space(config)
+        bohb_config = ConfigSpace.ConfigurationSpace()
+        bohb_config.add_hyperparameters([
+            ConfigSpace.CategoricalHyperparameter("a", [2, 3, 4]),
+            ConfigSpace.UniformIntegerHyperparameter(
+                "b/x", lower=0, upper=4, q=2),
+            ConfigSpace.UniformFloatHyperparameter(
+                "b/z", lower=1e-4, upper=1e-2, log=True)
+        ])
+
+        converted_config.seed(1234)
+        bohb_config.seed(1234)
+
+        searcher1 = TuneBOHB(space=converted_config)
+        searcher2 = TuneBOHB(space=bohb_config)
+
+        config1 = searcher1.suggest("0")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertIn(config1["a"], [2, 3, 4])
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
+
+        searcher = TuneBOHB(metric="a", mode="max")
+        analysis = tune.run(
+            _mock_objective, config=config, search_alg=searcher, num_samples=1)
+        trial = analysis.trials[0]
+        self.assertIn(trial.config["a"], [2, 3, 4])
+        self.assertEqual(trial.config["b"]["y"], 4)
+
+    def testConvertDragonfly(self):
+        from ray.tune.suggest.dragonfly import DragonflySearch
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        with self.assertRaises(ValueError):
+            converted_config = DragonflySearch.convert_search_space(config)
+
+        config = {
+            "a": 4,
+            "b": {
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        dragonfly_config = [{
+            "name": "b/z",
+            "type": "float",
+            "min": 1e-4,
+            "max": 1e-2
+        }]
+        converted_config = DragonflySearch.convert_search_space(config)
+
+        np.random.seed(1234)
+        searcher1 = DragonflySearch(
+            optimizer="bandit",
+            domain="euclidean",
+            space=converted_config,
+            metric="none")
+
+        config1 = searcher1.suggest("0")
+
+        np.random.seed(1234)
+        searcher2 = DragonflySearch(
+            optimizer="bandit",
+            domain="euclidean",
+            space=dragonfly_config,
+            metric="none")
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertLess(config2["point"], 1e-2)
+
+        searcher = DragonflySearch()
+        invalid_config = {"a/b": tune.uniform(4.0, 8.0)}
+        with self.assertRaises(ValueError):
+            searcher.set_search_properties("none", "max", invalid_config)
+        invalid_config = {"a": {"b/c": tune.uniform(4.0, 8.0)}}
+        with self.assertRaises(ValueError):
+            searcher.set_search_properties("none", "max", invalid_config)
+
+        searcher = DragonflySearch(
+            optimizer="bandit", domain="euclidean", metric="a", mode="max")
+        analysis = tune.run(
+            _mock_objective, config=config, search_alg=searcher, num_samples=1)
+        trial = analysis.trials[0]
+        self.assertLess(trial.config["point"], 1e-2)
+
     def testConvertHyperOpt(self):
         from ray.tune.suggest.hyperopt import HyperOptSearch
         from hyperopt import hp
@@ -301,6 +408,48 @@ class SearchSpaceTest(unittest.TestCase):
         trial = analysis.trials[0]
         assert trial.config["a"] in [2, 3, 4]
 
+    def testConvertNevergrad(self):
+        from ray.tune.suggest.nevergrad import NevergradSearch
+        import nevergrad as ng
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        converted_config = NevergradSearch.convert_search_space(config)
+        nevergrad_config = ng.p.Dict(
+            a=ng.p.Choice([2, 3, 4]),
+            b=ng.p.Dict(
+                x=ng.p.Scalar(lower=0, upper=5).set_integer_casting(),
+                z=ng.p.Log(lower=1e-4, upper=1e-2)))
+
+        searcher1 = NevergradSearch(
+            optimizer=ng.optimizers.OnePlusOne, space=converted_config)
+        searcher2 = NevergradSearch(
+            optimizer=ng.optimizers.OnePlusOne, space=nevergrad_config)
+
+        np.random.seed(1234)
+        config1 = searcher1.suggest("0")
+        np.random.seed(1234)
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertIn(config1["a"], [2, 3, 4])
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
+
+        searcher = NevergradSearch(
+            optimizer=ng.optimizers.OnePlusOne, metric="a", mode="max")
+        analysis = tune.run(
+            _mock_objective, config=config, search_alg=searcher, num_samples=1)
+        trial = analysis.trials[0]
+        assert trial.config["a"] in [2, 3, 4]
+
     def testConvertOptuna(self):
         from ray.tune.suggest.optuna import OptunaSearch, param
         from optuna.samplers import RandomSampler
@@ -341,6 +490,90 @@ class SearchSpaceTest(unittest.TestCase):
         trial = analysis.trials[0]
         assert trial.config["a"] in [2, 3, 4]
 
+    def testConvertSkOpt(self):
+        from ray.tune.suggest.skopt import SkOptSearch
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        converted_config = SkOptSearch.convert_search_space(config)
+        skopt_config = {"a": [2, 3, 4], "b/x": (0, 5), "b/z": (1e-4, 1e-2)}
+
+        searcher1 = SkOptSearch(space=converted_config)
+        searcher2 = SkOptSearch(space=skopt_config)
+
+        np.random.seed(1234)
+        config1 = searcher1.suggest("0")
+        np.random.seed(1234)
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertIn(config1["a"], [2, 3, 4])
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertLess(1e-4, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 1e-2)
+
+        searcher = SkOptSearch(metric="a", mode="max")
+        analysis = tune.run(
+            _mock_objective, config=config, search_alg=searcher, num_samples=1)
+        trial = analysis.trials[0]
+        self.assertIn(trial.config["a"], [2, 3, 4])
+        self.assertEqual(trial.config["b"]["y"], 4)
+
+    def testConvertZOOpt(self):
+        from ray.tune.suggest.zoopt import ZOOptSearch
+        from zoopt import ValueType
+
+        config = {
+            "a": tune.sample.Categorical([2, 3, 4]).uniform(),
+            "b": {
+                "x": tune.sample.Integer(0, 5).quantized(2),
+                "y": 4,
+                "z": tune.sample.Float(1e-4, 1e-2).loguniform()
+            }
+        }
+        # Does not support categorical variables
+        with self.assertRaises(ValueError):
+            converted_config = ZOOptSearch.convert_search_space(config)
+        config = {
+            "a": 2,
+            "b": {
+                "x": tune.sample.Integer(0, 5).uniform(),
+                "y": 4,
+                "z": tune.sample.Float(-3, 7).uniform().quantized(1e-4)
+            }
+        }
+        converted_config = ZOOptSearch.convert_search_space(config)
+
+        zoopt_config = {
+            "b/x": (ValueType.DISCRETE, [0, 5], True),
+            "b/z": (ValueType.CONTINUOUS, [-3, 7], 1e-4)
+        }
+
+        searcher1 = ZOOptSearch(dim_dict=converted_config, budget=5)
+        searcher2 = ZOOptSearch(dim_dict=zoopt_config, budget=5)
+
+        np.random.seed(1234)
+        config1 = searcher1.suggest("0")
+        np.random.seed(1234)
+        config2 = searcher2.suggest("0")
+
+        self.assertEqual(config1, config2)
+        self.assertIn(config1["b"]["x"], list(range(5)))
+        self.assertLess(-3, config1["b"]["z"])
+        self.assertLess(config1["b"]["z"], 7)
+
+        searcher = ZOOptSearch(budget=5, metric="a", mode="max")
+        analysis = tune.run(
+            _mock_objective, config=config, search_alg=searcher, num_samples=1)
+        trial = analysis.trials[0]
+        self.assertEqual(trial.config["b"]["y"], 4)
+
 
 if __name__ == "__main__":
     import pytest