diff --git a/python/ray/tune/examples/zoopt_example.py b/python/ray/tune/examples/zoopt_example.py index b582a0978..53da3c5e3 100644 --- a/python/ray/tune/examples/zoopt_example.py +++ b/python/ray/tune/examples/zoopt_example.py @@ -51,14 +51,21 @@ if __name__ == "__main__": # "height": (ValueType.CONTINUOUS, [-10, 10], 1e-2), # # for discrete dimensions: (discrete, search_range, has_order) # "width": (ValueType.DISCRETE, [0, 10], True) + # # for grid dimensions: (grid, grid_list) + # "layers": (ValueType.GRID, [4, 8, 16]) # } + zoopt_search_config = { + "parallel_num": 8, + } + zoopt_search = ZOOptSearch( algo="Asracos", # only support ASRacos currently budget=tune_kwargs["num_samples"], # dim_dict=space, # If you want to set the space yourself metric="mean_loss", - mode="min") + mode="min", + **zoopt_search_config) scheduler = AsyncHyperBandScheduler(metric="mean_loss", mode="min") diff --git a/python/ray/tune/suggest/zoopt.py b/python/ray/tune/suggest/zoopt.py index 38351d6cd..a913670b9 100644 --- a/python/ray/tune/suggest/zoopt.py +++ b/python/ray/tune/suggest/zoopt.py @@ -2,6 +2,7 @@ import copy import logging from typing import Dict, Optional, Tuple +import ray import ray.cloudpickle as pickle from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized, \ Uniform @@ -27,7 +28,7 @@ class ZOOptSearch(Searcher): Asynchronous Sequential RAndomized COordinate Shrinking (ASRacos) is implemented in Tune. - To use ZOOptSearch, install zoopt (>=0.4.0): ``pip install -U zoopt``. + To use ZOOptSearch, install zoopt (>=0.4.1): ``pip install -U zoopt``. Tune automatically converts search spaces to ZOOpt"s format: @@ -67,19 +68,26 @@ class ZOOptSearch(Searcher): dim_dict = { "height": (ValueType.CONTINUOUS, [-10, 10], 1e-2), - "width": (ValueType.DISCRETE, [-10, 10], False) + "width": (ValueType.DISCRETE, [-10, 10], False), + "layers": (ValueType.GRID, [4, 8, 16]) } "config": { "iterations": 10, # evaluation times } + zoopt_search_config = { + "parallel_num": 8, # how many workers to parallel + } + zoopt_search = ZOOptSearch( algo="Asracos", # only support Asracos currently budget=20, # must match `num_samples` in `tune.run()`. dim_dict=dim_dict, metric="mean_loss", - mode="min") + mode="min", + **zoopt_search_config + ) tune.run(my_objective, config=config, @@ -94,14 +102,17 @@ class ZOOptSearch(Searcher): budget (int): Number of samples. dim_dict (dict): Dimension dictionary. For continuous dimensions: (continuous, search_range, precision); - For discrete dimensions: (discrete, search_range, has_order). + For discrete dimensions: (discrete, search_range, has_order); + For grid dimensions: (grid, grid_list). More details can be found in zoopt package. metric (str): The training result objective value attribute. Defaults to "episode_reward_mean". mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. Defaults to "min". - + parallel_num (int): How many workers to parallel. Note that initial + phase may start less workers than this number. More details can + be found in zoopt package. """ optimizer = None @@ -113,7 +124,8 @@ class ZOOptSearch(Searcher): metric: Optional[str] = None, mode: Optional[str] = None, **kwargs): - assert zoopt is not None, "Zoopt not found - please install zoopt." + assert zoopt is not None, "ZOOpt not found - please install zoopt " \ + "by `pip install -U zoopt`." assert budget is not None, "`budget` should not be None!" if mode: assert mode in ["min", "max"], "`mode` must be 'min' or 'max'." @@ -137,8 +149,9 @@ class ZOOptSearch(Searcher): self.best_solution_list = [] self.optimizer = None - super(ZOOptSearch, self).__init__( - metric=self._metric, mode=mode, **kwargs) + self.kwargs = kwargs + + super(ZOOptSearch, self).__init__(metric=self._metric, mode=mode) if self._dim_dict: self.setup_zoopt() @@ -153,7 +166,8 @@ class ZOOptSearch(Searcher): par = zoopt.Parameter(budget=self._budget) if self._algo == "sracos" or self._algo == "asracos": from zoopt.algos.opt_algorithms.racos.sracos import SRacosTune - self.optimizer = SRacosTune(dimension=dim, parameter=par) + self.optimizer = SRacosTune( + dimension=dim, parameter=par, **self.kwargs) def set_search_properties(self, metric: Optional[str], mode: Optional[str], config: Dict) -> bool: @@ -184,6 +198,13 @@ class ZOOptSearch(Searcher): "`tune.run()`.".format(self.__class__.__name__, "space")) _solution = self.optimizer.suggest() + + if _solution == "FINISHED": + if ray.__version__ >= "0.8.7": + return Searcher.FINISHED + else: + return None + if _solution: self.solution_dict[str(trial_id)] = _solution _x = _solution.get_x() @@ -248,14 +269,12 @@ class ZOOptSearch(Searcher): True) elif isinstance(domain, Categorical): - # Categorical variables would use ValjeType.DISCRETE with + # Categorical variables would use ValueType.DISCRETE with # has_partial_order=False, however, currently we do not # keep track of category values and cannot automatically # translate back and forth between them. - raise ValueError( - "ZOOpt does not support automatic conversion for " - "categorical variables. Please instantiate ZOOpt with " - "a manually defined search space.") + if isinstance(sampler, Uniform): + return (ValueType.GRID, domain.categories) raise ValueError("ZOOpt does not support parameters of type " "`{}` with samplers of type `{}`".format( diff --git a/python/ray/tune/tests/test_sample.py b/python/ray/tune/tests/test_sample.py index e2f9bff50..ca61627fd 100644 --- a/python/ray/tune/tests/test_sample.py +++ b/python/ray/tune/tests/test_sample.py @@ -533,7 +533,7 @@ class SearchSpaceTest(unittest.TestCase): "a": tune.sample.Categorical([2, 3, 4]).uniform(), "b": { "x": tune.sample.Integer(0, 5).quantized(2), - "y": 4, + "y": tune.sample.Categorical([2, 4, 6, 8]).uniform(), "z": tune.sample.Float(1e-4, 1e-2).loguniform() } } @@ -544,7 +544,7 @@ class SearchSpaceTest(unittest.TestCase): "a": 2, "b": { "x": tune.sample.Integer(0, 5).uniform(), - "y": 4, + "y": tune.sample.Categorical([2, 4, 6, 8]).uniform(), "z": tune.sample.Float(-3, 7).uniform().quantized(1e-4) } } @@ -552,11 +552,16 @@ class SearchSpaceTest(unittest.TestCase): zoopt_config = { "b/x": (ValueType.DISCRETE, [0, 5], True), - "b/z": (ValueType.CONTINUOUS, [-3, 7], 1e-4) + "b/y": (ValueType.GRID, [2, 4, 6, 8]), + "b/z": (ValueType.CONTINUOUS, [-3, 7], 1e-4), } - searcher1 = ZOOptSearch(dim_dict=converted_config, budget=5) - searcher2 = ZOOptSearch(dim_dict=zoopt_config, budget=5) + zoopt_search_config = {"parallel_num": 4} + + searcher1 = ZOOptSearch( + dim_dict=converted_config, budget=5, **zoopt_search_config) + searcher2 = ZOOptSearch( + dim_dict=zoopt_config, budget=5, **zoopt_search_config) np.random.seed(1234) config1 = searcher1.suggest("0") @@ -565,14 +570,16 @@ class SearchSpaceTest(unittest.TestCase): self.assertEqual(config1, config2) self.assertIn(config1["b"]["x"], list(range(5))) + self.assertIn(config1["b"]["y"], [2, 4, 6, 8]) self.assertLess(-3, config1["b"]["z"]) self.assertLess(config1["b"]["z"], 7) - searcher = ZOOptSearch(budget=5, metric="a", mode="max") + searcher = ZOOptSearch( + budget=5, metric="a", mode="max", **zoopt_search_config) analysis = tune.run( _mock_objective, config=config, search_alg=searcher, num_samples=1) trial = analysis.trials[0] - self.assertEqual(trial.config["b"]["y"], 4) + self.assertIn(trial.config["b"]["y"], [2, 4, 6, 8]) if __name__ == "__main__": diff --git a/python/requirements_tune.txt b/python/requirements_tune.txt index 5954f9811..9e6261a97 100644 --- a/python/requirements_tune.txt +++ b/python/requirements_tune.txt @@ -30,4 +30,4 @@ git+git://github.com/huggingface/transformers.git@bdcc4b78a27775d1ec8f3fd297cb67 git+git://github.com/ray-project/tune-sklearn@master#tune-sklearn wandb xgboost -zoopt>=0.4.0 +zoopt>=0.4.1