From 6999075c7570655c416b2a35b911ba7610a043ef Mon Sep 17 00:00:00 2001 From: Kai Fricke Date: Thu, 19 Nov 2020 19:27:16 +0100 Subject: [PATCH] [tune] Add `seed` parameter to BOHB (#12160) --- python/ray/tune/suggest/bohb.py | 10 ++++++- python/ray/tune/tests/test_searchers.py | 36 ++++++++++++++++++------- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/python/ray/tune/suggest/bohb.py b/python/ray/tune/suggest/bohb.py index 745122d31..33f22640f 100644 --- a/python/ray/tune/suggest/bohb.py +++ b/python/ray/tune/suggest/bohb.py @@ -48,6 +48,9 @@ class TuneBOHB(Searcher): metric (str): The training result objective value attribute. mode (str): One of {min, max}. Determines whether objective is minimizing or maximizing the metric attribute. + seed (int): Optional random seed to initialize the random number + generator. Setting this should lead to identical initial + configurations at each run. Tune automatically converts search spaces to TuneBOHB's format: @@ -100,7 +103,8 @@ class TuneBOHB(Searcher): bohb_config: Optional[Dict] = None, max_concurrent: int = 10, metric: Optional[str] = None, - mode: Optional[str] = None): + mode: Optional[str] = None, + seed: Optional[int] = None): from hpbandster.optimizers.config_generators.bohb import BOHB assert BOHB is not None, "HpBandSter must be installed!" if mode: @@ -122,6 +126,7 @@ class TuneBOHB(Searcher): space = self.convert_search_space(space) self._space = space + self._seed = seed super(TuneBOHB, self).__init__(metric=self._metric, mode=mode) @@ -136,6 +141,9 @@ class TuneBOHB(Searcher): elif self._mode == "min": self._metric_op = 1. + if self._seed is not None: + self._space.seed(self._seed) + bohb_config = self._bohb_config or {} self.bohber = BOHB(self._space, **bohb_config) diff --git a/python/ray/tune/tests/test_searchers.py b/python/ray/tune/tests/test_searchers.py index ad6d31fee..102111573 100644 --- a/python/ray/tune/tests/test_searchers.py +++ b/python/ray/tune/tests/test_searchers.py @@ -35,14 +35,35 @@ class InvalidValuesTest(unittest.TestCase): def tearDownClass(cls): ray.shutdown() - def testBayesOpt(self): - from ray.tune.suggest.bayesopt import BayesOptSearch + def testAx(self): + from ray.tune.suggest.ax import AxSearch + from ax.service.ax_client import AxClient - np.random.seed(1234) # At least one nan, inf, -inf and float + converted_config = AxSearch.convert_search_space(self.config) + # At least one nan, inf, -inf and float + client = AxClient(random_seed=4321) + client.create_experiment( + parameters=converted_config, objective_name="_metric") + searcher = AxSearch(ax_client=client, metric="_metric", mode="max") out = tune.run( _invalid_objective, - search_alg=BayesOptSearch(), + search_alg=searcher, + metric="_metric", + mode="max", + num_samples=4, + reuse_actors=False) + + best_trial = out.best_trial + self.assertLessEqual(best_trial.config["report"], 2.0) + + def testBayesOpt(self): + from ray.tune.suggest.bayesopt import BayesOptSearch + + out = tune.run( + _invalid_objective, + # At least one nan, inf, -inf and float + search_alg=BayesOptSearch(random_state=1234), config=self.config, metric="_metric", mode="max", @@ -55,13 +76,10 @@ class InvalidValuesTest(unittest.TestCase): def testBOHB(self): from ray.tune.suggest.bohb import TuneBOHB - converted_config = TuneBOHB.convert_search_space(self.config) - converted_config.seed(1000) # At least one nan, inf, -inf and float - out = tune.run( _invalid_objective, - search_alg=TuneBOHB( - space=converted_config, metric="_metric", mode="max"), + search_alg=TuneBOHB(seed=1000), + config=self.config, metric="_metric", mode="max", num_samples=8,