diff --git a/ci/long_running_tests/workloads/pbt.py b/ci/long_running_tests/workloads/pbt.py index 86473d86e..5e63596c4 100644 --- a/ci/long_running_tests/workloads/pbt.py +++ b/ci/long_running_tests/workloads/pbt.py @@ -37,7 +37,8 @@ ray.init(redis_address=cluster.redis_address) pbt = PopulationBasedTraining( time_attr="training_iteration", - reward_attr="episode_reward_mean", + metric="episode_reward_mean", + mode="max", perturbation_interval=10, hyperparam_mutations={ "lr": [0.1, 0.01, 0.001, 0.0001], diff --git a/doc/source/tune-schedulers.rst b/doc/source/tune-schedulers.rst index cbb105ff5..2f6957f3f 100644 --- a/doc/source/tune-schedulers.rst +++ b/doc/source/tune-schedulers.rst @@ -7,7 +7,7 @@ By default, Tune schedules trials in serial order with the ``FIFOScheduler`` cla tune.run( ... , scheduler=AsyncHyperBandScheduler()) -Tune includes distributed implementations of early stopping algorithms such as `Median Stopping Rule `__, `HyperBand `__, and an `asynchronous version of HyperBand `__. These algorithms are very resource efficient and can outperform Bayesian Optimization methods in `many cases `__. Currently, all schedulers take in a ``reward_attr``, which is assumed to be maximized. +Tune includes distributed implementations of early stopping algorithms such as `Median Stopping Rule `__, `HyperBand `__, and an `asynchronous version of HyperBand `__. These algorithms are very resource efficient and can outperform Bayesian Optimization methods in `many cases `__. All schedulers take in a ``metric``, which is a value returned in the result dict of your Trainable and is maximized or minimized according to ``mode``. Current Available Trial Schedulers: @@ -25,7 +25,8 @@ Tune includes a distributed implementation of `Population Based Training (PBT) < pbt_scheduler = PopulationBasedTraining( time_attr='time_total_s', - reward_attr='mean_accuracy', + metric='mean_accuracy', + mode='max', perturbation_interval=600.0, hyperparam_mutations={ "lr": [1e-3, 5e-4, 1e-4, 5e-5, 1e-5], @@ -52,7 +53,8 @@ The `asynchronous version of HyperBand 0, "grace_period must be positive!" assert reduction_factor > 1, "Reduction Factor not valid!" assert brackets > 0, "brackets must be positive!" + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" + + if reward_attr is not None: + mode = "max" + metric = reward_attr + logger.warning( + "`reward_attr` is deprecated and will be removed in a future " + "version of Tune. " + "Setting `metric={}` and `mode=max`.".format(reward_attr)) + FIFOScheduler.__init__(self) self._reduction_factor = reduction_factor self._max_t = max_t @@ -63,7 +76,11 @@ class AsyncHyperBandScheduler(FIFOScheduler): ] self._counter = 0 # for self._num_stopped = 0 - self._reward_attr = reward_attr + self._metric = metric + if mode == "max": + self._metric_op = 1. + elif mode == "min": + self._metric_op = -1. self._time_attr = time_attr def on_trial_add(self, trial_runner, trial): @@ -80,7 +97,7 @@ class AsyncHyperBandScheduler(FIFOScheduler): else: bracket = self._trial_info[trial.trial_id] action = bracket.on_result(trial, result[self._time_attr], - result[self._reward_attr]) + self._metric_op * result[self._metric]) if action == TrialScheduler.STOP: self._num_stopped += 1 return action @@ -88,7 +105,7 @@ class AsyncHyperBandScheduler(FIFOScheduler): def on_trial_complete(self, trial_runner, trial, result): bracket = self._trial_info[trial.trial_id] bracket.on_result(trial, result[self._time_attr], - result[self._reward_attr]) + self._metric_op * result[self._metric]) del self._trial_info[trial.trial_id] def on_trial_remove(self, trial_runner, trial): diff --git a/python/ray/tune/schedulers/hyperband.py b/python/ray/tune/schedulers/hyperband.py index c9bdde8ab..b1ca1deec 100644 --- a/python/ray/tune/schedulers/hyperband.py +++ b/python/ray/tune/schedulers/hyperband.py @@ -43,8 +43,9 @@ class HyperBandScheduler(FIFOScheduler): To use this implementation of HyperBand with Tune, all you need to do is specify the max length of time a trial can run `max_t`, the time - units `time_attr`, and the name of the reported objective value - `reward_attr`. We automatically determine reasonable values for the other + units `time_attr`, the name of the reported objective value `metric`, + and if `metric` is to be maximized or minimized (`mode`). + We automatically determine reasonable values for the other HyperBand parameters based on the given values. For example, to limit trials to 10 minutes and early stop based on the @@ -62,9 +63,10 @@ class HyperBandScheduler(FIFOScheduler): Note that you can pass in something non-temporal such as `training_iteration` as a measure of progress, the only requirement is that the attribute should increase monotonically. - reward_attr (str): The training result objective value attribute. As - with `time_attr`, this may refer to any objective value. Stopping + metric (str): The training result objective value attribute. Stopping procedures will use this attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. max_t (int): max time units per trial. Trials will be stopped after max_t time units (determined by time_attr) have passed. The scheduler will terminate trials after this time has passed. @@ -74,16 +76,28 @@ class HyperBandScheduler(FIFOScheduler): def __init__(self, time_attr="training_iteration", - reward_attr="episode_reward_mean", + reward_attr=None, + metric="episode_reward_mean", + mode="max", max_t=81): assert max_t > 0, "Max (time_attr) not valid!" + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" + + if reward_attr is not None: + mode = "max" + metric = reward_attr + logger.warning( + "`reward_attr` is deprecated and will be removed in a future " + "version of Tune. " + "Setting `metric={}` and `mode=max`.".format(reward_attr)) + FIFOScheduler.__init__(self) self._eta = 3 self._s_max_1 = 5 self._max_t_attr = max_t # bracket max trials self._get_n0 = lambda s: int( - np.ceil(self._s_max_1/(s+1) * self._eta**s)) + np.ceil(self._s_max_1 / (s + 1) * self._eta**s)) # bracket initial iterations self._get_r0 = lambda s: int((max_t * self._eta**(-s))) self._hyperbands = [[]] # list of hyperband iterations @@ -92,7 +106,11 @@ class HyperBandScheduler(FIFOScheduler): # Tracks state for new trial add self._state = {"bracket": None, "band_idx": 0} self._num_stopped = 0 - self._reward_attr = reward_attr + self._metric = metric + if mode == "max": + self._metric_op = 1. + elif mode == "min": + self._metric_op = -1. self._time_attr = time_attr def on_trial_add(self, trial_runner, trial): @@ -173,7 +191,8 @@ class HyperBandScheduler(FIFOScheduler): bracket.cleanup_full(trial_runner) return TrialScheduler.STOP - good, bad = bracket.successive_halving(self._reward_attr) + good, bad = bracket.successive_halving(self._metric, + self._metric_op) # kill bad trials self._num_stopped += len(bad) for t in bad: @@ -322,7 +341,7 @@ class Bracket(): return len(self._live_trials) == self._n - def successive_halving(self, reward_attr): + def successive_halving(self, metric, metric_op): assert self._halves > 0 self._halves -= 1 self._n /= self._eta @@ -332,7 +351,8 @@ class Bracket(): self._r = int(min(self._r, self._max_t_attr - self._cumul_r)) self._cumul_r += self._r sorted_trials = sorted( - self._live_trials, key=lambda t: self._live_trials[t][reward_attr]) + self._live_trials, + key=lambda t: metric_op * self._live_trials[t][metric]) good, bad = sorted_trials[-self._n:], sorted_trials[:-self._n] return good, bad diff --git a/python/ray/tune/schedulers/median_stopping_rule.py b/python/ray/tune/schedulers/median_stopping_rule.py index e554a69f0..36276273c 100644 --- a/python/ray/tune/schedulers/median_stopping_rule.py +++ b/python/ray/tune/schedulers/median_stopping_rule.py @@ -22,9 +22,10 @@ class MedianStoppingRule(FIFOScheduler): Note that you can pass in something non-temporal such as `training_iteration` as a measure of progress, the only requirement is that the attribute should increase monotonically. - reward_attr (str): The training result objective value attribute. As - with `time_attr`, this may refer to any objective value that - is supposed to increase with time. + metric (str): The training result objective value attribute. Stopping + procedures will use this attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. grace_period (float): Only stop trials at least this old in time. The units are the same as the attribute named by `time_attr`. min_samples_required (int): Min samples to compute median over. @@ -37,18 +38,34 @@ class MedianStoppingRule(FIFOScheduler): def __init__(self, time_attr="time_total_s", - reward_attr="episode_reward_mean", + reward_attr=None, + metric="episode_reward_mean", + mode="max", grace_period=60.0, min_samples_required=3, hard_stop=True, verbose=True): + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" + + if reward_attr is not None: + mode = "max" + metric = reward_attr + logger.warning( + "`reward_attr` is deprecated and will be removed in a future " + "version of Tune. " + "Setting `metric={}` and `mode=max`.".format(reward_attr)) + FIFOScheduler.__init__(self) self._stopped_trials = set() self._completed_trials = set() self._results = collections.defaultdict(list) self._grace_period = grace_period self._min_samples_required = min_samples_required - self._reward_attr = reward_attr + self._metric = metric + if mode == "max": + self._metric_op = 1. + elif mode == "min": + self._metric_op = -1. self._time_attr = time_attr self._hard_stop = hard_stop self._verbose = verbose @@ -110,11 +127,9 @@ class MedianStoppingRule(FIFOScheduler): results = self._results[trial] # TODO(ekl) we could do interpolation to be more precise, but for now # assume len(results) is large and the time diffs are roughly equal - return np.mean([ - r[self._reward_attr] for r in results - if r[self._time_attr] <= t_max - ]) + return self._metric_op * np.mean( + [r[self._metric] for r in results if r[self._time_attr] <= t_max]) def _best_result(self, trial): results = self._results[trial] - return max(r[self._reward_attr] for r in results) + return max(self._metric_op * r[self._metric] for r in results) diff --git a/python/ray/tune/schedulers/pbt.py b/python/ray/tune/schedulers/pbt.py index e5d793fd1..b6d1b4e80 100644 --- a/python/ray/tune/schedulers/pbt.py +++ b/python/ray/tune/schedulers/pbt.py @@ -120,9 +120,10 @@ class PopulationBasedTraining(FIFOScheduler): Note that you can pass in something non-temporal such as `training_iteration` as a measure of progress, the only requirement is that the attribute should increase monotonically. - reward_attr (str): The training result objective value attribute. As - with `time_attr`, this may refer to any objective value. Stopping + metric (str): The training result objective value attribute. Stopping procedures will use this attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. perturbation_interval (float): Models will be considered for perturbation at this interval of `time_attr`. Note that perturbation incurs checkpoint overhead, so you shouldn't set this @@ -149,7 +150,8 @@ class PopulationBasedTraining(FIFOScheduler): Example: >>> pbt = PopulationBasedTraining( >>> time_attr="training_iteration", - >>> reward_attr="episode_reward_mean", + >>> metric="episode_reward_mean", + >>> mode="max", >>> perturbation_interval=10, # every 10 `time_attr` units >>> # (training_iterations in this case) >>> hyperparam_mutations={ @@ -165,7 +167,9 @@ class PopulationBasedTraining(FIFOScheduler): def __init__(self, time_attr="time_total_s", - reward_attr="episode_reward_mean", + reward_attr=None, + metric="episode_reward_mean", + mode="max", perturbation_interval=60.0, hyperparam_mutations={}, resample_probability=0.25, @@ -175,8 +179,23 @@ class PopulationBasedTraining(FIFOScheduler): raise TuneError( "You must specify at least one of `hyperparam_mutations` or " "`custom_explore_fn` to use PBT.") + + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" + + if reward_attr is not None: + mode = "max" + metric = reward_attr + logger.warning( + "`reward_attr` is deprecated and will be removed in a future " + "version of Tune. " + "Setting `metric={}` and `mode=max`.".format(reward_attr)) + FIFOScheduler.__init__(self) - self._reward_attr = reward_attr + self._metric = metric + if mode == "max": + self._metric_op = 1. + elif mode == "min": + self._metric_op = -1. self._time_attr = time_attr self._perturbation_interval = perturbation_interval self._hyperparam_mutations = hyperparam_mutations @@ -199,7 +218,7 @@ class PopulationBasedTraining(FIFOScheduler): if time - state.last_perturbation_time < self._perturbation_interval: return TrialScheduler.CONTINUE # avoid checkpoint overhead - score = result[self._reward_attr] + score = self._metric_op * result[self._metric] state.last_score = score state.last_perturbation_time = time lower_quantile, upper_quantile = self._quantiles() diff --git a/python/ray/tune/suggest/bayesopt.py b/python/ray/tune/suggest/bayesopt.py index 029ee4ddd..7b2530411 100644 --- a/python/ray/tune/suggest/bayesopt.py +++ b/python/ray/tune/suggest/bayesopt.py @@ -3,6 +3,7 @@ from __future__ import division from __future__ import print_function import copy +import logging try: # Python 3 only -- needed for lint test. import bayes_opt as byo except ImportError: @@ -10,6 +11,8 @@ except ImportError: from ray.tune.suggest.suggestion import SuggestionAlgorithm +logger = logging.getLogger(__name__) + class BayesOptSearch(SuggestionAlgorithm): """A wrapper around BayesOpt to provide trial suggestions. @@ -22,8 +25,9 @@ class BayesOptSearch(SuggestionAlgorithm): this space which will be used to run trials. max_concurrent (int): Number of maximum concurrent trials. Defaults to 10. - reward_attr (str): The training result objective value attribute. - This refers to an increasing value. + metric (str): The training result objective value attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. utility_kwargs (dict): Parameters to define the utility function. Must provide values for the keys `kind`, `kappa`, and `xi`. random_state (int): Used to initialize BayesOpt. @@ -35,13 +39,15 @@ class BayesOptSearch(SuggestionAlgorithm): >>> 'height': (-100, 100), >>> } >>> algo = BayesOptSearch( - >>> space, max_concurrent=4, reward_attr="neg_mean_loss") + >>> space, max_concurrent=4, metric="mean_loss", mode="min") """ def __init__(self, space, max_concurrent=10, - reward_attr="episode_reward_mean", + reward_attr=None, + metric="episode_reward_mean", + mode="max", utility_kwargs=None, random_state=1, verbose=0, @@ -52,8 +58,22 @@ class BayesOptSearch(SuggestionAlgorithm): assert type(max_concurrent) is int and max_concurrent > 0 assert utility_kwargs is not None, ( "Must define arguments for the utiliy function!") + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" + + if reward_attr is not None: + mode = "max" + metric = reward_attr + logger.warning( + "`reward_attr` is deprecated and will be removed in a future " + "version of Tune. " + "Setting `metric={}` and `mode=max`.".format(reward_attr)) + self._max_concurrent = max_concurrent - self._reward_attr = reward_attr + self._metric = metric + if mode == "max": + self._metric_op = 1. + elif mode == "min": + self._metric_op = -1. self._live_trial_mapping = {} self.optimizer = byo.BayesianOptimization( @@ -85,7 +105,7 @@ class BayesOptSearch(SuggestionAlgorithm): if result: self.optimizer.register( params=self._live_trial_mapping[trial_id], - target=result[self._reward_attr]) + target=self._metric_op * result[self._metric]) del self._live_trial_mapping[trial_id] diff --git a/python/ray/tune/suggest/hyperopt.py b/python/ray/tune/suggest/hyperopt.py index 533e320c0..efae5d031 100644 --- a/python/ray/tune/suggest/hyperopt.py +++ b/python/ray/tune/suggest/hyperopt.py @@ -15,6 +15,8 @@ except ImportError: from ray.tune.error import TuneError from ray.tune.suggest.suggestion import SuggestionAlgorithm +logger = logging.getLogger(__name__) + class HyperOptSearch(SuggestionAlgorithm): """A wrapper around HyperOpt to provide trial suggestions. @@ -30,8 +32,9 @@ class HyperOptSearch(SuggestionAlgorithm): parameters generated in the variant generation process. max_concurrent (int): Number of maximum concurrent trials. Defaults to 10. - reward_attr (str): The training result objective value attribute. - This refers to an increasing value. + metric (str): The training result objective value attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. points_to_evaluate (list): Initial parameter suggestions to be run first. This is for when you already have some good parameters you want hyperopt to run first to help the TPE algorithm @@ -52,21 +55,38 @@ class HyperOptSearch(SuggestionAlgorithm): >>> 'activation': 0, # The index of "relu" >>> }] >>> algo = HyperOptSearch( - >>> space, max_concurrent=4, reward_attr="neg_mean_loss", + >>> space, max_concurrent=4, metric="mean_loss", mode="min", >>> points_to_evaluate=current_best_params) """ def __init__(self, space, max_concurrent=10, - reward_attr="episode_reward_mean", + reward_attr=None, + metric="episode_reward_mean", + mode="max", points_to_evaluate=None, **kwargs): assert hpo is not None, "HyperOpt must be installed!" from hyperopt.fmin import generate_trials_to_calculate assert type(max_concurrent) is int and max_concurrent > 0 + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" + + if reward_attr is not None: + mode = "max" + metric = reward_attr + logger.warning( + "`reward_attr` is deprecated and will be removed in a future " + "version of Tune. " + "Setting `metric={}` and `mode=max`.".format(reward_attr)) + self._max_concurrent = max_concurrent - self._reward_attr = reward_attr + self._metric = metric + # hyperopt internally minimizes, so "max" => -1 + if mode == "max": + self._metric_op = -1. + elif mode == "min": + self._metric_op = 1. self.algo = hpo.tpe.suggest self.domain = hpo.Domain(lambda spc: spc, space) if points_to_evaluate is None: @@ -151,7 +171,7 @@ class HyperOptSearch(SuggestionAlgorithm): del self._live_trial_mapping[trial_id] def _to_hyperopt_result(self, result): - return {"loss": -result[self._reward_attr], "status": "ok"} + return {"loss": self._metric_op * result[self._metric], "status": "ok"} def _get_hyperopt_trial(self, trial_id): if trial_id not in self._live_trial_mapping: diff --git a/python/ray/tune/suggest/nevergrad.py b/python/ray/tune/suggest/nevergrad.py index 284311a36..2ad8eed37 100644 --- a/python/ray/tune/suggest/nevergrad.py +++ b/python/ray/tune/suggest/nevergrad.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import logging try: import nevergrad as ng except ImportError: @@ -9,6 +10,8 @@ except ImportError: from ray.tune.suggest.suggestion import SuggestionAlgorithm +logger = logging.getLogger(__name__) + class NevergradSearch(SuggestionAlgorithm): """A wrapper around Nevergrad to provide trial suggestions. @@ -28,15 +31,16 @@ class NevergradSearch(SuggestionAlgorithm): (see nevergrad v0.2.0+). max_concurrent (int): Number of maximum concurrent trials. Defaults to 10. - reward_attr (str): The training result objective value attribute. - This refers to an increasing value. + metric (str): The training result objective value attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. Example: >>> from nevergrad.optimization import optimizerlib >>> instrumentation = 1 >>> optimizer = optimizerlib.OnePlusOne(instrumentation, budget=100) >>> algo = NevergradSearch(optimizer, ["lr"], max_concurrent=4, - >>> reward_attr="neg_mean_loss") + >>> metric="mean_loss", mode="min") Note: In nevergrad v0.2.0+, optimizers can be instrumented. @@ -49,7 +53,7 @@ class NevergradSearch(SuggestionAlgorithm): >>> instrumentation = inst.Instrumentation(lr=lr) >>> optimizer = optimizerlib.OnePlusOne(instrumentation, budget=100) >>> algo = NevergradSearch(optimizer, None, max_concurrent=4, - >>> reward_attr="neg_mean_loss") + >>> metric="mean_loss", mode="min") """ @@ -57,13 +61,30 @@ class NevergradSearch(SuggestionAlgorithm): optimizer, parameter_names, max_concurrent=10, - reward_attr="episode_reward_mean", + reward_attr=None, + metric="episode_reward_mean", + mode="max", **kwargs): assert ng is not None, "Nevergrad must be installed!" assert type(max_concurrent) is int and max_concurrent > 0 + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" + + if reward_attr is not None: + mode = "max" + metric = reward_attr + logger.warning( + "`reward_attr` is deprecated and will be removed in a future " + "version of Tune. " + "Setting `metric={}` and `mode=max`.".format(reward_attr)) + self._max_concurrent = max_concurrent self._parameters = parameter_names - self._reward_attr = reward_attr + self._metric = metric + # nevergrad.tell internally minimizes, so "max" => -1 + if mode == "max": + self._metric_op = -1. + elif mode == "min": + self._metric_op = 1. self._nevergrad_opt = optimizer self._live_trial_mapping = {} super(NevergradSearch, self).__init__(**kwargs) @@ -119,7 +140,8 @@ class NevergradSearch(SuggestionAlgorithm): """ ng_trial_info = self._live_trial_mapping.pop(trial_id) if result: - self._nevergrad_opt.tell(ng_trial_info, -result[self._reward_attr]) + self._nevergrad_opt.tell(ng_trial_info, + self._metric_op * result[self._metric]) def _num_live_trials(self): return len(self._live_trial_mapping) diff --git a/python/ray/tune/suggest/sigopt.py b/python/ray/tune/suggest/sigopt.py index 72d2d0afc..9aaf593f1 100644 --- a/python/ray/tune/suggest/sigopt.py +++ b/python/ray/tune/suggest/sigopt.py @@ -4,6 +4,7 @@ from __future__ import print_function import copy import os +import logging try: import sigopt as sgo except ImportError: @@ -11,6 +12,8 @@ except ImportError: from ray.tune.suggest.suggestion import SuggestionAlgorithm +logger = logging.getLogger(__name__) + class SigOptSearch(SuggestionAlgorithm): """A wrapper around SigOpt to provide trial suggestions. @@ -25,8 +28,9 @@ class SigOptSearch(SuggestionAlgorithm): name (str): Name of experiment. Required by SigOpt. max_concurrent (int): Number of maximum concurrent trials supported based on the user's SigOpt plan. Defaults to 1. - reward_attr (str): The training result objective value attribute. - This refers to an increasing value. + metric (str): The training result objective value attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. Example: >>> space = [ @@ -49,21 +53,37 @@ class SigOptSearch(SuggestionAlgorithm): >>> ] >>> algo = SigOptSearch( >>> space, name="SigOpt Example Experiment", - >>> max_concurrent=1, reward_attr="neg_mean_loss") + >>> max_concurrent=1, metric="mean_loss", mode="min") """ def __init__(self, space, name="Default Tune Experiment", max_concurrent=1, - reward_attr="episode_reward_mean", + reward_attr=None, + metric="episode_reward_mean", + mode="max", **kwargs): assert sgo is not None, "SigOpt must be installed!" assert type(max_concurrent) is int and max_concurrent > 0 assert "SIGOPT_KEY" in os.environ, \ "SigOpt API key must be stored as environ variable at SIGOPT_KEY" + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" + + if reward_attr is not None: + mode = "max" + metric = reward_attr + logger.warning( + "`reward_attr` is deprecated and will be removed in a future " + "version of Tune. " + "Setting `metric={}` and `mode=max`.".format(reward_attr)) + self._max_concurrent = max_concurrent - self._reward_attr = reward_attr + self._metric = metric + if mode == "max": + self._metric_op = 1. + elif mode == "min": + self._metric_op = -1. self._live_trial_mapping = {} # Create a connection with SigOpt API, requires API key @@ -108,7 +128,7 @@ class SigOptSearch(SuggestionAlgorithm): if result: self.conn.experiments(self.experiment.id).observations().create( suggestion=self._live_trial_mapping[trial_id].id, - value=result[self._reward_attr], + value=self._metric_op * result[self._metric], ) # Update the experiment object self.experiment = self.conn.experiments(self.experiment.id).fetch() diff --git a/python/ray/tune/suggest/skopt.py b/python/ray/tune/suggest/skopt.py index 26457c5fa..f60a0856e 100644 --- a/python/ray/tune/suggest/skopt.py +++ b/python/ray/tune/suggest/skopt.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import logging try: import skopt as sko except ImportError: @@ -9,6 +10,8 @@ except ImportError: from ray.tune.suggest.suggestion import SuggestionAlgorithm +logger = logging.getLogger(__name__) + def _validate_warmstart(parameter_names, points_to_evaluate, evaluated_rewards): @@ -52,8 +55,9 @@ class SkOptSearch(SuggestionAlgorithm): the dimension of the optimizer output. max_concurrent (int): Number of maximum concurrent trials. Defaults to 10. - reward_attr (str): The training result objective value attribute. - This refers to an increasing value. + metric (str): The training result objective value attribute. + mode (str): One of {min, max}. Determines whether objective is + minimizing or maximizing the metric attribute. points_to_evaluate (list of lists): A list of points you'd like to run first before sampling from the optimiser, e.g. these could be parameter configurations you already know work well to help @@ -73,7 +77,8 @@ class SkOptSearch(SuggestionAlgorithm): >>> algo = SkOptSearch(optimizer, >>> ["width", "height"], >>> max_concurrent=4, - >>> reward_attr="neg_mean_loss", + >>> metric="mean_loss", + >>> mode="min", >>> points_to_evaluate=current_best_params) """ @@ -81,7 +86,9 @@ class SkOptSearch(SuggestionAlgorithm): optimizer, parameter_names, max_concurrent=10, - reward_attr="episode_reward_mean", + reward_attr=None, + metric="episode_reward_mean", + mode="max", points_to_evaluate=None, evaluated_rewards=None, **kwargs): @@ -91,6 +98,15 @@ class SkOptSearch(SuggestionAlgorithm): assert type(max_concurrent) is int and max_concurrent > 0 _validate_warmstart(parameter_names, points_to_evaluate, evaluated_rewards) + assert mode in ["min", "max"], "`mode` must be 'min' or 'max'!" + + if reward_attr is not None: + mode = "max" + metric = reward_attr + logger.warning( + "`reward_attr` is deprecated and will be removed in a future " + "version of Tune. " + "Setting `metric={}` and `mode=max`.".format(reward_attr)) self._initial_points = [] if points_to_evaluate and evaluated_rewards: @@ -99,7 +115,12 @@ class SkOptSearch(SuggestionAlgorithm): self._initial_points = points_to_evaluate self._max_concurrent = max_concurrent self._parameters = parameter_names - self._reward_attr = reward_attr + self._metric = metric + # Skopt internally minimizes, so "max" => -1 + if mode == "max": + self._metric_op = -1. + elif mode == "min": + self._metric_op = 1. self._skopt_opt = optimizer self._live_trial_mapping = {} super(SkOptSearch, self).__init__(**kwargs) @@ -131,7 +152,8 @@ class SkOptSearch(SuggestionAlgorithm): """ skopt_trial_info = self._live_trial_mapping.pop(trial_id) if result: - self._skopt_opt.tell(skopt_trial_info, -result[self._reward_attr]) + self._skopt_opt.tell(skopt_trial_info, + self._metric_op * result[self._metric]) def _num_live_trials(self): return len(self._live_trial_mapping) diff --git a/python/ray/tune/tests/test_experiment_analysis.py b/python/ray/tune/tests/test_experiment_analysis.py index 96b08c0a6..a0721abc5 100644 --- a/python/ray/tune/tests/test_experiment_analysis.py +++ b/python/ray/tune/tests/test_experiment_analysis.py @@ -36,7 +36,8 @@ class ExperimentAnalysisSuite(unittest.TestCase): def run_test_exp(self): ahb = AsyncHyperBandScheduler( time_attr="training_iteration", - reward_attr=self.metric, + metric=self.metric, + mode="max", grace_period=5, max_t=100) diff --git a/python/ray/tune/tests/test_trial_scheduler.py b/python/ray/tune/tests/test_trial_scheduler.py index 1f8ab8025..1c3337554 100644 --- a/python/ray/tune/tests/test_trial_scheduler.py +++ b/python/ray/tune/tests/test_trial_scheduler.py @@ -135,33 +135,43 @@ class EarlyStoppingSuite(unittest.TestCase): rule.on_trial_result(None, t3, result(2, 260)), TrialScheduler.PAUSE) - def testAlternateMetrics(self): - def result2(t, rew): - return dict(training_iteration=t, neg_mean_loss=rew) - + def _test_metrics(self, result_func, metric, mode): rule = MedianStoppingRule( grace_period=0, min_samples_required=1, time_attr="training_iteration", - reward_attr="neg_mean_loss") + metric=metric, + mode=mode) t1 = Trial("PPO") # mean is 450, max 900, t_max=10 t2 = Trial("PPO") # mean is 450, max 450, t_max=5 for i in range(10): self.assertEqual( - rule.on_trial_result(None, t1, result2(i, i * 100)), + rule.on_trial_result(None, t1, result_func(i, i * 100)), TrialScheduler.CONTINUE) for i in range(5): self.assertEqual( - rule.on_trial_result(None, t2, result2(i, 450)), + rule.on_trial_result(None, t2, result_func(i, 450)), TrialScheduler.CONTINUE) - rule.on_trial_complete(None, t1, result2(10, 1000)) + rule.on_trial_complete(None, t1, result_func(10, 1000)) self.assertEqual( - rule.on_trial_result(None, t2, result2(5, 450)), + rule.on_trial_result(None, t2, result_func(5, 450)), TrialScheduler.CONTINUE) self.assertEqual( - rule.on_trial_result(None, t2, result2(6, 0)), + rule.on_trial_result(None, t2, result_func(6, 0)), TrialScheduler.CONTINUE) + def testAlternateMetrics(self): + def result2(t, rew): + return dict(training_iteration=t, neg_mean_loss=rew) + + self._test_metrics(result2, "neg_mean_loss", "max") + + def testAlternateMetricsMin(self): + def result2(t, rew): + return dict(training_iteration=t, mean_loss=-rew) + + self._test_metrics(result2, "mean_loss", "min") + class _MockTrialExecutor(TrialExecutor): def start_trial(self, trial, checkpoint_obj=None): @@ -495,14 +505,9 @@ class HyperbandSuite(unittest.TestCase): TrialScheduler.PAUSE, sched.on_trial_result(mock_runner, t, result(new_units, 12))) - def testAlternateMetrics(self): - """Checking that alternate metrics will pass.""" - - def result2(t, rew): - return dict(time_total_s=t, neg_mean_loss=rew) - + def _test_metrics(self, result_func, metric, mode): sched = HyperBandScheduler( - time_attr="time_total_s", reward_attr="neg_mean_loss") + time_attr="time_total_s", metric=metric, mode=mode) stats = self.default_statistics() for i in range(stats["max_trials"]): @@ -518,13 +523,29 @@ class HyperbandSuite(unittest.TestCase): # Provides results from 0 to 8 in order, keeping the last one running for i, trl in enumerate(big_bracket.current_trials()): - action = sched.on_trial_result(runner, trl, result2(1, i)) + action = sched.on_trial_result(runner, trl, result_func(1, i)) runner.process_action(trl, action) new_length = len(big_bracket.current_trials()) self.assertEqual(action, TrialScheduler.CONTINUE) self.assertEqual(new_length, self.downscale(current_length, sched)) + def testAlternateMetrics(self): + """Checking that alternate metrics will pass.""" + + def result2(t, rew): + return dict(time_total_s=t, neg_mean_loss=rew) + + self._test_metrics(result2, "neg_mean_loss", "max") + + def testAlternateMetricsMin(self): + """Checking that alternate metrics will pass.""" + + def result2(t, rew): + return dict(time_total_s=t, mean_loss=-rew) + + self._test_metrics(result2, "mean_loss", "min") + def testJumpingTime(self): sched, mock_runner = self.schedulerSetup(81) big_bracket = sched._hyperbands[0][-1] @@ -1015,14 +1036,12 @@ class AsyncHyperBandSuite(unittest.TestCase): scheduler.on_trial_result(None, t3, result(2, 260)), TrialScheduler.STOP) - def testAlternateMetrics(self): - def result2(t, rew): - return dict(training_iteration=t, neg_mean_loss=rew) - + def _test_metrics(self, result_func, metric, mode): scheduler = AsyncHyperBandScheduler( grace_period=1, time_attr="training_iteration", - reward_attr="neg_mean_loss", + metric=metric, + mode=mode, brackets=1) t1 = Trial("PPO") # mean is 450, max 900, t_max=10 t2 = Trial("PPO") # mean is 450, max 450, t_max=5 @@ -1030,20 +1049,32 @@ class AsyncHyperBandSuite(unittest.TestCase): scheduler.on_trial_add(None, t2) for i in range(10): self.assertEqual( - scheduler.on_trial_result(None, t1, result2(i, i * 100)), + scheduler.on_trial_result(None, t1, result_func(i, i * 100)), TrialScheduler.CONTINUE) for i in range(5): self.assertEqual( - scheduler.on_trial_result(None, t2, result2(i, 450)), + scheduler.on_trial_result(None, t2, result_func(i, 450)), TrialScheduler.CONTINUE) - scheduler.on_trial_complete(None, t1, result2(10, 1000)) + scheduler.on_trial_complete(None, t1, result_func(10, 1000)) self.assertEqual( - scheduler.on_trial_result(None, t2, result2(5, 450)), + scheduler.on_trial_result(None, t2, result_func(5, 450)), TrialScheduler.CONTINUE) self.assertEqual( - scheduler.on_trial_result(None, t2, result2(6, 0)), + scheduler.on_trial_result(None, t2, result_func(6, 0)), TrialScheduler.CONTINUE) + def testAlternateMetrics(self): + def result2(t, rew): + return dict(training_iteration=t, neg_mean_loss=rew) + + self._test_metrics(result2, "neg_mean_loss", "max") + + def testAlternateMetricsMin(self): + def result2(t, rew): + return dict(training_iteration=t, mean_loss=-rew) + + self._test_metrics(result2, "mean_loss", "min") + if __name__ == "__main__": unittest.main(verbosity=2)