[tune] use default anonymous metric _metric if at least a mode is set (#12159)

Co-authored-by: Richard Liaw <rliaw@berkeley.edu>

python/ray/tune/suggest/ax.py

@@ -1,6 +1,7 @@
 from typing import Dict, List, Optional, Union
 
 from ax.service.ax_client import AxClient
+from ray.tune.result import DEFAULT_METRIC
 from ray.tune.sample import Categorical, Float, Integer, LogUniform, \
     Quantized, Uniform
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
@@ -45,7 +46,8 @@ class AxSearch(Searcher):
         metric (str): Name of the metric used as objective in this
             experiment. This metric must be present in `raw_data` argument
             to `log_data`. This metric must also be present in the dict
-            reported/returned by the Trainable.
+            reported/returned by the Trainable. If None but a mode was passed,
+            the `ray.tune.result.DEFAULT_METRIC` will be used per default.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute. Defaults to "max".
         parameter_constraints (list[str]): Parameter constraints, such as
@@ -146,9 +148,13 @@ class AxSearch(Searcher):
         self._live_trial_mapping = {}
 
         if self._ax or self._space:
-            self.setup_experiment()
+            self._setup_experiment()
+
+    def _setup_experiment(self):
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
 
-    def setup_experiment(self):
         if not self._ax:
             self._ax = AxClient()
 
@@ -200,7 +206,8 @@ class AxSearch(Searcher):
             self._metric = metric
         if mode:
             self._mode = mode
-        self.setup_experiment()
+
+        self._setup_experiment()
         return True
 
     def suggest(self, trial_id: str) -> Optional[Dict]:

python/ray/tune/suggest/bayesopt.py

@@ -5,6 +5,7 @@ import json
 from typing import Dict, Optional, Tuple
 
 from ray.tune import ExperimentAnalysis
+from ray.tune.result import DEFAULT_METRIC
 from ray.tune.sample import Domain, Float, Quantized
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
@@ -53,7 +54,9 @@ class BayesOptSearch(Searcher):
     Args:
         space (dict): Continuous search space. Parameters will be sampled from
             this space which will be used to run trials.
-        metric (str): The training result objective value attribute.
+        metric (str): The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
         utility_kwargs (dict): Parameters to define the utility function.
@@ -205,9 +208,13 @@ class BayesOptSearch(Searcher):
 
         self.optimizer = None
         if space:
-            self.setup_optimizer()
+            self._setup_optimizer()
+
+    def _setup_optimizer(self):
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
 
-    def setup_optimizer(self):
         self.optimizer = byo.BayesianOptimization(
             f=None,
             pbounds=self._space,
@@ -230,7 +237,7 @@ class BayesOptSearch(Searcher):
         elif self._mode == "min":
             self._metric_op = -1.
 
-        self.setup_optimizer()
+        self._setup_optimizer()
         return True
 
     def suggest(self, trial_id: str) -> Optional[Dict]:

python/ray/tune/suggest/bohb.py

@@ -6,6 +6,7 @@ import math
 from typing import Dict, Optional, Union
 
 import ConfigSpace
+from ray.tune.result import DEFAULT_METRIC
 from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
     Normal, \
     Quantized, \
@@ -45,7 +46,9 @@ class TuneBOHB(Searcher):
         bohb_config (dict): configuration for HpBandSter BOHB algorithm
         max_concurrent (int): Number of maximum concurrent trials. Defaults
             to 10.
-        metric (str): The training result objective value attribute.
+        metric (str): The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
         seed (int): Optional random seed to initialize the random number
@@ -133,11 +136,15 @@ class TuneBOHB(Searcher):
         super(TuneBOHB, self).__init__(metric=self._metric, mode=mode)
 
         if self._space:
-            self.setup_bohb()
+            self._setup_bohb()
 
-    def setup_bohb(self):
+    def _setup_bohb(self):
         from hpbandster.optimizers.config_generators.bohb import BOHB
 
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
         if self._mode == "max":
             self._metric_op = -1.
         elif self._mode == "min":
@@ -161,7 +168,7 @@ class TuneBOHB(Searcher):
         if mode:
             self._mode = mode
 
-        self.setup_bohb()
+        self._setup_bohb()
         return True
 
     def suggest(self, trial_id: str) -> Optional[Dict]:

python/ray/tune/suggest/dragonfly.py

@@ -7,6 +7,7 @@ import logging
 import pickle
 from typing import Dict, List, Optional, Union
 
+from ray.tune.result import DEFAULT_METRIC
 from ray.tune.sample import Domain, Float, Quantized
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
@@ -62,7 +63,9 @@ class DragonflySearch(Searcher):
             an optimizer as the `optimizer` argument. Defines the search space
             and requires a `domain` to be set. Can be automatically converted
             from the `config` dict passed to `tune.run()`.
-        metric (str): The training result objective value attribute.
+        metric (str): The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
         points_to_evaluate (list of lists): A list of points you'd like to run
@@ -177,9 +180,9 @@ class DragonflySearch(Searcher):
             self._opt = optimizer
             self.init_dragonfly()
         elif self._space:
-            self.setup_dragonfly()
+            self._setup_dragonfly()
 
-    def setup_dragonfly(self):
+    def _setup_dragonfly(self):
         """Setup dragonfly when no optimizer has been passed."""
         assert not self._opt, "Optimizer already set."
 
@@ -259,6 +262,10 @@ class DragonflySearch(Searcher):
         elif self._mode == "max":
             self._metric_op = 1.
 
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
     def set_search_properties(self, metric: Optional[str], mode: Optional[str],
                               config: Dict) -> bool:
         if self._opt:
@@ -270,7 +277,7 @@ class DragonflySearch(Searcher):
         if mode:
             self._mode = mode
 
-        self.setup_dragonfly()
+        self._setup_dragonfly()
         return True
 
     def suggest(self, trial_id: str) -> Optional[Dict]:

python/ray/tune/suggest/hyperopt.py

@@ -6,6 +6,7 @@ import logging
 from functools import partial
 import pickle
 
+from ray.tune.result import DEFAULT_METRIC
 from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
     Normal, \
     Quantized, \
@@ -50,7 +51,9 @@ class HyperOptSearch(Searcher):
         space (dict): HyperOpt configuration. Parameters will be sampled
             from this configuration and will be used to override
             parameters generated in the variant generation process.
-        metric (str): The training result objective value attribute.
+        metric (str): The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
         points_to_evaluate (list): Initial parameter suggestions to be run
@@ -177,14 +180,22 @@ class HyperOptSearch(Searcher):
                     UNRESOLVED_SEARCH_SPACE.format(
                         par="space", cls=type(self)))
                 space = self.convert_search_space(space)
-            self.domain = hpo.Domain(lambda spc: spc, space)
+            self._space = space
+            self._setup_hyperopt()
+
+    def _setup_hyperopt(self):
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
+        self.domain = hpo.Domain(lambda spc: spc, self._space)
 
     def set_search_properties(self, metric: Optional[str], mode: Optional[str],
                               config: Dict) -> bool:
         if self.domain:
             return False
         space = self.convert_search_space(config)
-        self.domain = hpo.Domain(lambda spc: spc, space)
+        self._space = space
 
         if metric:
             self._metric = metric
@@ -196,6 +207,7 @@ class HyperOptSearch(Searcher):
         elif self._mode == "min":
             self.metric_op = 1.
 
+        self._setup_hyperopt()
         return True
 
     def suggest(self, trial_id: str) -> Optional[Dict]:

python/ray/tune/suggest/nevergrad.py

@@ -2,6 +2,7 @@ import logging
 import pickle
 from typing import Dict, Optional, Union, List, Sequence
 
+from ray.tune.result import DEFAULT_METRIC
 from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
     Quantized
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
@@ -45,7 +46,9 @@ class NevergradSearch(Searcher):
         space (list|nevergrad.parameter.Parameter): Nevergrad parametrization
             to be passed to optimizer on instantiation, or list of parameter
             names if you passed an optimizer object.
-        metric (str): The training result objective value attribute.
+        metric (str): The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
         points_to_evaluate (list): Initial parameter suggestions to be run
@@ -165,9 +168,9 @@ class NevergradSearch(Searcher):
         self.max_concurrent = max_concurrent
 
         if self._nevergrad_opt or self._space:
-            self.setup_nevergrad()
+            self._setup_nevergrad()
 
-    def setup_nevergrad(self):
+    def _setup_nevergrad(self):
         if self._opt_factory:
             self._nevergrad_opt = self._opt_factory(self._space)
 
@@ -177,6 +180,10 @@ class NevergradSearch(Searcher):
         elif self._mode == "min":
             self._metric_op = 1.
 
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
         if hasattr(self._nevergrad_opt, "instrumentation"):  # added in v0.2.0
             if self._nevergrad_opt.instrumentation.kwargs:
                 if self._nevergrad_opt.instrumentation.args:
@@ -209,7 +216,7 @@ class NevergradSearch(Searcher):
         if mode:
             self._mode = mode
 
-        self.setup_nevergrad()
+        self._setup_nevergrad()
         return True
 
     def suggest(self, trial_id: str) -> Optional[Dict]:

python/ray/tune/suggest/optuna.py

@@ -2,7 +2,7 @@ import logging
 import pickle
 from typing import Dict, List, Optional, Tuple, Union
 
-from ray.tune.result import TRAINING_ITERATION
+from ray.tune.result import DEFAULT_METRIC, TRAINING_ITERATION
 from ray.tune.sample import Categorical, Domain, Float, Integer, LogUniform, \
     Quantized, Uniform
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
@@ -57,8 +57,9 @@ class OptunaSearch(Searcher):
         space (list): Hyperparameter search space definition for Optuna's
             sampler. This is a list, and samples for the parameters will
             be obtained in order.
-        metric (str): Metric that is reported back to Optuna on trial
-            completion.
+        metric (str): The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
         sampler (optuna.samplers.BaseSampler): Optuna sampler used to
@@ -139,9 +140,13 @@ class OptunaSearch(Searcher):
         self._ot_trials = {}
         self._ot_study = None
         if self._space:
-            self.setup_study(mode)
+            self._setup_study(mode)
+
+    def _setup_study(self, mode: str):
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
 
-    def setup_study(self, mode: str):
         self._ot_study = ot.study.create_study(
             storage=self._storage,
             sampler=self._sampler,
@@ -160,7 +165,8 @@ class OptunaSearch(Searcher):
             self._metric = metric
         if mode:
             self._mode = mode
-        self.setup_study(mode)
+
+        self._setup_study(mode)
         return True
 
     def suggest(self, trial_id: str) -> Optional[Dict]:

python/ray/tune/suggest/skopt.py

@@ -2,6 +2,7 @@ import logging
 import pickle
 from typing import Dict, List, Optional, Tuple, Union
 
+from ray.tune.result import DEFAULT_METRIC
 from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
@@ -75,7 +76,9 @@ class SkOptSearch(Searcher):
             parameters. If you passed an optimizer instance as the
             `optimizer` argument, this should be a list of parameter names
             instead.
-        metric (str): The training result objective value attribute.
+        metric (str): The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
         points_to_evaluate (list of lists): A list of points you'd like to run
@@ -184,11 +187,11 @@ class SkOptSearch(Searcher):
 
         self._skopt_opt = optimizer
         if self._skopt_opt or self._space:
-            self.setup_skopt()
+            self._setup_skopt()
 
         self._live_trial_mapping = {}
 
-    def setup_skopt(self):
+    def _setup_skopt(self):
         _validate_warmstart(self._parameter_names, self._points_to_evaluate,
                             self._evaluated_rewards)
 
@@ -213,6 +216,10 @@ class SkOptSearch(Searcher):
         elif self._mode == "min":
             self._metric_op = 1.
 
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
+
     def set_search_properties(self, metric: Optional[str], mode: Optional[str],
                               config: Dict) -> bool:
         if self._skopt_opt:
@@ -228,7 +235,7 @@ class SkOptSearch(Searcher):
         if mode:
             self._mode = mode
 
-        self.setup_skopt()
+        self._setup_skopt()
         return True
 
     def suggest(self, trial_id: str) -> Optional[Dict]:

python/ray/tune/suggest/zoopt.py

@@ -4,6 +4,7 @@ from typing import Dict, Optional, Tuple
 
 import ray
 import ray.cloudpickle as pickle
+from ray.tune.result import DEFAULT_METRIC
 from ray.tune.sample import Categorical, Domain, Float, Integer, Quantized, \
     Uniform
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
@@ -113,11 +114,11 @@ class ZOOptSearch(Searcher):
             For discrete dimensions: (discrete, search_range, has_order);
             For grid dimensions: (grid, grid_list).
             More details can be found in zoopt package.
-        metric (str): The training result objective value attribute.
-            Defaults to "episode_reward_mean".
+        metric (str): The training result objective value attribute. If None
+            but a mode was passed, the anonymous metric `_metric` will be used
+            per default.
         mode (str): One of {min, max}. Determines whether objective is
             minimizing or maximizing the metric attribute.
-            Defaults to "min".
         parallel_num (int): How many workers to parallel. Note that initial
             phase may start less workers than this number. More details can
             be found in zoopt package.
@@ -171,9 +172,13 @@ class ZOOptSearch(Searcher):
         super(ZOOptSearch, self).__init__(metric=self._metric, mode=mode)
 
         if self._dim_dict:
-            self.setup_zoopt()
+            self._setup_zoopt()
+
+    def _setup_zoopt(self):
+        if self._metric is None and self._mode:
+            # If only a mode was passed, use anonymous metric
+            self._metric = DEFAULT_METRIC
 
-    def setup_zoopt(self):
         _dim_list = []
         for k in self._dim_dict:
             self._dim_keys.append(k)
@@ -203,7 +208,7 @@ class ZOOptSearch(Searcher):
         elif self._mode == "min":
             self._metric_op = 1.
 
-        self.setup_zoopt()
+        self._setup_zoopt()
         return True
 
     def suggest(self, trial_id: str) -> Optional[Dict]: