[tune] Handle infinite and NaN values (#11835)

python/ray/tune/BUILD

@@ -163,6 +163,14 @@ py_test(
     tags = ["exclusive"],
 )
 
+py_test(
+    name = "test_searchers",
+    size = "medium",
+    srcs = ["tests/test_searchers.py"],
+    deps = [":tune_lib"],
+    tags = ["exclusive"],
+)
+
 py_test(
     name = "test_sync",
     size = "medium",

python/ray/tune/analysis/experiment_analysis.py

@@ -5,6 +5,7 @@ from numbers import Number
 from typing import Any, Dict, List, Optional, Tuple
 
 from ray.tune.utils import flatten_dict
+from ray.tune.utils.util import is_nan_or_inf
 
 try:
     import pandas as pd
@@ -495,7 +496,8 @@ class ExperimentAnalysis(Analysis):
     def get_best_trial(self,
                        metric: Optional[str] = None,
                        mode: Optional[str] = None,
-                       scope: str = "last") -> Optional[Trial]:
+                       scope: str = "last",
+                       filter_nan_and_inf: bool = True) -> Optional[Trial]:
         """Retrieve the best trial object.
 
         Compares all trials' scores on ``metric``.
@@ -518,6 +520,9 @@ class ExperimentAnalysis(Analysis):
                 `metric` and compare across trials based on `mode=[min,max]`.
                 If `scope=all`, find each trial's min/max score for `metric`
                 based on `mode`, and compare trials based on `mode=[min,max]`.
+            filter_nan_and_inf (bool): If True (default), NaN or infinite
+                values are disregarded and these trials are never selected as
+                the best trial.
         """
         metric = self._validate_metric(metric)
         mode = self._validate_mode(mode)
@@ -541,6 +546,9 @@ class ExperimentAnalysis(Analysis):
             else:
                 metric_score = trial.metric_analysis[metric][mode]
 
+            if filter_nan_and_inf and is_nan_or_inf(metric_score):
+                continue
+
             if best_metric_score is None:
                 best_metric_score = metric_score
                 best_trial = trial
@@ -555,7 +563,7 @@ class ExperimentAnalysis(Analysis):
 
         if not best_trial:
             logger.warning(
-                "Could not find best trial. Did you pass the correct `metric`"
+                "Could not find best trial. Did you pass the correct `metric` "
                 "parameter?")
         return best_trial
 

python/ray/tune/schedulers/hyperband.py

@@ -244,12 +244,14 @@ class HyperBandScheduler(FIFOScheduler):
                     bracket.cleanup_trial(t)
                     action = TrialScheduler.STOP
                 else:
-                    raise TuneError("Trial with unexpected status encountered")
+                    raise TuneError(f"Trial with unexpected bad status "
+                                    f"encountered: {t.status}")
 
             # ready the good trials - if trial is too far ahead, don't continue
             for t in good:
                 if t.status not in [Trial.PAUSED, Trial.RUNNING]:
-                    raise TuneError("Trial with unexpected status encountered")
+                    raise TuneError(f"Trial with unexpected good status "
+                                    f"encountered: {t.status}")
                 if bracket.continue_trial(t):
                     if t.status == Trial.PAUSED:
                         self._unpause_trial(trial_runner, t)

python/ray/tune/suggest/bayesopt.py

@@ -9,7 +9,7 @@ from ray.tune.sample import Domain, Float, Quantized
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
-from ray.tune.utils.util import unflatten_dict
+from ray.tune.utils.util import is_nan_or_inf, unflatten_dict
 
 try:  # Python 3 only -- needed for lint test.
     import bayes_opt as byo
@@ -38,6 +38,9 @@ class BayesOptSearch(Searcher):
     fmfn/BayesianOptimization is a library for Bayesian Optimization. More
     info can be found here: https://github.com/fmfn/BayesianOptimization.
 
+    This searcher will automatically filter out any NaN, inf or -inf
+    results.
+
     You will need to install fmfn/BayesianOptimization via the following:
 
     .. code-block:: bash
@@ -352,6 +355,8 @@ class BayesOptSearch(Searcher):
 
     def _register_result(self, params: Tuple[str], result: Dict):
         """Register given tuple of params and results."""
+        if is_nan_or_inf(result[self.metric]):
+            return
         self.optimizer.register(params, self._metric_op * result[self.metric])
 
     def save(self, checkpoint_path: str):

python/ray/tune/suggest/dragonfly.py

@@ -11,7 +11,7 @@ from ray.tune.sample import Domain, Float, Quantized
 from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
-from ray.tune.utils.util import flatten_dict
+from ray.tune.utils.util import flatten_dict, is_nan_or_inf
 
 try:  # Python 3 only -- needed for lint test.
     import dragonfly
@@ -47,6 +47,9 @@ class DragonflySearch(Searcher):
     This interface requires using FunctionCallers and optimizers provided by
     Dragonfly.
 
+    This searcher will automatically filter out any NaN, inf or -inf
+    results.
+
     Parameters:
         optimizer (dragonfly.opt.BlackboxOptimiser|str): Optimizer provided
             from dragonfly. Choose an optimiser that extends BlackboxOptimiser.
@@ -131,7 +134,7 @@ class DragonflySearch(Searcher):
     """
 
     def __init__(self,
-                 optimizer: Optional[BlackboxOptimiser] = None,
+                 optimizer: Optional[Union[str, BlackboxOptimiser]] = None,
                  domain: Optional[str] = None,
                  space: Optional[Union[Dict, List[Dict]]] = None,
                  metric: Optional[str] = None,
@@ -304,7 +307,7 @@ class DragonflySearch(Searcher):
                           error: bool = False):
         """Passes result to Dragonfly unless early terminated or errored."""
         trial_info = self._live_trial_mapping.pop(trial_id)
-        if result:
+        if result and not is_nan_or_inf(result[self._metric]):
             self._opt.tell([(trial_info,
                              self._metric_op * result[self._metric])])
 
@@ -357,5 +360,4 @@ class DragonflySearch(Searcher):
             resolve_value("/".join(path), domain)
             for path, domain in domain_vars
         ]
-
         return space

python/ray/tune/suggest/skopt.py

@@ -7,7 +7,7 @@ from ray.tune.suggest.suggestion import UNRESOLVED_SEARCH_SPACE, \
     UNDEFINED_METRIC_MODE, UNDEFINED_SEARCH_SPACE
 from ray.tune.suggest.variant_generator import parse_spec_vars
 from ray.tune.utils import flatten_dict
-from ray.tune.utils.util import unflatten_dict
+from ray.tune.utils.util import is_nan_or_inf, unflatten_dict
 
 try:
     import skopt as sko
@@ -64,6 +64,9 @@ class SkOptSearch(Searcher):
 
     This Search Algorithm requires you to pass in a `skopt Optimizer object`_.
 
+    This searcher will automatically filter out any NaN, inf or -inf
+    results.
+
     Parameters:
         optimizer (skopt.optimizer.Optimizer): Optimizer provided
             from skopt.
@@ -268,8 +271,9 @@ class SkOptSearch(Searcher):
 
     def _process_result(self, trial_id: str, result: Dict):
         skopt_trial_info = self._live_trial_mapping[trial_id]
-        self._skopt_opt.tell(skopt_trial_info,
-                             self._metric_op * result[self._metric])
+        if result and not is_nan_or_inf(result[self._metric]):
+            self._skopt_opt.tell(skopt_trial_info,
+                                 self._metric_op * result[self._metric])
 
     def save(self, checkpoint_path: str):
         trials_object = (self._initial_points, self._skopt_opt)

python/ray/tune/tests/test_searchers.py

@@ -0,0 +1,180 @@
+import unittest
+
+import numpy as np
+
+import ray
+from ray import tune
+
+
+def _invalid_objective(config):
+    # DragonFly uses `point`
+    metric = "point" if "point" in config else "report"
+
+    if config[metric] > 4:
+        tune.report(float("inf"))
+    elif config[metric] > 3:
+        tune.report(float("-inf"))
+    elif config[metric] > 2:
+        tune.report(np.nan)
+    else:
+        tune.report(float(config[metric]) or 0.1)
+
+
+class InvalidValuesTest(unittest.TestCase):
+    def setUp(self):
+        self.config = {"report": tune.uniform(0.0, 5.0)}
+
+    def tearDown(self):
+        pass
+
+    @classmethod
+    def setUpClass(cls):
+        ray.init(num_cpus=4, num_gpus=0, include_dashboard=False)
+
+    @classmethod
+    def tearDownClass(cls):
+        ray.shutdown()
+
+    def testBayesOpt(self):
+        from ray.tune.suggest.bayesopt import BayesOptSearch
+
+        np.random.seed(1234)  # At least one nan, inf, -inf and float
+
+        out = tune.run(
+            _invalid_objective,
+            search_alg=BayesOptSearch(),
+            config=self.config,
+            metric="_metric",
+            mode="max",
+            num_samples=8,
+            reuse_actors=False)
+
+        best_trial = out.best_trial
+        self.assertLessEqual(best_trial.config["report"], 2.0)
+
+    def testBOHB(self):
+        from ray.tune.suggest.bohb import TuneBOHB
+
+        converted_config = TuneBOHB.convert_search_space(self.config)
+        converted_config.seed(1000)  # At least one nan, inf, -inf and float
+
+        out = tune.run(
+            _invalid_objective,
+            search_alg=TuneBOHB(
+                space=converted_config, metric="_metric", mode="max"),
+            metric="_metric",
+            mode="max",
+            num_samples=8,
+            reuse_actors=False)
+
+        best_trial = out.best_trial
+        self.assertLessEqual(best_trial.config["report"], 2.0)
+
+    def testDragonfly(self):
+        from ray.tune.suggest.dragonfly import DragonflySearch
+
+        np.random.seed(1000)  # At least one nan, inf, -inf and float
+
+        out = tune.run(
+            _invalid_objective,
+            search_alg=DragonflySearch(domain="euclidean", optimizer="random"),
+            config=self.config,
+            metric="_metric",
+            mode="max",
+            num_samples=8,
+            reuse_actors=False)
+
+        best_trial = out.best_trial
+        self.assertLessEqual(best_trial.config["point"], 2.0)
+
+    def testHyperopt(self):
+        from ray.tune.suggest.hyperopt import HyperOptSearch
+
+        out = tune.run(
+            _invalid_objective,
+            # At least one nan, inf, -inf and float
+            search_alg=HyperOptSearch(random_state_seed=1234),
+            config=self.config,
+            metric="_metric",
+            mode="max",
+            num_samples=8,
+            reuse_actors=False)
+
+        best_trial = out.best_trial
+        self.assertLessEqual(best_trial.config["report"], 2.0)
+
+    def testNevergrad(self):
+        from ray.tune.suggest.nevergrad import NevergradSearch
+        import nevergrad as ng
+
+        np.random.seed(2020)  # At least one nan, inf, -inf and float
+
+        out = tune.run(
+            _invalid_objective,
+            search_alg=NevergradSearch(optimizer=ng.optimizers.RandomSearch),
+            config=self.config,
+            metric="_metric",
+            mode="max",
+            num_samples=16,
+            reuse_actors=False)
+
+        best_trial = out.best_trial
+        self.assertLessEqual(best_trial.config["report"], 2.0)
+
+    def testOptuna(self):
+        from ray.tune.suggest.optuna import OptunaSearch
+        from optuna.samplers import RandomSampler
+
+        np.random.seed(1000)  # At least one nan, inf, -inf and float
+
+        out = tune.run(
+            _invalid_objective,
+            search_alg=OptunaSearch(sampler=RandomSampler(seed=1234)),
+            config=self.config,
+            metric="_metric",
+            mode="max",
+            num_samples=8,
+            reuse_actors=False)
+
+        best_trial = out.best_trial
+        self.assertLessEqual(best_trial.config["report"], 2.0)
+
+    def testSkopt(self):
+        from ray.tune.suggest.skopt import SkOptSearch
+
+        np.random.seed(1234)  # At least one nan, inf, -inf and float
+
+        out = tune.run(
+            _invalid_objective,
+            search_alg=SkOptSearch(),
+            config=self.config,
+            metric="_metric",
+            mode="max",
+            num_samples=8,
+            reuse_actors=False)
+
+        best_trial = out.best_trial
+        self.assertLessEqual(best_trial.config["report"], 2.0)
+
+    def testZOOpt(self):
+        from ray.tune.suggest.zoopt import ZOOptSearch
+
+        np.random.seed(1000)  # At least one nan, inf, -inf and float
+
+        out = tune.run(
+            _invalid_objective,
+            search_alg=ZOOptSearch(budget=100, parallel_num=4),
+            config=self.config,
+            metric="_metric",
+            mode="max",
+            num_samples=8,
+            reuse_actors=False)
+
+        best_trial = out.best_trial
+        self.assertLessEqual(best_trial.config["report"], 2.0)
+
+
+if __name__ == "__main__":
+    import pytest
+    import sys
+    sys.exit(pytest.main(["-v", __file__]))

python/ray/tune/tests/test_trial_scheduler.py

@@ -1836,6 +1836,21 @@ class AsyncHyperBandSuite(unittest.TestCase):
                 TrialScheduler.CONTINUE)
         return t1, t2
 
+    def nanInfSetup(self, scheduler, runner=None):
+        t1 = Trial("PPO")
+        t2 = Trial("PPO")
+        t3 = Trial("PPO")
+        scheduler.on_trial_add(runner, t1)
+        scheduler.on_trial_add(runner, t2)
+        scheduler.on_trial_add(runner, t3)
+        for i in range(10):
+            scheduler.on_trial_result(runner, t1, result(i, np.nan))
+        for i in range(10):
+            scheduler.on_trial_result(runner, t2, result(i, float("inf")))
+        for i in range(10):
+            scheduler.on_trial_result(runner, t3, result(i, float("-inf")))
+        return t1, t2, t3
+
     def testAsyncHBOnComplete(self):
         scheduler = AsyncHyperBandScheduler(
             metric="episode_reward_mean", mode="max", max_t=10, brackets=1)
@@ -1921,6 +1936,41 @@ class AsyncHyperBandSuite(unittest.TestCase):
             scheduler.on_trial_result(None, t3, result(2, 260)),
             TrialScheduler.STOP)
 
+    def testMedianStoppingNanInf(self):
+        scheduler = MedianStoppingRule(
+            metric="episode_reward_mean", mode="max")
+
+        t1, t2, t3 = self.nanInfSetup(scheduler)
+        scheduler.on_trial_complete(None, t1, result(10, np.nan))
+        scheduler.on_trial_complete(None, t2, result(10, float("inf")))
+        scheduler.on_trial_complete(None, t3, result(10, float("-inf")))
+
+    def testHyperbandNanInf(self):
+        scheduler = HyperBandScheduler(
+            metric="episode_reward_mean", mode="max")
+        t1, t2, t3 = self.nanInfSetup(scheduler)
+        scheduler.on_trial_complete(None, t1, result(10, np.nan))
+        scheduler.on_trial_complete(None, t2, result(10, float("inf")))
+        scheduler.on_trial_complete(None, t3, result(10, float("-inf")))
+
+    def testBOHBNanInf(self):
+        scheduler = HyperBandForBOHB(metric="episode_reward_mean", mode="max")
+
+        runner = _MockTrialRunner(scheduler)
+        runner._search_alg = MagicMock()
+        runner._search_alg.searcher = MagicMock()
+
+        t1, t2, t3 = self.nanInfSetup(scheduler, runner)
+        # skip trial complete in this mock setting
+
+    def testPBTNanInf(self):
+        scheduler = PopulationBasedTraining(
+            metric="episode_reward_mean", mode="max")
+        t1, t2, t3 = self.nanInfSetup(scheduler)
+        scheduler.on_trial_complete(None, t1, result(10, np.nan))
+        scheduler.on_trial_complete(None, t2, result(10, float("inf")))
+        scheduler.on_trial_complete(None, t3, result(10, float("-inf")))
+
     def _test_metrics(self, result_func, metric, mode):
         scheduler = AsyncHyperBandScheduler(
             grace_period=1,

python/ray/tune/utils/util.py

@@ -161,6 +161,10 @@ def date_str():
     return datetime.today().strftime("%Y-%m-%d_%H-%M-%S")
 
 
+def is_nan_or_inf(value):
+    return np.isnan(value) or np.isinf(value)
+
+
 def env_integer(key, default):
     # TODO(rliaw): move into ray.constants
     if key in os.environ: