[tune] handling nan values (#9381)

python/ray/tune/analysis/experiment_analysis.py

@@ -65,6 +65,13 @@ class Analysis:
             mode (str): One of [min, max].
         """
         rows = self._retrieve_rows(metric=metric, mode=mode)
+        if not rows:
+            # only nans encountered when retrieving rows
+            logger.warning("Not able to retrieve the best config for {} "
+                           "according to the specified metric "
+                           "(only nans encountered).".format(
+                               self._experiment_dir))
+            return None
         all_configs = self.get_all_configs()
         compare_op = max if mode == "max" else min
         best_path = compare_op(rows, key=lambda k: rows[k][metric])
@@ -77,11 +84,20 @@ class Analysis:
             metric (str): Key for trial info to order on.
             mode (str): One of [min, max].
         """
+        assert mode in ["max", "min"]
         df = self.dataframe(metric=metric, mode=mode)
-        if mode == "max":
-            return df.iloc[df[metric].idxmax()].logdir
-        elif mode == "min":
-            return df.iloc[df[metric].idxmin()].logdir
+        mode_idx = pd.Series.idxmax if mode == "max" else pd.Series.idxmin
+        try:
+            return df.iloc[mode_idx(df[metric])].logdir
+        except KeyError:
+            # all dirs contains only nan values
+            # for the specified metric
+            # -> df is an empty dataframe
+            logger.warning("Not able to retrieve the best logdir for {} "
+                           "according to the specified metric "
+                           "(only nans encountered).".format(
+                               self._experiment_dir))
+            return None
 
     def fetch_trial_dataframes(self):
         fail_count = 0
@@ -161,7 +177,13 @@ class Analysis:
                 idx = df[metric].idxmin()
             else:
                 idx = -1
-            rows[path] = df.iloc[idx].to_dict()
+            try:
+                rows[path] = df.iloc[idx].to_dict()
+            except TypeError:
+                # idx is nan
+                logger.warning(
+                    "Warning: Non-numerical value(s) encountered for {}".
+                    format(path))
 
         return rows
 

python/ray/tune/tests/test_experiment_analysis.py

@@ -4,6 +4,7 @@ import tempfile
 import random
 import os
 import pandas as pd
+from numpy import nan
 
 import ray
 from ray.tune import run, sample_from
@@ -38,6 +39,21 @@ class ExperimentAnalysisSuite(unittest.TestCase):
                 "height": sample_from(lambda spec: int(100 * random.random())),
             })
 
+    def nan_test_exp(self):
+        nan_ea = run(
+            lambda x: nan,
+            name="testing_nan",
+            local_dir=self.test_dir,
+            stop={"training_iteration": 1},
+            checkpoint_freq=1,
+            num_samples=self.num_samples,
+            config={
+                "width": sample_from(
+                    lambda spec: 10 + int(90 * random.random())),
+                "height": sample_from(lambda spec: int(100 * random.random())),
+            })
+        return nan_ea
+
     def testDataframe(self):
         df = self.ea.dataframe()
 
@@ -58,11 +74,15 @@ class ExperimentAnalysisSuite(unittest.TestCase):
 
     def testBestConfig(self):
         best_config = self.ea.get_best_config(self.metric)
-
         self.assertTrue(isinstance(best_config, dict))
         self.assertTrue("width" in best_config)
         self.assertTrue("height" in best_config)
 
+    def testBestConfigNan(self):
+        nan_ea = self.nan_test_exp()
+        best_config = nan_ea.get_best_config(self.metric)
+        self.assertIsNone(best_config)
+
     def testBestLogdir(self):
         logdir = self.ea.get_best_logdir(self.metric)
         self.assertTrue(logdir.startswith(self.test_path))
@@ -70,6 +90,11 @@ class ExperimentAnalysisSuite(unittest.TestCase):
         self.assertTrue(logdir2.startswith(self.test_path))
         self.assertNotEquals(logdir, logdir2)
 
+    def testBestLogdirNan(self):
+        nan_ea = self.nan_test_exp()
+        logdir = nan_ea.get_best_logdir(self.metric)
+        self.assertIsNone(logdir)
+
     def testGetTrialCheckpointsPathsByTrial(self):
         best_trial = self.ea.get_best_trial(self.metric)
         checkpoints_metrics = self.ea.get_trial_checkpoints_paths(best_trial)