[tune] Added EarlyStopping and relative test suite (#8459)

python/ray/tune/tests/test_api.py

@@ -10,7 +10,8 @@ import ray
 from ray.rllib import _register_all
 
 from ray import tune
-from ray.tune import DurableTrainable, Trainable, TuneError, Stopper
+from ray.tune import (DurableTrainable, Trainable, TuneError, Stopper,
+                      EarlyStopping)
 from ray.tune import register_env, register_trainable, run_experiments
 from ray.tune.schedulers import TrialScheduler, FIFOScheduler
 from ray.tune.trial import Trial
@@ -487,6 +488,37 @@ class TrainableFunctionApiTest(unittest.TestCase):
                 t.last_result.get("training_iteration") is None
                 for t in trials))
 
+    def testEarlyStopping(self):
+        def train(config, reporter):
+            reporter(test=0)
+
+        top = 3
+
+        with self.assertRaises(ValueError):
+            EarlyStopping("test", top=0)
+        with self.assertRaises(ValueError):
+            EarlyStopping("test", top="0")
+        with self.assertRaises(ValueError):
+            EarlyStopping("test", std=0)
+        with self.assertRaises(ValueError):
+            EarlyStopping("test", std="0")
+        with self.assertRaises(ValueError):
+            EarlyStopping("test", mode="0")
+
+        stopper = EarlyStopping("test", top=top, mode="min")
+
+        analysis = tune.run(train, num_samples=10, stop=stopper)
+        self.assertTrue(
+            all(t.status == Trial.TERMINATED for t in analysis.trials))
+        self.assertTrue(len(analysis.dataframe()) <= top)
+
+        stopper = EarlyStopping("test", top=top, mode="min")
+
+        analysis = tune.run(train, num_samples=10, stop=stopper)
+        self.assertTrue(
+            all(t.status == Trial.TERMINATED for t in analysis.trials))
+        self.assertTrue(len(analysis.dataframe()) <= top)
+
     def testBadStoppingFunction(self):
         def train(config, reporter):
             for i in range(10):