[tune] logger refactor part 3: Add ExperimentLogger class (#11749)

python/ray/tune/tests/test_api.py

@@ -12,7 +12,7 @@ from ray.rllib import _register_all
 
 from ray import tune
 from ray.tune import (DurableTrainable, Trainable, TuneError, Stopper,
-                      EarlyStopping)
+                      EarlyStopping, run)
 from ray.tune import register_env, register_trainable, run_experiments
 from ray.tune.schedulers import (TrialScheduler, FIFOScheduler,
                                  AsyncHyperBandScheduler)
@@ -90,19 +90,19 @@ class TrainableFunctionApiTest(unittest.TestCase):
         class_trainable_name = "class_trainable"
         register_trainable(class_trainable_name, _WrappedTrainable)
 
-        trials = run_experiments(
-            {
-                "function_api": {
-                    "run": _function_trainable,
-                    "loggers": [FunctionAPILogger],
-                },
-                "class_api": {
-                    "run": class_trainable_name,
-                    "loggers": [ClassAPILogger],
-                },
-            },
+        [trial1] = run(
+            _function_trainable,
+            loggers=[FunctionAPILogger],
             raise_on_failed_trial=False,
-            scheduler=MockScheduler())
+            scheduler=MockScheduler()).trials
+
+        [trial2] = run(
+            class_trainable_name,
+            loggers=[ClassAPILogger],
+            raise_on_failed_trial=False,
+            scheduler=MockScheduler()).trials
+
+        trials = [trial1, trial2]
 
         # Ignore these fields
         NO_COMPARE_FIELDS = {

python/ray/tune/tests/test_run_experiment.py

@@ -7,7 +7,7 @@ from ray.rllib import _register_all
 from ray.tune.result import TIMESTEPS_TOTAL
 from ray.tune import Trainable, TuneError
 from ray.tune import register_trainable, run_experiments
-from ray.tune.logger import Logger
+from ray.tune.logger import LegacyExperimentLogger, Logger
 from ray.tune.experiment import Experiment
 from ray.tune.trial import Trial, ExportFormat
 
@@ -173,21 +173,25 @@ class RunExperimentTest(unittest.TestCase):
         for trial in trials:
             self.assertEqual(trial.status, Trial.TERMINATED)
 
-    def testCustomLogger(self):
+    def testCustomLoggerNoAutoLogging(self):
+        """Does not create CSV/JSON logger callbacks automatically"""
+        os.environ["TUNE_DISABLE_AUTO_CALLBACK_LOGGERS"] = "1"
+
         class CustomLogger(Logger):
             def on_result(self, result):
                 with open(os.path.join(self.logdir, "test.log"), "w") as f:
                     f.write("hi")
 
-        [trial] = run_experiments({
-            "foo": {
-                "run": "__fake",
-                "stop": {
-                    "training_iteration": 1
-                },
-                "loggers": [CustomLogger]
-            }
-        })
+        [trial] = run_experiments(
+            {
+                "foo": {
+                    "run": "__fake",
+                    "stop": {
+                        "training_iteration": 1
+                    }
+                }
+            },
+            callbacks=[LegacyExperimentLogger(logger_classes=[CustomLogger])])
         self.assertTrue(os.path.exists(os.path.join(trial.logdir, "test.log")))
         self.assertFalse(
             os.path.exists(os.path.join(trial.logdir, "params.json")))
@@ -203,16 +207,65 @@ class RunExperimentTest(unittest.TestCase):
         self.assertTrue(
             os.path.exists(os.path.join(trial.logdir, "params.json")))
 
+        [trial] = run_experiments(
+            {
+                "foo": {
+                    "run": "__fake",
+                    "stop": {
+                        "training_iteration": 1
+                    }
+                }
+            },
+            callbacks=[LegacyExperimentLogger(logger_classes=[])])
+        self.assertFalse(
+            os.path.exists(os.path.join(trial.logdir, "params.json")))
+
+    def testCustomLoggerWithAutoLogging(self):
+        """Creates CSV/JSON logger callbacks automatically"""
+        if "TUNE_DISABLE_AUTO_CALLBACK_LOGGERS" in os.environ:
+            del os.environ["TUNE_DISABLE_AUTO_CALLBACK_LOGGERS"]
+
+        class CustomLogger(Logger):
+            def on_result(self, result):
+                with open(os.path.join(self.logdir, "test.log"), "w") as f:
+                    f.write("hi")
+
+        [trial] = run_experiments(
+            {
+                "foo": {
+                    "run": "__fake",
+                    "stop": {
+                        "training_iteration": 1
+                    }
+                }
+            },
+            callbacks=[LegacyExperimentLogger(logger_classes=[CustomLogger])])
+        self.assertTrue(os.path.exists(os.path.join(trial.logdir, "test.log")))
+        self.assertTrue(
+            os.path.exists(os.path.join(trial.logdir, "params.json")))
+
         [trial] = run_experiments({
             "foo": {
                 "run": "__fake",
                 "stop": {
                     "training_iteration": 1
-                },
-                "loggers": []
+                }
             }
         })
-        self.assertFalse(
+        self.assertTrue(
+            os.path.exists(os.path.join(trial.logdir, "params.json")))
+
+        [trial] = run_experiments(
+            {
+                "foo": {
+                    "run": "__fake",
+                    "stop": {
+                        "training_iteration": 1
+                    }
+                }
+            },
+            callbacks=[LegacyExperimentLogger(logger_classes=[])])
+        self.assertTrue(
             os.path.exists(os.path.join(trial.logdir, "params.json")))
 
     def testCustomTrialString(self):

python/ray/tune/tests/test_trial_runner.py

@@ -263,7 +263,7 @@ class TrialRunnerTest(unittest.TestCase):
             def on_trial_result(self, trial_runner, trial, result):
                 if result["training_iteration"] == 1:
                     executor = trial_runner.trial_executor
-                    executor.stop_trial(trial, stop_logger=False)
+                    executor.stop_trial(trial)
                     trial.update_resources(2, 0)
                     executor.start_trial(trial)
                 return TrialScheduler.CONTINUE

python/ray/tune/tests/test_trial_runner_callbacks.py

@@ -9,12 +9,16 @@ import ray
 from ray import tune
 from ray.rllib import _register_all
 from ray.tune.checkpoint_manager import Checkpoint
+from ray.tune.logger import DEFAULT_LOGGERS, ExperimentLogger, \
+    LegacyExperimentLogger
 from ray.tune.ray_trial_executor import RayTrialExecutor
 from ray.tune.result import TRAINING_ITERATION
+from ray.tune.syncer import SyncConfig, SyncerCallback
 
 from ray.tune.trial import Trial
-from ray.tune.callback import Callback
 from ray.tune.trial_runner import TrialRunner
+from ray.tune import Callback
+from ray.tune.utils.callback import create_default_callbacks
 
 
 class TestCallback(Callback):
@@ -200,6 +204,63 @@ class TrialRunnerCallbacks(unittest.TestCase):
             self.callback.state["trial_complete"]["trial"].config["do"],
             "delay")
 
+    def testCallbackReordering(self):
+        """SyncerCallback should come after ExperimentLogger callbacks"""
+
+        def get_positions(callbacks):
+            first_logger_pos = None
+            last_logger_pos = None
+            syncer_pos = None
+            for i, callback in enumerate(callbacks):
+                if isinstance(callback, ExperimentLogger):
+                    if first_logger_pos is None:
+                        first_logger_pos = i
+                    last_logger_pos = i
+                elif isinstance(callback, SyncerCallback):
+                    syncer_pos = i
+            return first_logger_pos, last_logger_pos, syncer_pos
+
+        # Auto creation of loggers, no callbacks, no syncer
+        callbacks = create_default_callbacks(None, SyncConfig(), None)
+        first_logger_pos, last_logger_pos, syncer_pos = get_positions(
+            callbacks)
+        self.assertLess(last_logger_pos, syncer_pos)
+
+        # Auto creation of loggers with callbacks
+        callbacks = create_default_callbacks([Callback()], SyncConfig(), None)
+        first_logger_pos, last_logger_pos, syncer_pos = get_positions(
+            callbacks)
+        self.assertLess(last_logger_pos, syncer_pos)
+
+        # Auto creation of loggers with existing logger (but no CSV/JSON)
+        callbacks = create_default_callbacks([ExperimentLogger()],
+                                             SyncConfig(), None)
+        first_logger_pos, last_logger_pos, syncer_pos = get_positions(
+            callbacks)
+        self.assertLess(last_logger_pos, syncer_pos)
+
+        # This should throw an error as the syncer comes before the logger
+        with self.assertRaises(ValueError):
+            callbacks = create_default_callbacks(
+                [SyncerCallback(None),
+                 ExperimentLogger()], SyncConfig(), None)
+
+        # This should be reordered but preserve the regular callback order
+        [mc1, mc2, mc3] = [Callback(), Callback(), Callback()]
+        # Has to be legacy logger to avoid logger callback creation
+        lc = LegacyExperimentLogger(logger_classes=DEFAULT_LOGGERS)
+        callbacks = create_default_callbacks([mc1, mc2, lc, mc3], SyncConfig(),
+                                             None)
+        print(callbacks)
+        first_logger_pos, last_logger_pos, syncer_pos = get_positions(
+            callbacks)
+        self.assertLess(last_logger_pos, syncer_pos)
+        self.assertLess(callbacks.index(mc1), callbacks.index(mc2))
+        self.assertLess(callbacks.index(mc2), callbacks.index(mc3))
+        self.assertLess(callbacks.index(lc), callbacks.index(mc3))
+        # Syncer callback is appended
+        self.assertLess(callbacks.index(mc3), syncer_pos)
+
 
 if __name__ == "__main__":
     import pytest

python/ray/tune/tests/test_trial_scheduler.py

@@ -217,10 +217,8 @@ class _MockTrialExecutor(TrialExecutor):
         trial.restored_checkpoint = checkpoint_obj.value
         trial.status = Trial.RUNNING
 
-    def stop_trial(self, trial, error=False, error_msg=None, stop_logger=True):
+    def stop_trial(self, trial, error=False, error_msg=None):
         trial.status = Trial.ERROR if error else Trial.TERMINATED
-        if stop_logger:
-            trial.logger_running = False
 
     def restore(self, trial, checkpoint=None, block=False):
         pass