diff --git a/doc/source/tune/user-guide.rst b/doc/source/tune/user-guide.rst index 7b3c7faf9..14218dd35 100644 --- a/doc/source/tune/user-guide.rst +++ b/doc/source/tune/user-guide.rst @@ -686,6 +686,7 @@ These are the environment variables Ray Tune currently considers: with the parameter values in them) * **TUNE_RESULT_DIR**: Directory where Tune trial results are stored. If this is not set, ``~/ray_results`` will be used. +* **TUNE_SYNCER_VERBOSITY**: Amount of command output when using Tune with Docker Syncer. Defaults to 0. There are some environment variables that are mostly relevant for integrated libraries: diff --git a/python/ray/autoscaler/_private/cli_logger.py b/python/ray/autoscaler/_private/cli_logger.py index fa7577419..b0890599e 100644 --- a/python/ray/autoscaler/_private/cli_logger.py +++ b/python/ray/autoscaler/_private/cli_logger.py @@ -283,6 +283,7 @@ class _CliLogger(): self.indent_level = 0 self._verbosity = 0 + self._verbosity_overriden = False self._color_mode = "auto" self._log_style = "record" self.pretty = False @@ -340,12 +341,15 @@ class _CliLogger(): @property def verbosity(self): - if not self.pretty: + if self._verbosity_overriden: + return self._verbosity + elif not self.pretty: return 999 return self._verbosity def _set_verbosity(self, x): self._verbosity = x + self._verbosity_overriden = True def detect_colors(self): """Update color output settings. diff --git a/python/ray/autoscaler/_private/commands.py b/python/ray/autoscaler/_private/commands.py index 922c2332b..487297390 100644 --- a/python/ray/autoscaler/_private/commands.py +++ b/python/ray/autoscaler/_private/commands.py @@ -972,8 +972,9 @@ def rsync(config_file: str, if source and target: # print rsync progress for single file rsync - cmd_output_util.set_output_redirected(False) - set_rsync_silent(False) + if cli_logger.verbosity > 0: + cmd_output_util.set_output_redirected(False) + set_rsync_silent(False) rsync(source, target, is_file_mount) else: updater.sync_file_mounts(rsync) diff --git a/python/ray/autoscaler/sdk.py b/python/ray/autoscaler/sdk.py index bf190fe69..2f67cce7b 100644 --- a/python/ray/autoscaler/sdk.py +++ b/python/ray/autoscaler/sdk.py @@ -10,6 +10,7 @@ from ray.autoscaler._private import commands from ray.autoscaler._private.event_system import ( # noqa: F401 CreateClusterEvent, # noqa: F401 global_event_system) +from ray.autoscaler._private.cli_logger import cli_logger def create_or_update_cluster(cluster_config: Union[dict, str], @@ -206,6 +207,32 @@ def request_resources(num_cpus: Optional[int] = None, return commands.request_resources(num_cpus, bundles) +def configure_logging(log_style: Optional[str] = None, + color_mode: Optional[str] = None, + verbosity: Optional[int] = None): + """Configures logging for cluster command calls. + + Args: + log_style (str): If 'pretty', outputs with formatting and color. + If 'record', outputs record-style without formatting. + 'auto' defaults to 'pretty', and disables pretty logging + if stdin is *not* a TTY. Defaults to "auto". + color_mode (str): + Can be "true", "false", or "auto". + + Enables or disables `colorful`. + + If `color_mode` is "auto", is set to `not stdout.isatty()` + vebosity (int): + Output verbosity (0, 1, 2, 3). + + Low verbosity will disable `verbose` and `very_verbose` messages. + + """ + cli_logger.configure( + log_style=log_style, color_mode=color_mode, verbosity=verbosity) + + @contextmanager def _as_config_file(cluster_config: Union[dict, str]) -> Iterator[str]: if isinstance(cluster_config, dict): diff --git a/python/ray/tune/examples/pbt_example.py b/python/ray/tune/examples/pbt_example.py index 768d05ec2..389516174 100755 --- a/python/ray/tune/examples/pbt_example.py +++ b/python/ray/tune/examples/pbt_example.py @@ -85,8 +85,15 @@ if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( "--smoke-test", action="store_true", help="Finish quickly for testing") + parser.add_argument( + "--cluster", + action="store_true", + help="Distribute tuning on a cluster") args, _ = parser.parse_known_args() - if args.smoke_test: + + if args.cluster: + ray.init(address="auto") + elif args.smoke_test: ray.init(num_cpus=2) # force pausing to happen for test else: ray.init() diff --git a/python/ray/tune/integration/docker.py b/python/ray/tune/integration/docker.py index 3b1290de7..b3cb1d3ea 100644 --- a/python/ray/tune/integration/docker.py +++ b/python/ray/tune/integration/docker.py @@ -2,9 +2,10 @@ import os from typing import Optional, Tuple from ray import services -from ray.autoscaler.sdk import rsync +from ray.autoscaler.sdk import rsync, configure_logging from ray.tune.syncer import NodeSyncer from ray.tune.sync_client import SyncClient +from ray.tune.utils import env_integer class DockerSyncer(NodeSyncer): @@ -13,6 +14,9 @@ class DockerSyncer(NodeSyncer): without a custom sync client. The sync client defaults to ``DockerSyncClient`` instead. + Set the env var `TUNE_SYNCER_VERBOSITY` to increase verbosity + of syncing operations (0, 1, 2, 3). Defaults to 0. + .. note:: This syncer only works with the Ray cluster launcher. If you use your own Docker setup, make sure the nodes can connect @@ -35,6 +39,9 @@ class DockerSyncer(NodeSyncer): local_dir: str, remote_dir: str, sync_client: Optional[SyncClient] = None): + configure_logging( + log_style="record", + verbosity=env_integer("TUNE_SYNCER_VERBOSITY", 0)) self.local_ip = services.get_node_ip_address() self.worker_ip = None diff --git a/python/ray/tune/ray_trial_executor.py b/python/ray/tune/ray_trial_executor.py index 2174abfdc..285dc5505 100644 --- a/python/ray/tune/ray_trial_executor.py +++ b/python/ray/tune/ray_trial_executor.py @@ -398,8 +398,8 @@ class RayTrialExecutor(TrialExecutor): trial (Trial): Trial to be reset. new_config (dict): New configuration for Trial trainable. new_experiment_tag (str): New experiment name for trial. - logger_creator (Callable[[Dict], Logger]): A function that - instantiates a logger on the actor process. + logger_creator (Optional[Callable[[Dict], Logger]]): Function + that instantiates a logger on the actor process. Returns: True if `reset_config` is successful else False. diff --git a/python/ray/tune/trainable.py b/python/ray/tune/trainable.py index ccae6d287..aa08fd03b 100644 --- a/python/ray/tune/trainable.py +++ b/python/ray/tune/trainable.py @@ -391,7 +391,12 @@ class Trainable: self._result_logger.flush() self._result_logger.close() - self._create_logger(new_config.copy(), logger_creator) + if logger_creator: + logger.debug("Logger reset.") + self._create_logger(new_config.copy(), logger_creator) + else: + logger.debug("Did not reset logger. Got: " + f"trainable.reset(logger_creator={logger_creator}).") stdout_file = new_config.pop(STDOUT_FILE, None) stderr_file = new_config.pop(STDERR_FILE, None)