mirror of
https://github.com/wassname/ray.git
synced 2026-09-12 12:51:15 +08:00
[autoscaler] make commands very explicit on logs (#10713)
This commit is contained in:
@@ -187,6 +187,9 @@ def _isatty():
|
|||||||
class _CliLogger():
|
class _CliLogger():
|
||||||
"""Singleton class for CLI logging.
|
"""Singleton class for CLI logging.
|
||||||
|
|
||||||
|
Without calling 'cli_logger.configure', the CLILogger will default
|
||||||
|
to 'record' style logging.
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
old_style (bool):
|
old_style (bool):
|
||||||
If `old_style` is `True`, the old logging calls are used instead
|
If `old_style` is `True`, the old logging calls are used instead
|
||||||
@@ -219,13 +222,13 @@ class _CliLogger():
|
|||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.old_style = os.environ.get("RAY_LOG_NEWSTYLE", "1") == "0"
|
self.old_style = os.environ.get("RAY_LOG_NEWSTYLE", "1") == "0"
|
||||||
self.pretty = True
|
|
||||||
self.interactive = _isatty()
|
|
||||||
self.indent_level = 0
|
self.indent_level = 0
|
||||||
|
|
||||||
self._verbosity = 0
|
self._verbosity = 0
|
||||||
self._color_mode = "auto"
|
self._color_mode = "auto"
|
||||||
self._log_style = "auto"
|
self._log_style = "record"
|
||||||
|
self.pretty = False
|
||||||
|
self.interactive = False
|
||||||
|
|
||||||
# store whatever colorful has detected for future use if
|
# store whatever colorful has detected for future use if
|
||||||
# the color ouput is toggled (colorful detects # of supported colors,
|
# the color ouput is toggled (colorful detects # of supported colors,
|
||||||
@@ -239,13 +242,27 @@ class _CliLogger():
|
|||||||
format_tmpl = ray_constants.LOGGER_FORMAT
|
format_tmpl = ray_constants.LOGGER_FORMAT
|
||||||
self._formatter = logging.Formatter(format_tmpl)
|
self._formatter = logging.Formatter(format_tmpl)
|
||||||
|
|
||||||
|
def configure(self, log_style=None, color_mode=None, verbosity=None):
|
||||||
|
"""Configures the logger according to values."""
|
||||||
|
if log_style is not None:
|
||||||
|
self._set_log_style(log_style)
|
||||||
|
|
||||||
|
if color_mode is not None:
|
||||||
|
self._set_color_mode(color_mode)
|
||||||
|
|
||||||
|
if verbosity is not None:
|
||||||
|
self._set_verbosity(verbosity)
|
||||||
|
|
||||||
|
self.detect_colors()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def log_style(self):
|
def log_style(self):
|
||||||
return self._log_style
|
return self._log_style
|
||||||
|
|
||||||
@log_style.setter
|
def _set_log_style(self, x):
|
||||||
def log_style(self, x):
|
"""Configures interactivity and formatting."""
|
||||||
self._log_style = x.lower()
|
self._log_style = x.lower()
|
||||||
|
self.interactive = _isatty()
|
||||||
|
|
||||||
if self._log_style == "auto":
|
if self._log_style == "auto":
|
||||||
self.pretty = _isatty()
|
self.pretty = _isatty()
|
||||||
@@ -259,8 +276,7 @@ class _CliLogger():
|
|||||||
def color_mode(self):
|
def color_mode(self):
|
||||||
return self._color_mode
|
return self._color_mode
|
||||||
|
|
||||||
@color_mode.setter
|
def _set_color_mode(self, x):
|
||||||
def color_mode(self, x):
|
|
||||||
self._color_mode = x.lower()
|
self._color_mode = x.lower()
|
||||||
self.detect_colors()
|
self.detect_colors()
|
||||||
|
|
||||||
@@ -270,8 +286,7 @@ class _CliLogger():
|
|||||||
return 999
|
return 999
|
||||||
return self._verbosity
|
return self._verbosity
|
||||||
|
|
||||||
@verbosity.setter
|
def _set_verbosity(self, x):
|
||||||
def verbosity(self, x):
|
|
||||||
self._verbosity = x
|
self._verbosity = x
|
||||||
|
|
||||||
def detect_colors(self):
|
def detect_colors(self):
|
||||||
|
|||||||
@@ -397,10 +397,7 @@ def start(node_ip_address, redis_address, address, redis_port, port,
|
|||||||
system_config, lru_evict, enable_object_reconstruction,
|
system_config, lru_evict, enable_object_reconstruction,
|
||||||
metrics_export_port, log_style, log_color, verbose):
|
metrics_export_port, log_style, log_color, verbose):
|
||||||
"""Start Ray processes manually on the local machine."""
|
"""Start Ray processes manually on the local machine."""
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
if gcs_server_port and not head:
|
if gcs_server_port and not head:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
@@ -755,10 +752,7 @@ def start(node_ip_address, redis_address, address, redis_port, port,
|
|||||||
def stop(force, verbose, log_style, log_color):
|
def stop(force, verbose, log_style, log_color):
|
||||||
"""Stop Ray processes manually on the local machine."""
|
"""Stop Ray processes manually on the local machine."""
|
||||||
|
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
# Note that raylet needs to exit before object store, otherwise
|
# Note that raylet needs to exit before object store, otherwise
|
||||||
# it cannot exit gracefully.
|
# it cannot exit gracefully.
|
||||||
@@ -923,10 +917,7 @@ def up(cluster_config_file, min_workers, max_workers, no_restart, restart_only,
|
|||||||
yes, cluster_name, no_config_cache, redirect_command_output,
|
yes, cluster_name, no_config_cache, redirect_command_output,
|
||||||
use_login_shells, log_style, log_color, verbose):
|
use_login_shells, log_style, log_color, verbose):
|
||||||
"""Create or update a Ray cluster."""
|
"""Create or update a Ray cluster."""
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
if restart_only or no_restart:
|
if restart_only or no_restart:
|
||||||
cli_logger.doassert(restart_only != no_restart,
|
cli_logger.doassert(restart_only != no_restart,
|
||||||
@@ -989,10 +980,7 @@ def up(cluster_config_file, min_workers, max_workers, no_restart, restart_only,
|
|||||||
def down(cluster_config_file, yes, workers_only, cluster_name,
|
def down(cluster_config_file, yes, workers_only, cluster_name,
|
||||||
keep_min_workers, log_style, log_color, verbose):
|
keep_min_workers, log_style, log_color, verbose):
|
||||||
"""Tear down a Ray cluster."""
|
"""Tear down a Ray cluster."""
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
teardown_cluster(cluster_config_file, yes, workers_only, cluster_name,
|
teardown_cluster(cluster_config_file, yes, workers_only, cluster_name,
|
||||||
keep_min_workers)
|
keep_min_workers)
|
||||||
@@ -1042,10 +1030,7 @@ def kill_random_node(cluster_config_file, yes, hard, cluster_name):
|
|||||||
def monitor(cluster_config_file, lines, cluster_name, log_style, log_color,
|
def monitor(cluster_config_file, lines, cluster_name, log_style, log_color,
|
||||||
verbose):
|
verbose):
|
||||||
"""Tails the autoscaler logs of a Ray cluster."""
|
"""Tails the autoscaler logs of a Ray cluster."""
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
monitor_cluster(cluster_config_file, lines, cluster_name)
|
monitor_cluster(cluster_config_file, lines, cluster_name)
|
||||||
|
|
||||||
@@ -1085,10 +1070,7 @@ def monitor(cluster_config_file, lines, cluster_name, log_style, log_color,
|
|||||||
def attach(cluster_config_file, start, screen, tmux, cluster_name,
|
def attach(cluster_config_file, start, screen, tmux, cluster_name,
|
||||||
no_config_cache, new, port_forward, log_style, log_color, verbose):
|
no_config_cache, new, port_forward, log_style, log_color, verbose):
|
||||||
"""Create or attach to a SSH session to a Ray cluster."""
|
"""Create or attach to a SSH session to a Ray cluster."""
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
port_forward = [(port, port) for port in list(port_forward)]
|
port_forward = [(port, port) for port in list(port_forward)]
|
||||||
attach_cluster(
|
attach_cluster(
|
||||||
@@ -1116,10 +1098,7 @@ def attach(cluster_config_file, start, screen, tmux, cluster_name,
|
|||||||
def rsync_down(cluster_config_file, source, target, cluster_name, log_style,
|
def rsync_down(cluster_config_file, source, target, cluster_name, log_style,
|
||||||
log_color, verbose):
|
log_color, verbose):
|
||||||
"""Download specific files from a Ray cluster."""
|
"""Download specific files from a Ray cluster."""
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
rsync(cluster_config_file, source, target, cluster_name, down=True)
|
rsync(cluster_config_file, source, target, cluster_name, down=True)
|
||||||
|
|
||||||
@@ -1144,10 +1123,7 @@ def rsync_down(cluster_config_file, source, target, cluster_name, log_style,
|
|||||||
def rsync_up(cluster_config_file, source, target, cluster_name, all_nodes,
|
def rsync_up(cluster_config_file, source, target, cluster_name, all_nodes,
|
||||||
log_style, log_color, verbose):
|
log_style, log_color, verbose):
|
||||||
"""Upload specific files to a Ray cluster."""
|
"""Upload specific files to a Ray cluster."""
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
rsync(
|
rsync(
|
||||||
cluster_config_file,
|
cluster_config_file,
|
||||||
@@ -1215,10 +1191,7 @@ def submit(cluster_config_file, screen, tmux, stop, start, cluster_name,
|
|||||||
Example:
|
Example:
|
||||||
>>> ray submit [CLUSTER.YAML] experiment.py -- --smoke-test
|
>>> ray submit [CLUSTER.YAML] experiment.py -- --smoke-test
|
||||||
"""
|
"""
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
cli_logger.doassert(not (screen and tmux),
|
cli_logger.doassert(not (screen and tmux),
|
||||||
"`{}` and `{}` are incompatible.", cf.bold("--screen"),
|
"`{}` and `{}` are incompatible.", cf.bold("--screen"),
|
||||||
@@ -1339,10 +1312,7 @@ def exec(cluster_config_file, cmd, run_env, screen, tmux, stop, start,
|
|||||||
cluster_name, no_config_cache, port_forward, log_style, log_color,
|
cluster_name, no_config_cache, port_forward, log_style, log_color,
|
||||||
verbose):
|
verbose):
|
||||||
"""Execute a command via SSH on a Ray cluster."""
|
"""Execute a command via SSH on a Ray cluster."""
|
||||||
cli_logger.log_style = log_style
|
cli_logger.configure(log_style, log_color, verbose)
|
||||||
cli_logger.color_mode = log_color
|
|
||||||
cli_logger.verbosity = verbose
|
|
||||||
cli_logger.detect_colors()
|
|
||||||
|
|
||||||
port_forward = [(port, port) for port in list(port_forward)]
|
port_forward = [(port, port) for port in list(port_forward)]
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user