diff --git a/doc/source/autoscaling.rst b/doc/source/autoscaling.rst index 370618944..54ebcc350 100644 --- a/doc/source/autoscaling.rst +++ b/doc/source/autoscaling.rst @@ -70,6 +70,9 @@ You can use ``ray exec`` to conveniently run commands on clusters. Note that scr $ ray exec cluster.yaml 'echo "hello world"' \ --start --stop --cluster-name experiment-1 + # Run a command in a detached tmux session + $ ray exec cluster.yaml 'echo "hello world"' --tmux + # Run a command in a screen (experimental) $ ray exec cluster.yaml 'echo "hello world"' --screen @@ -86,6 +89,10 @@ You can use ``ray attach`` to attach to an interactive console on the cluster. # Open a screen on a new cluster called 'session-1' $ ray attach cluster.yaml --start --cluster-name=session-1 + # Attach to tmux session on cluster (creates a new one if none available) + $ ray attach cluster.yaml --tmux + + Port-forwarding applications ---------------------------- diff --git a/python/ray/autoscaler/commands.py b/python/ray/autoscaler/commands.py index 2410c20d6..a2be4a803 100644 --- a/python/ray/autoscaler/commands.py +++ b/python/ray/autoscaler/commands.py @@ -210,27 +210,29 @@ def get_or_create_head_node(config, config_file, no_restart, restart_only, yes, provider.external_ip(head_node))) -def attach_cluster(config_file, start, override_cluster_name): +def attach_cluster(config_file, start, use_tmux, override_cluster_name): """Attaches to a screen for the specified cluster. Arguments: config_file: path to the cluster yaml start: whether to start the cluster if it isn't up + use_tmux: whether to use tmux as multiplexer override_cluster_name: set the name of the cluster """ - - exec_cluster(config_file, "screen -L -xRR", False, False, start, + cmd = "tmux attach || tmux new" if use_tmux else "screen -L -xRR" + exec_cluster(config_file, cmd, False, False, False, start, override_cluster_name, None) -def exec_cluster(config_file, cmd, screen, stop, start, override_cluster_name, - port_forward): +def exec_cluster(config_file, cmd, screen, tmux, stop, start, + override_cluster_name, port_forward): """Runs a command on the specified cluster. Arguments: config_file: path to the cluster yaml cmd: command to run screen: whether to run in a screen + tmux: whether to run in a tmux session stop: whether to stop the cluster after command run start: whether to start the cluster if it isn't up override_cluster_name: set the name of the cluster @@ -254,10 +256,16 @@ def exec_cluster(config_file, cmd, screen, stop, start, override_cluster_name, if stop: cmd += ("; ray stop; ray teardown ~/ray_bootstrap_config.yaml --yes " "--workers-only; sudo shutdown -h now") - _exec(updater, cmd, screen, expect_error=stop, port_forward=port_forward) + _exec( + updater, + cmd, + screen, + tmux, + expect_error=stop, + port_forward=port_forward) -def _exec(updater, cmd, screen, expect_error=False, port_forward=None): +def _exec(updater, cmd, screen, tmux, expect_error=False, port_forward=None): if cmd: if screen: cmd = [ @@ -265,6 +273,13 @@ def _exec(updater, cmd, screen, expect_error=False, port_forward=None): quote(cmd + "; exec bash") ] cmd = " ".join(cmd) + elif tmux: + # TODO: Consider providing named session functionality + cmd = [ + "tmux", "new", "-d", "bash", "-c", + quote(cmd + "; exec bash") + ] + cmd = " ".join(cmd) updater.ssh_cmd( cmd, verbose=False, diff --git a/python/ray/autoscaler/local/node_provider.py b/python/ray/autoscaler/local/node_provider.py index 432c7b0ab..f20da8d22 100644 --- a/python/ray/autoscaler/local/node_provider.py +++ b/python/ray/autoscaler/local/node_provider.py @@ -12,6 +12,8 @@ from ray.autoscaler.node_provider import NodeProvider from ray.autoscaler.tags import TAG_RAY_NODE_TYPE logger = logging.getLogger(__name__) +filelock_logger = logging.getLogger("filelock") +filelock_logger.setLevel(logging.WARNING) class ClusterState(object): diff --git a/python/ray/scripts/scripts.py b/python/ray/scripts/scripts.py index e942797de..09ef158dd 100644 --- a/python/ray/scripts/scripts.py +++ b/python/ray/scripts/scripts.py @@ -470,14 +470,16 @@ def teardown(cluster_config_file, yes, workers_only, cluster_name): is_flag=True, default=False, help=("Start the cluster if needed.")) +@click.option( + "--tmux", is_flag=True, default=False, help=("Run the command in tmux.")) @click.option( "--cluster-name", "-n", required=False, type=str, help=("Override the configured cluster name.")) -def attach(cluster_config_file, start, cluster_name): - attach_cluster(cluster_config_file, start, cluster_name) +def attach(cluster_config_file, start, tmux, cluster_name): + attach_cluster(cluster_config_file, start, tmux, cluster_name) @cli.command() @@ -526,6 +528,8 @@ def rsync_up(cluster_config_file, source, target, cluster_name): is_flag=True, default=False, help=("Run the command in a screen.")) +@click.option( + "--tmux", is_flag=True, default=False, help=("Run the command in tmux.")) @click.option( "--cluster-name", "-n", @@ -534,10 +538,14 @@ def rsync_up(cluster_config_file, source, target, cluster_name): help=("Override the configured cluster name.")) @click.option( "--port-forward", required=False, type=int, help=("Port to forward.")) -def exec_cmd(cluster_config_file, cmd, screen, stop, start, cluster_name, +def exec_cmd(cluster_config_file, cmd, screen, tmux, stop, start, cluster_name, port_forward): - exec_cluster(cluster_config_file, cmd, screen, stop, start, cluster_name, - port_forward) + assert not (screen and tmux), "Can specify only one of `screen` or `tmux`." + exec_cluster(cluster_config_file, cmd, screen, tmux, stop, start, + cluster_name, port_forward) + if tmux: + logger.info("Use `ray attach {} --tmux` " + "to check on command status.".format(cluster_config_file)) @cli.command()