[autoscaler] Add tmux support for attach and exec (#2907)

Adds a tmux flag that can be used to support background execution of experiments. Cannot be used together with screen. Seems to be useful feature that has shown up with different users.

doc/source/autoscaling.rst

@@ -70,6 +70,9 @@ You can use ``ray exec`` to conveniently run commands on clusters. Note that scr
     $ ray exec cluster.yaml 'echo "hello world"' \
         --start --stop --cluster-name experiment-1
 
+    # Run a command in a detached tmux session
+    $ ray exec cluster.yaml 'echo "hello world"' --tmux
+
     # Run a command in a screen (experimental)
     $ ray exec cluster.yaml 'echo "hello world"' --screen
 
@@ -86,6 +89,10 @@ You can use ``ray attach`` to attach to an interactive console on the cluster.
     # Open a screen on a new cluster called 'session-1'
     $ ray attach cluster.yaml --start --cluster-name=session-1
 
+    # Attach to tmux session on cluster (creates a new one if none available)
+    $ ray attach cluster.yaml --tmux
+
+
 Port-forwarding applications
 ----------------------------
 

python/ray/autoscaler/commands.py

@@ -210,27 +210,29 @@ def get_or_create_head_node(config, config_file, no_restart, restart_only, yes,
                                        provider.external_ip(head_node)))
 
 
-def attach_cluster(config_file, start, override_cluster_name):
+def attach_cluster(config_file, start, use_tmux, override_cluster_name):
     """Attaches to a screen for the specified cluster.
 
     Arguments:
         config_file: path to the cluster yaml
         start: whether to start the cluster if it isn't up
+        use_tmux: whether to use tmux as multiplexer
         override_cluster_name: set the name of the cluster
     """
-
-    exec_cluster(config_file, "screen -L -xRR", False, False, start,
+    cmd = "tmux attach || tmux new" if use_tmux else "screen -L -xRR"
+    exec_cluster(config_file, cmd, False, False, False, start,
                  override_cluster_name, None)
 
 
-def exec_cluster(config_file, cmd, screen, stop, start, override_cluster_name,
-                 port_forward):
+def exec_cluster(config_file, cmd, screen, tmux, stop, start,
+                 override_cluster_name, port_forward):
     """Runs a command on the specified cluster.
 
     Arguments:
         config_file: path to the cluster yaml
         cmd: command to run
         screen: whether to run in a screen
+        tmux: whether to run in a tmux session
         stop: whether to stop the cluster after command run
         start: whether to start the cluster if it isn't up
         override_cluster_name: set the name of the cluster
@@ -254,10 +256,16 @@ def exec_cluster(config_file, cmd, screen, stop, start, override_cluster_name,
     if stop:
         cmd += ("; ray stop; ray teardown ~/ray_bootstrap_config.yaml --yes "
                 "--workers-only; sudo shutdown -h now")
-    _exec(updater, cmd, screen, expect_error=stop, port_forward=port_forward)
+    _exec(
+        updater,
+        cmd,
+        screen,
+        tmux,
+        expect_error=stop,
+        port_forward=port_forward)
 
 
-def _exec(updater, cmd, screen, expect_error=False, port_forward=None):
+def _exec(updater, cmd, screen, tmux, expect_error=False, port_forward=None):
     if cmd:
         if screen:
             cmd = [
@@ -265,6 +273,13 @@ def _exec(updater, cmd, screen, expect_error=False, port_forward=None):
                 quote(cmd + "; exec bash")
             ]
             cmd = " ".join(cmd)
+        elif tmux:
+            # TODO: Consider providing named session functionality
+            cmd = [
+                "tmux", "new", "-d", "bash", "-c",
+                quote(cmd + "; exec bash")
+            ]
+            cmd = " ".join(cmd)
         updater.ssh_cmd(
             cmd,
             verbose=False,

python/ray/autoscaler/local/node_provider.py

@@ -12,6 +12,8 @@ from ray.autoscaler.node_provider import NodeProvider
 from ray.autoscaler.tags import TAG_RAY_NODE_TYPE
 
 logger = logging.getLogger(__name__)
+filelock_logger = logging.getLogger("filelock")
+filelock_logger.setLevel(logging.WARNING)
 
 
 class ClusterState(object):

python/ray/scripts/scripts.py

@@ -470,14 +470,16 @@ def teardown(cluster_config_file, yes, workers_only, cluster_name):
     is_flag=True,
     default=False,
     help=("Start the cluster if needed."))
+@click.option(
+    "--tmux", is_flag=True, default=False, help=("Run the command in tmux."))
 @click.option(
     "--cluster-name",
     "-n",
     required=False,
     type=str,
     help=("Override the configured cluster name."))
-def attach(cluster_config_file, start, cluster_name):
-    attach_cluster(cluster_config_file, start, cluster_name)
+def attach(cluster_config_file, start, tmux, cluster_name):
+    attach_cluster(cluster_config_file, start, tmux, cluster_name)
 
 
 @cli.command()
@@ -526,6 +528,8 @@ def rsync_up(cluster_config_file, source, target, cluster_name):
     is_flag=True,
     default=False,
     help=("Run the command in a screen."))
+@click.option(
+    "--tmux", is_flag=True, default=False, help=("Run the command in tmux."))
 @click.option(
     "--cluster-name",
     "-n",
@@ -534,10 +538,14 @@ def rsync_up(cluster_config_file, source, target, cluster_name):
     help=("Override the configured cluster name."))
 @click.option(
     "--port-forward", required=False, type=int, help=("Port to forward."))
-def exec_cmd(cluster_config_file, cmd, screen, stop, start, cluster_name,
+def exec_cmd(cluster_config_file, cmd, screen, tmux, stop, start, cluster_name,
              port_forward):
-    exec_cluster(cluster_config_file, cmd, screen, stop, start, cluster_name,
-                 port_forward)
+    assert not (screen and tmux), "Can specify only one of `screen` or `tmux`."
+    exec_cluster(cluster_config_file, cmd, screen, tmux, stop, start,
+                 cluster_name, port_forward)
+    if tmux:
+        logger.info("Use `ray attach {} --tmux` "
+                    "to check on command status.".format(cluster_config_file))
 
 
 @cli.command()