[autoscaler] Add tmux support for attach and exec (#2907)

Adds a tmux flag that can be used to support background execution of experiments. Cannot be used together with screen. Seems to be useful feature that has shown up with different users.

python/ray/autoscaler/commands.py

@@ -210,27 +210,29 @@ def get_or_create_head_node(config, config_file, no_restart, restart_only, yes,
                                        provider.external_ip(head_node)))
 
 
-def attach_cluster(config_file, start, override_cluster_name):
+def attach_cluster(config_file, start, use_tmux, override_cluster_name):
     """Attaches to a screen for the specified cluster.
 
     Arguments:
         config_file: path to the cluster yaml
         start: whether to start the cluster if it isn't up
+        use_tmux: whether to use tmux as multiplexer
         override_cluster_name: set the name of the cluster
     """
-
-    exec_cluster(config_file, "screen -L -xRR", False, False, start,
+    cmd = "tmux attach || tmux new" if use_tmux else "screen -L -xRR"
+    exec_cluster(config_file, cmd, False, False, False, start,
                  override_cluster_name, None)
 
 
-def exec_cluster(config_file, cmd, screen, stop, start, override_cluster_name,
-                 port_forward):
+def exec_cluster(config_file, cmd, screen, tmux, stop, start,
+                 override_cluster_name, port_forward):
     """Runs a command on the specified cluster.
 
     Arguments:
         config_file: path to the cluster yaml
         cmd: command to run
         screen: whether to run in a screen
+        tmux: whether to run in a tmux session
         stop: whether to stop the cluster after command run
         start: whether to start the cluster if it isn't up
         override_cluster_name: set the name of the cluster
@@ -254,10 +256,16 @@ def exec_cluster(config_file, cmd, screen, stop, start, override_cluster_name,
     if stop:
         cmd += ("; ray stop; ray teardown ~/ray_bootstrap_config.yaml --yes "
                 "--workers-only; sudo shutdown -h now")
-    _exec(updater, cmd, screen, expect_error=stop, port_forward=port_forward)
+    _exec(
+        updater,
+        cmd,
+        screen,
+        tmux,
+        expect_error=stop,
+        port_forward=port_forward)
 
 
-def _exec(updater, cmd, screen, expect_error=False, port_forward=None):
+def _exec(updater, cmd, screen, tmux, expect_error=False, port_forward=None):
     if cmd:
         if screen:
             cmd = [
@@ -265,6 +273,13 @@ def _exec(updater, cmd, screen, expect_error=False, port_forward=None):
                 quote(cmd + "; exec bash")
             ]
             cmd = " ".join(cmd)
+        elif tmux:
+            # TODO: Consider providing named session functionality
+            cmd = [
+                "tmux", "new", "-d", "bash", "-c",
+                quote(cmd + "; exec bash")
+            ]
+            cmd = " ".join(cmd)
         updater.ssh_cmd(
             cmd,
             verbose=False,

python/ray/autoscaler/local/node_provider.py

@@ -12,6 +12,8 @@ from ray.autoscaler.node_provider import NodeProvider
 from ray.autoscaler.tags import TAG_RAY_NODE_TYPE
 
 logger = logging.getLogger(__name__)
+filelock_logger = logging.getLogger("filelock")
+filelock_logger.setLevel(logging.WARNING)
 
 
 class ClusterState(object):

python/ray/scripts/scripts.py

@@ -470,14 +470,16 @@ def teardown(cluster_config_file, yes, workers_only, cluster_name):
     is_flag=True,
     default=False,
     help=("Start the cluster if needed."))
+@click.option(
+    "--tmux", is_flag=True, default=False, help=("Run the command in tmux."))
 @click.option(
     "--cluster-name",
     "-n",
     required=False,
     type=str,
     help=("Override the configured cluster name."))
-def attach(cluster_config_file, start, cluster_name):
-    attach_cluster(cluster_config_file, start, cluster_name)
+def attach(cluster_config_file, start, tmux, cluster_name):
+    attach_cluster(cluster_config_file, start, tmux, cluster_name)
 
 
 @cli.command()
@@ -526,6 +528,8 @@ def rsync_up(cluster_config_file, source, target, cluster_name):
     is_flag=True,
     default=False,
     help=("Run the command in a screen."))
+@click.option(
+    "--tmux", is_flag=True, default=False, help=("Run the command in tmux."))
 @click.option(
     "--cluster-name",
     "-n",
@@ -534,10 +538,14 @@ def rsync_up(cluster_config_file, source, target, cluster_name):
     help=("Override the configured cluster name."))
 @click.option(
     "--port-forward", required=False, type=int, help=("Port to forward."))
-def exec_cmd(cluster_config_file, cmd, screen, stop, start, cluster_name,
+def exec_cmd(cluster_config_file, cmd, screen, tmux, stop, start, cluster_name,
              port_forward):
-    exec_cluster(cluster_config_file, cmd, screen, stop, start, cluster_name,
-                 port_forward)
+    assert not (screen and tmux), "Can specify only one of `screen` or `tmux`."
+    exec_cluster(cluster_config_file, cmd, screen, tmux, stop, start,
+                 cluster_name, port_forward)
+    if tmux:
+        logger.info("Use `ray attach {} --tmux` "
+                    "to check on command status.".format(cluster_config_file))
 
 
 @cli.command()