[autoscaler] Experimental support for local / on-prem clusters (#2678)

This adds some experimental (undocumented) support for launching Ray on existing nodes. You have to provide the head ip, and the list of worker ips.

There are also a couple additional utils added for rsyncing files and port-forward.

python/ray/autoscaler/local/config.py

@@ -0,0 +1,7 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+def bootstrap_local(config):
+    return config

python/ray/autoscaler/local/example-full.yaml

@@ -0,0 +1,32 @@
+cluster_name: default
+min_workers: 0
+max_workers: 0
+docker:
+    image: ""
+    container_name: ""
+target_utilization_fraction: 0.8
+idle_timeout_minutes: 5
+provider:
+    type: local
+    head_ip: YOUR_HEAD_NODE_HOSTNAME
+    worker_ips: []
+auth:
+    ssh_user: YOUR_USERNAME
+    ssh_private_key: ~/.ssh/id_rsa
+head_node: {}
+worker_nodes: {}
+file_mounts:
+     "/tmp/ray_sha": "/YOUR/LOCAL/RAY/REPO/.git/refs/heads/YOUR_BRANCH"
+setup_commands: []
+head_setup_commands: []
+worker_setup_commands: []
+setup_commands:
+    - source activate ray && test -e ray || git clone https://github.com/YOUR_GITHUB/ray.git
+    - source activate ray && cd ray && git fetch && git reset --hard `cat /tmp/ray_sha`
+#    - source activate ray && cd ray/python && pip install -e .
+head_start_ray_commands:
+    - source activate ray && ray stop
+    - source activate ray && ulimit -c unlimited && ray start --head --redis-port=6379 --autoscaling-config=~/ray_bootstrap_config.yaml
+worker_start_ray_commands:
+    - source activate ray && ray stop
+    - source activate ray && ray start --redis-address=$RAY_HEAD_IP:6379

python/ray/autoscaler/local/node_provider.py

@@ -0,0 +1,126 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from filelock import FileLock
+import json
+import os
+import socket
+
+from ray.autoscaler.node_provider import NodeProvider
+from ray.autoscaler.tags import TAG_RAY_NODE_TYPE
+
+
+class ClusterState(object):
+    def __init__(self, lock_path, save_path, provider_config):
+        self.file_lock = FileLock(lock_path)
+        self.save_path = save_path
+
+        with self.file_lock:
+            if os.path.exists(self.save_path):
+                workers = json.loads(open(self.save_path).read())
+            else:
+                workers = {}
+            print("Loaded cluster state", workers)
+            for worker_ip in provider_config["worker_ips"]:
+                if worker_ip not in workers:
+                    workers[worker_ip] = {
+                        "tags": {
+                            TAG_RAY_NODE_TYPE: "worker"
+                        },
+                        "state": "terminated",
+                    }
+                else:
+                    assert workers[worker_ip]["tags"][
+                        TAG_RAY_NODE_TYPE] == "worker"
+            if provider_config["head_ip"] not in workers:
+                workers[provider_config["head_ip"]] = {
+                    "tags": {
+                        TAG_RAY_NODE_TYPE: "head"
+                    },
+                    "state": "terminated",
+                }
+            else:
+                assert workers[provider_config["head_ip"]]["tags"][
+                    TAG_RAY_NODE_TYPE] == "head"
+            assert len(workers) == len(provider_config["worker_ips"]) + 1
+            with open(self.save_path, "w") as f:
+                print("Writing cluster state", workers)
+                f.write(json.dumps(workers))
+
+    def get(self):
+        with self.file_lock:
+            workers = json.loads(open(self.save_path).read())
+            return workers
+
+    def put(self, worker_id, info):
+        assert "tags" in info
+        assert "state" in info
+        with self.file_lock:
+            workers = self.get()
+            workers[worker_id] = info
+            with open(self.save_path, "w") as f:
+                print("Writing cluster state", workers)
+                f.write(json.dumps(workers))
+
+
+class LocalNodeProvider(NodeProvider):
+    def __init__(self, provider_config, cluster_name):
+        NodeProvider.__init__(self, provider_config, cluster_name)
+        self.state = ClusterState("/tmp/cluster-{}.lock".format(cluster_name),
+                                  "/tmp/cluster-{}.state".format(cluster_name),
+                                  provider_config)
+
+    def nodes(self, tag_filters):
+        workers = self.state.get()
+        matching_ips = []
+        for worker_ip, info in workers.items():
+            if info["state"] == "terminated":
+                continue
+            ok = True
+            for k, v in tag_filters.items():
+                if info["tags"].get(k) != v:
+                    ok = False
+                    break
+            if ok:
+                matching_ips.append(worker_ip)
+        return matching_ips
+
+    def is_running(self, node_id):
+        return self.state.get()[node_id]["state"] == "running"
+
+    def is_terminated(self, node_id):
+        return not self.is_running(node_id)
+
+    def node_tags(self, node_id):
+        return self.state.get()[node_id]["tags"]
+
+    def external_ip(self, node_id):
+        return socket.gethostbyname(node_id)
+
+    def internal_ip(self, node_id):
+        return socket.gethostbyname(node_id)
+
+    def set_node_tags(self, node_id, tags):
+        with self.state.file_lock:
+            info = self.state.get()[node_id]
+            info["tags"].update(tags)
+            self.state.put(node_id, info)
+
+    def create_node(self, node_config, tags, count):
+        node_type = tags[TAG_RAY_NODE_TYPE]
+        with self.state.file_lock:
+            workers = self.state.get()
+            for node_id, info in workers.items():
+                if (info["state"] == "terminated"
+                        and info["tags"][TAG_RAY_NODE_TYPE] == node_type):
+                    info["tags"] = tags
+                    info["state"] = "running"
+                    self.state.put(node_id, info)
+                    return
+
+    def terminate_node(self, node_id):
+        workers = self.state.get()
+        info = workers[node_id]
+        info["state"] = "terminated"
+        self.state.put(node_id, info)