Fixes and cleanups for the multinode setting. (#143)

* Add function for driver to get address info from Redis.

* Use Redis address instead of Redis port.

* Configure Redis to run in unprotected mode.

* Add method for starting Ray processes on non-head node.

* Pass in correct node ip address to start_plasma_manager.

* Script for starting Ray processes.

* Handle the case where an object already exists in the store. Maybe this should also compare the object hashes.

* Have driver get info from Redis when start_ray_local=False.

* Fix.

* Script for killing ray processes.

* Catch some errors when the main_loop in a worker throws an exception.

* Allow redirecting stdout and stderr to /dev/null.

* Wrap start_ray.py in a shell script.

* More helpful error messages.

* Fixes.

* Wait for redis server to start up before configuring it.

* Allow seeding of deterministic object ID generation.

* Small change.

scripts/start_ray.py

@@ -0,0 +1,40 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+
+import ray.services as services
+
+parser = argparse.ArgumentParser(description="Parse addresses for the worker to connect to.")
+parser.add_argument("--node-ip-address", required=True, type=str, help="the ip address of the worker's node")
+parser.add_argument("--redis-address", required=False, type=str, help="the address to use for Redis")
+parser.add_argument("--num-workers", default=10, required=False, type=int, help="the number of workers to start on this node")
+parser.add_argument("--head", action="store_true", help="provide this argument for the head node")
+
+if __name__ == "__main__":
+  args = parser.parse_args()
+
+  # Note that we redirect stdout and stderr to /dev/null because otherwise
+  # attempts to print may cause exceptions if a process is started inside of an
+  # SSH connection and the SSH connection dies. TODO(rkn): This is a temporary
+  # fix. We should actually redirect stdout and stderr to Redis in some way.
+
+  if args.head:
+    # Start Ray on the head node.
+    if args.redis_address is not None:
+      raise Exception("If --head is passed in, a Redis server will be started, so a Redis address should not be provided.")
+    address_info = services.start_ray_local(node_ip_address=args.node_ip_address,
+                                            num_workers=args.num_workers,
+                                            cleanup=False,
+                                            redirect_output=True)
+  else:
+    # Start Ray on a non-head node.
+    if args.redis_address is None:
+      raise Exception("If --head is not passed in, --redis-address must be provided.")
+    address_info = services.start_ray_node(node_ip_address=args.node_ip_address,
+                                           redis_address=args.redis_address,
+                                           num_workers=args.num_workers,
+                                           cleanup=False,
+                                           redirect_output=True)
+  print(address_info)

scripts/start_ray.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+python "$ROOT_DIR/start_ray.py" "$@"

scripts/stop_ray.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+killall redis-server global_scheduler plasma_store plasma_manager photon_scheduler