Fixes and cleanups for the multinode setting. (#143)

* Add function for driver to get address info from Redis.

* Use Redis address instead of Redis port.

* Configure Redis to run in unprotected mode.

* Add method for starting Ray processes on non-head node.

* Pass in correct node ip address to start_plasma_manager.

* Script for starting Ray processes.

* Handle the case where an object already exists in the store. Maybe this should also compare the object hashes.

* Have driver get info from Redis when start_ray_local=False.

* Fix.

* Script for killing ray processes.

* Catch some errors when the main_loop in a worker throws an exception.

* Allow redirecting stdout and stderr to /dev/null.

* Wrap start_ray.py in a shell script.

* More helpful error messages.

* Fixes.

* Wait for redis server to start up before configuring it.

* Allow seeding of deterministic object ID generation.

* Small change.

src/photon/photon/photon_services.py

@@ -10,7 +10,7 @@ import time
 def random_name():
   return str(random.randint(0, 99999999))
 
-def start_local_scheduler(plasma_store_name, plasma_manager_name=None, plasma_address=None, node_ip_address="127.0.0.1", redis_address=None, use_valgrind=False, use_profiler=False):
+def start_local_scheduler(plasma_store_name, plasma_manager_name=None, plasma_address=None, node_ip_address="127.0.0.1", redis_address=None, use_valgrind=False, use_profiler=False, redirect_output=False):
   """Start a local scheduler process.
 
   Args:
@@ -29,6 +29,8 @@ def start_local_scheduler(plasma_store_name, plasma_manager_name=None, plasma_ad
       valgrind. If this is True, use_profiler must be False.
     use_profiler (bool): True if the local scheduler should be started inside a
       profiler. If this is True, use_valgrind must be False.
+    redirect_output (bool): True if stdout and stderr should be redirected to
+      /dev/null.
 
   Return:
     A tuple of the name of the local scheduler socket and the process ID of the
@@ -47,13 +49,16 @@ def start_local_scheduler(plasma_store_name, plasma_manager_name=None, plasma_ad
     command += ["-r", redis_address]
   if plasma_address is not None:
     command += ["-a", plasma_address]
-  if use_valgrind:
-    pid = subprocess.Popen(["valgrind", "--track-origins=yes", "--leak-check=full", "--show-leak-kinds=all", "--error-exitcode=1"] + command)
-    time.sleep(1.0)
-  elif use_profiler:
-    pid = subprocess.Popen(["valgrind", "--tool=callgrind"] + command)
-    time.sleep(1.0)
-  else:
-    pid = subprocess.Popen(command)
-    time.sleep(0.1)
+  with open(os.devnull, "w") as FNULL:
+    stdout = FNULL if redirect_output else None
+    stderr = FNULL if redirect_output else None
+    if use_valgrind:
+      pid = subprocess.Popen(["valgrind", "--track-origins=yes", "--leak-check=full", "--show-leak-kinds=all", "--error-exitcode=1"] + command, stdout=stdout, stderr=stderr)
+      time.sleep(1.0)
+    elif use_profiler:
+      pid = subprocess.Popen(["valgrind", "--tool=callgrind"] + command, stdout=stdout, stderr=stderr)
+      time.sleep(1.0)
+    else:
+      pid = subprocess.Popen(command, stdout=stdout, stderr=stderr)
+      time.sleep(0.1)
   return local_scheduler_name, pid

src/photon/photon_scheduler.c

@@ -29,6 +29,7 @@ local_scheduler_state *init_local_scheduler(
     event_loop *loop,
     const char *redis_addr,
     int redis_port,
+    const char *local_scheduler_socket_name,
     const char *plasma_store_socket_name,
     const char *plasma_manager_socket_name,
     const char *plasma_manager_address,
@@ -40,19 +41,24 @@ local_scheduler_state *init_local_scheduler(
   utarray_new(state->workers, &worker_icd);
   /* Connect to Redis if a Redis address is provided. */
   if (redis_addr != NULL) {
-    int num_args = 0;
+    int num_args;
     const char **db_connect_args = NULL;
     if (plasma_manager_address != NULL) {
+      num_args = 4;
+      db_connect_args = malloc(sizeof(char *) * num_args);
+      db_connect_args[0] = "local_scheduler_socket_name";
+      db_connect_args[1] = local_scheduler_socket_name;
+      db_connect_args[2] = "aux_address";
+      db_connect_args[3] = plasma_manager_address;
+    } else {
       num_args = 2;
       db_connect_args = malloc(sizeof(char *) * num_args);
-      db_connect_args[0] = "aux_address";
-      db_connect_args[1] = plasma_manager_address;
+      db_connect_args[0] = "local_scheduler_socket_name";
+      db_connect_args[1] = local_scheduler_socket_name;
     }
     state->db = db_connect(redis_addr, redis_port, "photon", node_ip_address,
                            num_args, db_connect_args);
-    if (num_args != 0) {
-      free(db_connect_args);
-    };
+    free(db_connect_args);
     db_attach(state->db, loop, false);
   } else {
     state->db = NULL;
@@ -295,10 +301,10 @@ void start_server(const char *node_ip_address,
                   bool global_scheduler_exists) {
   int fd = bind_ipc_sock(socket_name, true);
   event_loop *loop = event_loop_create();
-  g_state =
-      init_local_scheduler(node_ip_address, loop, redis_addr, redis_port,
-                           plasma_store_socket_name, plasma_manager_socket_name,
-                           plasma_manager_address, global_scheduler_exists);
+  g_state = init_local_scheduler(
+      node_ip_address, loop, redis_addr, redis_port, socket_name,
+      plasma_store_socket_name, plasma_manager_socket_name,
+      plasma_manager_address, global_scheduler_exists);
 
   /* Register a callback for registering new clients. */
   event_loop_add_file(loop, fd, EVENT_LOOP_READ, new_client_connection,

src/photon/photon_scheduler.h

@@ -69,6 +69,7 @@ local_scheduler_state *init_local_scheduler(
     event_loop *loop,
     const char *redis_addr,
     int redis_port,
+    const char *local_scheduler_socket_name,
     const char *plasma_manager_socket_name,
     const char *plasma_store_socket_name,
     const char *plasma_manager_address,

src/photon/test/photon_tests.c

@@ -59,6 +59,7 @@ photon_mock *init_photon_mock() {
   CHECK(mock->plasma_fd >= 0 && mock->photon_fd >= 0);
   mock->photon_state = init_local_scheduler(
       "127.0.0.1", mock->loop, redis_addr, redis_port,
+      utstring_body(photon_socket_name),
       utstring_body(plasma_manager_socket_name),
       utstring_body(plasma_store_socket_name), NULL, false);
   /* Connect a Photon client. */