Clean up when a driver disconnects. (#462)

* Clean up state when drivers exit.

* Remove unnecessary field in ActorMapEntry struct.

* Have monitor release GPU resources in Redis when driver exits.

* Enable multiple drivers in multi-node tests and test driver cleanup.

* Make redis GPU allocation a redis transaction and small cleanups.

* Fix multi-node test.

* Small cleanups.

* Make global scheduler take node_ip_address so it appears in the right place in the client table.

* Cleanups.

* Fix linting and cleanups in local scheduler.

* Fix removed_driver_test.

* Fix bug related to vector -> list.

* Fix linting.

* Cleanup.

* Fix multi node tests.

* Fix jenkins tests.

* Add another multi node test with many drivers.

* Fix linting.

* Make the actor creation notification a flatbuffer message.

* Revert "Make the actor creation notification a flatbuffer message."

This reverts commit af99099c8084dbf9177fb4e34c0c9b1a12c78f39.

* Add comment explaining flatbuffer problems.

python/ray/global_scheduler/global_scheduler_services.py

@@ -7,13 +7,15 @@ import subprocess
 import time
 
 
-def start_global_scheduler(redis_address, use_valgrind=False,
+def start_global_scheduler(redis_address, node_ip_address, use_valgrind=False,
                            use_profiler=False, stdout_file=None,
                            stderr_file=None):
   """Start a global scheduler process.
 
   Args:
     redis_address (str): The address of the Redis instance.
+    node_ip_address: The IP address of the node that this scheduler will run
+      on.
     use_valgrind (bool): True if the global scheduler should be started inside
       of valgrind. If this is True, use_profiler must be False.
     use_profiler (bool): True if the global scheduler should be started inside
@@ -31,7 +33,9 @@ def start_global_scheduler(redis_address, use_valgrind=False,
   global_scheduler_executable = os.path.join(
       os.path.abspath(os.path.dirname(__file__)),
       "../core/src/global_scheduler/global_scheduler")
-  command = [global_scheduler_executable, "-r", redis_address]
+  command = [global_scheduler_executable,
+             "-r", redis_address,
+             "-h", node_ip_address]
   if use_valgrind:
     pid = subprocess.Popen(["valgrind",
                             "--track-origins=yes",

python/ray/global_scheduler/test/test.py

@@ -71,7 +71,7 @@ class TestGlobalScheduler(unittest.TestCase):
                                           port=redis_port)
     # Start one global scheduler.
     self.p1 = global_scheduler.start_global_scheduler(
-        redis_address, use_valgrind=USE_VALGRIND)
+        redis_address, node_ip_address, use_valgrind=USE_VALGRIND)
     self.plasma_store_pids = []
     self.plasma_manager_pids = []
     self.local_scheduler_pids = []