Local scheduler filters out dead clients during reconstruction (#1182)

* Object table lookup returns vector of DBClientID instead of address strings

* Add node IP address to DBClient notification

* DB client cache stores entire DB client, convert addresses to std::string

* get cached db client returns the client

* Expose a call to initialize the redis cache

* Local scheduler filters out dead clients during reconstruction

* Remove node ip address from dbclient, use aux_address for plasma managers

* Get entire db client entry when not found in cache

* Fix common tests

* Fix address in tests

* Push error to driver if driver task did the put

* Address Robert's comments and cleanup

* Remove unused Redis command

* Fix db test

python/ray/experimental/state.py

@@ -353,9 +353,9 @@ class GlobalState(object):
                 "Deleted": bool(int(decode(client_info[b"deleted"]))),
                 "DBClientID": binary_to_hex(client_info[b"ray_client_id"])
             }
-            if b"aux_address" in client_info:
+            if b"manager_address" in client_info:
                 client_info_parsed["AuxAddress"] = decode(
-                    client_info[b"aux_address"])
+                    client_info[b"manager_address"])
             if b"num_cpus" in client_info:
                 client_info_parsed["NumCPUs"] = float(
                     decode(client_info[b"num_cpus"]))

python/ray/worker.py

@@ -1096,7 +1096,7 @@ def get_address_info_from_redis_helper(redis_address, node_ip_address):
     # Build the address information.
     object_store_addresses = []
     for manager in plasma_managers:
-        address = manager[b"address"].decode("ascii")
+        address = manager[b"manager_address"].decode("ascii")
         port = services.get_port(address)
         object_store_addresses.append(
             services.ObjectStoreAddress(