Shard Redis. (#539)

* Implement sharding in the Ray core

* Single node Python modifications to do sharding

* Do the sharding in redis.cc

* Pipe num_redis_shards through start_ray.py and worker.py.

* Use multiple redis shards in multinode tests.

* first steps for sharding ray.global_state

* Fix problem in multinode docker test.

* fix runtest.py

* fix some tests

* fix redis shard startup

* fix redis sharding

* fix

* fix bug introduced by the map-iterator being consumed

* fix sharding bug

* shard event table

* update number of Redis clients to be 64K

* Fix object table tests by flushing shards in between unit tests

* Fix local scheduler tests

* Documentation

* Register shard locations in the primary shard

* Add plasma unit tests back to build

* lint

* lint and fix build

* Fix

* Address Robert's comments

* Refactor start_ray_processes to start Redis shard

* lint

* Fix global scheduler python tests

* Fix redis module test

* Fix plasma test

* Fix component failure test

* Fix local scheduler test

* Fix runtest.py

* Fix global scheduler test for python3

* Fix task_table_test_and_update bug, from actor task table submission race

* Fix jenkins tests.

* Retry Redis shard connections

* Fix test cases

* Convert database clients to DBClient struct

* Fix race condition when subscribing to db client table

* Remove unused lines, add APITest for sharded Ray

* Fix

* Fix memory leak

* Suppress ReconstructionTests output

* Suppress output for APITestSharded

* Reissue task table add/update commands if initial command does not publish to any subscribers.

* fix

* Fix linting.

* fix tests

* fix linting

* fix python test

* fix linting

test/jenkins_tests/multi_node_docker_test.py

@@ -86,8 +86,8 @@ class DockerRunner(object):
     else:
       return m.group(1)
 
-  def _start_head_node(self, docker_image, mem_size, shm_size, num_cpus,
-                       num_gpus, development_mode):
+  def _start_head_node(self, docker_image, mem_size, shm_size,
+                       num_redis_shards, num_cpus, num_gpus, development_mode):
     """Start the Ray head node inside a docker container."""
     mem_arg = ["--memory=" + mem_size] if mem_size else []
     shm_arg = ["--shm-size=" + shm_size] if shm_size else []
@@ -99,6 +99,7 @@ class DockerRunner(object):
     command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg +
                [docker_image, "/ray/scripts/start_ray.sh", "--head",
                 "--redis-port=6379",
+                "--num-redis-shards={}".format(num_redis_shards),
                 "--num-cpus={}".format(num_cpus),
                 "--num-gpus={}".format(num_gpus)])
     print("Starting head node with command:{}".format(command))
@@ -137,8 +138,8 @@ class DockerRunner(object):
     self.worker_container_ids.append(container_id)
 
   def start_ray(self, docker_image=None, mem_size=None, shm_size=None,
-                num_nodes=None, num_cpus=None, num_gpus=None,
-                development_mode=None):
+                num_nodes=None, num_redis_shards=1, num_cpus=None,
+                num_gpus=None, development_mode=None):
     """Start a Ray cluster within docker.
 
     This starts one docker container running the head node and num_nodes - 1
@@ -153,6 +154,7 @@ class DockerRunner(object):
         with. This will be passed into `docker run` as the `--shm-size` flag.
       num_nodes: The number of nodes to use in the cluster (this counts the
         head node as well).
+      num_redis_shards: The number of Redis shards to use on the head node.
       num_cpus: A list of the number of CPUs to start each node with.
       num_gpus: A list of the number of GPUs to start each node with.
       development_mode: True if you want to mount the local copy of
@@ -163,8 +165,8 @@ class DockerRunner(object):
     assert len(num_gpus) == num_nodes
 
     # Launch the head node.
-    self._start_head_node(docker_image, mem_size, shm_size, num_cpus[0],
-                          num_gpus[0], development_mode)
+    self._start_head_node(docker_image, mem_size, shm_size, num_redis_shards,
+                          num_cpus[0], num_gpus[0], development_mode)
     # Start the worker nodes.
     for i in range(num_nodes - 1):
       self._start_worker_node(docker_image, mem_size, shm_size,
@@ -252,6 +254,9 @@ if __name__ == "__main__":
   parser.add_argument("--shm-size", default="1G", help="shared memory size")
   parser.add_argument("--num-nodes", default=1, type=int,
                       help="number of nodes to use in the cluster")
+  parser.add_argument("--num-redis-shards", default=1, type=int,
+                      help=("the number of Redis shards to start on the head "
+                            "node"))
   parser.add_argument("--num-cpus", type=str,
                       help=("a comma separated list of values representing "
                             "the number of CPUs to start each node with"))
@@ -282,8 +287,8 @@ if __name__ == "__main__":
   d = DockerRunner()
   d.start_ray(docker_image=args.docker_image, mem_size=args.mem_size,
               shm_size=args.shm_size, num_nodes=num_nodes,
-              num_cpus=num_cpus, num_gpus=num_gpus,
-              development_mode=args.development_mode)
+              num_redis_shards=args.num_redis_shards, num_cpus=num_cpus,
+              num_gpus=num_gpus, development_mode=args.development_mode)
   try:
     run_results = d.run_test(args.test_script, args.num_drivers,
                              driver_locations=driver_locations)