Integrate credis with Ray & route task table entries into credis. (#1841)

test/credis_test.py

@@ -1,14 +1,18 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 
 import os
-import redis
 import unittest
 
+import redis
+
 import ray
 
 
+def parse_client(addr_port_str):
+    redis_address, redis_port = addr_port_str.split(":")
+    return redis.StrictRedis(host=redis_address, port=redis_port)
+
+
 @unittest.skipIf(not os.environ.get('RAY_USE_NEW_GCS', False),
                  "Tests functionality of the new GCS.")
 class CredisTest(unittest.TestCase):
@@ -19,15 +23,19 @@ class CredisTest(unittest.TestCase):
         ray.worker.cleanup()
 
     def test_credis_started(self):
-        assert "credis_address" in self.config
-        credis_address, credis_port = self.config["credis_address"].split(":")
-        credis_client = redis.StrictRedis(
-            host=credis_address, port=credis_port)
-        assert credis_client.ping() is True
+        assert "redis_address" in self.config
+        primary = parse_client(self.config['redis_address'])
+        assert primary.ping() is True
 
-        redis_client = ray.worker.global_state.redis_client
-        addr = redis_client.get("credis_address").decode("ascii")
-        assert addr == self.config["credis_address"]
+        # Check that primary has loaded credis' master module.
+        chain = primary.execute_command('MASTER.GET_CHAIN')
+        assert len(chain) == 1
+
+        # Check that the shard has loaded credis' member module.
+        member = primary.lrange('RedisShards', 0, -1)[0]
+        assert chain[0] == member
+        shard = parse_client(member.decode())
+        assert shard.execute_command('MEMBER.SN') == -1
 
 
 if __name__ == "__main__":

test/multi_node_test.py

@@ -1,15 +1,13 @@
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+from __future__ import absolute_import, division, print_function
 
 import os
-import ray
 import subprocess
 import sys
 import tempfile
 import time
 import unittest
 
+import ray
 from ray.test.test_utils import run_and_get_output
 
 
@@ -215,12 +213,6 @@ class StartRayScriptTest(unittest.TestCase):
         run_and_get_output(["ray", "start", "--head", "--redis-port", "6379"])
         subprocess.Popen(["ray", "stop"]).wait()
 
-        # Test starting Ray with redis shard ports specified.
-        run_and_get_output([
-            "ray", "start", "--head", "--redis-shard-ports", "6380,6381,6382"
-        ])
-        subprocess.Popen(["ray", "stop"]).wait()
-
         # Test starting Ray with a node IP address specified.
         run_and_get_output(
             ["ray", "start", "--head", "--node-ip-address", "127.0.0.1"])
@@ -244,15 +236,23 @@ class StartRayScriptTest(unittest.TestCase):
             ["ray", "start", "--head", "--redis-max-clients", "100"])
         subprocess.Popen(["ray", "stop"]).wait()
 
-        # Test starting Ray with all arguments specified.
-        run_and_get_output([
-            "ray", "start", "--head", "--num-workers", "20", "--redis-port",
-            "6379", "--redis-shard-ports", "6380,6381,6382",
-            "--object-manager-port", "12345", "--num-cpus", "100",
-            "--num-gpus", "0", "--redis-max-clients", "100", "--resources",
-            "{\"Custom\": 1}"
-        ])
-        subprocess.Popen(["ray", "stop"]).wait()
+        if "RAY_USE_NEW_GCS" not in os.environ:
+            # Test starting Ray with redis shard ports specified.
+            run_and_get_output([
+                "ray", "start", "--head", "--redis-shard-ports",
+                "6380,6381,6382"
+            ])
+            subprocess.Popen(["ray", "stop"]).wait()
+
+            # Test starting Ray with all arguments specified.
+            run_and_get_output([
+                "ray", "start", "--head", "--num-workers", "20",
+                "--redis-port", "6379", "--redis-shard-ports",
+                "6380,6381,6382", "--object-manager-port", "12345",
+                "--num-cpus", "100", "--num-gpus", "0", "--redis-max-clients",
+                "100", "--resources", "{\"Custom\": 1}"
+            ])
+            subprocess.Popen(["ray", "stop"]).wait()
 
         # Test starting Ray with invalid arguments.
         with self.assertRaises(Exception):

test/runtest.py

@@ -1066,6 +1066,10 @@ class APITest(unittest.TestCase):
             ray.get(3)
 
 
+@unittest.skipIf(
+    os.environ.get('RAY_USE_NEW_GCS', False),
+    "For now, RAY_USE_NEW_GCS supports 1 shard, and credis "
+    "supports 1-node chain for that shard only.")
 class APITestSharded(APITest):
     def init_ray(self, **kwargs):
         if kwargs is None: