wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-08 11:25:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

Shard Redis. (#539)

Browse Source 
* Implement sharding in the Ray core

* Single node Python modifications to do sharding

* Do the sharding in redis.cc

* Pipe num_redis_shards through start_ray.py and worker.py.

* Use multiple redis shards in multinode tests.

* first steps for sharding ray.global_state

* Fix problem in multinode docker test.

* fix runtest.py

* fix some tests

* fix redis shard startup

* fix redis sharding

* fix

* fix bug introduced by the map-iterator being consumed

* fix sharding bug

* shard event table

* update number of Redis clients to be 64K

* Fix object table tests by flushing shards in between unit tests

* Fix local scheduler tests

* Documentation

* Register shard locations in the primary shard

* Add plasma unit tests back to build

* lint

* lint and fix build

* Fix

* Address Robert's comments

* Refactor start_ray_processes to start Redis shard

* lint

* Fix global scheduler python tests

* Fix redis module test

* Fix plasma test

* Fix component failure test

* Fix local scheduler test

* Fix runtest.py

* Fix global scheduler test for python3

* Fix task_table_test_and_update bug, from actor task table submission race

* Fix jenkins tests.

* Retry Redis shard connections

* Fix test cases

* Convert database clients to DBClient struct

* Fix race condition when subscribing to db client table

* Remove unused lines, add APITest for sharded Ray

* Fix

* Fix memory leak

* Suppress ReconstructionTests output

* Suppress output for APITestSharded

* Reissue task table add/update commands if initial command does not publish to any subscribers.

* fix

* Fix linting.

* fix tests

* fix linting

* fix python test

* fix linting
This commit is contained in:
Stephanie Wang
2017-05-18 17:40:41 -07:00
committed by Philipp Moritz
parent 0a4304725f
commit ee08c8274b
39 changed files with 1336 additions and 651 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/jenkins_tests/multi_node_docker_test.py
+13 -8
View File
@@ -86,8 +86,8 @@ class DockerRunner(object):
else:
return m.group(1)
def _start_head_node(self, docker_image, mem_size, shm_size, num_cpus,
num_gpus, development_mode):
def _start_head_node(self, docker_image, mem_size, shm_size,
num_redis_shards, num_cpus, num_gpus, development_mode):
"""Start the Ray head node inside a docker container."""
mem_arg = ["--memory=" + mem_size] if mem_size else []
shm_arg = ["--shm-size=" + shm_size] if shm_size else []
@@ -99,6 +99,7 @@ class DockerRunner(object):
command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg +
[docker_image, "/ray/scripts/start_ray.sh", "--head",
"--redis-port=6379",
"--num-redis-shards={}".format(num_redis_shards),
"--num-cpus={}".format(num_cpus),
"--num-gpus={}".format(num_gpus)])
print("Starting head node with command:{}".format(command))
@@ -137,8 +138,8 @@ class DockerRunner(object):
self.worker_container_ids.append(container_id)
def start_ray(self, docker_image=None, mem_size=None, shm_size=None,
num_nodes=None, num_cpus=None, num_gpus=None,
development_mode=None):
num_nodes=None, num_redis_shards=1, num_cpus=None,
num_gpus=None, development_mode=None):
"""Start a Ray cluster within docker.
This starts one docker container running the head node and num_nodes - 1
@@ -153,6 +154,7 @@ class DockerRunner(object):
with. This will be passed into `docker run` as the `--shm-size` flag.
num_nodes: The number of nodes to use in the cluster (this counts the
head node as well).
num_redis_shards: The number of Redis shards to use on the head node.
num_cpus: A list of the number of CPUs to start each node with.
num_gpus: A list of the number of GPUs to start each node with.
development_mode: True if you want to mount the local copy of
@@ -163,8 +165,8 @@ class DockerRunner(object):
assert len(num_gpus) == num_nodes
# Launch the head node.
self._start_head_node(docker_image, mem_size, shm_size, num_cpus[0],
num_gpus[0], development_mode)
self._start_head_node(docker_image, mem_size, shm_size, num_redis_shards,
num_cpus[0], num_gpus[0], development_mode)
# Start the worker nodes.
for i in range(num_nodes - 1):
self._start_worker_node(docker_image, mem_size, shm_size,
@@ -252,6 +254,9 @@ if __name__ == "__main__":
parser.add_argument("--shm-size", default="1G", help="shared memory size")
parser.add_argument("--num-nodes", default=1, type=int,
help="number of nodes to use in the cluster")
parser.add_argument("--num-redis-shards", default=1, type=int,
help=("the number of Redis shards to start on the head "
"node"))
parser.add_argument("--num-cpus", type=str,
help=("a comma separated list of values representing "
"the number of CPUs to start each node with"))
@@ -282,8 +287,8 @@ if __name__ == "__main__":
d = DockerRunner()
d.start_ray(docker_image=args.docker_image, mem_size=args.mem_size,
shm_size=args.shm_size, num_nodes=num_nodes,
num_cpus=num_cpus, num_gpus=num_gpus,
development_mode=args.development_mode)
num_redis_shards=args.num_redis_shards, num_cpus=num_cpus,
num_gpus=num_gpus, development_mode=args.development_mode)
try:
run_results = d.run_test(args.test_script, args.num_drivers,
driver_locations=driver_locations)
test/jenkins_tests/run_multi_node_tests.sh
+3
View File
@@ -14,11 +14,13 @@ echo "Using Docker image" $DOCKER_SHA
python $ROOT_DIR/multi_node_docker_test.py \
--docker-image=$DOCKER_SHA \
--num-nodes=5 \
--num-redis-shards=10 \
--test-script=/ray/test/jenkins_tests/multi_node_tests/test_0.py
python $ROOT_DIR/multi_node_docker_test.py \
--docker-image=$DOCKER_SHA \
--num-nodes=5 \
--num-redis-shards=5 \
--num-gpus=0,1,2,3,4 \
--num-drivers=7 \
--driver-locations=0,1,0,1,2,3,4 \
@@ -27,6 +29,7 @@ python $ROOT_DIR/multi_node_docker_test.py \
python $ROOT_DIR/multi_node_docker_test.py \
--docker-image=$DOCKER_SHA \
--num-nodes=5 \
--num-redis-shards=2 \
--num-gpus=0,0,5,6,50 \
--num-drivers=100 \
--test-script=/ray/test/jenkins_tests/multi_node_tests/many_drivers_test.py
test/runtest.py
+34 -42
View File
@@ -293,8 +293,16 @@ class WorkerTest(unittest.TestCase):
class APITest(unittest.TestCase):
def init_ray(self, kwargs=None):
if kwargs is None:
kwargs = {}
ray.init(**kwargs)
def tearDown(self):
ray.worker.cleanup()
def testRegisterClass(self):
ray.init(num_workers=2)
self.init_ray({"num_workers": 2})
# Check that putting an object of a class that has not been registered
# throws an exception.
@@ -417,11 +425,9 @@ class APITest(unittest.TestCase):
self.assertFalse(hasattr(c2, "method0"))
self.assertFalse(hasattr(c2, "method1"))
ray.worker.cleanup()
def testKeywordArgs(self):
reload(test_functions)
ray.init(num_workers=1)
self.init_ray()
x = test_functions.keyword_fct1.remote(1)
self.assertEqual(ray.get(x), "1 hello")
@@ -483,11 +489,9 @@ class APITest(unittest.TestCase):
self.assertEqual(ray.get(f3.remote(4)), 4)
ray.worker.cleanup()
def testVariableNumberOfArgs(self):
reload(test_functions)
ray.init(num_workers=1)
self.init_ray()
x = test_functions.varargs_fct1.remote(0, 1, 2)
self.assertEqual(ray.get(x), "0 1 2")
@@ -516,18 +520,14 @@ class APITest(unittest.TestCase):
self.assertEqual(ray.get(f2.remote(1, 2, 3)), (1, 2, (3,)))
self.assertEqual(ray.get(f2.remote(1, 2, 3, 4)), (1, 2, (3, 4)))
ray.worker.cleanup()
def testNoArgs(self):
reload(test_functions)
ray.init(num_workers=1)
self.init_ray()
ray.get(test_functions.no_op.remote())
ray.worker.cleanup()
def testDefiningRemoteFunctions(self):
ray.init(num_workers=3, num_cpus=3)
self.init_ray({"num_cpus": 3})
# Test that we can define a remote function in the shell.
@ray.remote
@@ -584,10 +584,8 @@ class APITest(unittest.TestCase):
self.assertEqual(ray.get(l.remote(1)), 2)
self.assertEqual(ray.get(m.remote(1)), 2)
ray.worker.cleanup()
def testGetMultiple(self):
ray.init(num_workers=0)
self.init_ray()
object_ids = [ray.put(i) for i in range(10)]
self.assertEqual(ray.get(object_ids), list(range(10)))
@@ -597,10 +595,8 @@ class APITest(unittest.TestCase):
results = ray.get([object_ids[i] for i in indices])
self.assertEqual(results, indices)
ray.worker.cleanup()
def testWait(self):
ray.init(num_workers=1, num_cpus=1)
self.init_ray({"num_cpus": 1})
@ray.remote
def f(delay):
@@ -633,12 +629,10 @@ class APITest(unittest.TestCase):
x = ray.put(1)
self.assertRaises(Exception, lambda: ray.wait([x, x]))
ray.worker.cleanup()
def testMultipleWaitsAndGets(self):
# It is important to use three workers here, so that the three tasks
# launched in this experiment can run at the same time.
ray.init(num_workers=3)
self.init_ray()
@ray.remote
def f(delay):
@@ -665,8 +659,6 @@ class APITest(unittest.TestCase):
x = f.remote(1)
ray.get([h.remote([x]), h.remote([x])])
ray.worker.cleanup()
def testCachingEnvironmentVariables(self):
# Test that we can define environment variables before the driver is
# connected.
@@ -690,15 +682,13 @@ class APITest(unittest.TestCase):
ray.env.bar.append(1)
return ray.env.bar
ray.init(num_workers=2)
self.init_ray()
self.assertEqual(ray.get(use_foo.remote()), 1)
self.assertEqual(ray.get(use_foo.remote()), 1)
self.assertEqual(ray.get(use_bar.remote()), [1])
self.assertEqual(ray.get(use_bar.remote()), [1])
ray.worker.cleanup()
def testCachingFunctionsToRun(self):
# Test that we export functions to run on all workers before the driver is
# connected.
@@ -718,7 +708,7 @@ class APITest(unittest.TestCase):
sys.path.append(4)
ray.worker.global_worker.run_function_on_all_workers(f)
ray.init(num_workers=2)
self.init_ray()
@ray.remote
def get_state():
@@ -738,10 +728,8 @@ class APITest(unittest.TestCase):
sys.path.pop()
ray.worker.global_worker.run_function_on_all_workers(f)
ray.worker.cleanup()
def testRunningFunctionOnAllWorkers(self):
ray.init(num_workers=1)
self.init_ray()
def f(worker_info):
sys.path.append("fake_directory")
@@ -764,10 +752,8 @@ class APITest(unittest.TestCase):
return sys.path
self.assertTrue("fake_directory" not in ray.get(get_path2.remote()))
ray.worker.cleanup()
def testLoggingAPI(self):
ray.init(num_workers=1, driver_mode=ray.SILENT_MODE)
self.init_ray({"driver_mode": ray.SILENT_MODE})
def events():
# This is a hack for getting the event log. It is not part of the API.
@@ -815,12 +801,10 @@ class APITest(unittest.TestCase):
wait_for_num_events(3)
self.assertEqual(len(events()), 3)
ray.worker.cleanup()
def testIdenticalFunctionNames(self):
# Define a bunch of remote functions and make sure that we don't
# accidentally call an older version.
ray.init(num_workers=2)
self.init_ray()
num_calls = 200
@@ -878,10 +862,8 @@ class APITest(unittest.TestCase):
result_values = ray.get([g.remote() for _ in range(num_calls)])
self.assertEqual(result_values, num_calls * [5])
ray.worker.cleanup()
def testIllegalAPICalls(self):
ray.init(num_workers=0)
self.init_ray()
# Verify that we cannot call put on an ObjectID.
x = ray.put(1)
@@ -891,7 +873,16 @@ class APITest(unittest.TestCase):
with self.assertRaises(Exception):
ray.get(3)
ray.worker.cleanup()
class APITestSharded(APITest):
def init_ray(self, kwargs=None):
if kwargs is None:
kwargs = {}
kwargs["start_ray_local"] = True
kwargs["num_redis_shards"] = 20
kwargs["redirect_output"] = True
ray.worker._init(**kwargs)
class PythonModeTest(unittest.TestCase):
@@ -1619,7 +1610,8 @@ class GlobalStateAPI(unittest.TestCase):
task_table = ray.global_state.task_table()
self.assertEqual(len(task_table), 1)
self.assertEqual(driver_task_id, list(task_table.keys())[0])
self.assertEqual(task_table[driver_task_id]["State"], "RUNNING")
self.assertEqual(task_table[driver_task_id]["State"],
ray.experimental.state.TASK_STATUS_RUNNING)
self.assertEqual(task_table[driver_task_id]["TaskSpec"]["TaskID"],
driver_task_id)
self.assertEqual(task_table[driver_task_id]["TaskSpec"]["ActorID"],
test/stress_tests.py
+26 -21
View File
@@ -6,10 +6,6 @@ import unittest
import ray
import numpy as np
import time
import redis
# Import flatbuffer bindings.
from ray.core.generated.TaskReply import TaskReply
class TaskTests(unittest.TestCase):
@@ -137,26 +133,38 @@ class ReconstructionTests(unittest.TestCase):
num_local_schedulers = 1
def setUp(self):
# Start a Redis instance and Plasma store instances with a total of 1GB
# memory.
# Start the Redis global state store.
node_ip_address = "127.0.0.1"
self.redis_port = ray.services.new_port()
print(self.redis_port)
redis_address = ray.services.address(node_ip_address, self.redis_port)
redis_address, redis_shards = ray.services.start_redis(node_ip_address)
self.redis_ip_address = ray.services.get_ip_address(redis_address)
self.redis_port = ray.services.get_port(redis_address)
time.sleep(0.1)
# Start the Plasma store instances with a total of 1GB memory.
self.plasma_store_memory = 10 ** 9
plasma_addresses = []
objstore_memory = (self.plasma_store_memory // self.num_local_schedulers)
for i in range(self.num_local_schedulers):
store_stdout_file, store_stderr_file = ray.services.new_log_files(
"plasma_store_{}".format(i), True)
manager_stdout_file, manager_stderr_file = ray.services.new_log_files(
"plasma_manager_{}".format(i), True)
plasma_addresses.append(ray.services.start_objstore(
node_ip_address, redis_address, objstore_memory=objstore_memory))
address_info = {"redis_address": redis_address,
"object_store_addresses": plasma_addresses}
node_ip_address, redis_address, objstore_memory=objstore_memory,
store_stdout_file=store_stdout_file,
store_stderr_file=store_stderr_file,
manager_stdout_file=manager_stdout_file,
manager_stderr_file=manager_stderr_file))
# Start the rest of the services in the Ray cluster.
address_info = {"redis_address": redis_address,
"redis_shards": redis_shards,
"object_store_addresses": plasma_addresses}
ray.worker._init(address_info=address_info, start_ray_local=True,
num_workers=1,
num_local_schedulers=self.num_local_schedulers,
num_cpus=[1] * self.num_local_schedulers,
redirect_output=True,
driver_mode=ray.SILENT_MODE)
def tearDown(self):
@@ -164,14 +172,11 @@ class ReconstructionTests(unittest.TestCase):
# Determine the IDs of all local schedulers that had a task scheduled or
# submitted.
r = redis.StrictRedis(port=self.redis_port)
task_ids = r.keys("TT:*")
task_ids = [task_id[3:] for task_id in task_ids]
local_scheduler_ids = []
for task_id in task_ids:
message = r.execute_command("ray.task_table_get", task_id)
task_reply_object = TaskReply.GetRootAsTaskReply(message, 0)
local_scheduler_ids.append(task_reply_object.LocalSchedulerId())
state = ray.experimental.state.GlobalState()
state._initialize_global_state(self.redis_ip_address, self.redis_port)
tasks = state.task_table()
local_scheduler_ids = set(task["LocalSchedulerID"] for task in
tasks.values())
# Make sure that all nodes in the cluster were used by checking that the
# set of local scheduler IDs that had a task scheduled or submitted is
@@ -179,7 +184,7 @@ class ReconstructionTests(unittest.TestCase):
# total number of local schedulers to account for NIL_LOCAL_SCHEDULER_ID.
# This is the local scheduler ID associated with the driver task, since it
# is not scheduled by a particular local scheduler.
self.assertEqual(len(set(local_scheduler_ids)),
self.assertEqual(len(local_scheduler_ids),
self.num_local_schedulers + 1)
# Clean up the Ray cluster.