wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-14 12:40:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Lint Python files with Yapf (#1872)

Browse Source
This commit is contained in:
Philipp Moritz
2018-04-11 10:11:35 -07:00
committed by Robert Nishihara
parent a3ddde398c
commit 74162d1492
97 changed files with 3927 additions and 3139 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/actor_test.py
+197 -154
View File
@@ -15,7 +15,6 @@ import ray.test.test_utils
class ActorAPI(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@@ -39,20 +38,22 @@ class ActorAPI(unittest.TestCase):
self.assertEqual(ray.get(actor.get_values.remote(2, 3)), (3, 5, "ab"))
actor = Actor.remote(1, 2, "c")
self.assertEqual(ray.get(actor.get_values.remote(2, 3, "d")),
(3, 5, "cd"))
self.assertEqual(
ray.get(actor.get_values.remote(2, 3, "d")), (3, 5, "cd"))
actor = Actor.remote(1, arg2="c")
self.assertEqual(ray.get(actor.get_values.remote(0, arg2="d")),
(1, 3, "cd"))
self.assertEqual(ray.get(actor.get_values.remote(0, arg2="d", arg1=0)),
(1, 1, "cd"))
self.assertEqual(
ray.get(actor.get_values.remote(0, arg2="d")), (1, 3, "cd"))
self.assertEqual(
ray.get(actor.get_values.remote(0, arg2="d", arg1=0)),
(1, 1, "cd"))
actor = Actor.remote(1, arg2="c", arg1=2)
self.assertEqual(ray.get(actor.get_values.remote(0, arg2="d")),
(1, 4, "cd"))
self.assertEqual(ray.get(actor.get_values.remote(0, arg2="d", arg1=0)),
(1, 2, "cd"))
self.assertEqual(
ray.get(actor.get_values.remote(0, arg2="d")), (1, 4, "cd"))
self.assertEqual(
ray.get(actor.get_values.remote(0, arg2="d", arg1=0)),
(1, 2, "cd"))
# Make sure we get an exception if the constructor is called
# incorrectly.
@@ -84,16 +85,18 @@ class ActorAPI(unittest.TestCase):
self.assertEqual(ray.get(actor.get_values.remote(1)), (1, 3, (), ()))
actor = Actor.remote(1, 2)
self.assertEqual(ray.get(actor.get_values.remote(2, 3)),
(3, 5, (), ()))
self.assertEqual(
ray.get(actor.get_values.remote(2, 3)), (3, 5, (), ()))
actor = Actor.remote(1, 2, "c")
self.assertEqual(ray.get(actor.get_values.remote(2, 3, "d")),
(3, 5, ("c",), ("d",)))
self.assertEqual(
ray.get(actor.get_values.remote(2, 3, "d")), (3, 5, ("c", ),
("d", )))
actor = Actor.remote(1, 2, "a", "b", "c", "d")
self.assertEqual(ray.get(actor.get_values.remote(2, 3, 1, 2, 3, 4)),
(3, 5, ("a", "b", "c", "d"), (1, 2, 3, 4)))
self.assertEqual(
ray.get(actor.get_values.remote(2, 3, 1, 2, 3, 4)),
(3, 5, ("a", "b", "c", "d"), (1, 2, 3, 4)))
@ray.remote
class Actor(object):
@@ -106,7 +109,7 @@ class ActorAPI(unittest.TestCase):
a = Actor.remote()
self.assertEqual(ray.get(a.get_values.remote()), ((), ()))
a = Actor.remote(1)
self.assertEqual(ray.get(a.get_values.remote(2)), ((1,), (2,)))
self.assertEqual(ray.get(a.get_values.remote(2)), ((1, ), (2, )))
a = Actor.remote(1, 2)
self.assertEqual(ray.get(a.get_values.remote(3, 4)), ((1, 2), (3, 4)))
@@ -191,6 +194,7 @@ class ActorAPI(unittest.TestCase):
# This is an invalid way of using the actor decorator.
with self.assertRaises(Exception):
@ray.remote()
class Actor(object):
def __init__(self):
@@ -198,6 +202,7 @@ class ActorAPI(unittest.TestCase):
# This is an invalid way of using the actor decorator.
with self.assertRaises(Exception):
@ray.remote(invalid_kwarg=0) # noqa: F811
class Actor(object):
def __init__(self):
@@ -205,6 +210,7 @@ class ActorAPI(unittest.TestCase):
# This is an invalid way of using the actor decorator.
with self.assertRaises(Exception):
@ray.remote(num_cpus=0, invalid_kwarg=0) # noqa: F811
class Actor(object):
def __init__(self):
@@ -300,7 +306,6 @@ class ActorAPI(unittest.TestCase):
class ActorMethods(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@@ -417,8 +422,9 @@ class ActorMethods(unittest.TestCase):
results = []
# Call each actor's method a bunch of times.
for i in range(num_actors):
results += [actors[i].increase.remote()
for _ in range(num_increases)]
results += [
actors[i].increase.remote() for _ in range(num_increases)
]
result_values = ray.get(results)
for i in range(num_actors):
self.assertEqual(
@@ -440,7 +446,6 @@ class ActorMethods(unittest.TestCase):
class ActorNesting(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@@ -510,6 +515,7 @@ class ActorNesting(unittest.TestCase):
def get_value(self):
return self.x
self.actor2 = Actor2.remote(z)
def get_values(self, z):
@@ -556,12 +562,14 @@ class ActorNesting(unittest.TestCase):
def get_value(self):
return self.x
actor = Actor1.remote(x)
return ray.get([actor.get_value.remote() for _ in range(n)])
self.assertEqual(ray.get(f.remote(3, 1)), [3])
self.assertEqual(ray.get([f.remote(i, 20) for i in range(10)]),
[20 * [i] for i in range(10)])
self.assertEqual(
ray.get([f.remote(i, 20) for i in range(10)]),
[20 * [i] for i in range(10)])
def testUseActorWithinRemoteFunction(self):
# Make sure we can create and use actors within remote funtions.
@@ -591,6 +599,7 @@ class ActorNesting(unittest.TestCase):
# Export a bunch of remote functions.
num_remote_functions = 50
for i in range(num_remote_functions):
@ray.remote
def f():
return i
@@ -613,7 +622,6 @@ class ActorNesting(unittest.TestCase):
class ActorInheritance(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@@ -646,7 +654,6 @@ class ActorInheritance(unittest.TestCase):
class ActorSchedulingProperties(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@@ -674,7 +681,6 @@ class ActorSchedulingProperties(unittest.TestCase):
class ActorsOnMultipleNodes(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@@ -692,8 +698,10 @@ class ActorsOnMultipleNodes(unittest.TestCase):
def testActorLoadBalancing(self):
num_local_schedulers = 3
ray.worker._init(start_ray_local=True, num_workers=0,
num_local_schedulers=num_local_schedulers)
ray.worker._init(
start_ray_local=True,
num_workers=0,
num_local_schedulers=num_local_schedulers)
@ray.remote
class Actor1(object):
@@ -712,13 +720,13 @@ class ActorsOnMultipleNodes(unittest.TestCase):
attempts = 0
while attempts < num_attempts:
actors = [Actor1.remote() for _ in range(num_actors)]
locations = ray.get([actor.get_location.remote()
for actor in actors])
locations = ray.get(
[actor.get_location.remote() for actor in actors])
names = set(locations)
counts = [locations.count(name) for name in names]
print("Counts are {}.".format(counts))
if (len(names) == num_local_schedulers and
all([count >= minimum_count for count in counts])):
if (len(names) == num_local_schedulers
and all([count >= minimum_count for count in counts])):
break
attempts += 1
self.assertLess(attempts, num_attempts)
@@ -732,18 +740,17 @@ class ActorsOnMultipleNodes(unittest.TestCase):
class ActorsWithGPUs(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Crashing with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Crashing with new GCS API.")
def testActorGPUs(self):
num_local_schedulers = 3
num_gpus_per_scheduler = 4
ray.worker._init(
start_ray_local=True, num_workers=0,
start_ray_local=True,
num_workers=0,
num_local_schedulers=num_local_schedulers,
num_cpus=(num_local_schedulers * [10 * num_gpus_per_scheduler]),
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
@@ -760,19 +767,21 @@ class ActorsWithGPUs(unittest.TestCase):
tuple(self.gpu_ids))
# Create one actor per GPU.
actors = [Actor1.remote() for _
in range(num_local_schedulers * num_gpus_per_scheduler)]
actors = [
Actor1.remote()
for _ in range(num_local_schedulers * num_gpus_per_scheduler)
]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get([actor.get_location_and_ids.remote()
for actor in actors])
locations_and_ids = ray.get(
[actor.get_location_and_ids.remote() for actor in actors])
node_names = set([location for location, gpu_id in locations_and_ids])
self.assertEqual(len(node_names), num_local_schedulers)
location_actor_combinations = []
for node_name in node_names:
for gpu_id in range(num_gpus_per_scheduler):
location_actor_combinations.append((node_name, (gpu_id,)))
self.assertEqual(set(locations_and_ids),
set(location_actor_combinations))
location_actor_combinations.append((node_name, (gpu_id, )))
self.assertEqual(
set(locations_and_ids), set(location_actor_combinations))
# Creating a new actor should fail because all of the GPUs are being
# used.
@@ -784,7 +793,8 @@ class ActorsWithGPUs(unittest.TestCase):
num_local_schedulers = 3
num_gpus_per_scheduler = 5
ray.worker._init(
start_ray_local=True, num_workers=0,
start_ray_local=True,
num_workers=0,
num_local_schedulers=num_local_schedulers,
num_cpus=(num_local_schedulers * [10 * num_gpus_per_scheduler]),
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
@@ -803,8 +813,8 @@ class ActorsWithGPUs(unittest.TestCase):
# Create some actors.
actors1 = [Actor1.remote() for _ in range(num_local_schedulers * 2)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get([actor.get_location_and_ids.remote()
for actor in actors1])
locations_and_ids = ray.get(
[actor.get_location_and_ids.remote() for actor in actors1])
node_names = set([location for location, gpu_id in locations_and_ids])
self.assertEqual(len(node_names), num_local_schedulers)
@@ -835,11 +845,11 @@ class ActorsWithGPUs(unittest.TestCase):
# Create some actors.
actors2 = [Actor2.remote() for _ in range(num_local_schedulers)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get([actor.get_location_and_ids.remote()
for actor in actors2])
self.assertEqual(node_names,
set([location for location, gpu_id
in locations_and_ids]))
locations_and_ids = ray.get(
[actor.get_location_and_ids.remote() for actor in actors2])
self.assertEqual(
node_names,
set([location for location, gpu_id in locations_and_ids]))
for location, gpu_ids in locations_and_ids:
gpus_in_use[location].extend(gpu_ids)
for node_name in node_names:
@@ -855,9 +865,12 @@ class ActorsWithGPUs(unittest.TestCase):
def testActorDifferentNumbersOfGPUs(self):
# Test that we can create actors on two nodes that have different
# numbers of GPUs.
ray.worker._init(start_ray_local=True, num_workers=0,
num_local_schedulers=3, num_cpus=[10, 10, 10],
num_gpus=[0, 5, 10])
ray.worker._init(
start_ray_local=True,
num_workers=0,
num_local_schedulers=3,
num_cpus=[10, 10, 10],
num_gpus=[0, 5, 10])
@ray.remote(num_gpus=1)
class Actor1(object):
@@ -872,16 +885,19 @@ class ActorsWithGPUs(unittest.TestCase):
# Create some actors.
actors = [Actor1.remote() for _ in range(0 + 5 + 10)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get([actor.get_location_and_ids.remote()
for actor in actors])
locations_and_ids = ray.get(
[actor.get_location_and_ids.remote() for actor in actors])
node_names = set([location for location, gpu_id in locations_and_ids])
self.assertEqual(len(node_names), 2)
for node_name in node_names:
node_gpu_ids = [gpu_id for location, gpu_id in locations_and_ids
if location == node_name]
node_gpu_ids = [
gpu_id for location, gpu_id in locations_and_ids
if location == node_name
]
self.assertIn(len(node_gpu_ids), [5, 10])
self.assertEqual(set(node_gpu_ids),
set([(i,) for i in range(len(node_gpu_ids))]))
self.assertEqual(
set(node_gpu_ids),
set([(i, ) for i in range(len(node_gpu_ids))]))
# Creating a new actor should fail because all of the GPUs are being
# used.
@@ -893,8 +909,10 @@ class ActorsWithGPUs(unittest.TestCase):
num_local_schedulers = 10
num_gpus_per_scheduler = 10
ray.worker._init(
start_ray_local=True, num_workers=0,
num_local_schedulers=num_local_schedulers, redirect_output=True,
start_ray_local=True,
num_workers=0,
num_local_schedulers=num_local_schedulers,
redirect_output=True,
num_cpus=(num_local_schedulers * [10 * num_gpus_per_scheduler]),
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
@@ -906,15 +924,17 @@ class ActorsWithGPUs(unittest.TestCase):
self.gpu_ids = ray.get_gpu_ids()
def get_location_and_ids(self):
return ((ray.worker.global_worker.plasma_client
.store_socket_name),
tuple(self.gpu_ids))
return ((ray.worker.global_worker.plasma_client.
store_socket_name), tuple(self.gpu_ids))
# Create n actors.
for _ in range(n):
Actor.remote()
ray.get([create_actors.remote(num_gpus_per_scheduler)
for _ in range(num_local_schedulers)])
ray.get([
create_actors.remote(num_gpus_per_scheduler)
for _ in range(num_local_schedulers)
])
@ray.remote(num_gpus=1)
class Actor(object):
@@ -936,7 +956,8 @@ class ActorsWithGPUs(unittest.TestCase):
num_local_schedulers = 3
num_gpus_per_scheduler = 6
ray.worker._init(
start_ray_local=True, num_workers=0,
start_ray_local=True,
num_workers=0,
num_local_schedulers=num_local_schedulers,
num_cpus=num_gpus_per_scheduler,
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
@@ -951,11 +972,11 @@ class ActorsWithGPUs(unittest.TestCase):
self.assertLess(first_interval[0], first_interval[1])
self.assertLess(second_interval[0], second_interval[1])
intervals_nonoverlapping = (
first_interval[1] <= second_interval[0] or
second_interval[1] <= first_interval[0])
first_interval[1] <= second_interval[0]
or second_interval[1] <= first_interval[0])
assert intervals_nonoverlapping, (
"Intervals {} and {} are overlapping."
.format(first_interval, second_interval))
"Intervals {} and {} are overlapping.".format(
first_interval, second_interval))
@ray.remote(num_gpus=1)
def f1():
@@ -995,13 +1016,16 @@ class ActorsWithGPUs(unittest.TestCase):
def locations_to_intervals_for_many_tasks():
# Launch a bunch of GPU tasks.
locations_ids_and_intervals = ray.get(
[f1.remote() for _
in range(5 * num_local_schedulers * num_gpus_per_scheduler)] +
[f2.remote() for _
in range(5 * num_local_schedulers * num_gpus_per_scheduler)] +
[f1.remote() for _
in range(5 * num_local_schedulers * num_gpus_per_scheduler)])
locations_ids_and_intervals = ray.get([
f1.remote() for _ in range(
5 * num_local_schedulers * num_gpus_per_scheduler)
] + [
f2.remote() for _ in range(
5 * num_local_schedulers * num_gpus_per_scheduler)
] + [
f1.remote() for _ in range(
5 * num_local_schedulers * num_gpus_per_scheduler)
])
locations_to_intervals = collections.defaultdict(lambda: [])
for location, gpu_ids, interval in locations_ids_and_intervals:
@@ -1012,8 +1036,9 @@ class ActorsWithGPUs(unittest.TestCase):
# Run a bunch of GPU tasks.
locations_to_intervals = locations_to_intervals_for_many_tasks()
# Make sure that all GPUs were used.
self.assertEqual(len(locations_to_intervals),
num_local_schedulers * num_gpus_per_scheduler)
self.assertEqual(
len(locations_to_intervals),
num_local_schedulers * num_gpus_per_scheduler)
# For each GPU, verify that the set of tasks that used this specific
# GPU did not overlap in time.
for locations in locations_to_intervals:
@@ -1030,8 +1055,9 @@ class ActorsWithGPUs(unittest.TestCase):
# Run a bunch of GPU tasks.
locations_to_intervals = locations_to_intervals_for_many_tasks()
# Make sure that all but one of the GPUs were used.
self.assertEqual(len(locations_to_intervals),
num_local_schedulers * num_gpus_per_scheduler - 1)
self.assertEqual(
len(locations_to_intervals),
num_local_schedulers * num_gpus_per_scheduler - 1)
# For each GPU, verify that the set of tasks that used this specific
# GPU did not overlap in time.
for locations in locations_to_intervals:
@@ -1041,14 +1067,15 @@ class ActorsWithGPUs(unittest.TestCase):
# Create several more actors that use GPUs.
actors = [Actor1.remote() for _ in range(3)]
actor_locations = ray.get([actor.get_location_and_ids.remote()
for actor in actors])
actor_locations = ray.get(
[actor.get_location_and_ids.remote() for actor in actors])
# Run a bunch of GPU tasks.
locations_to_intervals = locations_to_intervals_for_many_tasks()
# Make sure that all but 11 of the GPUs were used.
self.assertEqual(len(locations_to_intervals),
num_local_schedulers * num_gpus_per_scheduler - 1 - 3)
self.assertEqual(
len(locations_to_intervals),
num_local_schedulers * num_gpus_per_scheduler - 1 - 3)
# For each GPU, verify that the set of tasks that used this specific
# GPU did not overlap in time.
for locations in locations_to_intervals:
@@ -1059,9 +1086,10 @@ class ActorsWithGPUs(unittest.TestCase):
self.assertNotIn(location, locations_to_intervals)
# Create more actors to fill up all the GPUs.
more_actors = [Actor1.remote() for _ in
range(num_local_schedulers *
num_gpus_per_scheduler - 1 - 3)]
more_actors = [
Actor1.remote() for _ in range(
num_local_schedulers * num_gpus_per_scheduler - 1 - 3)
]
# Wait for the actors to finish being created.
ray.get([actor.get_location_and_ids.remote() for actor in more_actors])
@@ -1195,16 +1223,17 @@ class ActorsWithGPUs(unittest.TestCase):
class ActorReconstruction(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testLocalSchedulerDying(self):
ray.worker._init(start_ray_local=True, num_local_schedulers=2,
num_workers=0, redirect_output=True)
ray.worker._init(
start_ray_local=True,
num_local_schedulers=2,
num_workers=0,
redirect_output=True)
@ray.remote
class Counter(object):
@@ -1243,8 +1272,7 @@ class ActorReconstruction(unittest.TestCase):
self.assertEqual(results, list(range(1, 1 + len(results))))
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testManyLocalSchedulersDying(self):
# This test can be made more stressful by increasing the numbers below.
# The total number of actors created will be
@@ -1253,9 +1281,11 @@ class ActorReconstruction(unittest.TestCase):
num_actors_at_a_time = 3
num_function_calls_at_a_time = 10
ray.worker._init(start_ray_local=True,
num_local_schedulers=num_local_schedulers,
num_workers=0, redirect_output=True)
ray.worker._init(
start_ray_local=True,
num_local_schedulers=num_local_schedulers,
num_workers=0,
redirect_output=True)
@ray.remote
class SlowCounter(object):
@@ -1281,14 +1311,13 @@ class ActorReconstruction(unittest.TestCase):
# a local scheduler, and run some more methods.
for i in range(num_local_schedulers - 1):
# Create some actors.
actors.extend([SlowCounter.remote()
for _ in range(num_actors_at_a_time)])
actors.extend(
[SlowCounter.remote() for _ in range(num_actors_at_a_time)])
# Run some methods.
for j in range(len(actors)):
actor = actors[j]
for _ in range(num_function_calls_at_a_time):
result_ids[actor].append(
actor.inc.remote(j ** 2 * 0.000001))
result_ids[actor].append(actor.inc.remote(j**2 * 0.000001))
# Kill a plasma store to get rid of the cached objects and trigger
# exit of the corresponding local scheduler. Don't kill the first
# local scheduler since that is the one that the driver is
@@ -1302,18 +1331,24 @@ class ActorReconstruction(unittest.TestCase):
for j in range(len(actors)):
actor = actors[j]
for _ in range(num_function_calls_at_a_time):
result_ids[actor].append(
actor.inc.remote(j ** 2 * 0.000001))
result_ids[actor].append(actor.inc.remote(j**2 * 0.000001))
# Get the results and check that they have the correct values.
for _, result_id_list in result_ids.items():
self.assertEqual(ray.get(result_id_list),
list(range(1, len(result_id_list) + 1)))
self.assertEqual(
ray.get(result_id_list), list(
range(1,
len(result_id_list) + 1)))
def setup_counter_actor(self, test_checkpoint=False, save_exception=False,
def setup_counter_actor(self,
test_checkpoint=False,
save_exception=False,
resume_exception=False):
ray.worker._init(start_ray_local=True, num_local_schedulers=2,
num_workers=0, redirect_output=True)
ray.worker._init(
start_ray_local=True,
num_local_schedulers=2,
num_workers=0,
redirect_output=True)
# Only set the checkpoint interval if we're testing with checkpointing.
checkpoint_interval = -1
@@ -1371,8 +1406,7 @@ class ActorReconstruction(unittest.TestCase):
return actor, ids
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testCheckpointing(self):
actor, ids = self.setup_counter_actor(test_checkpoint=True)
# Wait for the last task to finish running.
@@ -1397,8 +1431,7 @@ class ActorReconstruction(unittest.TestCase):
self.assertLess(num_inc_calls, x)
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testRemoteCheckpoint(self):
actor, ids = self.setup_counter_actor(test_checkpoint=True)
@@ -1424,8 +1457,7 @@ class ActorReconstruction(unittest.TestCase):
self.assertEqual(x, 101)
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testLostCheckpoint(self):
actor, ids = self.setup_counter_actor(test_checkpoint=True)
# Wait for the first fraction of tasks to finish running.
@@ -1451,11 +1483,10 @@ class ActorReconstruction(unittest.TestCase):
self.assertLess(5, num_inc_calls)
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testCheckpointException(self):
actor, ids = self.setup_counter_actor(test_checkpoint=True,
save_exception=True)
actor, ids = self.setup_counter_actor(
test_checkpoint=True, save_exception=True)
# Wait for the last task to finish running.
ray.get(ids[-1])
@@ -1481,11 +1512,10 @@ class ActorReconstruction(unittest.TestCase):
self.assertEqual(error[b"type"], b"checkpoint")
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testCheckpointResumeException(self):
actor, ids = self.setup_counter_actor(test_checkpoint=True,
resume_exception=True)
actor, ids = self.setup_counter_actor(
test_checkpoint=True, resume_exception=True)
# Wait for the last task to finish running.
ray.get(ids[-1])
@@ -1527,8 +1557,9 @@ class ActorReconstruction(unittest.TestCase):
count = ray.get(ids[-1])
num_incs = 100
num_iters = 10
forks = [fork_many_incs.remote(counter, num_incs) for _ in
range(num_iters)]
forks = [
fork_many_incs.remote(counter, num_incs) for _ in range(num_iters)
]
ray.wait(forks, num_returns=len(forks))
count += num_incs * num_iters
@@ -1547,8 +1578,7 @@ class ActorReconstruction(unittest.TestCase):
self.assertEqual(x, count + 1)
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testRemoteCheckpointDistributedHandle(self):
counter, ids = self.setup_counter_actor(test_checkpoint=True)
@@ -1564,8 +1594,9 @@ class ActorReconstruction(unittest.TestCase):
count = ray.get(ids[-1])
num_incs = 100
num_iters = 10
forks = [fork_many_incs.remote(counter, num_incs) for _ in
range(num_iters)]
forks = [
fork_many_incs.remote(counter, num_incs) for _ in range(num_iters)
]
ray.wait(forks, num_returns=len(forks))
ray.wait([counter.__ray_checkpoint__.remote()])
count += num_incs * num_iters
@@ -1605,8 +1636,9 @@ class ActorReconstruction(unittest.TestCase):
count = ray.get(ids[-1])
num_incs = 100
num_iters = 10
forks = [fork_many_incs.remote(counter, num_incs) for _ in
range(num_iters)]
forks = [
fork_many_incs.remote(counter, num_incs) for _ in range(num_iters)
]
ray.wait(forks, num_returns=len(forks))
count += num_incs * num_iters
@@ -1624,11 +1656,13 @@ class ActorReconstruction(unittest.TestCase):
x = ray.get(counter.inc.remote())
self.assertEqual(x, count + 1)
def _testNondeterministicReconstruction(self, num_forks,
num_items_per_fork,
num_forks_to_wait):
ray.worker._init(start_ray_local=True, num_local_schedulers=2,
num_workers=0, redirect_output=True)
def _testNondeterministicReconstruction(
self, num_forks, num_items_per_fork, num_forks_to_wait):
ray.worker._init(
start_ray_local=True,
num_local_schedulers=2,
num_workers=0,
redirect_output=True)
# Make a shared queue.
@ray.remote
@@ -1668,8 +1702,9 @@ class ActorReconstruction(unittest.TestCase):
# unique objects to push onto the shared queue.
enqueue_tasks = []
for fork in range(num_forks):
enqueue_tasks.append(enqueue.remote(
actor, [(fork, i) for i in range(num_items_per_fork)]))
enqueue_tasks.append(
enqueue.remote(actor,
[(fork, i) for i in range(num_items_per_fork)]))
# Wait for the forks to complete their tasks.
enqueue_tasks = ray.get(enqueue_tasks)
enqueue_tasks = [fork_ids[0] for fork_ids in enqueue_tasks]
@@ -1689,8 +1724,8 @@ class ActorReconstruction(unittest.TestCase):
ray.get(enqueue_tasks)
reconstructed_queue = ray.get(actor.read.remote())
# Make sure the final queue has all items from all forks.
self.assertEqual(len(reconstructed_queue), num_forks *
num_items_per_fork)
self.assertEqual(
len(reconstructed_queue), num_forks * num_items_per_fork)
# Make sure that the prefix of the final queue matches the queue from
# the initial execution.
self.assertEqual(queue, reconstructed_queue[:len(queue)])
@@ -1709,7 +1744,6 @@ class ActorReconstruction(unittest.TestCase):
class DistributedActorHandles(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@@ -1757,8 +1791,9 @@ class DistributedActorHandles(unittest.TestCase):
# Fork num_iters times.
num_forks = 10
num_items_per_fork = 100
ray.get([fork.remote(queue, i, num_items_per_fork) for i in
range(num_forks)])
ray.get([
fork.remote(queue, i, num_items_per_fork) for i in range(num_forks)
])
items = ray.get(queue.read.remote())
for i in range(num_forks):
filtered_items = [item[1] for item in items if item[0] == i]
@@ -1812,7 +1847,6 @@ class DistributedActorHandles(unittest.TestCase):
class ActorPlacementAndResources(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@@ -1836,14 +1870,20 @@ class ActorPlacementAndResources(unittest.TestCase):
actor2s = [Actor2.remote() for _ in range(2)]
results = [a.method.remote() for a in actor2s]
ready_ids, remaining_ids = ray.wait(results, num_returns=len(results),
timeout=1000)
ready_ids, remaining_ids = ray.wait(
results, num_returns=len(results), timeout=1000)
self.assertEqual(len(ready_ids), 1)
def testCustomLabelPlacement(self):
ray.worker._init(start_ray_local=True, num_local_schedulers=2,
num_workers=0, resources=[{"CustomResource1": 2},
{"CustomResource2": 2}])
ray.worker._init(
start_ray_local=True,
num_local_schedulers=2,
num_workers=0,
resources=[{
"CustomResource1": 2
}, {
"CustomResource2": 2
}])
@ray.remote(resources={"CustomResource1": 1})
class ResourceActor1(object):
@@ -1868,8 +1908,11 @@ class ActorPlacementAndResources(unittest.TestCase):
self.assertNotEqual(location, local_plasma)
def testCreatingMoreActorsThanResources(self):
ray.init(num_workers=0, num_cpus=10, num_gpus=2,
resources={"CustomResource1": 1})
ray.init(
num_workers=0,
num_cpus=10,
num_gpus=2,
resources={"CustomResource1": 1})
@ray.remote(num_gpus=1)
class ResourceActor1(object):
test/array_test.py
+37 -30
View File
@@ -20,8 +20,9 @@ class RemoteArrayTest(unittest.TestCase):
ray.worker.cleanup()
def testMethods(self):
for module in [ra.core, ra.random, ra.linalg, da.core, da.random,
da.linalg]:
for module in [
ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg
]:
reload(module)
ray.init()
@@ -56,8 +57,9 @@ class DistributedArrayTest(unittest.TestCase):
ray.worker.cleanup()
def testAssemble(self):
for module in [ra.core, ra.random, ra.linalg, da.core, da.random,
da.linalg]:
for module in [
ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg
]:
reload(module)
ray.init()
@@ -66,15 +68,18 @@ class DistributedArrayTest(unittest.TestCase):
x = da.DistArray([2 * da.BLOCK_SIZE, da.BLOCK_SIZE],
np.array([[a], [b]]))
assert_equal(x.assemble(),
np.vstack([np.ones([da.BLOCK_SIZE, da.BLOCK_SIZE]),
np.zeros([da.BLOCK_SIZE, da.BLOCK_SIZE])]))
np.vstack([
np.ones([da.BLOCK_SIZE, da.BLOCK_SIZE]),
np.zeros([da.BLOCK_SIZE, da.BLOCK_SIZE])
]))
def testMethods(self):
for module in [ra.core, ra.random, ra.linalg, da.core, da.random,
da.linalg]:
for module in [
ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg
]:
reload(module)
ray.worker._init(start_ray_local=True, num_local_schedulers=2,
num_cpus=[10, 10])
ray.worker._init(
start_ray_local=True, num_local_schedulers=2, num_cpus=[10, 10])
x = da.zeros.remote([9, 25, 51], "float")
assert_equal(ray.get(da.assemble.remote(x)), np.zeros([9, 25, 51]))
@@ -84,21 +89,23 @@ class DistributedArrayTest(unittest.TestCase):
x = da.random.normal.remote([11, 25, 49])
y = da.copy.remote(x)
assert_equal(ray.get(da.assemble.remote(x)),
ray.get(da.assemble.remote(y)))
assert_equal(
ray.get(da.assemble.remote(x)), ray.get(da.assemble.remote(y)))
x = da.eye.remote(25, dtype_name="float")
assert_equal(ray.get(da.assemble.remote(x)), np.eye(25))
x = da.random.normal.remote([25, 49])
y = da.triu.remote(x)
assert_equal(ray.get(da.assemble.remote(y)),
np.triu(ray.get(da.assemble.remote(x))))
assert_equal(
ray.get(da.assemble.remote(y)),
np.triu(ray.get(da.assemble.remote(x))))
x = da.random.normal.remote([25, 49])
y = da.tril.remote(x)
assert_equal(ray.get(da.assemble.remote(y)),
np.tril(ray.get(da.assemble.remote(x))))
assert_equal(
ray.get(da.assemble.remote(y)),
np.tril(ray.get(da.assemble.remote(x))))
x = da.random.normal.remote([25, 49])
y = da.random.normal.remote([49, 18])
@@ -113,31 +120,31 @@ class DistributedArrayTest(unittest.TestCase):
x = da.random.normal.remote([23, 42])
y = da.random.normal.remote([23, 42])
z = da.add.remote(x, y)
assert_almost_equal(ray.get(da.assemble.remote(z)),
ray.get(da.assemble.remote(x)) +
ray.get(da.assemble.remote(y)))
assert_almost_equal(
ray.get(da.assemble.remote(z)),
ray.get(da.assemble.remote(x)) + ray.get(da.assemble.remote(y)))
# test subtract
x = da.random.normal.remote([33, 40])
y = da.random.normal.remote([33, 40])
z = da.subtract.remote(x, y)
assert_almost_equal(ray.get(da.assemble.remote(z)),
ray.get(da.assemble.remote(x)) -
ray.get(da.assemble.remote(y)))
assert_almost_equal(
ray.get(da.assemble.remote(z)),
ray.get(da.assemble.remote(x)) - ray.get(da.assemble.remote(y)))
# test transpose
x = da.random.normal.remote([234, 432])
y = da.transpose.remote(x)
assert_equal(ray.get(da.assemble.remote(x)).T,
ray.get(da.assemble.remote(y)))
assert_equal(
ray.get(da.assemble.remote(x)).T, ray.get(da.assemble.remote(y)))
# test numpy_to_dist
x = da.random.normal.remote([23, 45])
y = da.assemble.remote(x)
z = da.numpy_to_dist.remote(y)
w = da.assemble.remote(z)
assert_equal(ray.get(da.assemble.remote(x)),
ray.get(da.assemble.remote(z)))
assert_equal(
ray.get(da.assemble.remote(x)), ray.get(da.assemble.remote(z)))
assert_equal(ray.get(y), ray.get(w))
# test da.tsqr
@@ -157,8 +164,8 @@ class DistributedArrayTest(unittest.TestCase):
# test da.linalg.modified_lu
def test_modified_lu(d1, d2):
print("testing dist_modified_lu with d1 = " + str(d1) +
", d2 = " + str(d2))
print("testing dist_modified_lu with d1 = " + str(d1) + ", d2 = " +
str(d2))
assert d1 >= d2
m = ra.random.normal.remote([d1, d2])
q, r = ra.linalg.qr.remote(m)
@@ -178,8 +185,8 @@ class DistributedArrayTest(unittest.TestCase):
# Check that l is lower triangular.
assert_equal(np.tril(l_val), l_val)
for d1, d2 in [(100, 100), (99, 98), (7, 5), (7, 7), (20, 7),
(20, 10)]:
for d1, d2 in [(100, 100), (99, 98), (7, 5), (7, 7), (20, 7), (20,
10)]:
test_modified_lu(d1, d2)
# test dist_tsqr_hr
test/autoscaler_test.py
+52 -23
View File
@@ -56,7 +56,8 @@ class MockProvider(NodeProvider):
raise Exception("oops")
return [
n.node_id for n in self.mock_nodes.values()
if n.matches(tag_filters) and n.state != "terminated"]
if n.matches(tag_filters) and n.state != "terminated"
]
def is_running(self, node_id):
return self.mock_nodes[node_id].state == "running"
@@ -101,7 +102,6 @@ SMALL_CLUSTER = {
"docker": {
"image": "example",
"container_name": "mock",
},
"auth": {
"ssh_user": "ubuntu",
@@ -269,8 +269,11 @@ class AutoscalingTest(unittest.TestCase):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_concurrent_launches=5,
max_failures=0, update_interval_s=0)
config_path,
LoadMetrics(),
max_concurrent_launches=5,
max_failures=0,
update_interval_s=0)
self.assertEqual(len(self.provider.nodes({})), 0)
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
@@ -295,8 +298,11 @@ class AutoscalingTest(unittest.TestCase):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_concurrent_launches=5,
max_failures=0, update_interval_s=10)
config_path,
LoadMetrics(),
max_concurrent_launches=5,
max_failures=0,
update_interval_s=10)
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
new_config = SMALL_CLUSTER.copy()
@@ -328,8 +334,11 @@ class AutoscalingTest(unittest.TestCase):
config_path = self.write_config(SMALL_CLUSTER)
self.provider = MockProvider()
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_concurrent_launches=10,
max_failures=0, update_interval_s=0)
config_path,
LoadMetrics(),
max_concurrent_launches=10,
max_failures=0,
update_interval_s=0)
autoscaler.update()
# Write a corrupted config
@@ -383,16 +392,22 @@ class AutoscalingTest(unittest.TestCase):
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread,
config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
verbose_updates=True,
node_updater_cls=NodeUpdaterThread,
update_interval_s=0)
autoscaler.update()
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
for node in self.provider.mock_nodes.values():
node.state = "running"
assert len(self.provider.nodes(
{TAG_RAY_NODE_STATUS: "Uninitialized"})) == 2
assert len(
self.provider.nodes({
TAG_RAY_NODE_STATUS: "Uninitialized"
})) == 2
autoscaler.update()
self.waitFor(
lambda: len(self.provider.nodes(
@@ -403,16 +418,22 @@ class AutoscalingTest(unittest.TestCase):
self.provider = MockProvider()
runner = MockProcessRunner(fail_cmds=["cmd1"])
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread,
config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
verbose_updates=True,
node_updater_cls=NodeUpdaterThread,
update_interval_s=0)
autoscaler.update()
autoscaler.update()
self.assertEqual(len(self.provider.nodes({})), 2)
for node in self.provider.mock_nodes.values():
node.state = "running"
assert len(self.provider.nodes(
{TAG_RAY_NODE_STATUS: "Uninitialized"})) == 2
assert len(
self.provider.nodes({
TAG_RAY_NODE_STATUS: "Uninitialized"
})) == 2
autoscaler.update()
self.waitFor(
lambda: len(self.provider.nodes(
@@ -423,8 +444,12 @@ class AutoscalingTest(unittest.TestCase):
self.provider = MockProvider()
runner = MockProcessRunner()
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread,
config_path,
LoadMetrics(),
max_failures=0,
process_runner=runner,
verbose_updates=True,
node_updater_cls=NodeUpdaterThread,
update_interval_s=0)
autoscaler.update()
autoscaler.update()
@@ -490,8 +515,12 @@ class AutoscalingTest(unittest.TestCase):
runner = MockProcessRunner()
lm = LoadMetrics()
autoscaler = StandardAutoscaler(
config_path, lm, max_failures=0, process_runner=runner,
verbose_updates=True, node_updater_cls=NodeUpdaterThread,
config_path,
lm,
max_failures=0,
process_runner=runner,
verbose_updates=True,
node_updater_cls=NodeUpdaterThread,
update_interval_s=0)
autoscaler.update()
for node in self.provider.mock_nodes.values():
@@ -512,7 +541,7 @@ class AutoscalingTest(unittest.TestCase):
config["provider"] = {
"type": "external",
"module": "ray.autoscaler.node_provider.NodeProvider",
}
}
config_path = self.write_config(config)
autoscaler = StandardAutoscaler(
config_path, LoadMetrics(), max_failures=0, update_interval_s=0)
@@ -523,7 +552,7 @@ class AutoscalingTest(unittest.TestCase):
config["provider"] = {
"type": "external",
"module": "mymodule.provider_class",
}
}
invalid_provider = self.write_config(config)
self.assertRaises(
ImportError,
@@ -535,7 +564,7 @@ class AutoscalingTest(unittest.TestCase):
config["provider"] = {
"type": "external",
"module": "does-not-exist",
}
}
invalid_provider = self.write_config(config)
self.assertRaises(
ValueError,
test/component_failures_test.py
+75 -69
View File
@@ -11,7 +11,6 @@ import pyarrow as pa
class ComponentFailureTest(unittest.TestCase):
def tearDown(self):
ray.worker.cleanup()
@@ -24,54 +23,20 @@ class ComponentFailureTest(unittest.TestCase):
def f():
ray.worker.global_worker.plasma_client.get(obj_id)
ray.worker._init(num_workers=1,
driver_mode=ray.SILENT_MODE,
start_workers_from_local_scheduler=False,
start_ray_local=True,
redirect_output=True)
ray.worker._init(
num_workers=1,
driver_mode=ray.SILENT_MODE,
start_workers_from_local_scheduler=False,
start_ray_local=True,
redirect_output=True)
# Have the worker wait in a get call.
f.remote()
# Kill the worker.
time.sleep(1)
(ray.services
.all_processes[ray.services.PROCESS_TYPE_WORKER][0].terminate())
time.sleep(0.1)
# Seal the object so the store attempts to notify the worker that the
# get has been fulfilled.
ray.worker.global_worker.plasma_client.create(
pa.plasma.ObjectID(obj_id), 100)
ray.worker.global_worker.plasma_client.seal(pa.plasma.ObjectID(obj_id))
time.sleep(0.1)
# Make sure that nothing has died.
self.assertTrue(ray.services.all_processes_alive(
exclude=[ray.services.PROCESS_TYPE_WORKER]))
# This test checks that when a worker dies in the middle of a wait, the
# plasma store and manager will not die.
def testDyingWorkerWait(self):
obj_id = 20 * b"a"
@ray.remote
def f():
ray.worker.global_worker.plasma_client.wait([obj_id])
ray.worker._init(num_workers=1,
driver_mode=ray.SILENT_MODE,
start_workers_from_local_scheduler=False,
start_ray_local=True,
redirect_output=True)
# Have the worker wait in a get call.
f.remote()
# Kill the worker.
time.sleep(1)
(ray.services
.all_processes[ray.services.PROCESS_TYPE_WORKER][0].terminate())
(ray.services.all_processes[ray.services.PROCESS_TYPE_WORKER][0]
.terminate())
time.sleep(0.1)
# Seal the object so the store attempts to notify the worker that the
@@ -82,8 +47,46 @@ class ComponentFailureTest(unittest.TestCase):
time.sleep(0.1)
# Make sure that nothing has died.
self.assertTrue(ray.services.all_processes_alive(
exclude=[ray.services.PROCESS_TYPE_WORKER]))
self.assertTrue(
ray.services.all_processes_alive(
exclude=[ray.services.PROCESS_TYPE_WORKER]))
# This test checks that when a worker dies in the middle of a wait, the
# plasma store and manager will not die.
def testDyingWorkerWait(self):
obj_id = 20 * b"a"
@ray.remote
def f():
ray.worker.global_worker.plasma_client.wait([obj_id])
ray.worker._init(
num_workers=1,
driver_mode=ray.SILENT_MODE,
start_workers_from_local_scheduler=False,
start_ray_local=True,
redirect_output=True)
# Have the worker wait in a get call.
f.remote()
# Kill the worker.
time.sleep(1)
(ray.services.all_processes[ray.services.PROCESS_TYPE_WORKER][0]
.terminate())
time.sleep(0.1)
# Seal the object so the store attempts to notify the worker that the
# get has been fulfilled.
ray.worker.global_worker.plasma_client.create(
pa.plasma.ObjectID(obj_id), 100)
ray.worker.global_worker.plasma_client.seal(pa.plasma.ObjectID(obj_id))
time.sleep(0.1)
# Make sure that nothing has died.
self.assertTrue(
ray.services.all_processes_alive(
exclude=[ray.services.PROCESS_TYPE_WORKER]))
def _testWorkerFailed(self, num_local_schedulers):
@ray.remote
@@ -92,23 +95,25 @@ class ComponentFailureTest(unittest.TestCase):
return x
num_initial_workers = 4
ray.worker._init(num_workers=(num_initial_workers *
num_local_schedulers),
num_local_schedulers=num_local_schedulers,
start_workers_from_local_scheduler=False,
start_ray_local=True,
num_cpus=[num_initial_workers] * num_local_schedulers,
redirect_output=True)
ray.worker._init(
num_workers=(num_initial_workers * num_local_schedulers),
num_local_schedulers=num_local_schedulers,
start_workers_from_local_scheduler=False,
start_ray_local=True,
num_cpus=[num_initial_workers] * num_local_schedulers,
redirect_output=True)
# Submit more tasks than there are workers so that all workers and
# cores are utilized.
object_ids = [f.remote(i) for i
in range(num_initial_workers * num_local_schedulers)]
object_ids = [
f.remote(i)
for i in range(num_initial_workers * num_local_schedulers)
]
object_ids += [f.remote(object_id) for object_id in object_ids]
# Allow the tasks some time to begin executing.
time.sleep(0.1)
# Kill the workers as the tasks execute.
for worker in (ray.services
.all_processes[ray.services.PROCESS_TYPE_WORKER]):
for worker in (
ray.services.all_processes[ray.services.PROCESS_TYPE_WORKER]):
worker.terminate()
time.sleep(0.1)
# Make sure that we can still get the objects after the executing tasks
@@ -123,6 +128,7 @@ class ComponentFailureTest(unittest.TestCase):
def _testComponentFailed(self, component_type):
"""Kill a component on all worker nodes and check workload succeeds."""
@ray.remote
def f(x, j):
time.sleep(0.2)
@@ -140,9 +146,10 @@ class ComponentFailureTest(unittest.TestCase):
# Submit more tasks than there are workers so that all workers and
# cores are utilized.
object_ids = [f.remote(i, 0) for i
in range(num_workers_per_scheduler *
num_local_schedulers)]
object_ids = [
f.remote(i, 0)
for i in range(num_workers_per_scheduler * num_local_schedulers)
]
object_ids += [f.remote(object_id, 1) for object_id in object_ids]
object_ids += [f.remote(object_id, 2) for object_id in object_ids]
@@ -162,8 +169,8 @@ class ComponentFailureTest(unittest.TestCase):
# Make sure that we can still get the objects after the executing tasks
# died.
results = ray.get(object_ids)
expected_results = 4 * list(range(
num_workers_per_scheduler * num_local_schedulers))
expected_results = 4 * list(
range(num_workers_per_scheduler * num_local_schedulers))
self.assertEqual(results, expected_results)
def check_components_alive(self, component_type, check_component_alive):
@@ -182,8 +189,7 @@ class ComponentFailureTest(unittest.TestCase):
self.assertTrue(not component.poll() is None)
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testLocalSchedulerFailed(self):
# Kill all local schedulers on worker nodes.
self._testComponentFailed(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER)
@@ -198,8 +204,7 @@ class ComponentFailureTest(unittest.TestCase):
False)
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testPlasmaManagerFailed(self):
# Kill all plasma managers on worker nodes.
self._testComponentFailed(ray.services.PROCESS_TYPE_PLASMA_MANAGER)
@@ -214,8 +219,7 @@ class ComponentFailureTest(unittest.TestCase):
False)
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testPlasmaStoreFailed(self):
# Kill all plasma stores on worker nodes.
self._testComponentFailed(ray.services.PROCESS_TYPE_PLASMA_STORE)
@@ -235,7 +239,8 @@ class ComponentFailureTest(unittest.TestCase):
all_processes[ray.services.PROCESS_TYPE_PLASMA_STORE][0],
all_processes[ray.services.PROCESS_TYPE_PLASMA_MANAGER][0],
all_processes[ray.services.PROCESS_TYPE_LOCAL_SCHEDULER][0],
all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER][0]]
all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER][0]
]
# Kill all the components sequentially.
for process in processes:
@@ -253,7 +258,8 @@ class ComponentFailureTest(unittest.TestCase):
all_processes[ray.services.PROCESS_TYPE_PLASMA_STORE][0],
all_processes[ray.services.PROCESS_TYPE_PLASMA_MANAGER][0],
all_processes[ray.services.PROCESS_TYPE_LOCAL_SCHEDULER][0],
all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER][0]]
all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER][0]
]
# Kill all the components in parallel.
for process in processes:
test/credis_test.py
+4 -5
View File
@@ -9,9 +9,8 @@ import unittest
import ray
@unittest.skipIf(
not os.environ.get('RAY_USE_NEW_GCS', False),
"Tests functionality of the new GCS.")
@unittest.skipIf(not os.environ.get('RAY_USE_NEW_GCS', False),
"Tests functionality of the new GCS.")
class CredisTest(unittest.TestCase):
def setUp(self):
self.config = ray.init(num_workers=0)
@@ -22,8 +21,8 @@ class CredisTest(unittest.TestCase):
def test_credis_started(self):
assert "credis_address" in self.config
credis_address, credis_port = self.config["credis_address"].split(":")
credis_client = redis.StrictRedis(host=credis_address,
port=credis_port)
credis_client = redis.StrictRedis(
host=credis_address, port=credis_port)
assert credis_client.ping() is True
redis_client = ray.worker.global_state.redis_client
test/failure_test.py
+16 -10
View File
@@ -108,6 +108,7 @@ def temporary_helper_function():
def f(worker):
if ray.worker.global_worker.mode == ray.WORKER_MODE:
raise Exception("Function to run failed.")
ray.worker.global_worker.run_function_on_all_workers(f)
wait_for_errors(b"function_to_run", 2)
# Check that the error message is in the task info.
@@ -348,12 +349,14 @@ class PutErrorTest(unittest.TestCase):
ray.worker.cleanup()
def testPutError1(self):
store_size = 10 ** 6
ray.worker._init(start_ray_local=True, driver_mode=ray.SILENT_MODE,
object_store_memory=store_size)
store_size = 10**6
ray.worker._init(
start_ray_local=True,
driver_mode=ray.SILENT_MODE,
object_store_memory=store_size)
num_objects = 3
object_size = 4 * 10 ** 5
object_size = 4 * 10**5
# Define a task with a single dependency, a numpy array, that returns
# another array.
@@ -369,8 +372,9 @@ class PutErrorTest(unittest.TestCase):
# on the one before it. The result of the first task should get
# evicted.
args = []
arg = single_dependency.remote(0, np.zeros(object_size,
dtype=np.uint8))
arg = single_dependency.remote(0,
np.zeros(
object_size, dtype=np.uint8))
for i in range(num_objects):
arg = single_dependency.remote(i, arg)
args.append(arg)
@@ -393,12 +397,14 @@ class PutErrorTest(unittest.TestCase):
def testPutError2(self):
# This is the same as the previous test, but it calls ray.put directly.
store_size = 10 ** 6
ray.worker._init(start_ray_local=True, driver_mode=ray.SILENT_MODE,
object_store_memory=store_size)
store_size = 10**6
ray.worker._init(
start_ray_local=True,
driver_mode=ray.SILENT_MODE,
object_store_memory=store_size)
num_objects = 3
object_size = 4 * 10 ** 5
object_size = 4 * 10**5
# Define a task with a single dependency, a numpy array, that returns
# another array.
test/jenkins_tests/multi_node_docker_test.py
+127 -82
View File
@@ -58,6 +58,7 @@ class DockerRunner(object):
head_container_ip: The IP address of the docker container that runs the
head node.
"""
def __init__(self):
"""Initialize the DockerRunner."""
self.head_container_id = None
@@ -91,11 +92,14 @@ class DockerRunner(object):
Returns:
The IP address of the container.
"""
proc = subprocess.Popen(["docker", "inspect",
"--format={{.NetworkSettings.Networks.bridge"
".IPAddress}}",
container_id],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc = subprocess.Popen(
[
"docker", "inspect",
"--format={{.NetworkSettings.Networks.bridge"
".IPAddress}}", container_id
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
p = re.compile("([0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3})")
m = p.match(stdout_data)
@@ -110,23 +114,23 @@ class DockerRunner(object):
"""Start the Ray head node inside a docker container."""
mem_arg = ["--memory=" + mem_size] if mem_size else []
shm_arg = ["--shm-size=" + shm_size] if shm_size else []
volume_arg = (["-v",
"{}:{}".format(os.path.dirname(
os.path.realpath(__file__)),
"/ray/test/jenkins_tests")]
if development_mode else [])
volume_arg = ([
"-v", "{}:{}".format(
os.path.dirname(os.path.realpath(__file__)),
"/ray/test/jenkins_tests")
] if development_mode else [])
command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg +
[docker_image, "ray", "start", "--head", "--block",
"--redis-port=6379",
"--num-redis-shards={}".format(num_redis_shards),
"--num-cpus={}".format(num_cpus),
"--num-gpus={}".format(num_gpus),
"--no-ui"])
command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg + [
docker_image, "ray", "start", "--head", "--block",
"--redis-port=6379",
"--num-redis-shards={}".format(num_redis_shards),
"--num-cpus={}".format(num_cpus), "--num-gpus={}".format(num_gpus),
"--no-ui"
])
print("Starting head node with command:{}".format(command))
proc = subprocess.Popen(command,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
container_id = self._get_container_id(stdout_data)
if container_id is None:
@@ -139,29 +143,34 @@ class DockerRunner(object):
"""Start a Ray worker node inside a docker container."""
mem_arg = ["--memory=" + mem_size] if mem_size else []
shm_arg = ["--shm-size=" + shm_size] if shm_size else []
volume_arg = (["-v",
"{}:{}".format(os.path.dirname(
os.path.realpath(__file__)),
"/ray/test/jenkins_tests")]
if development_mode else [])
command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg +
["--shm-size=" + shm_size, docker_image,
"ray", "start", "--block",
"--redis-address={:s}:6379".format(self.head_container_ip),
"--num-cpus={}".format(num_cpus),
"--num-gpus={}".format(num_gpus)])
volume_arg = ([
"-v", "{}:{}".format(
os.path.dirname(os.path.realpath(__file__)),
"/ray/test/jenkins_tests")
] if development_mode else [])
command = (["docker", "run", "-d"] + mem_arg + shm_arg + volume_arg + [
"--shm-size=" + shm_size, docker_image, "ray", "start", "--block",
"--redis-address={:s}:6379".format(self.head_container_ip),
"--num-cpus={}".format(num_cpus), "--num-gpus={}".format(num_gpus)
])
print("Starting worker node with command:{}".format(command))
proc = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
container_id = self._get_container_id(stdout_data)
if container_id is None:
raise RuntimeError("Failed to find container id")
self.worker_container_ids.append(container_id)
def start_ray(self, docker_image=None, mem_size=None, shm_size=None,
num_nodes=None, num_redis_shards=1, num_cpus=None,
num_gpus=None, development_mode=None):
def start_ray(self,
docker_image=None,
mem_size=None,
shm_size=None,
num_nodes=None,
num_redis_shards=1,
num_cpus=None,
num_gpus=None,
development_mode=None):
"""Start a Ray cluster within docker.
This starts one docker container running the head node and
@@ -200,24 +209,31 @@ class DockerRunner(object):
def _stop_node(self, container_id):
"""Stop a node in the Ray cluster."""
proc = subprocess.Popen(["docker", "kill", container_id],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc = subprocess.Popen(
["docker", "kill", container_id],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
stopped_container_id = self._get_container_id(stdout_data)
if not container_id == stopped_container_id:
raise Exception("Failed to stop container {}."
.format(container_id))
proc = subprocess.Popen(["docker", "rm", "-f", container_id],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc = subprocess.Popen(
["docker", "rm", "-f", container_id],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
removed_container_id = self._get_container_id(stdout_data)
if not container_id == removed_container_id:
raise Exception("Failed to remove container {}."
.format(container_id))
print("stop_node", {"container_id": container_id,
"is_head": container_id == self.head_container_id})
print(
"stop_node", {
"container_id": container_id,
"is_head": container_id == self.head_container_id
})
def stop_ray(self):
"""Stop the Ray cluster."""
@@ -236,7 +252,10 @@ class DockerRunner(object):
return success
def run_test(self, test_script, num_drivers, driver_locations=None,
def run_test(self,
test_script,
num_drivers,
driver_locations=None,
timeout_seconds=600):
"""Run a test script.
@@ -258,11 +277,13 @@ class DockerRunner(object):
Raises:
Exception: An exception is raised if the timeout expires.
"""
all_container_ids = ([self.head_container_id] +
self.worker_container_ids)
all_container_ids = (
[self.head_container_id] + self.worker_container_ids)
if driver_locations is None:
driver_locations = [np.random.randint(0, len(all_container_ids))
for _ in range(num_drivers)]
driver_locations = [
np.random.randint(0, len(all_container_ids))
for _ in range(num_drivers)
]
# Define a signal handler and set an alarm to go off in
# timeout_seconds.
@@ -278,13 +299,15 @@ class DockerRunner(object):
for i in range(len(driver_locations)):
# Get the container ID to run the ith driver in.
container_id = all_container_ids[driver_locations[i]]
command = ["docker", "exec", container_id, "/bin/bash", "-c",
("RAY_REDIS_ADDRESS={}:6379 RAY_DRIVER_INDEX={} python "
"{}".format(self.head_container_ip, i, test_script))]
command = [
"docker", "exec", container_id, "/bin/bash", "-c",
("RAY_REDIS_ADDRESS={}:6379 RAY_DRIVER_INDEX={} python "
"{}".format(self.head_container_ip, i, test_script))
]
print("Starting driver with command {}.".format(test_script))
# Start the driver.
p = subprocess.Popen(command, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
p = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
driver_processes.append(p)
# Wait for the drivers to finish.
@@ -295,8 +318,10 @@ class DockerRunner(object):
print(stdout_data)
print("STDERR:")
print(stderr_data)
results.append({"success": p.returncode == 0,
"return_code": p.returncode})
results.append({
"success": p.returncode == 0,
"return_code": p.returncode
})
# Disable the alarm.
signal.alarm(0)
@@ -307,29 +332,43 @@ class DockerRunner(object):
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Run multinode tests in Docker.")
parser.add_argument("--docker-image", default="ray-project/deploy",
help="docker image")
parser.add_argument(
"--docker-image", default="ray-project/deploy", help="docker image")
parser.add_argument("--mem-size", help="memory size")
parser.add_argument("--shm-size", default="1G", help="shared memory size")
parser.add_argument("--num-nodes", default=1, type=int,
help="number of nodes to use in the cluster")
parser.add_argument("--num-redis-shards", default=1, type=int,
help=("the number of Redis shards to start on the "
"head node"))
parser.add_argument("--num-cpus", type=str,
help=("a comma separated list of values representing "
"the number of CPUs to start each node with"))
parser.add_argument("--num-gpus", type=str,
help=("a comma separated list of values representing "
"the number of GPUs to start each node with"))
parser.add_argument("--num-drivers", default=1, type=int,
help="number of drivers to run")
parser.add_argument("--driver-locations", type=str,
help=("a comma separated list of indices of the "
"containers to run the drivers in"))
parser.add_argument(
"--num-nodes",
default=1,
type=int,
help="number of nodes to use in the cluster")
parser.add_argument(
"--num-redis-shards",
default=1,
type=int,
help=("the number of Redis shards to start on the "
"head node"))
parser.add_argument(
"--num-cpus",
type=str,
help=("a comma separated list of values representing "
"the number of CPUs to start each node with"))
parser.add_argument(
"--num-gpus",
type=str,
help=("a comma separated list of values representing "
"the number of GPUs to start each node with"))
parser.add_argument(
"--num-drivers", default=1, type=int, help="number of drivers to run")
parser.add_argument(
"--driver-locations",
type=str,
help=("a comma separated list of indices of the "
"containers to run the drivers in"))
parser.add_argument("--test-script", required=True, help="test script")
parser.add_argument("--development-mode", action="store_true",
help="use local copies of the test scripts")
parser.add_argument(
"--development-mode",
action="store_true",
help="use local copies of the test scripts")
args = parser.parse_args()
# Parse the number of CPUs and GPUs to use for each worker.
@@ -340,18 +379,24 @@ if __name__ == "__main__":
if args.num_gpus is not None else num_nodes * [0])
# Parse the driver locations.
driver_locations = (None if args.driver_locations is None
else [int(i) for i
in args.driver_locations.split(",")])
driver_locations = (None if args.driver_locations is None else
[int(i) for i in args.driver_locations.split(",")])
d = DockerRunner()
d.start_ray(docker_image=args.docker_image, mem_size=args.mem_size,
shm_size=args.shm_size, num_nodes=num_nodes,
num_redis_shards=args.num_redis_shards, num_cpus=num_cpus,
num_gpus=num_gpus, development_mode=args.development_mode)
d.start_ray(
docker_image=args.docker_image,
mem_size=args.mem_size,
shm_size=args.shm_size,
num_nodes=num_nodes,
num_redis_shards=args.num_redis_shards,
num_cpus=num_cpus,
num_gpus=num_gpus,
development_mode=args.development_mode)
try:
run_results = d.run_test(args.test_script, args.num_drivers,
driver_locations=driver_locations)
run_results = d.run_test(
args.test_script,
args.num_drivers,
driver_locations=driver_locations)
finally:
successfully_stopped = d.stop_ray()
test/jenkins_tests/multi_node_tests/large_memory_test.py
+3 -5
View File
@@ -6,19 +6,17 @@ import numpy as np
import ray
if __name__ == "__main__":
ray.init(num_workers=0)
A = np.ones(2 ** 31 + 1, dtype="int8")
A = np.ones(2**31 + 1, dtype="int8")
a = ray.put(A)
assert np.sum(ray.get(a)) == np.sum(A)
del A
del a
print("Successfully put A.")
B = {"hello": np.zeros(2 ** 30 + 1),
"world": np.ones(2 ** 30 + 1)}
B = {"hello": np.zeros(2**30 + 1), "world": np.ones(2**30 + 1)}
b = ray.put(B)
assert np.sum(ray.get(b)["hello"]) == np.sum(B["hello"])
assert np.sum(ray.get(b)["world"]) == np.sum(B["world"])
@@ -26,7 +24,7 @@ if __name__ == "__main__":
del b
print("Successfully put B.")
C = [np.ones(2 ** 30 + 1), 42.0 * np.ones(2 ** 30 + 1)]
C = [np.ones(2**30 + 1), 42.0 * np.ones(2**30 + 1)]
c = ray.put(C)
assert np.sum(ray.get(c)[0]) == np.sum(C[0])
assert np.sum(ray.get(c)[1]) == np.sum(C[1])
test/jenkins_tests/multi_node_tests/many_drivers_test.py
+1 -2
View File
@@ -6,8 +6,7 @@ import os
import time
import ray
from ray.test.test_utils import (_wait_for_nodes_to_join,
_broadcast_event,
from ray.test.test_utils import (_wait_for_nodes_to_join, _broadcast_event,
_wait_for_event)
# This test should be run with 5 nodes, which have 0, 0, 5, 6, and 50 GPUs for
test/jenkins_tests/multi_node_tests/remove_driver_test.py
+44 -39
View File
@@ -6,10 +6,8 @@ import os
import time
import ray
from ray.test.test_utils import (_wait_for_nodes_to_join,
_broadcast_event,
_wait_for_event,
wait_for_pid_to_exit)
from ray.test.test_utils import (_wait_for_nodes_to_join, _broadcast_event,
_wait_for_event, wait_for_pid_to_exit)
# This test should be run with 5 nodes, which have 0, 1, 2, 3, and 4 GPUs for a
# total of 10 GPUs. It should be run with 7 drivers. Drivers 2 through 6 must
@@ -28,9 +26,10 @@ def remote_function_event_name(driver_index, task_index):
@ray.remote
def long_running_task(driver_index, task_index, redis_address):
_broadcast_event(remote_function_event_name(driver_index, task_index),
redis_address,
data=(ray.services.get_node_ip_address(), os.getpid()))
_broadcast_event(
remote_function_event_name(driver_index, task_index),
redis_address,
data=(ray.services.get_node_ip_address(), os.getpid()))
# Loop forever.
while True:
time.sleep(100)
@@ -42,10 +41,10 @@ num_long_running_tasks_per_driver = 2
@ray.remote
class Actor0(object):
def __init__(self, driver_index, actor_index, redis_address):
_broadcast_event(actor_event_name(driver_index, actor_index),
redis_address,
data=(ray.services.get_node_ip_address(),
os.getpid()))
_broadcast_event(
actor_event_name(driver_index, actor_index),
redis_address,
data=(ray.services.get_node_ip_address(), os.getpid()))
assert len(ray.get_gpu_ids()) == 0
def check_ids(self):
@@ -60,10 +59,10 @@ class Actor0(object):
@ray.remote(num_gpus=1)
class Actor1(object):
def __init__(self, driver_index, actor_index, redis_address):
_broadcast_event(actor_event_name(driver_index, actor_index),
redis_address,
data=(ray.services.get_node_ip_address(),
os.getpid()))
_broadcast_event(
actor_event_name(driver_index, actor_index),
redis_address,
data=(ray.services.get_node_ip_address(), os.getpid()))
assert len(ray.get_gpu_ids()) == 1
def check_ids(self):
@@ -78,10 +77,10 @@ class Actor1(object):
@ray.remote(num_gpus=2)
class Actor2(object):
def __init__(self, driver_index, actor_index, redis_address):
_broadcast_event(actor_event_name(driver_index, actor_index),
redis_address,
data=(ray.services.get_node_ip_address(),
os.getpid()))
_broadcast_event(
actor_event_name(driver_index, actor_index),
redis_address,
data=(ray.services.get_node_ip_address(), os.getpid()))
assert len(ray.get_gpu_ids()) == 2
def check_ids(self):
@@ -110,11 +109,13 @@ def driver_0(redis_address, driver_index):
long_running_task.remote(driver_index, i, redis_address)
# Create some actors that require one GPU.
actors_one_gpu = [Actor1.remote(driver_index, i, redis_address)
for i in range(5)]
actors_one_gpu = [
Actor1.remote(driver_index, i, redis_address) for i in range(5)
]
# Create some actors that don't require any GPUs.
actors_no_gpus = [Actor0.remote(driver_index, 5 + i, redis_address)
for i in range(5)]
actors_no_gpus = [
Actor0.remote(driver_index, 5 + i, redis_address) for i in range(5)
]
for _ in range(1000):
ray.get([actor.check_ids.remote() for actor in actors_one_gpu])
@@ -145,14 +146,17 @@ def driver_1(redis_address, driver_index):
long_running_task.remote(driver_index, i, redis_address)
# Create an actor that requires two GPUs.
actors_two_gpus = [Actor2.remote(driver_index, i, redis_address)
for i in range(1)]
actors_two_gpus = [
Actor2.remote(driver_index, i, redis_address) for i in range(1)
]
# Create some actors that require one GPU.
actors_one_gpu = [Actor1.remote(driver_index, 1 + i, redis_address)
for i in range(3)]
actors_one_gpu = [
Actor1.remote(driver_index, 1 + i, redis_address) for i in range(3)
]
# Create some actors that don't require any GPUs.
actors_no_gpus = [Actor0.remote(driver_index, 1 + 3 + i, redis_address)
for i in range(5)]
actors_no_gpus = [
Actor0.remote(driver_index, 1 + 3 + i, redis_address) for i in range(5)
]
for _ in range(1000):
ray.get([actor.check_ids.remote() for actor in actors_two_gpus])
@@ -179,8 +183,9 @@ def cleanup_driver(redis_address, driver_index):
# We go ahead and create some actors that don't require any GPUs. We
# don't need to wait for the other drivers to finish. We call methods
# on these actors later to make sure they haven't been killed.
actors_no_gpus = [Actor0.remote(driver_index, i, redis_address)
for i in range(10)]
actors_no_gpus = [
Actor0.remote(driver_index, i, redis_address) for i in range(10)
]
_wait_for_event("DRIVER_0_DONE", redis_address)
_wait_for_event("DRIVER_1_DONE", redis_address)
@@ -206,13 +211,13 @@ def cleanup_driver(redis_address, driver_index):
# Create some actors that require two GPUs.
actors_two_gpus = []
for i in range(3):
actors_two_gpus.append(try_to_create_actor(Actor2, driver_index,
10 + i))
actors_two_gpus.append(
try_to_create_actor(Actor2, driver_index, 10 + i))
# Create some actors that require one GPU.
actors_one_gpu = []
for i in range(4):
actors_one_gpu.append(try_to_create_actor(Actor1, driver_index,
10 + 3 + i))
actors_one_gpu.append(
try_to_create_actor(Actor1, driver_index, 10 + 3 + i))
removed_workers = 0
@@ -233,14 +238,14 @@ def cleanup_driver(redis_address, driver_index):
# Make sure that the PIDs for the actors from driver 0 and driver 1 have
# been killed.
for i in range(10):
node_ip_address, pid = _wait_for_event(actor_event_name(0, i),
redis_address)
node_ip_address, pid = _wait_for_event(
actor_event_name(0, i), redis_address)
if node_ip_address == ray.services.get_node_ip_address():
wait_for_pid_to_exit(pid)
removed_workers += 1
for i in range(9):
node_ip_address, pid = _wait_for_event(actor_event_name(1, i),
redis_address)
node_ip_address, pid = _wait_for_event(
actor_event_name(1, i), redis_address)
if node_ip_address == ray.services.get_node_ip_address():
wait_for_pid_to_exit(pid)
removed_workers += 1
test/jenkins_tests/multi_node_tests/test_0.py
+3 -2
View File
@@ -25,8 +25,9 @@ if __name__ == "__main__":
for i in range(num_attempts):
ip_addresses = ray.get([f.remote() for i in range(1000)])
distinct_addresses = set(ip_addresses)
counts = [ip_addresses.count(address) for address
in distinct_addresses]
counts = [
ip_addresses.count(address) for address in distinct_addresses
]
print("Counts are {}".format(counts))
if len(counts) == 5:
break
test/multi_node_test.py
+30 -37
View File
@@ -25,19 +25,17 @@ def run_string_as_driver(driver_script):
with tempfile.NamedTemporaryFile() as f:
f.write(driver_script.encode("ascii"))
f.flush()
out = subprocess.check_output([sys.executable,
f.name]).decode("ascii")
out = subprocess.check_output([sys.executable, f.name]).decode("ascii")
return out
class MultiNodeTest(unittest.TestCase):
def setUp(self):
out = run_and_get_output(["ray", "start", "--head"])
# Get the redis address from the output.
redis_substring_prefix = "redis_address=\""
redis_address_location = (out.find(redis_substring_prefix) +
len(redis_substring_prefix))
redis_address_location = (
out.find(redis_substring_prefix) + len(redis_substring_prefix))
redis_address = out[redis_address_location:]
self.redis_address = redis_address.split("\"")[0]
@@ -196,7 +194,6 @@ print("success")
class StartRayScriptTest(unittest.TestCase):
def testCallingStartRayHead(self):
# Test that we can call start-ray.sh with various command line
# parameters. TODO(rkn): This test only tests the --head code path. We
@@ -207,69 +204,65 @@ class StartRayScriptTest(unittest.TestCase):
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with a number of workers specified.
run_and_get_output(["ray", "start", "--head", "--num-workers",
"20"])
run_and_get_output(["ray", "start", "--head", "--num-workers", "20"])
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with a redis port specified.
run_and_get_output(["ray", "start", "--head",
"--redis-port", "6379"])
run_and_get_output(["ray", "start", "--head", "--redis-port", "6379"])
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with redis shard ports specified.
run_and_get_output(["ray", "start", "--head",
"--redis-shard-ports", "6380,6381,6382"])
run_and_get_output([
"ray", "start", "--head", "--redis-shard-ports", "6380,6381,6382"
])
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with a node IP address specified.
run_and_get_output(["ray", "start", "--head",
"--node-ip-address", "127.0.0.1"])
run_and_get_output(
["ray", "start", "--head", "--node-ip-address", "127.0.0.1"])
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with an object manager port specified.
run_and_get_output(["ray", "start", "--head",
"--object-manager-port", "12345"])
run_and_get_output(
["ray", "start", "--head", "--object-manager-port", "12345"])
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with the number of CPUs specified.
run_and_get_output(["ray", "start", "--head",
"--num-cpus", "100"])
run_and_get_output(["ray", "start", "--head", "--num-cpus", "100"])
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with the number of GPUs specified.
run_and_get_output(["ray", "start", "--head",
"--num-gpus", "100"])
run_and_get_output(["ray", "start", "--head", "--num-gpus", "100"])
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with the max redis clients specified.
run_and_get_output(["ray", "start", "--head",
"--redis-max-clients", "100"])
run_and_get_output(
["ray", "start", "--head", "--redis-max-clients", "100"])
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with all arguments specified.
run_and_get_output(["ray", "start", "--head",
"--num-workers", "20",
"--redis-port", "6379",
"--redis-shard-ports", "6380,6381,6382",
"--object-manager-port", "12345",
"--num-cpus", "100",
"--num-gpus", "0",
"--redis-max-clients", "100",
"--resources", "{\"Custom\": 1}"])
run_and_get_output([
"ray", "start", "--head", "--num-workers", "20", "--redis-port",
"6379", "--redis-shard-ports", "6380,6381,6382",
"--object-manager-port", "12345", "--num-cpus", "100",
"--num-gpus", "0", "--redis-max-clients", "100", "--resources",
"{\"Custom\": 1}"
])
subprocess.Popen(["ray", "stop"]).wait()
# Test starting Ray with invalid arguments.
with self.assertRaises(Exception):
run_and_get_output(["ray", "start", "--head",
"--redis-address", "127.0.0.1:6379"])
run_and_get_output([
"ray", "start", "--head", "--redis-address", "127.0.0.1:6379"
])
subprocess.Popen(["ray", "stop"]).wait()
def testUsingHostnames(self):
# Start the Ray processes on this machine.
run_and_get_output(
["ray", "start", "--head",
"--node-ip-address=localhost",
"--redis-port=6379"])
run_and_get_output([
"ray", "start", "--head", "--node-ip-address=localhost",
"--redis-port=6379"
])
ray.init(node_ip_address="localhost", redis_address="localhost:6379")
test/runtest.py
+73 -59
View File
@@ -184,12 +184,12 @@ DICT_OBJECTS = (
[{
obj: obj
} for obj in PRIMITIVE_OBJECTS
if (obj.__hash__ is not None and type(obj).__module__ != "numpy")] + [{
0:
obj
} for obj in BASE_OBJECTS] + [{
Foo(123): Foo(456)
}])
if (obj.__hash__ is not None and type(obj).__module__ != "numpy")] +
[{
0: obj
} for obj in BASE_OBJECTS] + [{
Foo(123): Foo(456)
}])
RAY_TEST_OBJECTS = BASE_OBJECTS + LIST_OBJECTS + TUPLE_OBJECTS + DICT_OBJECTS
@@ -359,25 +359,29 @@ class APITest(unittest.TestCase):
def custom_deserializer(serialized_obj):
return serialized_obj, "string2"
ray.register_custom_serializer(Foo, serializer=custom_serializer,
deserializer=custom_deserializer)
ray.register_custom_serializer(
Foo,
serializer=custom_serializer,
deserializer=custom_deserializer)
self.assertEqual(ray.get(ray.put(Foo())),
((3, "string1", Foo.__name__), "string2"))
self.assertEqual(
ray.get(ray.put(Foo())), ((3, "string1", Foo.__name__), "string2"))
class Bar(object):
def __init__(self):
self.x = 3
ray.register_custom_serializer(Bar, serializer=custom_serializer,
deserializer=custom_deserializer)
ray.register_custom_serializer(
Bar,
serializer=custom_serializer,
deserializer=custom_deserializer)
@ray.remote
def f():
return Bar()
self.assertEqual(ray.get(f.remote()),
((3, "string1", Bar.__name__), "string2"))
self.assertEqual(
ray.get(f.remote()), ((3, "string1", Bar.__name__), "string2"))
def testRegisterClass(self):
self.init_ray(num_workers=2)
@@ -700,10 +704,10 @@ class APITest(unittest.TestCase):
assert ray.get(f._submit(args=[1], num_return_vals=1)) == [0]
assert ray.get(f._submit(args=[2], num_return_vals=2)) == [0, 1]
assert ray.get(f._submit(args=[3], num_return_vals=3)) == [0, 1, 2]
assert ray.get(g._submit(args=[],
num_cpus=1,
num_gpus=1,
resources={"Custom": 1})) == [0]
assert ray.get(
g._submit(
args=[], num_cpus=1, num_gpus=1, resources={"Custom":
1})) == [0]
def testGetMultiple(self):
self.init_ray()
@@ -1234,8 +1238,8 @@ class ResourcesTest(unittest.TestCase):
time.sleep(0.1)
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 0
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
for gpu_id in gpu_ids:
assert gpu_id in range(num_gpus)
return gpu_ids
@@ -1245,8 +1249,8 @@ class ResourcesTest(unittest.TestCase):
time.sleep(0.1)
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 1
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
for gpu_id in gpu_ids:
assert gpu_id in range(num_gpus)
return gpu_ids
@@ -1256,8 +1260,8 @@ class ResourcesTest(unittest.TestCase):
time.sleep(0.1)
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 2
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
for gpu_id in gpu_ids:
assert gpu_id in range(num_gpus)
return gpu_ids
@@ -1267,8 +1271,8 @@ class ResourcesTest(unittest.TestCase):
time.sleep(0.1)
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 3
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
for gpu_id in gpu_ids:
assert gpu_id in range(num_gpus)
return gpu_ids
@@ -1278,8 +1282,8 @@ class ResourcesTest(unittest.TestCase):
time.sleep(0.1)
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 4
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
for gpu_id in gpu_ids:
assert gpu_id in range(num_gpus)
return gpu_ids
@@ -1289,8 +1293,8 @@ class ResourcesTest(unittest.TestCase):
time.sleep(0.1)
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 5
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
for gpu_id in gpu_ids:
assert gpu_id in range(num_gpus)
return gpu_ids
@@ -1342,16 +1346,16 @@ class ResourcesTest(unittest.TestCase):
def __init__(self):
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 0
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
# Set self.x to make sure that we got here.
self.x = 1
def test(self):
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 0
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
return self.x
@ray.remote(num_gpus=1)
@@ -1359,16 +1363,16 @@ class ResourcesTest(unittest.TestCase):
def __init__(self):
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 1
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
# Set self.x to make sure that we got here.
self.x = 1
def test(self):
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 1
assert (os.environ["CUDA_VISIBLE_DEVICES"] ==
",".join([str(i) for i in gpu_ids]))
assert (os.environ["CUDA_VISIBLE_DEVICES"] == ",".join(
[str(i) for i in gpu_ids]))
return self.x
a0 = Actor0.remote()
@@ -1379,9 +1383,7 @@ class ResourcesTest(unittest.TestCase):
def testZeroCPUs(self):
ray.worker._init(
start_ray_local=True,
num_local_schedulers=2,
num_cpus=[0, 2])
start_ray_local=True, num_local_schedulers=2, num_cpus=[0, 2])
local_plasma = ray.worker.global_worker.plasma_client.store_socket_name
@@ -1484,9 +1486,9 @@ class ResourcesTest(unittest.TestCase):
elif name == "run_on_2":
self.assertIn(result, [store_names[2]])
elif name == "run_on_0_1_2":
self.assertIn(result, [
store_names[0], store_names[1], store_names[2]
])
self.assertIn(
result,
[store_names[0], store_names[1], store_names[2]])
elif name == "run_on_1_2":
self.assertIn(result, [store_names[1], store_names[2]])
elif name == "run_on_0_2":
@@ -1518,7 +1520,11 @@ class ResourcesTest(unittest.TestCase):
start_ray_local=True,
num_local_schedulers=2,
num_cpus=[3, 3],
resources=[{"CustomResource": 0}, {"CustomResource": 1}])
resources=[{
"CustomResource": 0
}, {
"CustomResource": 1
}])
@ray.remote
def f():
@@ -1554,8 +1560,13 @@ class ResourcesTest(unittest.TestCase):
start_ray_local=True,
num_local_schedulers=2,
num_cpus=[3, 3],
resources=[{"CustomResource1": 1, "CustomResource2": 2},
{"CustomResource1": 3, "CustomResource2": 4}])
resources=[{
"CustomResource1": 1,
"CustomResource2": 2
}, {
"CustomResource1": 3,
"CustomResource2": 4
}])
@ray.remote(resources={"CustomResource1": 1})
def f():
@@ -1595,14 +1606,16 @@ class ResourcesTest(unittest.TestCase):
# Make sure that tasks with unsatisfied custom resource requirements do
# not get scheduled.
ready_ids, remaining_ids = ray.wait([j.remote(), k.remote()],
timeout=500)
ready_ids, remaining_ids = ray.wait(
[j.remote(), k.remote()], timeout=500)
self.assertEqual(ready_ids, [])
def testManyCustomResources(self):
num_custom_resources = 10000
total_resources = {str(i): np.random.randint(1, 7)
for i in range(num_custom_resources)}
total_resources = {
str(i): np.random.randint(1, 7)
for i in range(num_custom_resources)
}
ray.init(num_cpus=5, resources=total_resources)
def f():
@@ -1612,9 +1625,11 @@ class ResourcesTest(unittest.TestCase):
for _ in range(20):
num_resources = np.random.randint(0, num_custom_resources + 1)
permuted_resources = np.random.permutation(
num_custom_resources)[:num_resources]
random_resources = {str(i): total_resources[str(i)]
for i in permuted_resources}
num_custom_resources)[:num_resources]
random_resources = {
str(i): total_resources[str(i)]
for i in permuted_resources
}
remote_function = ray.remote(resources=random_resources)(f)
remote_functions.append(remote_function)
@@ -1634,8 +1649,7 @@ class CudaVisibleDevicesTest(unittest.TestCase):
def setUp(self):
# Record the curent value of this environment variable so that we can
# reset it after the test.
self.original_gpu_ids = os.environ.get(
"CUDA_VISIBLE_DEVICES", None)
self.original_gpu_ids = os.environ.get("CUDA_VISIBLE_DEVICES", None)
def tearDown(self):
ray.worker.cleanup()
@@ -2095,9 +2109,9 @@ class GlobalStateAPI(unittest.TestCase):
for object_info in object_table.values():
if len(object_info) != 5:
tables_ready = False
if (object_info["ManagerIDs"] is None or
object_info["DataSize"] == -1 or
object_info["Hash"] == ""):
if (object_info["ManagerIDs"] is None
or object_info["DataSize"] == -1
or object_info["Hash"] == ""):
tables_ready = False
if len(task_table) != 10 + 1:
test/stress_tests.py
+63 -52
View File
@@ -10,14 +10,15 @@ import time
class TaskTests(unittest.TestCase):
def testSubmittingTasks(self):
for num_local_schedulers in [1, 4]:
for num_workers_per_scheduler in [4]:
num_workers = num_local_schedulers * num_workers_per_scheduler
ray.worker._init(start_ray_local=True, num_workers=num_workers,
num_local_schedulers=num_local_schedulers,
num_cpus=100)
ray.worker._init(
start_ray_local=True,
num_workers=num_workers,
num_local_schedulers=num_local_schedulers,
num_cpus=100)
@ray.remote
def f(x):
@@ -42,9 +43,11 @@ class TaskTests(unittest.TestCase):
for num_local_schedulers in [1, 4]:
for num_workers_per_scheduler in [4]:
num_workers = num_local_schedulers * num_workers_per_scheduler
ray.worker._init(start_ray_local=True, num_workers=num_workers,
num_local_schedulers=num_local_schedulers,
num_cpus=100)
ray.worker._init(
start_ray_local=True,
num_workers=num_workers,
num_local_schedulers=num_local_schedulers,
num_cpus=100)
@ray.remote
def f(x):
@@ -89,7 +92,7 @@ class TaskTests(unittest.TestCase):
ray.init(num_workers=1)
for n in range(8):
x = np.zeros(10 ** n)
x = np.zeros(10**n)
for _ in range(100):
ray.put(x)
@@ -108,7 +111,7 @@ class TaskTests(unittest.TestCase):
def f():
return 1
n = 10 ** 4 # TODO(pcm): replace by 10 ** 5 once this is faster.
n = 10**4 # TODO(pcm): replace by 10 ** 5 once this is faster.
lst = ray.get([f.remote() for _ in range(n)])
self.assertEqual(lst, n * [1])
@@ -119,9 +122,11 @@ class TaskTests(unittest.TestCase):
for num_local_schedulers in [1, 4]:
for num_workers_per_scheduler in [4]:
num_workers = num_local_schedulers * num_workers_per_scheduler
ray.worker._init(start_ray_local=True, num_workers=num_workers,
num_local_schedulers=num_local_schedulers,
num_cpus=100)
ray.worker._init(
start_ray_local=True,
num_workers=num_workers,
num_local_schedulers=num_local_schedulers,
num_cpus=100)
@ray.remote
def f(x):
@@ -138,8 +143,10 @@ class TaskTests(unittest.TestCase):
time.sleep(x)
for i in range(1, 5):
x_ids = [g.remote(np.random.uniform(0, i))
for _ in range(2 * num_workers)]
x_ids = [
g.remote(np.random.uniform(0, i))
for _ in range(2 * num_workers)
]
ray.wait(x_ids, num_returns=len(x_ids))
self.assertTrue(ray.services.all_processes_alive())
@@ -159,34 +166,40 @@ class ReconstructionTests(unittest.TestCase):
time.sleep(0.1)
# Start the Plasma store instances with a total of 1GB memory.
self.plasma_store_memory = 10 ** 9
self.plasma_store_memory = 10**9
plasma_addresses = []
objstore_memory = (self.plasma_store_memory //
self.num_local_schedulers)
objstore_memory = (
self.plasma_store_memory // self.num_local_schedulers)
for i in range(self.num_local_schedulers):
store_stdout_file, store_stderr_file = ray.services.new_log_files(
"plasma_store_{}".format(i), True)
manager_stdout_file, manager_stderr_file = (
ray.services.new_log_files("plasma_manager_{}"
.format(i), True))
plasma_addresses.append(ray.services.start_objstore(
node_ip_address, redis_address,
objstore_memory=objstore_memory,
store_stdout_file=store_stdout_file,
store_stderr_file=store_stderr_file,
manager_stdout_file=manager_stdout_file,
manager_stderr_file=manager_stderr_file))
ray.services.new_log_files("plasma_manager_{}".format(i),
True))
plasma_addresses.append(
ray.services.start_objstore(
node_ip_address,
redis_address,
objstore_memory=objstore_memory,
store_stdout_file=store_stdout_file,
store_stderr_file=store_stderr_file,
manager_stdout_file=manager_stdout_file,
manager_stderr_file=manager_stderr_file))
# Start the rest of the services in the Ray cluster.
address_info = {"redis_address": redis_address,
"redis_shards": redis_shards,
"object_store_addresses": plasma_addresses}
ray.worker._init(address_info=address_info, start_ray_local=True,
num_workers=1,
num_local_schedulers=self.num_local_schedulers,
num_cpus=[1] * self.num_local_schedulers,
redirect_output=True,
driver_mode=ray.SILENT_MODE)
address_info = {
"redis_address": redis_address,
"redis_shards": redis_shards,
"object_store_addresses": plasma_addresses
}
ray.worker._init(
address_info=address_info,
start_ray_local=True,
num_workers=1,
num_local_schedulers=self.num_local_schedulers,
num_cpus=[1] * self.num_local_schedulers,
redirect_output=True,
driver_mode=ray.SILENT_MODE)
def tearDown(self):
self.assertTrue(ray.services.all_processes_alive())
@@ -197,8 +210,8 @@ class ReconstructionTests(unittest.TestCase):
state._initialize_global_state(self.redis_ip_address, self.redis_port)
if os.environ.get('RAY_USE_NEW_GCS', False):
tasks = state.task_table()
local_scheduler_ids = set(task["LocalSchedulerID"] for task in
tasks.values())
local_scheduler_ids = set(
task["LocalSchedulerID"] for task in tasks.values())
# Make sure that all nodes in the cluster were used by checking that
# the set of local scheduler IDs that had a task scheduled or submitted
@@ -208,8 +221,8 @@ class ReconstructionTests(unittest.TestCase):
# with the driver task, since it is not scheduled by a particular local
# scheduler.
if os.environ.get('RAY_USE_NEW_GCS', False):
self.assertEqual(len(local_scheduler_ids),
self.num_local_schedulers + 1)
self.assertEqual(
len(local_scheduler_ids), self.num_local_schedulers + 1)
# Clean up the Ray cluster.
ray.worker.cleanup()
@@ -254,8 +267,7 @@ class ReconstructionTests(unittest.TestCase):
del values
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Failing with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Failing with new GCS API.")
def testRecursive(self):
# Define the size of one task's return argument so that the combined
# sum of all objects' sizes is at least twice the plasma stores'
@@ -308,8 +320,7 @@ class ReconstructionTests(unittest.TestCase):
del values
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Failing with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Failing with new GCS API.")
def testMultipleRecursive(self):
# Define the size of one task's return argument so that the combined
# sum of all objects' sizes is at least twice the plasma stores'
@@ -375,8 +386,7 @@ class ReconstructionTests(unittest.TestCase):
return errors
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testNondeterministicTask(self):
# Define the size of one task's return argument so that the combined
# sum of all objects' sizes is at least twice the plasma stores'
@@ -432,14 +442,14 @@ class ReconstructionTests(unittest.TestCase):
# reexecuted.
min_errors = 1
return len(errors) >= min_errors
errors = self.wait_for_errors(error_check)
# Make sure all the errors have the correct type.
self.assertTrue(all(error[b"type"] == b"object_hash_mismatch"
for error in errors))
self.assertTrue(
all(error[b"type"] == b"object_hash_mismatch" for error in errors))
@unittest.skipIf(
os.environ.get('RAY_USE_NEW_GCS', False),
"Hanging with new GCS API.")
os.environ.get('RAY_USE_NEW_GCS', False), "Hanging with new GCS API.")
def testDriverPutErrors(self):
# Define the size of one task's return argument so that the combined
# sum of all objects' sizes is at least twice the plasma stores'
@@ -479,9 +489,10 @@ class ReconstructionTests(unittest.TestCase):
def error_check(errors):
return len(errors) > 1
errors = self.wait_for_errors(error_check)
self.assertTrue(all(error[b"type"] == b"put_reconstruction"
for error in errors))
self.assertTrue(
all(error[b"type"] == b"put_reconstruction" for error in errors))
class ReconstructionTestsMultinode(ReconstructionTests):
@@ -490,6 +501,7 @@ class ReconstructionTestsMultinode(ReconstructionTests):
# one worker each.
num_local_schedulers = 4
# NOTE(swang): This test tries to launch 1000 workers and breaks.
# class WorkerPoolTests(unittest.TestCase):
#
@@ -512,6 +524,5 @@ class ReconstructionTestsMultinode(ReconstructionTests):
# ray.get([g.remote(i) for i in range(1000)])
# ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)
test/tensorflow_test.py
+27 -27
View File
@@ -23,7 +23,6 @@ def make_linear_network(w_name=None, b_name=None):
class LossActor(object):
def __init__(self, use_loss=True):
# Uses a separate graph for each network.
with tf.Graph().as_default():
@@ -32,10 +31,8 @@ class LossActor(object):
loss, init, _, _ = make_linear_network()
sess = tf.Session()
# Additional code for setting and getting the weights.
weights = ray.experimental.TensorFlowVariables(loss if use_loss
else None,
sess,
input_variables=var)
weights = ray.experimental.TensorFlowVariables(
loss if use_loss else None, sess, input_variables=var)
# Return all of the data needed to use the network.
self.values = [weights, init, sess]
sess.run(init)
@@ -49,7 +46,6 @@ class LossActor(object):
class NetActor(object):
def __init__(self):
# Uses a separate graph for each network.
with tf.Graph().as_default():
@@ -71,7 +67,6 @@ class NetActor(object):
class TrainActor(object):
def __init__(self):
# Almost the same as above, but now returns the placeholders and
# gradient.
@@ -82,16 +77,17 @@ class TrainActor(object):
optimizer = tf.train.GradientDescentOptimizer(0.9)
grads = optimizer.compute_gradients(loss)
train = optimizer.apply_gradients(grads)
self.values = [loss, variables, init, sess, grads, train,
[x_data, y_data]]
self.values = [
loss, variables, init, sess, grads, train, [x_data, y_data]
]
sess.run(init)
def training_step(self, weights):
_, variables, _, sess, grads, _, placeholders = self.values
variables.set_weights(weights)
return sess.run([grad[0] for grad in grads],
feed_dict=dict(zip(placeholders,
[[1] * 100, [2] * 100])))
return sess.run(
[grad[0] for grad in grads],
feed_dict=dict(zip(placeholders, [[1] * 100, [2] * 100])))
def get_weights(self):
return self.values[1].get_weights()
@@ -216,8 +212,8 @@ class TensorFlowTest(unittest.TestCase):
net2 = ray.remote(NetActor).remote()
weights2 = ray.get(net2.get_weights.remote())
new_weights2 = ray.get(net2.set_and_get_weights.remote(
net2.get_weights.remote()))
new_weights2 = ray.get(
net2.set_and_get_weights.remote(net2.get_weights.remote()))
self.assertEqual(weights2, new_weights2)
def testVariablesControlDependencies(self):
@@ -247,22 +243,26 @@ class TensorFlowTest(unittest.TestCase):
net_values = TrainActor().values
loss, variables, _, sess, grads, train, placeholders = net_values
before_acc = sess.run(loss, feed_dict=dict(zip(placeholders,
[[2] * 100,
[4] * 100])))
before_acc = sess.run(
loss, feed_dict=dict(zip(placeholders, [[2] * 100, [4] * 100])))
for _ in range(3):
gradients_list = ray.get(
[net.training_step.remote(variables.get_weights())
for _ in range(2)])
mean_grads = [sum([gradients[i] for gradients in gradients_list]) /
len(gradients_list) for i
in range(len(gradients_list[0]))]
feed_dict = {grad[0]: mean_grad for (grad, mean_grad)
in zip(grads, mean_grads)}
gradients_list = ray.get([
net.training_step.remote(variables.get_weights())
for _ in range(2)
])
mean_grads = [
sum([gradients[i]
for gradients in gradients_list]) / len(gradients_list)
for i in range(len(gradients_list[0]))
]
feed_dict = {
grad[0]: mean_grad
for (grad, mean_grad) in zip(grads, mean_grads)
}
sess.run(train, feed_dict=feed_dict)
after_acc = sess.run(loss, feed_dict=dict(zip(placeholders,
[[2] * 100, [4] * 100])))
after_acc = sess.run(
loss, feed_dict=dict(zip(placeholders, [[2] * 100, [4] * 100])))
self.assertTrue(before_acc < after_acc)
test/xray_test.py
+1 -2
View File
@@ -57,12 +57,11 @@ def test_put_api(ray_start):
# Test putting object IDs.
x_id = ray.put(0)
for obj in [[x_id], (x_id,), {x_id: x_id}]:
for obj in [[x_id], (x_id, ), {x_id: x_id}]:
assert ray.get(ray.put(obj)) == obj
def test_actor_api(ray_start):
@ray.remote
class Foo(object):
def __init__(self, val):