wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-07 11:27:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Run flake8 in Travis and make code PEP8 compliant. (#387)

Browse Source
This commit is contained in:
Robert Nishihara
2017-03-21 12:57:54 -07:00
committed by Philipp Moritz
parent 083e7a28ad
commit ba02fc0eb0
54 changed files with 2391 additions and 1313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/actor_test.py
+105 -46
View File
@@ -2,11 +2,12 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import unittest
import numpy as np
import time
import unittest
import ray
class ActorAPI(unittest.TestCase):
def testKeywordArgs(self):
@@ -18,6 +19,7 @@ class ActorAPI(unittest.TestCase):
self.arg0 = arg0
self.arg1 = arg1
self.arg2 = arg2
def get_values(self, arg0, arg1=2, arg2="b"):
return self.arg0 + arg0, self.arg1 + arg1, self.arg2 + arg2
@@ -53,6 +55,7 @@ class ActorAPI(unittest.TestCase):
self.arg0 = arg0
self.arg1 = arg1
self.args = args
def get_values(self, arg0, arg1=2, *args):
return self.arg0 + arg0, self.arg1 + arg1, self.args, args
@@ -63,10 +66,12 @@ class ActorAPI(unittest.TestCase):
self.assertEqual(ray.get(actor.get_values(2, 3)), (3, 5, (), ()))
actor = Actor(1, 2, "c")
self.assertEqual(ray.get(actor.get_values(2, 3, "d")), (3, 5, ("c",), ("d",)))
self.assertEqual(ray.get(actor.get_values(2, 3, "d")),
(3, 5, ("c",), ("d",)))
actor = Actor(1, 2, "a", "b", "c", "d")
self.assertEqual(ray.get(actor.get_values(2, 3, 1, 2, 3, 4)), (3, 5, ("a", "b", "c", "d"), (1, 2, 3, 4)))
self.assertEqual(ray.get(actor.get_values(2, 3, 1, 2, 3, 4)),
(3, 5, ("a", "b", "c", "d"), (1, 2, 3, 4)))
ray.worker.cleanup()
@@ -77,6 +82,7 @@ class ActorAPI(unittest.TestCase):
class Actor(object):
def __init__(self):
pass
def get_values(self):
pass
@@ -112,8 +118,10 @@ class ActorAPI(unittest.TestCase):
def __init__(self, f2):
self.f1 = Foo(1)
self.f2 = f2
def get_values1(self):
return self.f1, self.f2
def get_values2(self, f3):
return self.f1, self.f2, f3
@@ -144,38 +152,39 @@ class ActorAPI(unittest.TestCase):
# This is an invalid way of using the actor decorator.
with self.assertRaises(Exception):
@ray.actor(invalid_kwarg=0)
@ray.actor(invalid_kwarg=0) # noqa: F811
class Actor(object):
def __init__(self):
pass
# This is an invalid way of using the actor decorator.
with self.assertRaises(Exception):
@ray.actor(num_cpus=0, invalid_kwarg=0)
@ray.actor(num_cpus=0, invalid_kwarg=0) # noqa: F811
class Actor(object):
def __init__(self):
pass
# This is a valid way of using the decorator.
@ray.actor(num_cpus=1)
@ray.actor(num_cpus=1) # noqa: F811
class Actor(object):
def __init__(self):
pass
# This is a valid way of using the decorator.
@ray.actor(num_gpus=1)
@ray.actor(num_gpus=1) # noqa: F811
class Actor(object):
def __init__(self):
pass
# This is a valid way of using the decorator.
@ray.actor(num_cpus=1, num_gpus=1)
@ray.actor(num_cpus=1, num_gpus=1) # noqa: F811
class Actor(object):
def __init__(self):
pass
ray.worker.cleanup()
class ActorMethods(unittest.TestCase):
def testDefineActor(self):
@@ -185,6 +194,7 @@ class ActorMethods(unittest.TestCase):
class Test(object):
def __init__(self, x):
self.x = x
def f(self, y):
return self.x + y
@@ -200,8 +210,10 @@ class ActorMethods(unittest.TestCase):
class Counter(object):
def __init__(self):
self.value = 0
def increase(self):
self.value += 1
def value(self):
return self.value
@@ -224,9 +236,11 @@ class ActorMethods(unittest.TestCase):
class Counter(object):
def __init__(self, value):
self.value = value
def increase(self):
self.value += 1
return self.value
def reset(self):
self.value = 0
@@ -240,7 +254,9 @@ class ActorMethods(unittest.TestCase):
results += [actors[i].increase() for _ in range(num_increases)]
result_values = ray.get(results)
for i in range(num_actors):
self.assertEqual(result_values[(num_increases * i):(num_increases * (i + 1))], list(range(i + 1, num_increases + i + 1)))
self.assertEqual(
result_values[(num_increases * i):(num_increases * (i + 1))],
list(range(i + 1, num_increases + i + 1)))
# Reset the actor values.
[actor.reset() for actor in actors]
@@ -251,10 +267,12 @@ class ActorMethods(unittest.TestCase):
results += [actor.increase() for actor in actors]
result_values = ray.get(results)
for j in range(num_increases):
self.assertEqual(result_values[(num_actors * j):(num_actors * (j + 1))], num_actors * [j + 1])
self.assertEqual(result_values[(num_actors * j):(num_actors * (j + 1))],
num_actors * [j + 1])
ray.worker.cleanup()
class ActorNesting(unittest.TestCase):
def testRemoteFunctionWithinActor(self):
@@ -302,7 +320,8 @@ class ActorNesting(unittest.TestCase):
self.assertEqual(ray.get(ray.get(actor.f())), list(range(1, 6)))
self.assertEqual(ray.get(actor.g()), list(range(1, 6)))
self.assertEqual(ray.get(actor.h([f.remote(i) for i in range(5)])), list(range(1, 6)))
self.assertEqual(ray.get(actor.h([f.remote(i) for i in range(5)])),
list(range(1, 6)))
ray.worker.cleanup()
@@ -320,6 +339,7 @@ class ActorNesting(unittest.TestCase):
class Actor2(object):
def __init__(self, x):
self.x = x
def get_value(self):
return self.x
self.actor2 = Actor2(z)
@@ -370,13 +390,15 @@ class ActorNesting(unittest.TestCase):
class Actor1(object):
def __init__(self, x):
self.x = x
def get_value(self):
return self.x
actor = Actor1(x)
return ray.get([actor.get_value() for _ in range(n)])
self.assertEqual(ray.get(f.remote(3, 1)), [3])
self.assertEqual(ray.get([f.remote(i, 20) for i in range(10)]), [20 * [i] for i in range(10)])
self.assertEqual(ray.get([f.remote(i, 20) for i in range(10)]),
[20 * [i] for i in range(10)])
ray.worker.cleanup()
@@ -421,8 +443,10 @@ class ActorNesting(unittest.TestCase):
def __init__(self):
# This should use the last version of f.
self.x = ray.get(f.remote())
def get_val(self):
return self.x
actor = Actor()
return ray.get(actor.get_val())
@@ -430,6 +454,7 @@ class ActorNesting(unittest.TestCase):
ray.worker.cleanup()
class ActorInheritance(unittest.TestCase):
def testInheritActorFromClass(self):
@@ -440,8 +465,10 @@ class ActorInheritance(unittest.TestCase):
class Foo(object):
def __init__(self, x):
self.x = x
def f(self):
return self.x
def g(self, y):
return self.x + y
@@ -449,6 +476,7 @@ class ActorInheritance(unittest.TestCase):
class Actor(Foo):
def __init__(self, x):
Foo.__init__(self, x)
def get_value(self):
return self.f()
@@ -458,6 +486,7 @@ class ActorInheritance(unittest.TestCase):
ray.worker.cleanup()
class ActorSchedulingProperties(unittest.TestCase):
def testRemoteFunctionsNotScheduledOnActors(self):
@@ -469,7 +498,7 @@ class ActorSchedulingProperties(unittest.TestCase):
def __init__(self):
pass
actor = Actor()
Actor()
@ray.remote
def f():
@@ -477,22 +506,26 @@ class ActorSchedulingProperties(unittest.TestCase):
# Make sure that f cannot be scheduled on the worker created for the actor.
# The wait call should time out.
ready_ids, remaining_ids = ray.wait([f.remote() for _ in range(10)], timeout=3000)
ready_ids, remaining_ids = ray.wait([f.remote() for _ in range(10)],
timeout=3000)
self.assertEqual(ready_ids, [])
self.assertEqual(len(remaining_ids), 10)
ray.worker.cleanup()
class ActorsOnMultipleNodes(unittest.TestCase):
def testActorLoadBalancing(self):
num_local_schedulers = 3
ray.worker._init(start_ray_local=True, num_workers=0, num_local_schedulers=num_local_schedulers)
ray.worker._init(start_ray_local=True, num_workers=0,
num_local_schedulers=num_local_schedulers)
@ray.actor
class Actor1(object):
def __init__(self):
pass
def get_location(self):
return ray.worker.global_worker.plasma_client.store_socket_name
@@ -509,7 +542,8 @@ class ActorsOnMultipleNodes(unittest.TestCase):
names = set(locations)
counts = [locations.count(name) for name in names]
print("Counts are {}.".format(counts))
if len(names) == num_local_schedulers and all([count >= minimum_count for count in counts]):
if len(names) == num_local_schedulers and all([count >= minimum_count
for count in counts]):
break
attempts += 1
self.assertLess(attempts, num_attempts)
@@ -523,26 +557,32 @@ class ActorsOnMultipleNodes(unittest.TestCase):
ray.worker.cleanup()
class ActorsWithGPUs(unittest.TestCase):
def testActorGPUs(self):
num_local_schedulers = 3
num_gpus_per_scheduler = 4
ray.worker._init(start_ray_local=True, num_workers=0,
num_local_schedulers=num_local_schedulers,
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
ray.worker._init(
start_ray_local=True, num_workers=0,
num_local_schedulers=num_local_schedulers,
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
@ray.actor(num_gpus=1)
class Actor1(object):
def __init__(self):
self.gpu_ids = ray.get_gpu_ids()
def get_location_and_ids(self):
return ray.worker.global_worker.plasma_client.store_socket_name, tuple(self.gpu_ids)
return (ray.worker.global_worker.plasma_client.store_socket_name,
tuple(self.gpu_ids))
# Create one actor per GPU.
actors = [Actor1() for _ in range(num_local_schedulers * num_gpus_per_scheduler)]
actors = [Actor1() for _
in range(num_local_schedulers * num_gpus_per_scheduler)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get([actor.get_location_and_ids() for actor in actors])
locations_and_ids = ray.get([actor.get_location_and_ids()
for actor in actors])
node_names = set([location for location, gpu_id in locations_and_ids])
self.assertEqual(len(node_names), num_local_schedulers)
location_actor_combinations = []
@@ -553,28 +593,32 @@ class ActorsWithGPUs(unittest.TestCase):
# Creating a new actor should fail because all of the GPUs are being used.
with self.assertRaises(Exception):
a = Actor1()
Actor1()
ray.worker.cleanup()
def testActorMultipleGPUs(self):
num_local_schedulers = 3
num_gpus_per_scheduler = 5
ray.worker._init(start_ray_local=True, num_workers=0,
num_local_schedulers=num_local_schedulers,
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
ray.worker._init(
start_ray_local=True, num_workers=0,
num_local_schedulers=num_local_schedulers,
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
@ray.actor(num_gpus=2)
class Actor1(object):
def __init__(self):
self.gpu_ids = ray.get_gpu_ids()
def get_location_and_ids(self):
return ray.worker.global_worker.plasma_client.store_socket_name, tuple(self.gpu_ids)
return (ray.worker.global_worker.plasma_client.store_socket_name,
tuple(self.gpu_ids))
# Create some actors.
actors = [Actor1() for _ in range(num_local_schedulers * 2)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get([actor.get_location_and_ids() for actor in actors])
locations_and_ids = ray.get([actor.get_location_and_ids()
for actor in actors])
node_names = set([location for location, gpu_id in locations_and_ids])
self.assertEqual(len(node_names), num_local_schedulers)
location_actor_combinations = []
@@ -585,20 +629,23 @@ class ActorsWithGPUs(unittest.TestCase):
# Creating a new actor should fail because all of the GPUs are being used.
with self.assertRaises(Exception):
a = Actor1()
Actor1()
# We should be able to create more actors that use only a single GPU.
@ray.actor(num_gpus=1)
class Actor2(object):
def __init__(self):
self.gpu_ids = ray.get_gpu_ids()
def get_location_and_ids(self):
return ray.worker.global_worker.plasma_client.store_socket_name, tuple(self.gpu_ids)
return (ray.worker.global_worker.plasma_client.store_socket_name,
tuple(self.gpu_ids))
# Create some actors.
actors = [Actor2() for _ in range(num_local_schedulers)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get([actor.get_location_and_ids() for actor in actors])
locations_and_ids = ray.get([actor.get_location_and_ids()
for actor in actors])
node_names = set([location for location, gpu_id in locations_and_ids])
self.assertEqual(len(node_names), num_local_schedulers)
location_actor_combinations = []
@@ -608,13 +655,13 @@ class ActorsWithGPUs(unittest.TestCase):
# Creating a new actor should fail because all of the GPUs are being used.
with self.assertRaises(Exception):
a = Actor2()
Actor2()
ray.worker.cleanup()
def testActorDifferentNumbersOfGPUs(self):
# Test that we can create actors on two nodes that have different numbers of
# GPUs.
# Test that we can create actors on two nodes that have different numbers
# of GPUs.
ray.worker._init(start_ray_local=True, num_workers=0,
num_local_schedulers=3, num_gpus=[0, 5, 10])
@@ -622,32 +669,38 @@ class ActorsWithGPUs(unittest.TestCase):
class Actor1(object):
def __init__(self):
self.gpu_ids = ray.get_gpu_ids()
def get_location_and_ids(self):
return ray.worker.global_worker.plasma_client.store_socket_name, tuple(self.gpu_ids)
return (ray.worker.global_worker.plasma_client.store_socket_name,
tuple(self.gpu_ids))
# Create some actors.
actors = [Actor1() for _ in range(0 + 5 + 10)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get([actor.get_location_and_ids() for actor in actors])
locations_and_ids = ray.get([actor.get_location_and_ids()
for actor in actors])
node_names = set([location for location, gpu_id in locations_and_ids])
self.assertEqual(len(node_names), 2)
for node_name in node_names:
node_gpu_ids = [gpu_id for location, gpu_id in locations_and_ids if location == node_name]
node_gpu_ids = [gpu_id for location, gpu_id in locations_and_ids
if location == node_name]
self.assertIn(len(node_gpu_ids), [5, 10])
self.assertEqual(set(node_gpu_ids), set([(i,) for i in range(len(node_gpu_ids))]))
self.assertEqual(set(node_gpu_ids),
set([(i,) for i in range(len(node_gpu_ids))]))
# Creating a new actor should fail because all of the GPUs are being used.
with self.assertRaises(Exception):
a = Actor1()
Actor1()
ray.worker.cleanup()
def testActorMultipleGPUsFromMultipleTasks(self):
num_local_schedulers = 10
num_gpus_per_scheduler = 10
ray.worker._init(start_ray_local=True, num_workers=0,
num_local_schedulers=num_local_schedulers,
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
ray.worker._init(
start_ray_local=True, num_workers=0,
num_local_schedulers=num_local_schedulers,
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
@ray.remote
def create_actors(n):
@@ -655,20 +708,25 @@ class ActorsWithGPUs(unittest.TestCase):
class Actor(object):
def __init__(self):
self.gpu_ids = ray.get_gpu_ids()
def get_location_and_ids(self):
return ray.worker.global_worker.plasma_client.store_socket_name, tuple(self.gpu_ids)
return (ray.worker.global_worker.plasma_client.store_socket_name,
tuple(self.gpu_ids))
# Create n actors.
for _ in range(n):
Actor()
ray.get([create_actors.remote(num_gpus_per_scheduler) for _ in range(num_local_schedulers)])
ray.get([create_actors.remote(num_gpus_per_scheduler)
for _ in range(num_local_schedulers)])
@ray.actor(num_gpus=1)
class Actor(object):
def __init__(self):
self.gpu_ids = ray.get_gpu_ids()
def get_location_and_ids(self):
return ray.worker.global_worker.plasma_client.store_socket_name, tuple(self.gpu_ids)
return (ray.worker.global_worker.plasma_client.store_socket_name,
tuple(self.gpu_ids))
# All the GPUs should be used up now.
with self.assertRaises(Exception):
@@ -676,5 +734,6 @@ class ActorsWithGPUs(unittest.TestCase):
ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)
test/array_test.py
+55 -27
View File
@@ -5,20 +5,21 @@ from __future__ import print_function
import unittest
import ray
import numpy as np
import time
from numpy.testing import assert_equal, assert_almost_equal
import sys
if sys.version_info >= (3, 0):
from importlib import reload
import ray.experimental.array.remote as ra
import ray.experimental.array.distributed as da
if sys.version_info >= (3, 0):
from importlib import reload
class RemoteArrayTest(unittest.TestCase):
def testMethods(self):
for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
for module in [ra.core, ra.random, ra.linalg, da.core, da.random,
da.linalg]:
reload(module)
ray.init(num_workers=1)
@@ -49,24 +50,30 @@ class RemoteArrayTest(unittest.TestCase):
ray.worker.cleanup()
class DistributedArrayTest(unittest.TestCase):
def testAssemble(self):
for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
for module in [ra.core, ra.random, ra.linalg, da.core, da.random,
da.linalg]:
reload(module)
ray.init(num_workers=1)
a = ra.ones.remote([da.BLOCK_SIZE, da.BLOCK_SIZE])
b = ra.zeros.remote([da.BLOCK_SIZE, da.BLOCK_SIZE])
x = da.DistArray([2 * da.BLOCK_SIZE, da.BLOCK_SIZE], np.array([[a], [b]]))
assert_equal(x.assemble(), np.vstack([np.ones([da.BLOCK_SIZE, da.BLOCK_SIZE]), np.zeros([da.BLOCK_SIZE, da.BLOCK_SIZE])]))
assert_equal(x.assemble(),
np.vstack([np.ones([da.BLOCK_SIZE, da.BLOCK_SIZE]),
np.zeros([da.BLOCK_SIZE, da.BLOCK_SIZE])]))
ray.worker.cleanup()
def testMethods(self):
for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
for module in [ra.core, ra.random, ra.linalg, da.core, da.random,
da.linalg]:
reload(module)
ray.worker._init(start_ray_local=True, num_workers=10, num_local_schedulers=2, num_cpus=[10, 10])
ray.worker._init(start_ray_local=True, num_workers=10,
num_local_schedulers=2, num_cpus=[10, 10])
x = da.zeros.remote([9, 25, 51], "float")
assert_equal(ray.get(da.assemble.remote(x)), np.zeros([9, 25, 51]))
@@ -76,18 +83,21 @@ class DistributedArrayTest(unittest.TestCase):
x = da.random.normal.remote([11, 25, 49])
y = da.copy.remote(x)
assert_equal(ray.get(da.assemble.remote(x)), ray.get(da.assemble.remote(y)))
assert_equal(ray.get(da.assemble.remote(x)),
ray.get(da.assemble.remote(y)))
x = da.eye.remote(25, dtype_name="float")
assert_equal(ray.get(da.assemble.remote(x)), np.eye(25))
x = da.random.normal.remote([25, 49])
y = da.triu.remote(x)
assert_equal(ray.get(da.assemble.remote(y)), np.triu(ray.get(da.assemble.remote(x))))
assert_equal(ray.get(da.assemble.remote(y)),
np.triu(ray.get(da.assemble.remote(x))))
x = da.random.normal.remote([25, 49])
y = da.tril.remote(x)
assert_equal(ray.get(da.assemble.remote(y)), np.tril(ray.get(da.assemble.remote(x))))
assert_equal(ray.get(da.assemble.remote(y)),
np.tril(ray.get(da.assemble.remote(x))))
x = da.random.normal.remote([25, 49])
y = da.random.normal.remote([49, 18])
@@ -102,29 +112,37 @@ class DistributedArrayTest(unittest.TestCase):
x = da.random.normal.remote([23, 42])
y = da.random.normal.remote([23, 42])
z = da.add.remote(x, y)
assert_almost_equal(ray.get(da.assemble.remote(z)), ray.get(da.assemble.remote(x)) + ray.get(da.assemble.remote(y)))
assert_almost_equal(ray.get(da.assemble.remote(z)),
ray.get(da.assemble.remote(x)) +
ray.get(da.assemble.remote(y)))
# test subtract
x = da.random.normal.remote([33, 40])
y = da.random.normal.remote([33, 40])
z = da.subtract.remote(x, y)
assert_almost_equal(ray.get(da.assemble.remote(z)), ray.get(da.assemble.remote(x)) - ray.get(da.assemble.remote(y)))
assert_almost_equal(ray.get(da.assemble.remote(z)),
ray.get(da.assemble.remote(x)) -
ray.get(da.assemble.remote(y)))
# test transpose
x = da.random.normal.remote([234, 432])
y = da.transpose.remote(x)
assert_equal(ray.get(da.assemble.remote(x)).T, ray.get(da.assemble.remote(y)))
assert_equal(ray.get(da.assemble.remote(x)).T,
ray.get(da.assemble.remote(y)))
# test numpy_to_dist
x = da.random.normal.remote([23, 45])
y = da.assemble.remote(x)
z = da.numpy_to_dist.remote(y)
w = da.assemble.remote(z)
assert_equal(ray.get(da.assemble.remote(x)), ray.get(da.assemble.remote(z)))
assert_equal(ray.get(da.assemble.remote(x)),
ray.get(da.assemble.remote(z)))
assert_equal(ray.get(y), ray.get(w))
# test da.tsqr
for shape in [[123, da.BLOCK_SIZE], [7, da.BLOCK_SIZE], [da.BLOCK_SIZE, da.BLOCK_SIZE], [da.BLOCK_SIZE, 7], [10 * da.BLOCK_SIZE, da.BLOCK_SIZE]]:
for shape in [[123, da.BLOCK_SIZE], [7, da.BLOCK_SIZE],
[da.BLOCK_SIZE, da.BLOCK_SIZE], [da.BLOCK_SIZE, 7],
[10 * da.BLOCK_SIZE, da.BLOCK_SIZE]]:
x = da.random.normal.remote(shape)
K = min(shape)
q, r = da.linalg.tsqr.remote(x)
@@ -138,23 +156,26 @@ class DistributedArrayTest(unittest.TestCase):
# test da.linalg.modified_lu
def test_modified_lu(d1, d2):
print("testing dist_modified_lu with d1 = " + str(d1) + ", d2 = " + str(d2))
print("testing dist_modified_lu with d1 = " + str(d1) +
", d2 = " + str(d2))
assert d1 >= d2
k = min(d1, d2)
m = ra.random.normal.remote([d1, d2])
q, r = ra.linalg.qr.remote(m)
l, u, s = da.linalg.modified_lu.remote(da.numpy_to_dist.remote(q))
q_val = ray.get(q)
r_val = ray.get(r)
ray.get(r)
l_val = ray.get(da.assemble.remote(l))
u_val = ray.get(u)
s_val = ray.get(s)
s_mat = np.zeros((d1, d2))
for i in range(len(s_val)):
s_mat[i, i] = s_val[i]
assert_almost_equal(q_val - s_mat, np.dot(l_val, u_val)) # check that q - s = l * u
assert_equal(np.triu(u_val), u_val) # check that u is upper triangular
assert_equal(np.tril(l_val), l_val) # check that l is lower triangular
# Check that q - s = l * u.
assert_almost_equal(q_val - s_mat, np.dot(l_val, u_val))
# Check that u is upper triangular.
assert_equal(np.triu(u_val), u_val)
# Check that l is lower triangular.
assert_equal(np.tril(l_val), l_val)
for d1, d2 in [(100, 100), (99, 98), (7, 5), (7, 7), (20, 7), (20, 10)]:
test_modified_lu(d1, d2)
@@ -172,10 +193,14 @@ class DistributedArrayTest(unittest.TestCase):
tall_eye = np.zeros((d1, min(d1, d2)))
np.fill_diagonal(tall_eye, 1)
q = tall_eye - np.dot(y_val, np.dot(t_val, y_top_val.T))
assert_almost_equal(np.dot(q.T, q), np.eye(min(d1, d2))) # check that q.T * q = I
assert_almost_equal(np.dot(q, r_val), a_val) # check that a = (I - y * t * y_top.T) * r
# Check that q.T * q = I.
assert_almost_equal(np.dot(q.T, q), np.eye(min(d1, d2)))
# Check that a = (I - y * t * y_top.T) * r.
assert_almost_equal(np.dot(q, r_val), a_val)
for d1, d2 in [(123, da.BLOCK_SIZE), (7, da.BLOCK_SIZE), (da.BLOCK_SIZE, da.BLOCK_SIZE), (da.BLOCK_SIZE, 7), (10 * da.BLOCK_SIZE, da.BLOCK_SIZE)]:
for d1, d2 in [(123, da.BLOCK_SIZE), (7, da.BLOCK_SIZE),
(da.BLOCK_SIZE, da.BLOCK_SIZE), (da.BLOCK_SIZE, 7),
(10 * da.BLOCK_SIZE, da.BLOCK_SIZE)]:
test_dist_tsqr_hr(d1, d2)
def test_dist_qr(d1, d2):
@@ -192,7 +217,9 @@ class DistributedArrayTest(unittest.TestCase):
assert_equal(r_val, np.triu(r_val))
assert_almost_equal(a_val, np.dot(q_val, r_val))
for d1, d2 in [(123, da.BLOCK_SIZE), (7, da.BLOCK_SIZE), (da.BLOCK_SIZE, da.BLOCK_SIZE), (da.BLOCK_SIZE, 7), (13, 21), (34, 35), (8, 7)]:
for d1, d2 in [(123, da.BLOCK_SIZE), (7, da.BLOCK_SIZE),
(da.BLOCK_SIZE, da.BLOCK_SIZE), (da.BLOCK_SIZE, 7),
(13, 21), (34, 35), (8, 7)]:
test_dist_qr(d1, d2)
test_dist_qr(d2, d1)
for _ in range(20):
@@ -202,5 +229,6 @@ class DistributedArrayTest(unittest.TestCase):
ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)
test/component_failures_test.py
+46 -32
View File
@@ -3,10 +3,10 @@ from __future__ import division
from __future__ import print_function
import ray
import sys
import time
import unittest
class ComponentFailureTest(unittest.TestCase):
def tearDown(self):
@@ -16,6 +16,7 @@ class ComponentFailureTest(unittest.TestCase):
# store and manager will not die.
def testDyingWorkerGet(self):
obj_id = 20 * b"a"
@ray.remote
def f():
ray.worker.global_worker.plasma_client.get(obj_id)
@@ -40,12 +41,14 @@ class ComponentFailureTest(unittest.TestCase):
time.sleep(0.1)
# Make sure that nothing has died.
self.assertTrue(ray.services.all_processes_alive(exclude=[ray.services.PROCESS_TYPE_WORKER]))
self.assertTrue(ray.services.all_processes_alive(
exclude=[ray.services.PROCESS_TYPE_WORKER]))
# This test checks that when a worker dies in the middle of a wait, the plasma
# store and manager will not die.
# This test checks that when a worker dies in the middle of a wait, the
# plasma store and manager will not die.
def testDyingWorkerWait(self):
obj_id = 20 * b"a"
@ray.remote
def f():
ray.worker.global_worker.plasma_client.wait([obj_id])
@@ -70,7 +73,8 @@ class ComponentFailureTest(unittest.TestCase):
time.sleep(0.1)
# Make sure that nothing has died.
self.assertTrue(ray.services.all_processes_alive(exclude=[ray.services.PROCESS_TYPE_WORKER]))
self.assertTrue(ray.services.all_processes_alive(
exclude=[ray.services.PROCESS_TYPE_WORKER]))
def _testWorkerFailed(self, num_local_schedulers):
@ray.remote
@@ -86,7 +90,8 @@ class ComponentFailureTest(unittest.TestCase):
num_cpus=[num_initial_workers] * num_local_schedulers)
# Submit more tasks than there are workers so that all workers and cores
# are utilized.
object_ids = [f.remote(i) for i in range(num_initial_workers * num_local_schedulers)]
object_ids = [f.remote(i) for i
in range(num_initial_workers * num_local_schedulers)]
object_ids += [f.remote(object_id) for object_id in object_ids]
# Allow the tasks some time to begin executing.
time.sleep(0.1)
@@ -94,7 +99,8 @@ class ComponentFailureTest(unittest.TestCase):
for worker in ray.services.all_processes[ray.services.PROCESS_TYPE_WORKER]:
worker.terminate()
time.sleep(0.1)
# Make sure that we can still get the objects after the executing tasks died.
# Make sure that we can still get the objects after the executing tasks
# died.
ray.get(object_ids)
def testWorkerFailed(self):
@@ -104,8 +110,7 @@ class ComponentFailureTest(unittest.TestCase):
self._testWorkerFailed(4)
def _testComponentFailed(self, component_type):
"""Kill a component on all worker nodes and check that workload succeeds.
"""
"""Kill a component on all worker nodes and check workload succeeds."""
@ray.remote
def f(x, j):
time.sleep(0.2)
@@ -114,14 +119,16 @@ class ComponentFailureTest(unittest.TestCase):
# Start with 4 workers and 4 cores.
num_local_schedulers = 4
num_workers_per_scheduler = 8
address_info = ray.worker._init(num_workers=num_local_schedulers * num_workers_per_scheduler,
num_local_schedulers=num_local_schedulers,
start_ray_local=True,
num_cpus=[num_workers_per_scheduler] * num_local_schedulers)
ray.worker._init(
num_workers=num_local_schedulers * num_workers_per_scheduler,
num_local_schedulers=num_local_schedulers,
start_ray_local=True,
num_cpus=[num_workers_per_scheduler] * num_local_schedulers)
# Submit more tasks than there are workers so that all workers and cores are
# utilized.
object_ids = [f.remote(i, 0) for i in range(num_workers_per_scheduler * num_local_schedulers)]
# Submit more tasks than there are workers so that all workers and cores
# are utilized.
object_ids = [f.remote(i, 0) for i
in range(num_workers_per_scheduler * num_local_schedulers)]
object_ids += [f.remote(object_id, 1) for object_id in object_ids]
object_ids += [f.remote(object_id, 2) for object_id in object_ids]
@@ -140,7 +147,8 @@ class ComponentFailureTest(unittest.TestCase):
# Make sure that we can still get the objects after the executing tasks
# died.
results = ray.get(object_ids)
expected_results = 4 * list(range(num_workers_per_scheduler * num_local_schedulers))
expected_results = 4 * list(range(
num_workers_per_scheduler * num_local_schedulers))
self.assertEqual(results, expected_results)
def check_components_alive(self, component_type, check_component_alive):
@@ -161,7 +169,8 @@ class ComponentFailureTest(unittest.TestCase):
# nodes.
self.check_components_alive(ray.services.PROCESS_TYPE_PLASMA_STORE, True)
self.check_components_alive(ray.services.PROCESS_TYPE_PLASMA_MANAGER, True)
self.check_components_alive(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER, False)
self.check_components_alive(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER,
False)
def testPlasmaManagerFailed(self):
# Kill all plasma managers on worker nodes.
@@ -170,8 +179,10 @@ class ComponentFailureTest(unittest.TestCase):
# The plasma stores should still be alive (but unreachable) on the worker
# nodes.
self.check_components_alive(ray.services.PROCESS_TYPE_PLASMA_STORE, True)
self.check_components_alive(ray.services.PROCESS_TYPE_PLASMA_MANAGER, False)
self.check_components_alive(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER, False)
self.check_components_alive(ray.services.PROCESS_TYPE_PLASMA_MANAGER,
False)
self.check_components_alive(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER,
False)
def testPlasmaStoreFailed(self):
# Kill all plasma stores on worker nodes.
@@ -179,17 +190,19 @@ class ComponentFailureTest(unittest.TestCase):
# No processes should be left alive on the worker nodes.
self.check_components_alive(ray.services.PROCESS_TYPE_PLASMA_STORE, False)
self.check_components_alive(ray.services.PROCESS_TYPE_PLASMA_MANAGER, False)
self.check_components_alive(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER, False)
self.check_components_alive(ray.services.PROCESS_TYPE_PLASMA_MANAGER,
False)
self.check_components_alive(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER,
False)
def testDriverLivesSequential(self):
ray.worker.init()
all_processes = ray.services.all_processes
processes = [
ray.services.all_processes[ray.services.PROCESS_TYPE_PLASMA_STORE][0],
ray.services.all_processes[ray.services.PROCESS_TYPE_PLASMA_MANAGER][0],
ray.services.all_processes[ray.services.PROCESS_TYPE_LOCAL_SCHEDULER][0],
ray.services.all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER][0],
]
all_processes[ray.services.PROCESS_TYPE_PLASMA_STORE][0],
all_processes[ray.services.PROCESS_TYPE_PLASMA_MANAGER][0],
all_processes[ray.services.PROCESS_TYPE_LOCAL_SCHEDULER][0],
all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER][0]]
# Kill all the components sequentially.
for process in processes:
@@ -202,12 +215,12 @@ class ComponentFailureTest(unittest.TestCase):
def testDriverLivesParallel(self):
ray.worker.init()
all_processes = ray.services.all_processes
processes = [
ray.services.all_processes[ray.services.PROCESS_TYPE_PLASMA_STORE][0],
ray.services.all_processes[ray.services.PROCESS_TYPE_PLASMA_MANAGER][0],
ray.services.all_processes[ray.services.PROCESS_TYPE_LOCAL_SCHEDULER][0],
ray.services.all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER][0],
]
all_processes[ray.services.PROCESS_TYPE_PLASMA_STORE][0],
all_processes[ray.services.PROCESS_TYPE_PLASMA_MANAGER][0],
all_processes[ray.services.PROCESS_TYPE_LOCAL_SCHEDULER][0],
all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER][0]]
# Kill all the components in parallel.
for process in processes:
@@ -222,5 +235,6 @@ class ComponentFailureTest(unittest.TestCase):
# If the driver can reach the tearDown method, then it is still alive.
if __name__ == "__main__":
unittest.main(verbosity=2)
test/failure_test.py
+54 -24
View File
@@ -9,14 +9,16 @@ import tempfile
import time
import unittest
import ray.test.test_functions as test_functions
if sys.version_info >= (3, 0):
from importlib import reload
import ray.test.test_functions as test_functions
def relevant_errors(error_type):
return [info for info in ray.error_info() if info[b"type"] == error_type]
def wait_for_errors(error_type, num_errors, timeout=10):
start_time = time.time()
while time.time() - start_time < timeout:
@@ -25,6 +27,7 @@ def wait_for_errors(error_type, num_errors, timeout=10):
time.sleep(0.1)
print("Timing out of wait.")
class FailureTest(unittest.TestCase):
def testUnknownSerialization(self):
reload(test_functions)
@@ -32,32 +35,35 @@ class FailureTest(unittest.TestCase):
test_functions.test_unknown_type.remote()
wait_for_errors(b"task", 1)
error_info = ray.error_info()
self.assertEqual(len(relevant_errors(b"task")), 1)
ray.worker.cleanup()
class TaskSerializationTest(unittest.TestCase):
def testReturnAndPassUnknownType(self):
ray.init(num_workers=1, driver_mode=ray.SILENT_MODE)
class Foo(object):
pass
# Check that returning an unknown type from a remote function raises an
# exception.
@ray.remote
def f():
return Foo()
self.assertRaises(Exception, lambda : ray.get(f.remote()))
self.assertRaises(Exception, lambda: ray.get(f.remote()))
# Check that passing an unknown type into a remote function raises an
# exception.
@ray.remote
def g(x):
return 1
self.assertRaises(Exception, lambda : g.remote(Foo()))
self.assertRaises(Exception, lambda: g.remote(Foo()))
ray.worker.cleanup()
class TaskStatusTest(unittest.TestCase):
def testFailedTask(self):
reload(test_functions)
@@ -66,10 +72,10 @@ class TaskStatusTest(unittest.TestCase):
test_functions.throw_exception_fct1.remote()
test_functions.throw_exception_fct1.remote()
wait_for_errors(b"task", 2)
result = ray.error_info()
self.assertEqual(len(relevant_errors(b"task")), 2)
for task in relevant_errors(b"task"):
self.assertIn(b"Test function 1 intentionally failed.", task.get(b"message"))
self.assertIn(b"Test function 1 intentionally failed.",
task.get(b"message"))
x = test_functions.throw_exception_fct2.remote()
try:
@@ -77,7 +83,8 @@ class TaskStatusTest(unittest.TestCase):
except Exception as e:
self.assertIn("Test function 2 intentionally failed.", str(e))
else:
self.assertTrue(False) # ray.get should throw an exception
# ray.get should throw an exception.
self.assertTrue(False)
x, y, z = test_functions.throw_exception_fct3.remote(1.0)
for ref in [x, y, z]:
@@ -86,7 +93,8 @@ class TaskStatusTest(unittest.TestCase):
except Exception as e:
self.assertIn("Test function 3 intentionally failed.", str(e))
else:
self.assertTrue(False) # ray.get should throw an exception
# ray.get should throw an exception.
self.assertTrue(False)
ray.worker.cleanup()
@@ -108,8 +116,8 @@ def temporary_helper_function():
sys.path.append(directory)
module = __import__(module_name)
# Define a function that closes over this temporary module. This should fail
# when it is unpickled.
# Define a function that closes over this temporary module. This should
# fail when it is unpickled.
@ray.remote
def g():
return module.temporary_python_file()
@@ -121,7 +129,7 @@ def temporary_helper_function():
# Check that if we try to call the function it throws an exception and does
# not hang.
for _ in range(10):
self.assertRaises(Exception, lambda : ray.get(g.remote()))
self.assertRaises(Exception, lambda: ray.get(g.remote()))
f.close()
@@ -150,16 +158,19 @@ def temporary_helper_function():
def initializer():
return 0
def reinitializer(foo):
raise Exception("The reinitializer failed.")
ray.env.foo = ray.EnvironmentVariable(initializer, reinitializer)
@ray.remote
def use_foo():
ray.env.foo
use_foo.remote()
wait_for_errors(b"reinitialize_environment_variable", 1)
# Check that the error message is in the task info.
self.assertIn(b"The reinitializer failed.", ray.error_info()[0][b"message"])
self.assertIn(b"The reinitializer failed.",
ray.error_info()[0][b"message"])
ray.worker.cleanup()
@@ -202,6 +213,7 @@ def temporary_helper_function():
class Foo(object):
def __init__(self):
self.x = module.temporary_python_file()
def get_val(self):
return 1
@@ -217,7 +229,8 @@ def temporary_helper_function():
# Wait for the error from when the __init__ tries to run.
wait_for_errors(b"task", 1)
self.assertIn(b"failed to be imported, and so cannot execute this method", ray.error_info()[1][b"message"])
self.assertIn(b"failed to be imported, and so cannot execute this method",
ray.error_info()[1][b"message"])
# Check that if we try to get the function it throws an exception and does
# not hang.
@@ -226,7 +239,8 @@ def temporary_helper_function():
# Wait for the error from when the call to get_val.
wait_for_errors(b"task", 2)
self.assertIn(b"failed to be imported, and so cannot execute this method", ray.error_info()[2][b"message"])
self.assertIn(b"failed to be imported, and so cannot execute this method",
ray.error_info()[2][b"message"])
f.close()
@@ -234,6 +248,7 @@ def temporary_helper_function():
sys.path.pop(-1)
ray.worker.cleanup()
class ActorTest(unittest.TestCase):
def testFailedActorInit(self):
@@ -241,12 +256,15 @@ class ActorTest(unittest.TestCase):
error_message1 = "actor constructor failed"
error_message2 = "actor method failed"
@ray.actor
class FailedActor(object):
def __init__(self):
raise Exception(error_message1)
def get_val(self):
return 1
def fail_method(self):
raise Exception(error_message2)
@@ -255,13 +273,15 @@ class ActorTest(unittest.TestCase):
# Make sure that we get errors from a failed constructor.
wait_for_errors(b"task", 1)
self.assertEqual(len(ray.error_info()), 1)
self.assertIn(error_message1, ray.error_info()[0][b"message"].decode("ascii"))
self.assertIn(error_message1,
ray.error_info()[0][b"message"].decode("ascii"))
# Make sure that we get errors from a failed method.
a.fail_method()
wait_for_errors(b"task", 2)
self.assertEqual(len(ray.error_info()), 2)
self.assertIn(error_message2, ray.error_info()[1][b"message"].decode("ascii"))
self.assertIn(error_message2,
ray.error_info()[1][b"message"].decode("ascii"))
ray.worker.cleanup()
@@ -272,6 +292,7 @@ class ActorTest(unittest.TestCase):
class Actor(object):
def __init__(self, missing_variable_name):
pass
def get_val(self, x):
pass
@@ -284,18 +305,22 @@ class ActorTest(unittest.TestCase):
wait_for_errors(b"task", 1)
self.assertEqual(len(ray.error_info()), 1)
if sys.version_info >= (3, 0):
self.assertIn("missing 1 required", ray.error_info()[0][b"message"].decode("ascii"))
self.assertIn("missing 1 required",
ray.error_info()[0][b"message"].decode("ascii"))
else:
self.assertIn("takes exactly 2 arguments", ray.error_info()[0][b"message"].decode("ascii"))
self.assertIn("takes exactly 2 arguments",
ray.error_info()[0][b"message"].decode("ascii"))
# Create an actor with too many arguments.
a = Actor(1, 2)
wait_for_errors(b"task", 2)
self.assertEqual(len(ray.error_info()), 2)
if sys.version_info >= (3, 0):
self.assertIn("but 3 were given", ray.error_info()[1][b"message"].decode("ascii"))
self.assertIn("but 3 were given",
ray.error_info()[1][b"message"].decode("ascii"))
else:
self.assertIn("takes exactly 2 arguments", ray.error_info()[1][b"message"].decode("ascii"))
self.assertIn("takes exactly 2 arguments",
ray.error_info()[1][b"message"].decode("ascii"))
# Create an actor the correct number of arguments.
a = Actor(1)
@@ -305,23 +330,28 @@ class ActorTest(unittest.TestCase):
wait_for_errors(b"task", 3)
self.assertEqual(len(ray.error_info()), 3)
if sys.version_info >= (3, 0):
self.assertIn("missing 1 required", ray.error_info()[2][b"message"].decode("ascii"))
self.assertIn("missing 1 required",
ray.error_info()[2][b"message"].decode("ascii"))
else:
self.assertIn("takes exactly 2 arguments", ray.error_info()[2][b"message"].decode("ascii"))
self.assertIn("takes exactly 2 arguments",
ray.error_info()[2][b"message"].decode("ascii"))
# Call a method with too many arguments.
a.get_val(1, 2)
wait_for_errors(b"task", 4)
self.assertEqual(len(ray.error_info()), 4)
if sys.version_info >= (3, 0):
self.assertIn("but 3 were given", ray.error_info()[3][b"message"].decode("ascii"))
self.assertIn("but 3 were given",
ray.error_info()[3][b"message"].decode("ascii"))
else:
self.assertIn("takes exactly 2 arguments", ray.error_info()[3][b"message"].decode("ascii"))
self.assertIn("takes exactly 2 arguments",
ray.error_info()[3][b"message"].decode("ascii"))
# Call a method that doesn't exist.
with self.assertRaises(AttributeError):
a.nonexistent_method()
ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)
test/jenkins_tests/multi_node_docker_test.py
+26 -18
View File
@@ -7,7 +7,7 @@ import os
import re
import subprocess
import sys
import time
def wait_for_output(proc):
"""This is a convenience method to parse a process's stdout and stderr.
@@ -19,10 +19,13 @@ def wait_for_output(proc):
A tuple of the stdout and stderr of the process as strings.
"""
stdout_data, stderr_data = proc.communicate()
stdout_data = stdout_data.decode("ascii") if stdout_data is not None else None
stderr_data = stderr_data.decode("ascii") if stderr_data is not None else None
stdout_data = (stdout_data.decode("ascii") if stdout_data is not None
else None)
stderr_data = (stderr_data.decode("ascii") if stderr_data is not None
else None)
return stdout_data, stderr_data
class DockerRunner(object):
"""This class manages the logistics of running multiple nodes in Docker.
@@ -34,8 +37,8 @@ class DockerRunner(object):
head_container_id: The ID of the docker container that runs the head node.
worker_container_ids: A list of the docker container IDs of the Ray worker
nodes.
head_container_ip: The IP address of the docker container that runs the head
node.
head_container_ip: The IP address of the docker container that runs the
head node.
"""
def __init__(self):
"""Initialize the DockerRunner."""
@@ -47,8 +50,8 @@ class DockerRunner(object):
"""Parse the docker container ID from stdout_data.
Args:
stdout_data: This should be a string with the standard output of a call to
a docker command.
stdout_data: This should be a string with the standard output of a call
to a docker command.
Returns:
The container ID of the docker container.
@@ -70,7 +73,8 @@ class DockerRunner(object):
The IP address of the container.
"""
proc = subprocess.Popen(["docker", "inspect",
"--format={{.NetworkSettings.Networks.bridge.IPAddress}}",
"--format={{.NetworkSettings.Networks.bridge"
".IPAddress}}",
container_id],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
@@ -86,9 +90,10 @@ class DockerRunner(object):
"""Start the Ray head node inside a docker container."""
mem_arg = ["--memory=" + mem_size] if mem_size else []
shm_arg = ["--shm-size=" + shm_size] if shm_size else []
volume_arg = ["-v",
"{}:{}".format(os.path.dirname(os.path.realpath(__file__)),
"/ray/test/jenkins_tests")] if development_mode else []
volume_arg = (["-v",
"{}:{}".format(os.path.dirname(os.path.realpath(__file__)),
"/ray/test/jenkins_tests")]
if development_mode else [])
proc = subprocess.Popen(["docker", "run", "-d"] + mem_arg + shm_arg +
volume_arg +
[docker_image, "/ray/scripts/start_ray.sh",
@@ -113,7 +118,8 @@ class DockerRunner(object):
proc = subprocess.Popen(["docker", "run", "-d"] + mem_arg + shm_arg +
["--shm-size=" + shm_size, docker_image,
"/ray/scripts/start_ray.sh",
"--redis-address={:s}:6379".format(self.head_container_ip)],
"--redis-address={:s}:6379".format(
self.head_container_ip)],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
container_id = self._get_container_id(stdout_data)
@@ -136,10 +142,10 @@ class DockerRunner(object):
mem_size: The amount of memory to start each docker container with. This
will be passed into `docker run` as the --memory flag. If this is None,
then no --memory flag will be used.
shm_size: The amount of shared memory to start each docker container with.
This will be passed into `docker run` as the `--shm-size` flag.
num_nodes: The number of nodes to use in the cluster (this counts the head
node as well).
shm_size: The amount of shared memory to start each docker container
with. This will be passed into `docker run` as the `--shm-size` flag.
num_nodes: The number of nodes to use in the cluster (this counts the
head node as well).
development_mode: True if you want to mount the local copy of
test/jenkins_test on the head node so we can avoid rebuilding docker
images during development.
@@ -163,7 +169,7 @@ class DockerRunner(object):
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_data, _ = wait_for_output(proc)
removed_container_id = self._get_container_id(stdout_data)
if not container_id == stopped_container_id:
if not container_id == removed_container_id:
raise Exception("Failed to remove container {}.".format(container_id))
print("stop_node", {"container_id": container_id,
@@ -202,8 +208,10 @@ class DockerRunner(object):
print(stderr_data)
return {"success": proc.returncode == 0, "return_code": proc.returncode}
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Run multinode tests in Docker.")
parser = argparse.ArgumentParser(
description="Run multinode tests in Docker.")
parser.add_argument("--docker-image", default="ray-project/deploy",
help="docker image")
parser.add_argument("--mem-size", help="memory size")
test/jenkins_tests/multi_node_tests/test_0.py
+2
View File
@@ -3,11 +3,13 @@ import time
import ray
@ray.remote
def f():
time.sleep(0.1)
return ray.services.get_node_ip_address()
if __name__ == "__main__":
ray.init(redis_address=os.environ["RAY_REDIS_ADDRESS"])
# Check that tasks are scheduled on all nodes.
test/microbenchmarks.py
+19 -12
View File
@@ -9,10 +9,11 @@ import sys
import time
import numpy as np
import ray.test.test_functions as test_functions
if sys.version_info >= (3, 0):
from importlib import reload
import ray.test.test_functions as test_functions
class MicroBenchmarkTest(unittest.TestCase):
@@ -20,7 +21,7 @@ class MicroBenchmarkTest(unittest.TestCase):
reload(test_functions)
ray.init(num_workers=3)
# measure the time required to submit a remote task to the scheduler
# Measure the time required to submit a remote task to the scheduler.
elapsed_times = []
for _ in range(1000):
start_time = time.time()
@@ -34,9 +35,10 @@ class MicroBenchmarkTest(unittest.TestCase):
print(" 90th percentile: {}".format(elapsed_times[900]))
print(" 99th percentile: {}".format(elapsed_times[990]))
print(" worst: {}".format(elapsed_times[999]))
# average_elapsed_time should be about 0.00038
# average_elapsed_time should be about 0.00038.
# measure the time required to submit a remote task to the scheduler (where the remote task returns one value)
# Measure the time required to submit a remote task to the scheduler
# (where the remote task returns one value).
elapsed_times = []
for _ in range(1000):
start_time = time.time()
@@ -50,9 +52,10 @@ class MicroBenchmarkTest(unittest.TestCase):
print(" 90th percentile: {}".format(elapsed_times[900]))
print(" 99th percentile: {}".format(elapsed_times[990]))
print(" worst: {}".format(elapsed_times[999]))
# average_elapsed_time should be about 0.001
# average_elapsed_time should be about 0.001.
# measure the time required to submit a remote task to the scheduler and get the result
# Measure the time required to submit a remote task to the scheduler and
# get the result.
elapsed_times = []
for _ in range(1000):
start_time = time.time()
@@ -62,14 +65,15 @@ class MicroBenchmarkTest(unittest.TestCase):
elapsed_times.append(end_time - start_time)
elapsed_times = np.sort(elapsed_times)
average_elapsed_time = sum(elapsed_times) / 1000
print("Time required to submit a trivial function call and get the result:")
print("Time required to submit a trivial function call and get the "
"result:")
print(" Average: {}".format(average_elapsed_time))
print(" 90th percentile: {}".format(elapsed_times[900]))
print(" 99th percentile: {}".format(elapsed_times[990]))
print(" worst: {}".format(elapsed_times[999]))
# average_elapsed_time should be about 0.0013
# average_elapsed_time should be about 0.0013.
# measure the time required to do do a put
# Measure the time required to do do a put.
elapsed_times = []
for _ in range(1000):
start_time = time.time()
@@ -83,7 +87,7 @@ class MicroBenchmarkTest(unittest.TestCase):
print(" 90th percentile: {}".format(elapsed_times[900]))
print(" 99th percentile: {}".format(elapsed_times[990]))
print(" worst: {}".format(elapsed_times[999]))
# average_elapsed_time should be about 0.00087
# average_elapsed_time should be about 0.00087.
ray.worker.cleanup()
@@ -105,11 +109,14 @@ class MicroBenchmarkTest(unittest.TestCase):
if d > 1.5 * b:
if os.getenv("TRAVIS") is None:
raise Exception("The caching test was too slow. d = {}, b = {}".format(d, b))
raise Exception("The caching test was too slow. "
"d = {}, b = {}".format(d, b))
else:
print("WARNING: The caching test was too slow. d = {}, b = {}".format(d, b))
print("WARNING: The caching test was too slow. "
"d = {}, b = {}".format(d, b))
ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)
test/multi_node_test.py
+16 -9
View File
@@ -2,17 +2,18 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import numpy as np
import os
import unittest
import ray
import subprocess
import sys
import tempfile
import time
start_ray_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../scripts/start_ray.sh")
stop_ray_script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../scripts/stop_ray.sh")
start_ray_script = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../scripts/start_ray.sh")
stop_ray_script = os.path.join(os.path.dirname(os.path.abspath(__file__)),
"../scripts/stop_ray.sh")
class MultiNodeTest(unittest.TestCase):
@@ -21,7 +22,8 @@ class MultiNodeTest(unittest.TestCase):
out = subprocess.check_output([start_ray_script, "--head"]).decode("ascii")
# Get the redis address from the output.
redis_substring_prefix = "redis_address=\""
redis_address_location = out.find(redis_substring_prefix) + len(redis_substring_prefix)
redis_address_location = (out.find(redis_substring_prefix) +
len(redis_substring_prefix))
redis_address = out[redis_address_location:]
self.redis_address = redis_address.split("\"")[0]
@@ -54,7 +56,8 @@ class MultiNodeTest(unittest.TestCase):
# Make sure we got the error.
self.assertEqual(len(ray.error_info()), 1)
self.assertIn(error_string1, ray.error_info()[0][b"message"].decode("ascii"))
self.assertIn(error_string1,
ray.error_info()[0][b"message"].decode("ascii"))
# Start another driver and make sure that it does not receive this error.
# Make the other driver throw an error, and make sure it receives that
@@ -98,7 +101,8 @@ print("success")
# Make sure that the other error message doesn't show up for this driver.
self.assertEqual(len(ray.error_info()), 1)
self.assertIn(error_string1, ray.error_info()[0][b"message"].decode("ascii"))
self.assertIn(error_string1,
ray.error_info()[0][b"message"].decode("ascii"))
ray.worker.cleanup()
@@ -149,6 +153,7 @@ print("success")
ray.worker.cleanup()
class StartRayScriptTest(unittest.TestCase):
def testCallingStartRayHead(self):
@@ -157,11 +162,12 @@ class StartRayScriptTest(unittest.TestCase):
# the non-head node code path.
# Test starting Ray with no arguments.
out = subprocess.check_output([start_ray_script, "--head"]).decode("ascii")
subprocess.check_output([start_ray_script, "--head"]).decode("ascii")
subprocess.Popen([stop_ray_script]).wait()
# Test starting Ray with a number of workers specified.
subprocess.check_output([start_ray_script, "--head", "--num-workers", "20"])
subprocess.check_output([start_ray_script, "--head", "--num-workers",
"20"])
subprocess.Popen([stop_ray_script]).wait()
# Test starting Ray with a redis port specified.
@@ -204,5 +210,6 @@ class StartRayScriptTest(unittest.TestCase):
"--redis-address", "127.0.0.1:6379"])
subprocess.Popen([stop_ray_script]).wait()
if __name__ == "__main__":
unittest.main(verbosity=2)
test/runtest.py
+147 -60
View File
@@ -12,24 +12,33 @@ import string
import sys
from collections import namedtuple
import ray.test.test_functions as test_functions
if sys.version_info >= (3, 0):
from importlib import reload
import ray.test.test_functions as test_functions
import ray.experimental.array.remote as ra
import ray.experimental.array.distributed as da
def assert_equal(obj1, obj2):
if type(obj1).__module__ == np.__name__ or type(obj2).__module__ == np.__name__:
if (hasattr(obj1, "shape") and obj1.shape == ()) or (hasattr(obj2, "shape") and obj2.shape == ()):
module_numpy = (type(obj1).__module__ == np.__name__ or
type(obj2).__module__ == np.__name__)
if module_numpy:
empty_shape = ((hasattr(obj1, "shape") and obj1.shape == ()) or
(hasattr(obj2, "shape") and obj2.shape == ()))
if empty_shape:
# This is a special case because currently np.testing.assert_equal fails
# because we do not properly handle different numerical types.
assert obj1 == obj2, "Objects {} and {} are different.".format(obj1, obj2)
assert obj1 == obj2, ("Objects {} and {} are "
"different.".format(obj1, obj2))
else:
np.testing.assert_equal(obj1, obj2)
elif hasattr(obj1, "__dict__") and hasattr(obj2, "__dict__"):
special_keys = ["_pytype_"]
assert set(list(obj1.__dict__.keys()) + special_keys) == set(list(obj2.__dict__.keys()) + special_keys), "Objects {} and {} are different.".format(obj1, obj2)
assert (set(list(obj1.__dict__.keys()) + special_keys) ==
set(list(obj2.__dict__.keys()) + special_keys)), ("Objects {} and "
"{} are "
"different."
.format(obj1,
obj2))
for key in obj1.__dict__.keys():
if key not in special_keys:
assert_equal(obj1.__dict__[key], obj2.__dict__[key])
@@ -38,24 +47,29 @@ def assert_equal(obj1, obj2):
for key in obj1.keys():
assert_equal(obj1[key], obj2[key])
elif type(obj1) is list or type(obj2) is list:
assert len(obj1) == len(obj2), "Objects {} and {} are lists with different lengths.".format(obj1, obj2)
assert len(obj1) == len(obj2), ("Objects {} and {} are lists with "
"different lengths.".format(obj1, obj2))
for i in range(len(obj1)):
assert_equal(obj1[i], obj2[i])
elif type(obj1) is tuple or type(obj2) is tuple:
assert len(obj1) == len(obj2), "Objects {} and {} are tuples with different lengths.".format(obj1, obj2)
assert len(obj1) == len(obj2), ("Objects {} and {} are tuples with "
"different lengths.".format(obj1, obj2))
for i in range(len(obj1)):
assert_equal(obj1[i], obj2[i])
elif ray.serialization.is_named_tuple(type(obj1)) or ray.serialization.is_named_tuple(type(obj2)):
assert len(obj1) == len(obj2), "Objects {} and {} are named tuples with different lengths.".format(obj1, obj2)
elif (ray.serialization.is_named_tuple(type(obj1)) or
ray.serialization.is_named_tuple(type(obj2))):
assert len(obj1) == len(obj2), ("Objects {} and {} are named tuples with "
"different lengths.".format(obj1, obj2))
for i in range(len(obj1)):
assert_equal(obj1[i], obj2[i])
else:
assert obj1 == obj2, "Objects {} and {} are different.".format(obj1, obj2)
if sys.version_info >= (3, 0):
long_extras = [0, np.array([["hi", u"hi"], [1.3, 1]])]
else:
long_extras = [long(0), np.array([["hi", u"hi"], [1.3, long(1)]])]
long_extras = [long(0), np.array([["hi", u"hi"], [1.3, long(1)]])] # noqa: E501,F821
PRIMITIVE_OBJECTS = [0, 0.0, 0.9, 1 << 62, "a", string.printable, "\u262F",
u"hello world", u"\xff\xfe\x9c\x001\x000\x00", None, True,
@@ -65,45 +79,55 @@ PRIMITIVE_OBJECTS = [0, 0.0, 0.9, 1 << 62, "a", string.printable, "\u262F",
np.random.normal(size=[100, 100]), np.array(["hi", 3]),
np.array(["hi", 3], dtype=object)] + long_extras
COMPLEX_OBJECTS = [[[[[[[[[[[[[]]]]]]]]]]]],
{"obj{}".format(i): np.random.normal(size=[100, 100]) for i in range(10)},
#{(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {}}}}}}}}}}}}},
((((((((((),),),),),),),),),),
{"a": {"b": {"c": {"d": {}}}}}
]
COMPLEX_OBJECTS = [
[[[[[[[[[[[[]]]]]]]]]]]],
{"obj{}".format(i): np.random.normal(size=[100, 100]) for i in range(10)},
# {(): {(): {(): {(): {(): {(): {(): {(): {(): {(): {
# (): {(): {}}}}}}}}}}}}},
((((((((((),),),),),),),),),),
{"a": {"b": {"c": {"d": {}}}}}]
class Foo(object):
def __init__(self):
pass
class Bar(object):
def __init__(self):
for i, val in enumerate(PRIMITIVE_OBJECTS + COMPLEX_OBJECTS):
setattr(self, "field{}".format(i), val)
class Baz(object):
def __init__(self):
self.foo = Foo()
self.bar = Bar()
def method(self, arg):
pass
class Qux(object):
def __init__(self):
self.objs = [Foo(), Bar(), Baz()]
class SubQux(Qux):
def __init__(self):
Qux.__init__(self)
class CustomError(Exception):
pass
Point = namedtuple("Point", ["x", "y"])
NamedTupleExample = namedtuple("Example", "field1, field2, field3, field4, field5")
NamedTupleExample = namedtuple("Example",
"field1, field2, field3, field4, field5")
CUSTOM_OBJECTS = [Exception("Test object."), CustomError(), Point(11, y=22),
Foo(), Bar(), Baz(), # Qux(), SubQux(),
Foo(), Bar(), Baz(), # Qux(), SubQux(),
NamedTupleExample(1, 1.0, "hi", np.zeros([3, 5]), [1, 2, 3])]
BASE_OBJECTS = PRIMITIVE_OBJECTS + COMPLEX_OBJECTS + CUSTOM_OBJECTS
@@ -112,8 +136,9 @@ LIST_OBJECTS = [[obj] for obj in BASE_OBJECTS]
TUPLE_OBJECTS = [(obj,) for obj in BASE_OBJECTS]
# The check that type(obj).__module__ != "numpy" should be unnecessary, but
# otherwise this seems to fail on Mac OS X on Travis.
DICT_OBJECTS = ([{obj: obj} for obj in PRIMITIVE_OBJECTS if obj.__hash__ is not None and type(obj).__module__ != "numpy"] +
# DICT_OBJECTS = ([{obj: obj} for obj in BASE_OBJECTS if obj.__hash__ is not None] +
DICT_OBJECTS = ([{obj: obj} for obj in PRIMITIVE_OBJECTS
if (obj.__hash__ is not None and
type(obj).__module__ != "numpy")] +
[{0: obj} for obj in BASE_OBJECTS])
RAY_TEST_OBJECTS = BASE_OBJECTS + LIST_OBJECTS + TUPLE_OBJECTS + DICT_OBJECTS
@@ -124,7 +149,10 @@ try:
cloudpickle.dumps(Point)
except AttributeError:
cloudpickle_command = "pip install --upgrade cloudpickle"
raise Exception("You have an older version of cloudpickle that is not able to serialize namedtuples. Try running \n\n{}\n\n".format(cloudpickle_command))
raise Exception("You have an older version of cloudpickle that is not able "
"to serialize namedtuples. Try running "
"\n\n{}\n\n".format(cloudpickle_command))
class SerializationTest(unittest.TestCase):
@@ -155,7 +183,7 @@ class SerializationTest(unittest.TestCase):
# Check that exceptions are thrown when we serialize the recursive objects.
for obj in recursive_objects:
self.assertRaises(Exception, lambda : ray.put(obj))
self.assertRaises(Exception, lambda: ray.put(obj))
ray.worker.cleanup()
@@ -181,6 +209,7 @@ class SerializationTest(unittest.TestCase):
ray.worker.cleanup()
class WorkerTest(unittest.TestCase):
def testPythonWorkers(self):
@@ -228,6 +257,7 @@ class WorkerTest(unittest.TestCase):
ray.worker.cleanup()
class APITest(unittest.TestCase):
def testRegisterClass(self):
@@ -237,10 +267,10 @@ class APITest(unittest.TestCase):
# throws an exception.
class TempClass(object):
pass
self.assertRaises(Exception, lambda : ray.put(Foo))
self.assertRaises(Exception, lambda: ray.put(Foo))
# Check that registering a class that Ray cannot serialize efficiently
# raises an exception.
self.assertRaises(Exception, lambda : ray.register_class(type(True)))
self.assertRaises(Exception, lambda: ray.register_class(type(True)))
# Check that registering the same class with pickle works.
ray.register_class(type(float), pickle=True)
self.assertEqual(ray.get(ray.put(float)), float)
@@ -328,7 +358,9 @@ class APITest(unittest.TestCase):
print("Still using old definition of f, trying again.")
# Test that we can close over plain old data.
data = [np.zeros([3, 5]), (1, 2, "a"), [0.0, 1.0, 1 << 62], 1 << 60, {"a": np.zeros(3)}]
data = [np.zeros([3, 5]), (1, 2, "a"), [0.0, 1.0, 1 << 62], 1 << 60,
{"a": np.zeros(3)}]
@ray.remote
def g():
return data
@@ -339,18 +371,22 @@ class APITest(unittest.TestCase):
def h():
return np.zeros([3, 5])
assert_equal(ray.get(h.remote()), np.zeros([3, 5]))
@ray.remote
def j():
return time.time()
ray.get(j.remote())
# Test that we can define remote functions that call other remote functions.
# Test that we can define remote functions that call other remote
# functions.
@ray.remote
def k(x):
return x + 1
@ray.remote
def l(x):
return ray.get(k.remote(x))
@ray.remote
def m(x):
return ray.get(l.remote(x))
@@ -398,7 +434,7 @@ class APITest(unittest.TestCase):
# Verify that calling wait with duplicate object IDs throws an exception.
x = ray.put(1)
self.assertRaises(Exception, lambda : ray.wait([x, x]))
self.assertRaises(Exception, lambda: ray.wait([x, x]))
ray.worker.cleanup()
@@ -435,11 +471,14 @@ class APITest(unittest.TestCase):
ray.worker.cleanup()
def testCachingEnvironmentVariables(self):
# Test that we can define environment variables before the driver is connected.
# Test that we can define environment variables before the driver is
# connected.
def foo_initializer():
return 1
def bar_initializer():
return []
def bar_reinitializer(bar):
return []
ray.env.foo = ray.EnvironmentVariable(foo_initializer)
@@ -448,6 +487,7 @@ class APITest(unittest.TestCase):
@ray.remote
def use_foo():
return ray.env.foo
@ray.remote
def use_bar():
ray.env.bar.append(1)
@@ -463,16 +503,20 @@ class APITest(unittest.TestCase):
ray.worker.cleanup()
def testCachingFunctionsToRun(self):
# Test that we export functions to run on all workers before the driver is connected.
# Test that we export functions to run on all workers before the driver is
# connected.
def f(worker_info):
sys.path.append(1)
ray.worker.global_worker.run_function_on_all_workers(f)
def f(worker_info):
sys.path.append(2)
ray.worker.global_worker.run_function_on_all_workers(f)
def g(worker_info):
sys.path.append(3)
ray.worker.global_worker.run_function_on_all_workers(g)
def f(worker_info):
sys.path.append(4)
ray.worker.global_worker.run_function_on_all_workers(f)
@@ -505,13 +549,16 @@ class APITest(unittest.TestCase):
def f(worker_info):
sys.path.append("fake_directory")
ray.worker.global_worker.run_function_on_all_workers(f)
@ray.remote
def get_path1():
return sys.path
self.assertEqual("fake_directory", ray.get(get_path1.remote())[-1])
def f(worker_info):
sys.path.pop(-1)
ray.worker.global_worker.run_function_on_all_workers(f)
# Create a second remote function to guarantee that when we call
# get_path2.remote(), the second function to run will have been run on the
# worker.
@@ -528,6 +575,7 @@ class APITest(unittest.TestCase):
def f(worker_info):
sys.path.append(worker_info)
ray.worker.global_worker.run_function_on_all_workers(f)
@ray.remote
def get_path():
time.sleep(1)
@@ -542,6 +590,7 @@ class APITest(unittest.TestCase):
counters = [worker_info["counter"] for worker_info in worker_infos]
# We use range(11) because the driver also runs the function.
self.assertEqual(set(counters), set(range(11)))
# Clean up the worker paths.
def f(worker_info):
sys.path.pop(-1)
@@ -555,7 +604,8 @@ class APITest(unittest.TestCase):
def events():
# This is a hack for getting the event log. It is not part of the API.
keys = ray.worker.global_worker.redis_client.keys("event_log:*")
return [ray.worker.global_worker.redis_client.lrange(key, 0, -1) for key in keys]
return [ray.worker.global_worker.redis_client.lrange(key, 0, -1)
for key in keys]
def wait_for_num_events(num_events, timeout=10):
start_time = time.time()
@@ -604,25 +654,28 @@ class APITest(unittest.TestCase):
# accidentally call an older version.
ray.init(num_workers=2)
num_remote_functions = 100
num_calls = 200
@ray.remote
def f():
return 1
results1 = [f.remote() for _ in range(num_calls)]
@ray.remote
def f():
return 2
results2 = [f.remote() for _ in range(num_calls)]
@ray.remote
def f():
return 3
results3 = [f.remote() for _ in range(num_calls)]
@ray.remote
def f():
return 4
results4 = [f.remote() for _ in range(num_calls)]
@ray.remote
def f():
return 5
@@ -637,16 +690,20 @@ class APITest(unittest.TestCase):
@ray.remote
def g():
return 1
@ray.remote
@ray.remote # noqa: F811
def g():
return 2
@ray.remote
@ray.remote # noqa: F811
def g():
return 3
@ray.remote
@ray.remote # noqa: F811
def g():
return 4
@ray.remote
@ray.remote # noqa: F811
def g():
return 5
@@ -668,6 +725,7 @@ class APITest(unittest.TestCase):
ray.worker.cleanup()
class PythonModeTest(unittest.TestCase):
def testPythonMode(self):
@@ -678,17 +736,21 @@ class PythonModeTest(unittest.TestCase):
def f():
return np.ones([3, 4, 5])
xref = f.remote()
assert_equal(xref, np.ones([3, 4, 5])) # remote functions should return by value
assert_equal(xref, ray.get(xref)) # ray.get should be the identity
# Remote functions should return by value.
assert_equal(xref, np.ones([3, 4, 5]))
# Check that ray.get is the identity.
assert_equal(xref, ray.get(xref))
y = np.random.normal(size=[11, 12])
assert_equal(y, ray.put(y)) # ray.put should be the identity
# Check that ray.put is the identity.
assert_equal(y, ray.put(y))
# make sure objects are immutable, this example is why we need to copy
# Make sure objects are immutable, this example is why we need to copy
# arguments before passing them into remote functions in python mode
aref = test_functions.python_mode_f.remote()
assert_equal(aref, np.array([0, 0]))
bref = test_functions.python_mode_g.remote(aref)
assert_equal(aref, np.array([0, 0])) # python_mode_g should not mutate aref
# Make sure python_mode_g does not mutate aref.
assert_equal(aref, np.array([0, 0]))
assert_equal(bref, np.array([1, 0]))
ray.worker.cleanup()
@@ -699,6 +761,7 @@ class PythonModeTest(unittest.TestCase):
def l_init():
return []
def l_reinit(l):
return []
ray.env.l = ray.EnvironmentVariable(l_init, l_reinit)
@@ -717,7 +780,8 @@ class PythonModeTest(unittest.TestCase):
assert_equal(ray.get(use_l.remote()), [1])
assert_equal(ray.get(use_l.remote()), [1])
# Make sure the local copy of the environment variable has not been mutated.
# Make sure the local copy of the environment variable has not been
# mutated.
assert_equal(l, [])
l = ray.env.l
assert_equal(l, [])
@@ -730,6 +794,7 @@ class PythonModeTest(unittest.TestCase):
ray.worker.cleanup()
class EnvironmentVariablesTest(unittest.TestCase):
def testEnvironmentVariables(self):
@@ -739,6 +804,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
def foo_initializer():
return 1
def foo_reinitializer(foo):
return foo
@@ -752,7 +818,8 @@ class EnvironmentVariablesTest(unittest.TestCase):
self.assertEqual(ray.get(use_foo.remote()), 1)
self.assertEqual(ray.get(use_foo.remote()), 1)
# Test that we can add a variable to the key-value store, mutate it, and reset it.
# Test that we can add a variable to the key-value store, mutate it, and
# reset it.
def bar_initializer():
return [1, 2, 3]
@@ -771,6 +838,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
def baz_initializer():
return np.zeros([4])
def baz_reinitializer(baz):
for i in range(len(baz)):
baz[i] = 0
@@ -794,6 +862,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
def qux_initializer():
return 0
def qux_reinitializer(x):
return x + 1
@@ -815,6 +884,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
def foo_initializer():
return []
def foo_reinitializer(foo):
return []
@@ -846,6 +916,7 @@ class EnvironmentVariablesTest(unittest.TestCase):
ray.worker.cleanup()
class UtilsTest(unittest.TestCase):
def testCopyingDirectory(self):
@@ -894,6 +965,7 @@ class UtilsTest(unittest.TestCase):
ray.worker.cleanup()
class ResourcesTest(unittest.TestCase):
def testResourceConstraints(self):
@@ -901,13 +973,16 @@ class ResourcesTest(unittest.TestCase):
ray.init(num_workers=num_workers, num_cpus=10, num_gpus=2)
# Attempt to wait for all of the workers to start up.
ray.worker.global_worker.run_function_on_all_workers(lambda worker_info: sys.path.append(worker_info["counter"]))
ray.worker.global_worker.run_function_on_all_workers(
lambda worker_info: sys.path.append(worker_info["counter"]))
@ray.remote(num_cpus=0)
def get_worker_id():
time.sleep(1)
return sys.path[-1]
while True:
if len(set(ray.get([get_worker_id.remote() for _ in range(num_workers)]))) == num_workers:
if len(set(ray.get([get_worker_id.remote()
for _ in range(num_workers)]))) == num_workers:
break
time_buffer = 0.3
@@ -974,13 +1049,16 @@ class ResourcesTest(unittest.TestCase):
ray.init(num_workers=num_workers, num_cpus=10, num_gpus=10)
# Attempt to wait for all of the workers to start up.
ray.worker.global_worker.run_function_on_all_workers(lambda worker_info: sys.path.append(worker_info["counter"]))
ray.worker.global_worker.run_function_on_all_workers(
lambda worker_info: sys.path.append(worker_info["counter"]))
@ray.remote(num_cpus=0)
def get_worker_id():
time.sleep(1)
return sys.path[-1]
while True:
if len(set(ray.get([get_worker_id.remote() for _ in range(num_workers)]))) == num_workers:
if len(set(ray.get([get_worker_id.remote()
for _ in range(num_workers)]))) == num_workers:
break
@ray.remote(num_cpus=1, num_gpus=9)
@@ -1021,8 +1099,8 @@ class ResourcesTest(unittest.TestCase):
def testMultipleLocalSchedulers(self):
# This test will define a bunch of tasks that can only be assigned to
# specific local schedulers, and we will check that they are assigned to the
# correct local schedulers.
# specific local schedulers, and we will check that they are assigned to
# the correct local schedulers.
address_info = ray.worker._init(start_ray_local=True,
num_local_schedulers=3,
num_cpus=[100, 5, 10],
@@ -1088,7 +1166,8 @@ class ResourcesTest(unittest.TestCase):
results.append(run_on_0_2.remote())
return names, results
store_names = [object_store_address.name for object_store_address in address_info["object_store_addresses"]]
store_names = [object_store_address.name for object_store_address
in address_info["object_store_addresses"]]
def validate_names_and_results(names, results):
for name, result in zip(names, ray.get(results)):
@@ -1099,7 +1178,8 @@ class ResourcesTest(unittest.TestCase):
elif name == "run_on_2":
self.assertIn(result, [store_names[2]])
elif name == "run_on_0_1_2":
self.assertIn(result, [store_names[0], store_names[1], store_names[2]])
self.assertIn(result, [store_names[0], store_names[1],
store_names[2]])
elif name == "run_on_1_2":
self.assertIn(result, [store_names[1], store_names[2]])
elif name == "run_on_0_2":
@@ -1128,6 +1208,7 @@ class ResourcesTest(unittest.TestCase):
ray.worker.cleanup()
class WorkerPoolTests(unittest.TestCase):
def tearDown(self):
@@ -1177,6 +1258,7 @@ class WorkerPoolTests(unittest.TestCase):
ray.worker.cleanup()
class SchedulingAlgorithm(unittest.TestCase):
def attempt_to_load_balance(self, remote_function, args, total_tasks,
@@ -1184,21 +1266,24 @@ class SchedulingAlgorithm(unittest.TestCase):
num_attempts=20):
attempts = 0
while attempts < num_attempts:
locations = ray.get([remote_function.remote(*args) for _ in range(total_tasks)])
locations = ray.get([remote_function.remote(*args)
for _ in range(total_tasks)])
names = set(locations)
counts = [locations.count(name) for name in names]
print("Counts are {}.".format(counts))
if len(names) == num_local_schedulers and all([count >= minimum_count for count in counts]):
if len(names) == num_local_schedulers and all([count >= minimum_count
for count in counts]):
break
attempts += 1
self.assertLess(attempts, num_attempts)
def testLoadBalancing(self):
# This test ensures that tasks are being assigned to all local schedulers in
# a roughly equal manner.
# This test ensures that tasks are being assigned to all local schedulers
# in a roughly equal manner.
num_workers = 21
num_local_schedulers = 3
ray.worker._init(start_ray_local=True, num_workers=num_workers, num_local_schedulers=num_local_schedulers)
ray.worker._init(start_ray_local=True, num_workers=num_workers,
num_local_schedulers=num_local_schedulers)
@ray.remote
def f():
@@ -1211,11 +1296,12 @@ class SchedulingAlgorithm(unittest.TestCase):
ray.worker.cleanup()
def testLoadBalancingWithDependencies(self):
# This test ensures that tasks are being assigned to all local schedulers in
# a roughly equal manner even when the tasks have dependencies.
# This test ensures that tasks are being assigned to all local schedulers
# in a roughly equal manner even when the tasks have dependencies.
num_workers = 3
num_local_schedulers = 3
ray.worker._init(start_ray_local=True, num_workers=num_workers, num_local_schedulers=num_local_schedulers)
ray.worker._init(start_ray_local=True, num_workers=num_workers,
num_local_schedulers=num_local_schedulers)
@ray.remote
def f(x):
@@ -1229,5 +1315,6 @@ class SchedulingAlgorithm(unittest.TestCase):
ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)
test/stress_tests.py
+49 -38
View File
@@ -11,6 +11,7 @@ import redis
# Import flatbuffer bindings.
from ray.core.generated.TaskReply import TaskReply
class TaskTests(unittest.TestCase):
def testSubmittingTasks(self):
@@ -93,7 +94,7 @@ class TaskTests(unittest.TestCase):
def f():
return 1
n = 10 ** 4 # TODO(pcm): replace by 10 ** 5 once this is faster
n = 10 ** 4 # TODO(pcm): replace by 10 ** 5 once this is faster.
l = ray.get([f.remote() for _ in range(n)])
self.assertEqual(l, n * [1])
@@ -123,12 +124,14 @@ class TaskTests(unittest.TestCase):
time.sleep(x)
for i in range(1, 5):
x_ids = [g.remote(np.random.uniform(0, i)) for _ in range(2 * num_workers)]
x_ids = [g.remote(np.random.uniform(0, i))
for _ in range(2 * num_workers)]
ray.wait(x_ids, num_returns=len(x_ids))
self.assertTrue(ray.services.all_processes_alive())
ray.worker.cleanup()
class ReconstructionTests(unittest.TestCase):
num_local_schedulers = 1
@@ -144,14 +147,10 @@ class ReconstructionTests(unittest.TestCase):
plasma_addresses = []
objstore_memory = (self.plasma_store_memory // self.num_local_schedulers)
for i in range(self.num_local_schedulers):
plasma_addresses.append(
ray.services.start_objstore(node_ip_address, redis_address,
objstore_memory=objstore_memory)
)
address_info = {
"redis_address": redis_address,
"object_store_addresses": plasma_addresses,
}
plasma_addresses.append(ray.services.start_objstore(
node_ip_address, redis_address, objstore_memory=objstore_memory))
address_info = {"redis_address": redis_address,
"object_store_addresses": plasma_addresses}
# Start the rest of the services in the Ray cluster.
ray.worker._init(address_info=address_info, start_ray_local=True,
@@ -180,7 +179,8 @@ class ReconstructionTests(unittest.TestCase):
# total number of local schedulers to account for NIL_LOCAL_SCHEDULER_ID.
# This is the local scheduler ID associated with the driver task, since it
# is not scheduled by a particular local scheduler.
self.assertEqual(len(set(local_scheduler_ids)), self.num_local_schedulers + 1)
self.assertEqual(len(set(local_scheduler_ids)),
self.num_local_schedulers + 1)
# Clean up the Ray cluster.
ray.worker.cleanup()
@@ -218,7 +218,7 @@ class ReconstructionTests(unittest.TestCase):
num_chunks = 4 * self.num_local_schedulers
chunk = num_objects // num_chunks
for i in range(num_chunks):
values = ray.get(args[i * chunk : (i + 1) * chunk])
values = ray.get(args[i * chunk:(i + 1) * chunk])
del values
def testRecursive(self):
@@ -261,14 +261,14 @@ class ReconstructionTests(unittest.TestCase):
self.assertEqual(value[0], i)
# Get 10 values randomly.
for _ in range(10):
i = np.random.randint(num_objects)
i = np.random.randint(num_objects)
value = ray.get(args[i])
self.assertEqual(value[0], i)
# Get values sequentially, in chunks.
num_chunks = 4 * self.num_local_schedulers
chunk = num_objects // num_chunks
for i in range(num_chunks):
values = ray.get(args[i * chunk : (i + 1) * chunk])
values = ray.get(args[i * chunk:(i + 1) * chunk])
del values
def testMultipleRecursive(self):
@@ -316,7 +316,7 @@ class ReconstructionTests(unittest.TestCase):
self.assertEqual(value[0], i)
# Get 10 values randomly.
for _ in range(10):
i = np.random.randint(num_objects)
i = np.random.randint(num_objects)
value = ray.get(args[i])
self.assertEqual(value[0], i)
@@ -391,7 +391,8 @@ class ReconstructionTests(unittest.TestCase):
return len(errors) >= min_errors
errors = self.wait_for_errors(error_check)
# Make sure all the errors have the correct type.
self.assertTrue(all(error[b"type"] == b"object_hash_mismatch" for error in errors))
self.assertTrue(all(error[b"type"] == b"object_hash_mismatch"
for error in errors))
# Make sure all the errors have the correct function name.
self.assertTrue(all(error[b"data"] == b"__main__.foo" for error in errors))
@@ -462,20 +463,26 @@ class ReconstructionTests(unittest.TestCase):
self.assertEqual(value[0], i)
put_arg_task.remote(size)
def error_check(errors):
return len(errors) > 1
errors = self.wait_for_errors(error_check)
# Make sure all the errors have the correct type.
self.assertTrue(all(error[b"type"] == b"put_reconstruction" for error in errors))
self.assertTrue(all(error[b"data"] == b"__main__.put_arg_task" for error in errors))
self.assertTrue(all(error[b"type"] == b"put_reconstruction"
for error in errors))
self.assertTrue(all(error[b"data"] == b"__main__.put_arg_task"
for error in errors))
put_task.remote(size)
def error_check(errors):
return any(error[b"data"] == b"__main__.put_task" for error in errors)
errors = self.wait_for_errors(error_check)
# Make sure all the errors have the correct type.
self.assertTrue(all(error[b"type"] == b"put_reconstruction" for error in errors))
self.assertTrue(any(error[b"data"] == b"__main__.put_task" for error in errors))
self.assertTrue(all(error[b"type"] == b"put_reconstruction"
for error in errors))
self.assertTrue(any(error[b"data"] == b"__main__.put_task"
for error in errors))
def testDriverPutErrors(self):
# Define the size of one task's return argument so that the combined sum of
@@ -511,11 +518,14 @@ class ReconstructionTests(unittest.TestCase):
# were evicted and whose originating tasks are still running, this
# for-loop should hang on its first iteration and push an error to the
# driver.
ray.worker.global_worker.local_scheduler_client.reconstruct_object(args[0].id())
ray.worker.global_worker.local_scheduler_client.reconstruct_object(
args[0].id())
def error_check(errors):
return len(errors) > 1
errors = self.wait_for_errors(error_check)
self.assertTrue(all(error[b"type"] == b"put_reconstruction" for error in errors))
self.assertTrue(all(error[b"type"] == b"put_reconstruction"
for error in errors))
self.assertTrue(all(error[b"data"] == b"Driver" for error in errors))
@@ -526,26 +536,27 @@ class ReconstructionTestsMultinode(ReconstructionTests):
num_local_schedulers = 4
# NOTE(swang): This test tries to launch 1000 workers and breaks.
#class WorkerPoolTests(unittest.TestCase):
# class WorkerPoolTests(unittest.TestCase):
#
# def tearDown(self):
# ray.worker.cleanup()
# def tearDown(self):
# ray.worker.cleanup()
#
# def testBlockingTasks(self):
# @ray.remote
# def f(i, j):
# return (i, j)
# def testBlockingTasks(self):
# @ray.remote
# def f(i, j):
# return (i, j)
#
# @ray.remote
# def g(i):
# # Each instance of g submits and blocks on the result of another remote
# # task.
# object_ids = [f.remote(i, j) for j in range(10)]
# return ray.get(object_ids)
# @ray.remote
# def g(i):
# # Each instance of g submits and blocks on the result of another remote
# # task.
# object_ids = [f.remote(i, j) for j in range(10)]
# return ray.get(object_ids)
#
# ray.init(num_workers=1)
# ray.get([g.remote(i) for i in range(1000)])
# ray.worker.cleanup()
# ray.init(num_workers=1)
# ray.get([g.remote(i) for i in range(1000)])
# ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)
test/tensorflow_test.py
+27 -34
View File
@@ -2,11 +2,12 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import unittest
import uuid
import tensorflow as tf
import ray
from numpy.testing import assert_almost_equal
import tensorflow as tf
import unittest
import ray
def make_linear_network(w_name=None, b_name=None):
# Define the inputs.
@@ -17,7 +18,9 @@ def make_linear_network(w_name=None, b_name=None):
b = tf.Variable(tf.zeros([1]), name=b_name)
y = w * x_data + b
# Return the loss and weight initializer.
return tf.reduce_mean(tf.square(y - y_data)), tf.global_variables_initializer(), x_data, y_data
return (tf.reduce_mean(tf.square(y - y_data)),
tf.global_variables_initializer(), x_data, y_data)
class NetActor(object):
@@ -40,6 +43,7 @@ class NetActor(object):
def get_weights(self):
return self.values[0].get_weights()
class TrainActor(object):
def __init__(self):
@@ -57,11 +61,13 @@ class TrainActor(object):
def training_step(self, weights):
_, variables, _, sess, grads, _, placeholders = self.values
variables.set_weights(weights)
return sess.run([grad[0] for grad in grads], feed_dict=dict(zip(placeholders, [[1]*100, [2]*100])))
return sess.run([grad[0] for grad in grads],
feed_dict=dict(zip(placeholders, [[1] * 100, [2] * 100])))
def get_weights(self):
return self.values[1].get_weights()
class TensorFlowTest(unittest.TestCase):
def testTensorFlowVariables(self):
@@ -113,9 +119,6 @@ class TensorFlowTest(unittest.TestCase):
net1 = NetActor()
net2 = NetActor()
net_vars1, init1, sess1 = net1.values
net_vars2, init2, sess2 = net2.values
# This is checking that the variable names of the two nets are the same,
# i.e. that the names in the weight dictionaries are the same
net1.values[0].set_weights(net2.values[0].get_weights())
@@ -125,7 +128,8 @@ class TensorFlowTest(unittest.TestCase):
# Test that different networks on the same worker are independent and
# we can get/set their weights without any interaction.
def testNetworksIndependent(self):
# Note we use only one worker to ensure that all of the remote functions run on the same worker.
# Note we use only one worker to ensure that all of the remote functions
# run on the same worker.
ray.init(num_workers=1)
net1 = NetActor()
net2 = NetActor()
@@ -151,15 +155,15 @@ class TensorFlowTest(unittest.TestCase):
ray.worker.cleanup()
# This test creates an additional network on the driver so that the tensorflow
# variables on the driver and the worker differ.
# This test creates an additional network on the driver so that the
# tensorflow variables on the driver and the worker differ.
def testNetworkDriverWorkerIndependent(self):
ray.init(num_workers=1)
# Create a network on the driver locally.
sess1 = tf.Session()
loss1, init1, _, _ = make_linear_network()
net_vars1 = ray.experimental.TensorFlowVariables(loss1, sess1)
ray.experimental.TensorFlowVariables(loss1, sess1)
sess1.run(init1)
net2 = ray.actor(NetActor)()
@@ -194,39 +198,28 @@ class TensorFlowTest(unittest.TestCase):
ray.worker.cleanup()
def testRemoteTrainingLoss(self):
ray.init(num_workers=2)
net = ray.actor(TrainActor)()
loss, variables, _, sess, grads, train, placeholders = TrainActor().values
before_acc = sess.run(loss, feed_dict=dict(zip(placeholders, [[2]*100, [4]*100])))
before_acc = sess.run(loss, feed_dict=dict(zip(placeholders,
[[2] * 100, [4] * 100])))
for _ in range(3):
gradients_list = ray.get([net.training_step(variables.get_weights()) for _ in range(2)])
mean_grads = [sum([gradients[i] for gradients in gradients_list]) / len(gradients_list) for i in range(len(gradients_list[0]))]
feed_dict = {grad[0]: mean_grad for (grad, mean_grad) in zip(grads, mean_grads)}
gradients_list = ray.get([net.training_step(variables.get_weights())
for _ in range(2)])
mean_grads = [sum([gradients[i] for gradients in gradients_list]) /
len(gradients_list) for i in range(len(gradients_list[0]))]
feed_dict = {grad[0]: mean_grad for (grad, mean_grad)
in zip(grads, mean_grads)}
sess.run(train, feed_dict=feed_dict)
after_acc = sess.run(loss, feed_dict=dict(zip(placeholders, [[2]*100, [4]*100])))
after_acc = sess.run(loss, feed_dict=dict(zip(placeholders,
[[2] * 100, [4] * 100])))
self.assertTrue(before_acc < after_acc)
ray.worker.cleanup()
def testVariablesControlDependencies(self):
ray.init(num_workers=1)
# Creates a network and appends a momentum optimizer.
sess = tf.Session()
loss, init, _, _ = make_linear_network()
minimizer = tf.train.MomentumOptimizer(0.9, 0.9).minimize(loss)
net_vars = ray.experimental.TensorFlowVariables(minimizer, sess)
sess.run(init)
# Tests if all variables are properly retrieved, 2 variables and 2 momentum
# variables.
self.assertEqual(len(net_vars.variables.items()), 4)
ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)