wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-09 11:32:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

push/pull -> put/get

Browse Source
This commit is contained in:
Robert Nishihara
2016-06-23 12:58:48 -07:00
parent fe918a9f8b
commit 188569be37
22 changed files with 203 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/arrays_test.py
+37 -37
View File
@@ -19,27 +19,27 @@ class ArraysSingleTest(unittest.TestCase):
# test eye
ref = ra.eye(3)
val = ray.pull(ref)
val = ray.get(ref)
self.assertTrue(np.alltrue(val == np.eye(3)))
# test zeros
ref = ra.zeros([3, 4, 5])
val = ray.pull(ref)
val = ray.get(ref)
self.assertTrue(np.alltrue(val == np.zeros([3, 4, 5])))
# test qr - pass by value
val_a = np.random.normal(size=[10, 13])
ref_q, ref_r = ra.linalg.qr(val_a)
val_q = ray.pull(ref_q)
val_r = ray.pull(ref_r)
val_q = ray.get(ref_q)
val_r = ray.get(ref_r)
self.assertTrue(np.allclose(np.dot(val_q, val_r), val_a))
# test qr - pass by objref
a = ra.random.normal([10, 13])
ref_q, ref_r = ra.linalg.qr(a)
val_a = ray.pull(a)
val_q = ray.pull(ref_q)
val_r = ray.pull(ref_r)
val_a = ray.get(a)
val_q = ray.get(ref_q)
val_r = ray.get(ref_r)
self.assertTrue(np.allclose(np.dot(val_q, val_r), val_a))
services.cleanup()
@@ -50,7 +50,7 @@ class ArraysDistTest(unittest.TestCase):
[w] = services.start_singlenode_cluster(return_drivers=True)
x = da.DistArray()
x.construct([2, 3, 4], np.array([[[ray.push(0, w)]]]))
x.construct([2, 3, 4], np.array([[[ray.put(0, w)]]]))
capsule, _ = serialization.serialize(w.handle, x) # TODO(rkn): THIS REQUIRES A WORKER_HANDLE
y = serialization.deserialize(w.handle, capsule) # TODO(rkn): THIS REQUIRES A WORKER_HANDLE
self.assertEqual(x.shape, y.shape)
@@ -75,25 +75,25 @@ class ArraysDistTest(unittest.TestCase):
services.start_singlenode_cluster(return_drivers=False, num_objstores=2, num_workers_per_objstore=5, worker_path=worker_path)
x = da.zeros([9, 25, 51], "float")
self.assertTrue(np.alltrue(ray.pull(da.assemble(x)) == np.zeros([9, 25, 51])))
self.assertTrue(np.alltrue(ray.get(da.assemble(x)) == np.zeros([9, 25, 51])))
x = da.ones([11, 25, 49], dtype_name="float")
self.assertTrue(np.alltrue(ray.pull(da.assemble(x)) == np.ones([11, 25, 49])))
self.assertTrue(np.alltrue(ray.get(da.assemble(x)) == np.ones([11, 25, 49])))
x = da.random.normal([11, 25, 49])
y = da.copy(x)
self.assertTrue(np.alltrue(ray.pull(da.assemble(x)) == ray.pull(da.assemble(y))))
self.assertTrue(np.alltrue(ray.get(da.assemble(x)) == ray.get(da.assemble(y))))
x = da.eye(25, dtype_name="float")
self.assertTrue(np.alltrue(ray.pull(da.assemble(x)) == np.eye(25)))
self.assertTrue(np.alltrue(ray.get(da.assemble(x)) == np.eye(25)))
x = da.random.normal([25, 49])
y = da.triu(x)
self.assertTrue(np.alltrue(ray.pull(da.assemble(y)) == np.triu(ray.pull(da.assemble(x)))))
self.assertTrue(np.alltrue(ray.get(da.assemble(y)) == np.triu(ray.get(da.assemble(x)))))
x = da.random.normal([25, 49])
y = da.tril(x)
self.assertTrue(np.alltrue(ray.pull(da.assemble(y)) == np.tril(ray.pull(da.assemble(x)))))
self.assertTrue(np.alltrue(ray.get(da.assemble(y)) == np.tril(ray.get(da.assemble(x)))))
x = da.random.normal([25, 49])
y = da.random.normal([49, 18])
@@ -101,42 +101,42 @@ class ArraysDistTest(unittest.TestCase):
w = da.assemble(z)
u = da.assemble(x)
v = da.assemble(y)
np.allclose(ray.pull(w), np.dot(ray.pull(u), ray.pull(v)))
self.assertTrue(np.allclose(ray.pull(w), np.dot(ray.pull(u), ray.pull(v))))
np.allclose(ray.get(w), np.dot(ray.get(u), ray.get(v)))
self.assertTrue(np.allclose(ray.get(w), np.dot(ray.get(u), ray.get(v))))
# test add
x = da.random.normal([23, 42])
y = da.random.normal([23, 42])
z = da.add(x, y)
self.assertTrue(np.allclose(ray.pull(da.assemble(z)), ray.pull(da.assemble(x)) + ray.pull(da.assemble(y))))
self.assertTrue(np.allclose(ray.get(da.assemble(z)), ray.get(da.assemble(x)) + ray.get(da.assemble(y))))
# test subtract
x = da.random.normal([33, 40])
y = da.random.normal([33, 40])
z = da.subtract(x, y)
self.assertTrue(np.allclose(ray.pull(da.assemble(z)), ray.pull(da.assemble(x)) - ray.pull(da.assemble(y))))
self.assertTrue(np.allclose(ray.get(da.assemble(z)), ray.get(da.assemble(x)) - ray.get(da.assemble(y))))
# test transpose
x = da.random.normal([234, 432])
y = da.transpose(x)
self.assertTrue(np.alltrue(ray.pull(da.assemble(x)).T == ray.pull(da.assemble(y))))
self.assertTrue(np.alltrue(ray.get(da.assemble(x)).T == ray.get(da.assemble(y))))
# test numpy_to_dist
x = da.random.normal([23, 45])
y = da.assemble(x)
z = da.numpy_to_dist(y)
w = da.assemble(z)
self.assertTrue(np.alltrue(ray.pull(da.assemble(x)) == ray.pull(da.assemble(z))))
self.assertTrue(np.alltrue(ray.pull(y) == ray.pull(w)))
self.assertTrue(np.alltrue(ray.get(da.assemble(x)) == ray.get(da.assemble(z))))
self.assertTrue(np.alltrue(ray.get(y) == ray.get(w)))
# test da.tsqr
for shape in [[123, da.BLOCK_SIZE], [7, da.BLOCK_SIZE], [da.BLOCK_SIZE, da.BLOCK_SIZE], [da.BLOCK_SIZE, 7], [10 * da.BLOCK_SIZE, da.BLOCK_SIZE]]:
x = da.random.normal(shape)
K = min(shape)
q, r = da.linalg.tsqr(x)
x_val = ray.pull(da.assemble(x))
q_val = ray.pull(da.assemble(q))
r_val = ray.pull(r)
x_val = ray.get(da.assemble(x))
q_val = ray.get(da.assemble(q))
r_val = ray.get(r)
self.assertTrue(r_val.shape == (K, shape[1]))
self.assertTrue(np.alltrue(r_val == np.triu(r_val)))
self.assertTrue(np.allclose(x_val, np.dot(q_val, r_val)))
@@ -150,11 +150,11 @@ class ArraysDistTest(unittest.TestCase):
m = ra.random.normal([d1, d2])
q, r = ra.linalg.qr(m)
l, u, s = da.linalg.modified_lu(da.numpy_to_dist(q))
q_val = ray.pull(q)
r_val = ray.pull(r)
l_val = ray.pull(da.assemble(l))
u_val = ray.pull(u)
s_val = ray.pull(s)
q_val = ray.get(q)
r_val = ray.get(r)
l_val = ray.get(da.assemble(l))
u_val = ray.get(u)
s_val = ray.get(s)
s_mat = np.zeros((d1, d2))
for i in range(len(s_val)):
s_mat[i, i] = s_val[i]
@@ -170,11 +170,11 @@ class ArraysDistTest(unittest.TestCase):
print "testing dist_tsqr_hr with d1 = " + str(d1) + ", d2 = " + str(d2)
a = da.random.normal([d1, d2])
y, t, y_top, r = da.linalg.tsqr_hr(a)
a_val = ray.pull(da.assemble(a))
y_val = ray.pull(da.assemble(y))
t_val = ray.pull(t)
y_top_val = ray.pull(y_top)
r_val = ray.pull(r)
a_val = ray.get(da.assemble(a))
y_val = ray.get(da.assemble(y))
t_val = ray.get(t)
y_top_val = ray.get(y_top)
r_val = ray.get(r)
tall_eye = np.zeros((d1, min(d1, d2)))
np.fill_diagonal(tall_eye, 1)
q = tall_eye - np.dot(y_val, np.dot(t_val, y_top_val.T))
@@ -189,9 +189,9 @@ class ArraysDistTest(unittest.TestCase):
a = da.random.normal([d1, d2])
K = min(d1, d2)
q, r = da.linalg.qr(a)
a_val = ray.pull(da.assemble(a))
q_val = ray.pull(da.assemble(q))
r_val = ray.pull(da.assemble(r))
a_val = ray.get(da.assemble(a))
q_val = ray.get(da.assemble(q))
r_val = ray.get(da.assemble(r))
self.assertTrue(q_val.shape == (d1, K))
self.assertTrue(r_val.shape == (K, d2))
self.assertTrue(np.allclose(np.dot(q_val.T, q_val), np.eye(K)))
test/microbenchmarks.py
+6 -6
View File
@@ -45,33 +45,33 @@ class MicroBenchmarkTest(unittest.TestCase):
print " worst: {}".format(elapsed_times[999])
self.assertTrue(average_elapsed_time < 0.002) # should take 0.001
# measure the time required to submit a remote task to the scheduler and pull the result
# measure the time required to submit a remote task to the scheduler and get the result
elapsed_times = []
for _ in range(1000):
start_time = time.time()
x = test_functions.trivial_function()
ray.pull(x)
ray.get(x)
end_time = time.time()
elapsed_times.append(end_time - start_time)
elapsed_times = np.sort(elapsed_times)
average_elapsed_time = sum(elapsed_times) / 1000
print "Time required to submit a trivial function call and pull the result:"
print "Time required to submit a trivial function call and get the result:"
print " Average: {}".format(average_elapsed_time)
print " 90th percentile: {}".format(elapsed_times[900])
print " 99th percentile: {}".format(elapsed_times[990])
print " worst: {}".format(elapsed_times[999])
self.assertTrue(average_elapsed_time < 0.002) # should take 0.0013
# measure the time required to do do a push
# measure the time required to do do a put
elapsed_times = []
for _ in range(1000):
start_time = time.time()
ray.push(1)
ray.put(1)
end_time = time.time()
elapsed_times.append(end_time - start_time)
elapsed_times = np.sort(elapsed_times)
average_elapsed_time = sum(elapsed_times) / 1000
print "Time required to push an int:"
print "Time required to put an int:"
print " Average: {}".format(average_elapsed_time)
print " 90th percentile: {}".format(elapsed_times[900])
print " 99th percentile: {}".format(elapsed_times[990])
test/runtest.py
+64 -64
View File
@@ -59,10 +59,10 @@ class SerializationTest(unittest.TestCase):
self.numpyTypeTest(w, 'float32')
self.numpyTypeTest(w, 'float64')
ref0 = ray.push(0, w)
ref1 = ray.push(0, w)
ref2 = ray.push(0, w)
ref3 = ray.push(0, w)
ref0 = ray.put(0, w)
ref1 = ray.put(0, w)
ref2 = ray.put(0, w)
ref3 = ray.put(0, w)
a = np.array([[ref0, ref1], [ref2, ref3]])
capsule, _ = serialization.serialize(w.handle, a)
@@ -82,45 +82,45 @@ class ObjStoreTest(unittest.TestCase):
def testObjStore(self):
[w1, w2] = services.start_singlenode_cluster(return_drivers=True, num_objstores=2, num_workers_per_objstore=0)
# pushing and pulling an object shouldn't change it
# putting and getting an object shouldn't change it
for data in ["h", "h" * 10000, 0, 0.0]:
objref = ray.push(data, w1)
result = ray.pull(objref, w1)
objref = ray.put(data, w1)
result = ray.get(objref, w1)
self.assertEqual(result, data)
# pushing an object, shipping it to another worker, and pulling it shouldn't change it
# putting an object, shipping it to another worker, and getting it shouldn't change it
for data in ["h", "h" * 10000, 0, 0.0, [1, 2, 3, "a", (1, 2)], ("a", ("b", 3))]:
objref = worker.push(data, w1)
result = worker.pull(objref, w2)
objref = worker.put(data, w1)
result = worker.get(objref, w2)
self.assertEqual(result, data)
# pushing an array, shipping it to another worker, and pulling it shouldn't change it
# putting an array, shipping it to another worker, and getting it shouldn't change it
for data in [np.zeros([10, 20]), np.random.normal(size=[45, 25])]:
objref = worker.push(data, w1)
result = worker.pull(objref, w2)
objref = worker.put(data, w1)
result = worker.get(objref, w2)
self.assertTrue(np.alltrue(result == data))
"""
# pulling multiple times shouldn't matter
# getting multiple times shouldn't matter
for data in [np.zeros([10, 20]), np.random.normal(size=[45, 25]), np.zeros([10, 20], dtype=np.dtype("float64")), np.zeros([10, 20], dtype=np.dtype("float32")), np.zeros([10, 20], dtype=np.dtype("int64")), np.zeros([10, 20], dtype=np.dtype("int32"))]:
objref = worker.push(data, w1)
result = worker.pull(objref, w2)
result = worker.pull(objref, w2)
result = worker.pull(objref, w2)
objref = worker.put(data, w1)
result = worker.get(objref, w2)
result = worker.get(objref, w2)
result = worker.get(objref, w2)
self.assertTrue(np.alltrue(result == data))
"""
# shipping a numpy array inside something else should be fine
data = ("a", np.random.normal(size=[10, 10]))
objref = worker.push(data, w1)
result = worker.pull(objref, w2)
objref = worker.put(data, w1)
result = worker.get(objref, w2)
self.assertTrue(data[0] == result[0])
self.assertTrue(np.alltrue(data[1] == result[1]))
# shipping a numpy array inside something else should be fine
data = ["a", np.random.normal(size=[10, 10])]
objref = worker.push(data, w1)
result = worker.pull(objref, w2)
objref = worker.put(data, w1)
result = worker.get(objref, w2)
self.assertTrue(data[0] == result[0])
self.assertTrue(np.alltrue(data[1] == result[1]))
@@ -128,31 +128,31 @@ class ObjStoreTest(unittest.TestCase):
class WorkerTest(unittest.TestCase):
def testPushPull(self):
def testPutGet(self):
[w] = services.start_singlenode_cluster(return_drivers=True)
for i in range(100):
value_before = i * 10 ** 6
objref = ray.push(value_before, w)
value_after = ray.pull(objref, w)
objref = ray.put(value_before, w)
value_after = ray.get(objref, w)
self.assertEqual(value_before, value_after)
for i in range(100):
value_before = i * 10 ** 6 * 1.0
objref = ray.push(value_before, w)
value_after = ray.pull(objref, w)
objref = ray.put(value_before, w)
value_after = ray.get(objref, w)
self.assertEqual(value_before, value_after)
for i in range(100):
value_before = "h" * i
objref = ray.push(value_before, w)
value_after = ray.pull(objref, w)
objref = ray.put(value_before, w)
value_after = ray.get(objref, w)
self.assertEqual(value_before, value_after)
for i in range(100):
value_before = [1] * i
objref = ray.push(value_before, w)
value_after = ray.pull(objref, w)
objref = ray.put(value_before, w)
value_after = ray.get(objref, w)
self.assertEqual(value_before, value_after)
services.cleanup()
@@ -164,11 +164,11 @@ class APITest(unittest.TestCase):
[w] = services.start_singlenode_cluster(return_drivers=True, num_workers_per_objstore=3, worker_path=worker_path)
objref = w.submit_task("test_functions.test_alias_f", [])
self.assertTrue(np.alltrue(ray.pull(objref[0], w) == np.ones([3, 4, 5])))
self.assertTrue(np.alltrue(ray.get(objref[0], w) == np.ones([3, 4, 5])))
objref = w.submit_task("test_functions.test_alias_g", [])
self.assertTrue(np.alltrue(ray.pull(objref[0], w) == np.ones([3, 4, 5])))
self.assertTrue(np.alltrue(ray.get(objref[0], w) == np.ones([3, 4, 5])))
objref = w.submit_task("test_functions.test_alias_h", [])
self.assertTrue(np.alltrue(ray.pull(objref[0], w) == np.ones([3, 4, 5])))
self.assertTrue(np.alltrue(ray.get(objref[0], w) == np.ones([3, 4, 5])))
services.cleanup()
@@ -177,35 +177,35 @@ class APITest(unittest.TestCase):
services.start_singlenode_cluster(return_drivers=False, num_workers_per_objstore=1, worker_path=worker_path)
x = test_functions.keyword_fct1(1)
self.assertEqual(ray.pull(x), "1 hello")
self.assertEqual(ray.get(x), "1 hello")
x = test_functions.keyword_fct1(1, "hi")
self.assertEqual(ray.pull(x), "1 hi")
self.assertEqual(ray.get(x), "1 hi")
x = test_functions.keyword_fct1(1, b="world")
self.assertEqual(ray.pull(x), "1 world")
self.assertEqual(ray.get(x), "1 world")
x = test_functions.keyword_fct2(a="w", b="hi")
self.assertEqual(ray.pull(x), "w hi")
self.assertEqual(ray.get(x), "w hi")
x = test_functions.keyword_fct2(b="hi", a="w")
self.assertEqual(ray.pull(x), "w hi")
self.assertEqual(ray.get(x), "w hi")
x = test_functions.keyword_fct2(a="w")
self.assertEqual(ray.pull(x), "w world")
self.assertEqual(ray.get(x), "w world")
x = test_functions.keyword_fct2(b="hi")
self.assertEqual(ray.pull(x), "hello hi")
self.assertEqual(ray.get(x), "hello hi")
x = test_functions.keyword_fct2("w")
self.assertEqual(ray.pull(x), "w world")
self.assertEqual(ray.get(x), "w world")
x = test_functions.keyword_fct2("w", "hi")
self.assertEqual(ray.pull(x), "w hi")
self.assertEqual(ray.get(x), "w hi")
x = test_functions.keyword_fct3(0, 1, c="w", d="hi")
self.assertEqual(ray.pull(x), "0 1 w hi")
self.assertEqual(ray.get(x), "0 1 w hi")
x = test_functions.keyword_fct3(0, 1, d="hi", c="w")
self.assertEqual(ray.pull(x), "0 1 w hi")
self.assertEqual(ray.get(x), "0 1 w hi")
x = test_functions.keyword_fct3(0, 1, c="w")
self.assertEqual(ray.pull(x), "0 1 w world")
self.assertEqual(ray.get(x), "0 1 w world")
x = test_functions.keyword_fct3(0, 1, d="hi")
self.assertEqual(ray.pull(x), "0 1 hello hi")
self.assertEqual(ray.get(x), "0 1 hello hi")
x = test_functions.keyword_fct3(0, 1)
self.assertEqual(ray.pull(x), "0 1 hello world")
self.assertEqual(ray.get(x), "0 1 hello world")
services.cleanup()
@@ -214,9 +214,9 @@ class APITest(unittest.TestCase):
services.start_singlenode_cluster(return_drivers=False, num_workers_per_objstore=1, worker_path=worker_path)
x = test_functions.varargs_fct1(0, 1, 2)
self.assertEqual(ray.pull(x), "0 1 2")
self.assertEqual(ray.get(x), "0 1 2")
x = test_functions.varargs_fct2(0, 1, 2)
self.assertEqual(ray.pull(x), "1 2")
self.assertEqual(ray.get(x), "1 2")
self.assertTrue(test_functions.kwargs_exception_thrown)
self.assertTrue(test_functions.varargs_and_kwargs_exception_thrown)
@@ -241,14 +241,14 @@ class TaskStatusTest(unittest.TestCase):
self.assertTrue(task['operationid'] not in task_ids)
task_ids.add(task['operationid'])
def check_pull_deallocated(data):
x = ray.push(data)
ray.pull(x)
def check_get_deallocated(data):
x = ray.put(data)
ray.get(x)
return x.val
def check_pull_not_deallocated(data):
x = ray.push(data)
y = ray.pull(x)
def check_get_not_deallocated(data):
x = ray.put(data)
y = ray.get(x)
return y, x.val
class ReferenceCountingTest(unittest.TestCase):
@@ -258,7 +258,7 @@ class ReferenceCountingTest(unittest.TestCase):
services.start_singlenode_cluster(return_drivers=False, num_workers_per_objstore=3, worker_path=worker_path)
x = test_functions.test_alias_f()
ray.pull(x)
ray.get(x)
time.sleep(0.1)
objref_val = x.val
self.assertTrue(ray.scheduler_info()["reference_counts"][objref_val] == 1)
@@ -267,7 +267,7 @@ class ReferenceCountingTest(unittest.TestCase):
self.assertTrue(ray.scheduler_info()["reference_counts"][objref_val] == -1) # -1 indicates deallocated
y = test_functions.test_alias_h()
ray.pull(y)
ray.get(y)
time.sleep(0.1)
objref_val = y.val
self.assertTrue(ray.scheduler_info()["reference_counts"][objref_val:(objref_val + 3)] == [1, 0, 0])
@@ -303,22 +303,22 @@ class ReferenceCountingTest(unittest.TestCase):
services.cleanup()
def testPull(self):
def testGet(self):
worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_worker.py")
services.start_singlenode_cluster(return_drivers=False, num_workers_per_objstore=3, worker_path=worker_path)
for val in RAY_TEST_OBJECTS + [np.zeros((2, 2)), UserDefinedType()]:
objref_val = check_pull_deallocated(val)
objref_val = check_get_deallocated(val)
self.assertEqual(ray.scheduler_info()["reference_counts"][objref_val], -1)
if not isinstance(val, bool) and val is not None:
x, objref_val = check_pull_not_deallocated(val)
x, objref_val = check_get_not_deallocated(val)
self.assertEqual(ray.scheduler_info()["reference_counts"][objref_val], 1)
services.cleanup()
@unittest.expectedFailure
def testPullFailing(self):
def testGetFailing(self):
worker_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "test_worker.py")
services.start_singlenode_cluster(return_drivers=False, num_workers_per_objstore=3, worker_path=worker_path)
@@ -326,10 +326,10 @@ class ReferenceCountingTest(unittest.TestCase):
# refcounts and therefore cannot keep the refcount up
# (see 5281bd414f6b404f61e1fe25ec5f6651defee206).
# The resulting behavior is still correct however because True, False and
# None are returned by pull "by value" and therefore can be reclaimed from
# None are returned by get "by value" and therefore can be reclaimed from
# the object store safely.
for val in [True, False, None]:
x, objref_val = check_pull_not_deallocated(val)
x, objref_val = check_get_not_deallocated(val)
self.assertEqual(ray.scheduler_info()["reference_counts"][objref_val], 1)
services.cleanup()