mirror of
https://github.com/wassname/ray.git
synced 2026-08-15 12:45:23 +08:00
Update worker.py and services.py to use plasma and the local scheduler. (#19)
* Update worker code and services code to use plasma and the local scheduler. * Cleanups. * Fix bug in which threads were started before the worker mode was set. This caused remote functions to be defined on workers before the worker knew it was in WORKER_MODE. * Fix bug in install-dependencies.sh. * Lengthen timeout in failure_test.py. * Cleanups. * Cleanup services.start_ray_local. * Clean up random name generation. * Cleanups.
This commit is contained in:
Robert Nishihara
committed by
Philipp Moritz
Philipp Moritz
parent
2068587af8
commit
072f442c1f
+21
-286
@@ -1,3 +1,5 @@
|
||||
from __future__ import print_function
|
||||
|
||||
import unittest
|
||||
import ray
|
||||
import numpy as np
|
||||
@@ -142,64 +144,6 @@ class SerializationTest(unittest.TestCase):
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
class ObjStoreTest(unittest.TestCase):
|
||||
|
||||
# Test setting up object stores, transfering data between them and retrieving data to a client
|
||||
def testObjStore(self):
|
||||
node_ip_address = "127.0.0.1"
|
||||
scheduler_address = ray.services.start_ray_local(num_objstores=2, num_workers=0, worker_path=None)
|
||||
ray.connect(node_ip_address, scheduler_address, mode=ray.SCRIPT_MODE)
|
||||
objstore_addresses = [objstore_info["address"] for objstore_info in ray.scheduler_info()["objstores"]]
|
||||
w1 = ray.worker.Worker()
|
||||
w2 = ray.worker.Worker()
|
||||
ray.reusables._cached_reusables = [] # This is a hack to make the test run.
|
||||
ray.connect(node_ip_address, scheduler_address, objstore_address=objstore_addresses[0], mode=ray.SCRIPT_MODE, worker=w1)
|
||||
ray.reusables._cached_reusables = [] # This is a hack to make the test run.
|
||||
ray.connect(node_ip_address, scheduler_address, objstore_address=objstore_addresses[1], mode=ray.SCRIPT_MODE, worker=w2)
|
||||
|
||||
for cls in [Foo, Bar, Baz, Qux, SubQux, Exception, CustomError, Point, NamedTupleExample]:
|
||||
ray.register_class(cls)
|
||||
|
||||
# putting and getting an object shouldn't change it
|
||||
for data in RAY_TEST_OBJECTS:
|
||||
objectid = ray.put(data, w1)
|
||||
result = ray.get(objectid, w1)
|
||||
assert_equal(result, data)
|
||||
|
||||
# putting an object, shipping it to another worker, and getting it shouldn't change it
|
||||
for data in RAY_TEST_OBJECTS:
|
||||
objectid = ray.put(data, w1)
|
||||
result = ray.get(objectid, w2)
|
||||
assert_equal(result, data)
|
||||
|
||||
# putting an object, shipping it to another worker, and getting it shouldn't change it
|
||||
for data in RAY_TEST_OBJECTS:
|
||||
objectid = ray.put(data, w2)
|
||||
result = ray.get(objectid, w1)
|
||||
assert_equal(result, data)
|
||||
|
||||
# This test fails. See https://github.com/ray-project/ray/issues/159.
|
||||
# getting multiple times shouldn't matter
|
||||
# for data in [np.zeros([10, 20]), np.random.normal(size=[45, 25]), np.zeros([10, 20], dtype=np.dtype("float64")), np.zeros([10, 20], dtype=np.dtype("float32")), np.zeros([10, 20], dtype=np.dtype("int64")), np.zeros([10, 20], dtype=np.dtype("int32"))]:
|
||||
# objectid = worker.put(data, w1)
|
||||
# result = worker.get(objectid, w2)
|
||||
# result = worker.get(objectid, w2)
|
||||
# result = worker.get(objectid, w2)
|
||||
# assert_equal(result, data)
|
||||
|
||||
# Getting a buffer after modifying it before it finishes should return updated buffer
|
||||
objectid = ray.libraylib.get_objectid(w1.handle)
|
||||
buf = ray.libraylib.allocate_buffer(w1.handle, objectid, 100)
|
||||
buf[0][0] = 1
|
||||
ray.libraylib.finish_buffer(w1.handle, objectid, buf[1], 0)
|
||||
completedbuffer = ray.libraylib.get_buffer(w1.handle, objectid)
|
||||
self.assertEqual(completedbuffer[0][0], 1)
|
||||
|
||||
# We started multiple drivers manually, so we will disconnect them manually.
|
||||
ray.disconnect(worker=w1)
|
||||
ray.disconnect(worker=w2)
|
||||
ray.worker.cleanup()
|
||||
|
||||
class WorkerTest(unittest.TestCase):
|
||||
|
||||
def testPutGet(self):
|
||||
@@ -233,29 +177,6 @@ class WorkerTest(unittest.TestCase):
|
||||
|
||||
class APITest(unittest.TestCase):
|
||||
|
||||
def testPassingArgumentsByValue(self):
|
||||
ray.init(start_ray_local=True, num_workers=0)
|
||||
|
||||
# The types that can be passed by value are defined by
|
||||
# is_argument_serializable in serialization.py.
|
||||
class Foo(object):
|
||||
pass
|
||||
CAN_PASS_BY_VALUE = [1, 1L, 1.0, True, False, None, [1L, 1.0, True, None],
|
||||
([1, 2, 3], {False: [1.0, u"hi", ()]}), 100 * ["a"]]
|
||||
CANNOT_PASS_BY_VALUE = [int, np.int64(0), np.float64(0), Foo(), [Foo()],
|
||||
(Foo()), {0: Foo()}, [[[int]]], 101 * [1],
|
||||
np.zeros(10)]
|
||||
|
||||
for obj in CAN_PASS_BY_VALUE:
|
||||
self.assertTrue(ray.serialization.is_argument_serializable(obj))
|
||||
self.assertEqual(obj, ray.serialization.deserialize_argument(ray.serialization.serialize_argument_if_possible(obj)))
|
||||
|
||||
for obj in CANNOT_PASS_BY_VALUE:
|
||||
self.assertFalse(ray.serialization.is_argument_serializable(obj))
|
||||
self.assertEqual(None, ray.serialization.serialize_argument_if_possible(obj))
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
def testRegisterClass(self):
|
||||
ray.init(start_ray_local=True, num_workers=0)
|
||||
|
||||
@@ -328,11 +249,7 @@ class APITest(unittest.TestCase):
|
||||
reload(test_functions)
|
||||
ray.init(start_ray_local=True, num_workers=1)
|
||||
|
||||
test_functions.no_op.remote()
|
||||
time.sleep(0.2)
|
||||
task_info = ray.task_info()
|
||||
self.assertEqual(len(task_info["failed_tasks"]), 0)
|
||||
self.assertEqual(len(task_info["running_tasks"]), 0)
|
||||
ray.get(test_functions.no_op.remote())
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
@@ -400,22 +317,22 @@ class APITest(unittest.TestCase):
|
||||
|
||||
objectids = [f.remote(1.0), f.remote(0.5), f.remote(0.5), f.remote(0.5)]
|
||||
ready_ids, remaining_ids = ray.wait(objectids)
|
||||
self.assertTrue(len(ready_ids) == 1)
|
||||
self.assertTrue(len(remaining_ids) == 3)
|
||||
self.assertEqual(len(ready_ids), 1)
|
||||
self.assertEqual(len(remaining_ids), 3)
|
||||
ready_ids, remaining_ids = ray.wait(objectids, num_returns=4)
|
||||
self.assertEqual(ready_ids, objectids)
|
||||
self.assertEqual(set(ready_ids), set([object_id.id() for object_id in objectids]))
|
||||
self.assertEqual(remaining_ids, [])
|
||||
|
||||
objectids = [f.remote(0.5), f.remote(0.5), f.remote(0.5), f.remote(0.5)]
|
||||
start_time = time.time()
|
||||
ready_ids, remaining_ids = ray.wait(objectids, timeout=1.75, num_returns=4)
|
||||
self.assertTrue(time.time() - start_time < 2)
|
||||
ready_ids, remaining_ids = ray.wait(objectids, timeout=1750, num_returns=4)
|
||||
self.assertLess(time.time() - start_time, 2)
|
||||
self.assertEqual(len(ready_ids), 3)
|
||||
self.assertEqual(len(remaining_ids), 1)
|
||||
ray.wait(objectids)
|
||||
objectids = [f.remote(1.0), f.remote(0.5), f.remote(0.5), f.remote(0.5)]
|
||||
start_time = time.time()
|
||||
ready_ids, remaining_ids = ray.wait(objectids, timeout=5)
|
||||
ready_ids, remaining_ids = ray.wait(objectids, timeout=5000)
|
||||
self.assertTrue(time.time() - start_time < 5)
|
||||
self.assertEqual(len(ready_ids), 1)
|
||||
self.assertEqual(len(remaining_ids), 3)
|
||||
@@ -504,150 +421,6 @@ class APITest(unittest.TestCase):
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
def testComputationGraph(self):
|
||||
ray.init(start_ray_local=True, num_workers=1)
|
||||
|
||||
@ray.remote
|
||||
def f(x):
|
||||
return x
|
||||
@ray.remote
|
||||
def g(x, y):
|
||||
return x, y
|
||||
a = f.remote(1)
|
||||
b = f.remote(1)
|
||||
c = g.remote(a, b)
|
||||
c = g.remote(a, 1)
|
||||
# Make sure that we can produce a computation_graph visualization.
|
||||
ray.visualize_computation_graph(view=False)
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
class ReferenceCountingTest(unittest.TestCase):
|
||||
|
||||
def testDeallocation(self):
|
||||
reload(test_functions)
|
||||
for module in [ra.core, ra.random, ra.linalg, da.core, da.random, da.linalg]:
|
||||
reload(module)
|
||||
ray.init(start_ray_local=True, num_workers=1)
|
||||
|
||||
def check_not_deallocated(object_ids):
|
||||
reference_counts = ray.scheduler_info()["reference_counts"]
|
||||
for object_id in object_ids:
|
||||
self.assertGreater(reference_counts[object_id.id], 0)
|
||||
|
||||
def check_everything_deallocated():
|
||||
reference_counts = ray.scheduler_info()["reference_counts"]
|
||||
self.assertEqual(reference_counts, len(reference_counts) * [-1])
|
||||
|
||||
z = da.zeros.remote([da.BLOCK_SIZE, 2 * da.BLOCK_SIZE])
|
||||
time.sleep(0.1)
|
||||
objectid_val = z.id
|
||||
time.sleep(0.1)
|
||||
check_not_deallocated([z])
|
||||
del z
|
||||
time.sleep(0.1)
|
||||
check_everything_deallocated()
|
||||
|
||||
x = ra.zeros.remote([10, 10])
|
||||
y = ra.zeros.remote([10, 10])
|
||||
z = ra.dot.remote(x, y)
|
||||
objectid_val = x.id
|
||||
time.sleep(0.1)
|
||||
check_not_deallocated([x, y, z])
|
||||
del x
|
||||
time.sleep(0.1)
|
||||
check_not_deallocated([y, z])
|
||||
del y
|
||||
time.sleep(0.1)
|
||||
check_not_deallocated([z])
|
||||
del z
|
||||
time.sleep(0.1)
|
||||
check_everything_deallocated()
|
||||
|
||||
z = da.zeros.remote([4 * da.BLOCK_SIZE])
|
||||
time.sleep(0.1)
|
||||
check_not_deallocated(ray.get(z).objectids.tolist())
|
||||
del z
|
||||
time.sleep(0.1)
|
||||
check_everything_deallocated()
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
def testGet(self):
|
||||
ray.init(start_ray_local=True, num_workers=3)
|
||||
|
||||
for cls in [Foo, Bar, Baz, Qux, SubQux, Exception, CustomError, Point, NamedTupleExample]:
|
||||
ray.register_class(cls)
|
||||
|
||||
# Remote objects should be deallocated when the corresponding ObjectID goes
|
||||
# out of scope, and all results of ray.get called on the ID go out of scope.
|
||||
for val in RAY_TEST_OBJECTS:
|
||||
x = ray.put(val)
|
||||
objectid = x.id
|
||||
xval = ray.get(x)
|
||||
del x, xval
|
||||
self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], -1)
|
||||
|
||||
# Remote objects that do not contain numpy arrays should be deallocated when
|
||||
# the corresponding ObjectID goes out of scope, even if ray.get has been
|
||||
# called on the ObjectID.
|
||||
for val in [True, False, None, 1, 1.0, 1L, "hi", u"hi", [1, 2, 3], (1, 2, 3), [(), {(): ()}]]:
|
||||
x = ray.put(val)
|
||||
objectid = x.id
|
||||
xval = ray.get(x)
|
||||
del x
|
||||
self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], -1)
|
||||
|
||||
# Remote objects that contain numpy arrays should not be deallocated when
|
||||
# the corresponding ObjectID goes out of scope, if ray.get has been called
|
||||
# on the ObjectID and the result of that call is still in scope.
|
||||
for val in [np.zeros(10), [np.zeros(10)], (((np.zeros(10)),),), {(): np.zeros(10)}, [1, 2, 3, np.zeros(1)]]:
|
||||
x = ray.put(val)
|
||||
objectid = x.id
|
||||
xval = ray.get(x)
|
||||
del x
|
||||
self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1)
|
||||
|
||||
# Getting an object multiple times should not be a problem. And the remote
|
||||
# object should not be deallocated until both of the results are out of scope.
|
||||
for val in [np.zeros(10), [np.zeros(10)], (((np.zeros(10)),),), {(): np.zeros(10)}, [1, 2, 3, np.zeros(1)]]:
|
||||
x = ray.put(val)
|
||||
objectid = x.id
|
||||
xval1 = ray.get(x)
|
||||
xval2 = ray.get(x)
|
||||
del xval1
|
||||
# Make sure we can still access xval2.
|
||||
xval2
|
||||
del xval2
|
||||
self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1)
|
||||
xval3 = ray.get(x)
|
||||
xval4 = ray.get(x)
|
||||
xval5 = ray.get(x)
|
||||
del x
|
||||
del xval4, xval5
|
||||
# Make sure we can still access xval3.
|
||||
xval3
|
||||
self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1)
|
||||
del xval3
|
||||
self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], -1)
|
||||
|
||||
# Getting an object multiple times and assigning it to the same name should
|
||||
# work. This was a problem in https://github.com/ray-project/ray/issues/159.
|
||||
for val in [np.zeros(10), [np.zeros(10)], (((np.zeros(10)),),), {(): np.zeros(10)}, [1, 2, 3, np.zeros(1)]]:
|
||||
x = ray.put(val)
|
||||
objectid = x.id
|
||||
xval = ray.get(x)
|
||||
xval = ray.get(x)
|
||||
xval = ray.get(x)
|
||||
xval = ray.get(x)
|
||||
self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1)
|
||||
del x
|
||||
self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], 1)
|
||||
del xval
|
||||
self.assertEqual(ray.scheduler_info()["reference_counts"][objectid], -1)
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
class PythonModeTest(unittest.TestCase):
|
||||
|
||||
def testPythonMode(self):
|
||||
@@ -712,18 +485,18 @@ class PythonModeTest(unittest.TestCase):
|
||||
|
||||
class PythonCExtensionTest(unittest.TestCase):
|
||||
|
||||
def testReferenceCountNone(self):
|
||||
ray.init(start_ray_local=True, num_workers=1)
|
||||
|
||||
# Make sure that we aren't accidentally messing up Python's reference counts.
|
||||
@ray.remote
|
||||
def f():
|
||||
return sys.getrefcount(None)
|
||||
first_count = ray.get(f.remote())
|
||||
second_count = ray.get(f.remote())
|
||||
self.assertEqual(first_count, second_count)
|
||||
|
||||
ray.worker.cleanup()
|
||||
# def testReferenceCountNone(self):
|
||||
# ray.init(start_ray_local=True, num_workers=1)
|
||||
#
|
||||
# # Make sure that we aren't accidentally messing up Python's reference counts.
|
||||
# @ray.remote
|
||||
# def f():
|
||||
# return sys.getrefcount(None)
|
||||
# first_count = ray.get(f.remote())
|
||||
# second_count = ray.get(f.remote())
|
||||
# self.assertEqual(first_count, second_count)
|
||||
#
|
||||
# ray.worker.cleanup()
|
||||
|
||||
def testReferenceCountTrue(self):
|
||||
ray.init(start_ray_local=True, num_workers=1)
|
||||
@@ -867,43 +640,5 @@ class ReusablesTest(unittest.TestCase):
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
class ClusterAttachingTest(unittest.TestCase):
|
||||
|
||||
def testAttachingToCluster(self):
|
||||
node_ip_address = "127.0.0.1"
|
||||
scheduler_port = np.random.randint(40000, 50000)
|
||||
scheduler_address = "{}:{}".format(node_ip_address, scheduler_port)
|
||||
ray.services.start_scheduler(scheduler_address, cleanup=True)
|
||||
time.sleep(0.1)
|
||||
ray.services.start_node(scheduler_address, node_ip_address, num_workers=1, cleanup=True)
|
||||
|
||||
ray.init(node_ip_address=node_ip_address, scheduler_address=scheduler_address)
|
||||
|
||||
@ray.remote
|
||||
def f(x):
|
||||
return x + 1
|
||||
self.assertEqual(ray.get(f.remote(0)), 1)
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
def testAttachingToClusterWithMultipleObjectStores(self):
|
||||
node_ip_address = "127.0.0.1"
|
||||
scheduler_port = np.random.randint(40000, 50000)
|
||||
scheduler_address = "{}:{}".format(node_ip_address, scheduler_port)
|
||||
ray.services.start_scheduler(scheduler_address, cleanup=True)
|
||||
time.sleep(0.1)
|
||||
ray.services.start_node(scheduler_address, node_ip_address, num_workers=5, cleanup=True)
|
||||
ray.services.start_node(scheduler_address, node_ip_address, num_workers=5, cleanup=True)
|
||||
ray.services.start_node(scheduler_address, node_ip_address, num_workers=5, cleanup=True)
|
||||
|
||||
ray.init(node_ip_address=node_ip_address, scheduler_address=scheduler_address)
|
||||
|
||||
@ray.remote
|
||||
def f(x):
|
||||
return x + 1
|
||||
self.assertEqual(ray.get(f.remote(0)), 1)
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
||||
Reference in New Issue
Block a user