mirror of
https://github.com/wassname/ray.git
synced 2026-07-07 06:55:53 +08:00
Move global state API out of global_state object. (#4857)
This commit is contained in:
committed by
Philipp Moritz
parent
ea8d7b4dc0
commit
6703519144
@@ -141,7 +141,7 @@ class Cluster(object):
|
||||
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
clients = ray.experimental.state.parse_client_table(redis_client)
|
||||
clients = ray.state._parse_client_table(redis_client)
|
||||
object_store_socket_names = [
|
||||
client["ObjectStoreSocketName"] for client in clients
|
||||
]
|
||||
@@ -174,7 +174,7 @@ class Cluster(object):
|
||||
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
clients = ray.experimental.state.parse_client_table(redis_client)
|
||||
clients = ray.state._parse_client_table(redis_client)
|
||||
live_clients = [
|
||||
client for client in clients
|
||||
if client["EntryType"] == EntryType.INSERTION
|
||||
|
||||
@@ -2439,7 +2439,7 @@ def test_checkpointing_save_exception(ray_start_regular,
|
||||
assert ray.get(actor.was_resumed_from_checkpoint.remote()) is False
|
||||
|
||||
# Check that checkpointing errors were pushed to the driver.
|
||||
errors = ray.error_info()
|
||||
errors = ray.errors()
|
||||
assert len(errors) > 0
|
||||
for error in errors:
|
||||
# An error for the actor process dying may also get pushed.
|
||||
@@ -2483,7 +2483,7 @@ def test_checkpointing_load_exception(ray_start_regular,
|
||||
assert ray.get(actor.was_resumed_from_checkpoint.remote()) is False
|
||||
|
||||
# Check that checkpointing errors were pushed to the driver.
|
||||
errors = ray.error_info()
|
||||
errors = ray.errors()
|
||||
assert len(errors) > 0
|
||||
for error in errors:
|
||||
# An error for the actor process dying may also get pushed.
|
||||
|
||||
@@ -935,7 +935,7 @@ def test_many_fractional_resources(shutdown_only):
|
||||
stop_time = time.time() + 10
|
||||
correct_available_resources = False
|
||||
while time.time() < stop_time:
|
||||
if ray.global_state.available_resources() == {
|
||||
if ray.available_resources() == {
|
||||
"CPU": 2.0,
|
||||
"GPU": 2.0,
|
||||
"Custom": 2.0,
|
||||
@@ -1176,7 +1176,7 @@ def test_profiling_api(ray_start_2_cpus):
|
||||
if time.time() - start_time > timeout_seconds:
|
||||
raise Exception("Timed out while waiting for information in "
|
||||
"profile table.")
|
||||
profile_data = ray.global_state.chrome_tracing_dump()
|
||||
profile_data = ray.timeline()
|
||||
event_types = {event["cat"] for event in profile_data}
|
||||
expected_types = [
|
||||
"worker_idle",
|
||||
@@ -1252,7 +1252,7 @@ def test_object_transfer_dump(ray_start_cluster):
|
||||
# The profiling information only flushes once every second.
|
||||
time.sleep(1.1)
|
||||
|
||||
transfer_dump = ray.global_state.chrome_tracing_object_transfer_dump()
|
||||
transfer_dump = ray.object_transfer_timeline()
|
||||
# Make sure the transfer dump can be serialized with JSON.
|
||||
json.loads(json.dumps(transfer_dump))
|
||||
assert len(transfer_dump) >= num_nodes**2
|
||||
@@ -1559,12 +1559,12 @@ def test_free_objects_multi_node(ray_start_cluster):
|
||||
|
||||
# Case3: These cases test the deleting creating tasks for the object.
|
||||
(a, b, c) = run_one_test(actors, False, False)
|
||||
task_table = ray.global_state.task_table()
|
||||
task_table = ray.tasks()
|
||||
for obj in [a, b, c]:
|
||||
assert ray._raylet.compute_task_id(obj).hex() in task_table
|
||||
|
||||
(a, b, c) = run_one_test(actors, False, True)
|
||||
task_table = ray.global_state.task_table()
|
||||
task_table = ray.tasks()
|
||||
for obj in [a, b, c]:
|
||||
assert ray._raylet.compute_task_id(obj).hex() not in task_table
|
||||
|
||||
@@ -2026,7 +2026,7 @@ def test_multiple_raylets(ray_start_cluster):
|
||||
results.append(run_on_0_2.remote())
|
||||
return names, results
|
||||
|
||||
client_table = ray.global_state.client_table()
|
||||
client_table = ray.nodes()
|
||||
store_names = []
|
||||
store_names += [
|
||||
client["ObjectStoreSocketName"] for client in client_table
|
||||
@@ -2214,13 +2214,13 @@ def test_zero_capacity_deletion_semantics(shutdown_only):
|
||||
ray.init(num_cpus=2, num_gpus=1, resources={"test_resource": 1})
|
||||
|
||||
def test():
|
||||
resources = ray.global_state.available_resources()
|
||||
resources = ray.available_resources()
|
||||
MAX_RETRY_ATTEMPTS = 5
|
||||
retry_count = 0
|
||||
|
||||
while resources and retry_count < MAX_RETRY_ATTEMPTS:
|
||||
time.sleep(0.1)
|
||||
resources = ray.global_state.available_resources()
|
||||
resources = ray.available_resources()
|
||||
retry_count += 1
|
||||
|
||||
if retry_count >= MAX_RETRY_ATTEMPTS:
|
||||
@@ -2394,7 +2394,7 @@ def test_load_balancing_with_dependencies(ray_start_cluster):
|
||||
def wait_for_num_tasks(num_tasks, timeout=10):
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
if len(ray.global_state.task_table()) >= num_tasks:
|
||||
if len(ray.tasks()) >= num_tasks:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise Exception("Timed out while waiting for global state.")
|
||||
@@ -2403,7 +2403,7 @@ def wait_for_num_tasks(num_tasks, timeout=10):
|
||||
def wait_for_num_objects(num_objects, timeout=10):
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
if len(ray.global_state.object_table()) >= num_objects:
|
||||
if len(ray.objects()) >= num_objects:
|
||||
return
|
||||
time.sleep(0.1)
|
||||
raise Exception("Timed out while waiting for global state.")
|
||||
@@ -2414,31 +2414,27 @@ def wait_for_num_objects(num_objects, timeout=10):
|
||||
reason="New GCS API doesn't have a Python API yet.")
|
||||
def test_global_state_api(shutdown_only):
|
||||
with pytest.raises(Exception):
|
||||
ray.global_state.object_table()
|
||||
ray.objects()
|
||||
|
||||
with pytest.raises(Exception):
|
||||
ray.global_state.task_table()
|
||||
ray.tasks()
|
||||
|
||||
with pytest.raises(Exception):
|
||||
ray.global_state.client_table()
|
||||
|
||||
with pytest.raises(Exception):
|
||||
ray.global_state.function_table()
|
||||
ray.nodes()
|
||||
|
||||
ray.init(num_cpus=5, num_gpus=3, resources={"CustomResource": 1})
|
||||
|
||||
resources = {"CPU": 5, "GPU": 3, "CustomResource": 1}
|
||||
assert ray.global_state.cluster_resources() == resources
|
||||
assert ray.cluster_resources() == resources
|
||||
|
||||
assert ray.global_state.object_table() == {}
|
||||
assert ray.objects() == {}
|
||||
|
||||
driver_id = ray.experimental.state.binary_to_hex(
|
||||
ray.worker.global_worker.worker_id)
|
||||
driver_id = ray.utils.binary_to_hex(ray.worker.global_worker.worker_id)
|
||||
driver_task_id = ray.worker.global_worker.current_task_id.hex()
|
||||
|
||||
# One task is put in the task table which corresponds to this driver.
|
||||
wait_for_num_tasks(1)
|
||||
task_table = ray.global_state.task_table()
|
||||
task_table = ray.tasks()
|
||||
assert len(task_table) == 1
|
||||
assert driver_task_id == list(task_table.keys())[0]
|
||||
task_spec = task_table[driver_task_id]["TaskSpec"]
|
||||
@@ -2451,7 +2447,7 @@ def test_global_state_api(shutdown_only):
|
||||
assert task_spec["FunctionID"] == nil_id_hex
|
||||
assert task_spec["ReturnObjectIDs"] == []
|
||||
|
||||
client_table = ray.global_state.client_table()
|
||||
client_table = ray.nodes()
|
||||
node_ip_address = ray.worker.global_worker.node_ip_address
|
||||
|
||||
assert len(client_table) == 1
|
||||
@@ -2466,24 +2462,19 @@ def test_global_state_api(shutdown_only):
|
||||
|
||||
# Wait for one additional task to complete.
|
||||
wait_for_num_tasks(1 + 1)
|
||||
task_table = ray.global_state.task_table()
|
||||
task_table = ray.tasks()
|
||||
assert len(task_table) == 1 + 1
|
||||
task_id_set = set(task_table.keys())
|
||||
task_id_set.remove(driver_task_id)
|
||||
task_id = list(task_id_set)[0]
|
||||
|
||||
function_table = ray.global_state.function_table()
|
||||
task_spec = task_table[task_id]["TaskSpec"]
|
||||
assert task_spec["ActorID"] == nil_id_hex
|
||||
assert task_spec["Args"] == [1, "hi", x_id]
|
||||
assert task_spec["DriverID"] == driver_id
|
||||
assert task_spec["ReturnObjectIDs"] == [result_id]
|
||||
function_table_entry = function_table[task_spec["FunctionID"]]
|
||||
assert function_table_entry["Name"] == "ray.tests.test_basic.f"
|
||||
assert function_table_entry["DriverID"] == driver_id
|
||||
assert function_table_entry["Module"] == "ray.tests.test_basic"
|
||||
|
||||
assert task_table[task_id] == ray.global_state.task_table(task_id)
|
||||
assert task_table[task_id] == ray.tasks(task_id)
|
||||
|
||||
# Wait for two objects, one for the x_id and one for result_id.
|
||||
wait_for_num_objects(2)
|
||||
@@ -2492,7 +2483,7 @@ def test_global_state_api(shutdown_only):
|
||||
timeout = 10
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
object_table = ray.global_state.object_table()
|
||||
object_table = ray.objects()
|
||||
tables_ready = (object_table[x_id]["ManagerIDs"] is not None and
|
||||
object_table[result_id]["ManagerIDs"] is not None)
|
||||
if tables_ready:
|
||||
@@ -2501,11 +2492,11 @@ def test_global_state_api(shutdown_only):
|
||||
raise Exception("Timed out while waiting for object table to "
|
||||
"update.")
|
||||
|
||||
object_table = ray.global_state.object_table()
|
||||
object_table = ray.objects()
|
||||
assert len(object_table) == 2
|
||||
|
||||
assert object_table[x_id] == ray.global_state.object_table(x_id)
|
||||
object_table_entry = ray.global_state.object_table(result_id)
|
||||
assert object_table[x_id] == ray.objects(x_id)
|
||||
object_table_entry = ray.objects(result_id)
|
||||
assert object_table[result_id] == object_table_entry
|
||||
|
||||
|
||||
@@ -2611,14 +2602,6 @@ def test_workers(shutdown_only):
|
||||
while len(worker_ids) != num_workers:
|
||||
worker_ids = set(ray.get([f.remote() for _ in range(10)]))
|
||||
|
||||
worker_info = ray.global_state.workers()
|
||||
assert len(worker_info) >= num_workers
|
||||
for worker_id, info in worker_info.items():
|
||||
assert "node_ip_address" in info
|
||||
assert "plasma_store_socket" in info
|
||||
assert "stderr_file" in info
|
||||
assert "stdout_file" in info
|
||||
|
||||
|
||||
def test_specific_driver_id():
|
||||
dummy_driver_id = ray.DriverID(b"00112233445566778899")
|
||||
@@ -2816,7 +2799,7 @@ def test_socket_dir_not_existing(shutdown_only):
|
||||
def test_raylet_is_robust_to_random_messages(ray_start_regular):
|
||||
node_manager_address = None
|
||||
node_manager_port = None
|
||||
for client in ray.global_state.client_table():
|
||||
for client in ray.nodes():
|
||||
if "NodeManagerAddress" in client:
|
||||
node_manager_address = client["NodeManagerAddress"]
|
||||
node_manager_port = client["NodeManagerPort"]
|
||||
@@ -2908,7 +2891,7 @@ def test_shutdown_disconnect_global_state():
|
||||
ray.shutdown()
|
||||
|
||||
with pytest.raises(Exception) as e:
|
||||
ray.global_state.object_table()
|
||||
ray.objects()
|
||||
assert str(e.value).endswith("ray.init has been called.")
|
||||
|
||||
|
||||
@@ -2922,8 +2905,8 @@ def test_redis_lru_with_set(ray_start_object_store_memory):
|
||||
removed = False
|
||||
start_time = time.time()
|
||||
while time.time() < start_time + 10:
|
||||
if ray.global_state.redis_clients[0].delete(b"OBJECT" +
|
||||
x_id.binary()) == 1:
|
||||
if ray.state.state.redis_clients[0].delete(b"OBJECT" +
|
||||
x_id.binary()) == 1:
|
||||
removed = True
|
||||
break
|
||||
assert removed
|
||||
|
||||
@@ -23,8 +23,8 @@ def test_dynamic_res_creation(ray_start_regular):
|
||||
|
||||
ray.get(set_res.remote(res_name, res_capacity))
|
||||
|
||||
available_res = ray.global_state.available_resources()
|
||||
cluster_res = ray.global_state.cluster_resources()
|
||||
available_res = ray.available_resources()
|
||||
cluster_res = ray.cluster_resources()
|
||||
|
||||
assert available_res[res_name] == res_capacity
|
||||
assert cluster_res[res_name] == res_capacity
|
||||
@@ -43,8 +43,8 @@ def test_dynamic_res_deletion(shutdown_only):
|
||||
|
||||
ray.get(delete_res.remote(res_name))
|
||||
|
||||
available_res = ray.global_state.available_resources()
|
||||
cluster_res = ray.global_state.cluster_resources()
|
||||
available_res = ray.available_resources()
|
||||
cluster_res = ray.cluster_resources()
|
||||
|
||||
assert res_name not in available_res
|
||||
assert res_name not in cluster_res
|
||||
@@ -69,7 +69,7 @@ def test_dynamic_res_infeasible_rescheduling(ray_start_regular):
|
||||
oid = remote_task.remote() # This is infeasible
|
||||
ray.get(set_res.remote(res_name, res_capacity)) # Now should be feasible
|
||||
|
||||
available_res = ray.global_state.available_resources()
|
||||
available_res = ray.available_resources()
|
||||
assert available_res[res_name] == res_capacity
|
||||
|
||||
successful, unsuccessful = ray.wait([oid], timeout=1)
|
||||
@@ -88,7 +88,7 @@ def test_dynamic_res_updation_clientid(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
target_clientid = ray.global_state.client_table()[1]["ClientID"]
|
||||
target_clientid = ray.nodes()[1]["ClientID"]
|
||||
|
||||
@ray.remote
|
||||
def set_res(resource_name, resource_capacity, client_id):
|
||||
@@ -102,7 +102,7 @@ def test_dynamic_res_updation_clientid(ray_start_cluster):
|
||||
new_capacity = res_capacity + 1
|
||||
ray.get(set_res.remote(res_name, new_capacity, target_clientid))
|
||||
|
||||
target_client = next(client for client in ray.global_state.client_table()
|
||||
target_client = next(client for client in ray.nodes()
|
||||
if client["ClientID"] == target_clientid)
|
||||
resources = target_client["Resources"]
|
||||
|
||||
@@ -122,7 +122,7 @@ def test_dynamic_res_creation_clientid(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
target_clientid = ray.global_state.client_table()[1]["ClientID"]
|
||||
target_clientid = ray.nodes()[1]["ClientID"]
|
||||
|
||||
@ray.remote
|
||||
def set_res(resource_name, resource_capacity, res_client_id):
|
||||
@@ -130,7 +130,7 @@ def test_dynamic_res_creation_clientid(ray_start_cluster):
|
||||
resource_name, resource_capacity, client_id=res_client_id)
|
||||
|
||||
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
|
||||
target_client = next(client for client in ray.global_state.client_table()
|
||||
target_client = next(client for client in ray.nodes()
|
||||
if client["ClientID"] == target_clientid)
|
||||
resources = target_client["Resources"]
|
||||
|
||||
@@ -152,9 +152,7 @@ def test_dynamic_res_creation_clientid_multiple(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
target_clientids = [
|
||||
client["ClientID"] for client in ray.global_state.client_table()
|
||||
]
|
||||
target_clientids = [client["ClientID"] for client in ray.nodes()]
|
||||
|
||||
@ray.remote
|
||||
def set_res(resource_name, resource_capacity, res_client_id):
|
||||
@@ -172,9 +170,8 @@ def test_dynamic_res_creation_clientid_multiple(ray_start_cluster):
|
||||
while time.time() - start_time < TIMEOUT and not success:
|
||||
resources_created = []
|
||||
for cid in target_clientids:
|
||||
target_client = next(client
|
||||
for client in ray.global_state.client_table()
|
||||
if client["ClientID"] == cid)
|
||||
target_client = next(
|
||||
client for client in ray.nodes() if client["ClientID"] == cid)
|
||||
resources = target_client["Resources"]
|
||||
resources_created.append(resources[res_name] == res_capacity)
|
||||
success = all(resources_created)
|
||||
@@ -196,7 +193,7 @@ def test_dynamic_res_deletion_clientid(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
target_clientid = ray.global_state.client_table()[1]["ClientID"]
|
||||
target_clientid = ray.nodes()[1]["ClientID"]
|
||||
|
||||
# Launch the delete task
|
||||
@ray.remote
|
||||
@@ -206,10 +203,10 @@ def test_dynamic_res_deletion_clientid(ray_start_cluster):
|
||||
|
||||
ray.get(delete_res.remote(res_name, target_clientid))
|
||||
|
||||
target_client = next(client for client in ray.global_state.client_table()
|
||||
target_client = next(client for client in ray.nodes()
|
||||
if client["ClientID"] == target_clientid)
|
||||
resources = target_client["Resources"]
|
||||
print(ray.global_state.cluster_resources())
|
||||
print(ray.cluster_resources())
|
||||
assert res_name not in resources
|
||||
|
||||
|
||||
@@ -228,9 +225,7 @@ def test_dynamic_res_creation_scheduler_consistency(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
clientids = [
|
||||
client["ClientID"] for client in ray.global_state.client_table()
|
||||
]
|
||||
clientids = [client["ClientID"] for client in ray.nodes()]
|
||||
|
||||
@ray.remote
|
||||
def set_res(resource_name, resource_capacity, res_client_id):
|
||||
@@ -267,9 +262,7 @@ def test_dynamic_res_deletion_scheduler_consistency(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
clientids = [
|
||||
client["ClientID"] for client in ray.global_state.client_table()
|
||||
]
|
||||
clientids = [client["ClientID"] for client in ray.nodes()]
|
||||
|
||||
@ray.remote
|
||||
def delete_res(resource_name, res_client_id):
|
||||
@@ -284,7 +277,7 @@ def test_dynamic_res_deletion_scheduler_consistency(ray_start_cluster):
|
||||
# Create the resource on node1
|
||||
target_clientid = clientids[1]
|
||||
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
|
||||
assert ray.global_state.cluster_resources()[res_name] == res_capacity
|
||||
assert ray.cluster_resources()[res_name] == res_capacity
|
||||
|
||||
# Delete the resource
|
||||
ray.get(delete_res.remote(res_name, target_clientid))
|
||||
@@ -322,9 +315,7 @@ def test_dynamic_res_concurrent_res_increment(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
clientids = [
|
||||
client["ClientID"] for client in ray.global_state.client_table()
|
||||
]
|
||||
clientids = [client["ClientID"] for client in ray.nodes()]
|
||||
target_clientid = clientids[1]
|
||||
|
||||
@ray.remote
|
||||
@@ -334,7 +325,7 @@ def test_dynamic_res_concurrent_res_increment(ray_start_cluster):
|
||||
|
||||
# Create the resource on node 1
|
||||
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
|
||||
assert ray.global_state.cluster_resources()[res_name] == res_capacity
|
||||
assert ray.cluster_resources()[res_name] == res_capacity
|
||||
|
||||
# Task to hold the resource till the driver signals to finish
|
||||
@ray.remote
|
||||
@@ -376,7 +367,7 @@ def test_dynamic_res_concurrent_res_increment(ray_start_cluster):
|
||||
}) # This should be infeasible
|
||||
successful, unsuccessful = ray.wait([task_3], timeout=TIMEOUT_DURATION)
|
||||
assert unsuccessful # The task did not complete because it's infeasible
|
||||
assert ray.global_state.available_resources()[res_name] == updated_capacity
|
||||
assert ray.available_resources()[res_name] == updated_capacity
|
||||
|
||||
|
||||
def test_dynamic_res_concurrent_res_decrement(ray_start_cluster):
|
||||
@@ -403,9 +394,7 @@ def test_dynamic_res_concurrent_res_decrement(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
clientids = [
|
||||
client["ClientID"] for client in ray.global_state.client_table()
|
||||
]
|
||||
clientids = [client["ClientID"] for client in ray.nodes()]
|
||||
target_clientid = clientids[1]
|
||||
|
||||
@ray.remote
|
||||
@@ -415,7 +404,7 @@ def test_dynamic_res_concurrent_res_decrement(ray_start_cluster):
|
||||
|
||||
# Create the resource on node 1
|
||||
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
|
||||
assert ray.global_state.cluster_resources()[res_name] == res_capacity
|
||||
assert ray.cluster_resources()[res_name] == res_capacity
|
||||
|
||||
# Task to hold the resource till the driver signals to finish
|
||||
@ray.remote
|
||||
@@ -457,7 +446,7 @@ def test_dynamic_res_concurrent_res_decrement(ray_start_cluster):
|
||||
}) # This should be infeasible
|
||||
successful, unsuccessful = ray.wait([task_3], timeout=TIMEOUT_DURATION)
|
||||
assert unsuccessful # The task did not complete because it's infeasible
|
||||
assert ray.global_state.available_resources()[res_name] == updated_capacity
|
||||
assert ray.available_resources()[res_name] == updated_capacity
|
||||
|
||||
|
||||
def test_dynamic_res_concurrent_res_delete(ray_start_cluster):
|
||||
@@ -482,9 +471,7 @@ def test_dynamic_res_concurrent_res_delete(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
clientids = [
|
||||
client["ClientID"] for client in ray.global_state.client_table()
|
||||
]
|
||||
clientids = [client["ClientID"] for client in ray.nodes()]
|
||||
target_clientid = clientids[1]
|
||||
|
||||
@ray.remote
|
||||
@@ -499,7 +486,7 @@ def test_dynamic_res_concurrent_res_delete(ray_start_cluster):
|
||||
|
||||
# Create the resource on node 1
|
||||
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
|
||||
assert ray.global_state.cluster_resources()[res_name] == res_capacity
|
||||
assert ray.cluster_resources()[res_name] == res_capacity
|
||||
|
||||
# Task to hold the resource till the driver signals to finish
|
||||
@ray.remote
|
||||
@@ -534,7 +521,7 @@ def test_dynamic_res_concurrent_res_delete(ray_start_cluster):
|
||||
args=[], resources={res_name: 1}) # This should be infeasible
|
||||
successful, unsuccessful = ray.wait([task_2], timeout=TIMEOUT_DURATION)
|
||||
assert unsuccessful # The task did not complete because it's infeasible
|
||||
assert res_name not in ray.global_state.available_resources()
|
||||
assert res_name not in ray.available_resources()
|
||||
|
||||
|
||||
def test_dynamic_res_creation_stress(ray_start_cluster):
|
||||
@@ -553,9 +540,7 @@ def test_dynamic_res_creation_stress(ray_start_cluster):
|
||||
|
||||
ray.init(redis_address=cluster.redis_address)
|
||||
|
||||
clientids = [
|
||||
client["ClientID"] for client in ray.global_state.client_table()
|
||||
]
|
||||
clientids = [client["ClientID"] for client in ray.nodes()]
|
||||
target_clientid = clientids[1]
|
||||
|
||||
@ray.remote
|
||||
@@ -578,7 +563,7 @@ def test_dynamic_res_creation_stress(ray_start_cluster):
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < TIMEOUT and not success:
|
||||
resources = ray.global_state.cluster_resources()
|
||||
resources = ray.cluster_resources()
|
||||
all_resources_created = []
|
||||
for i in range(0, NUM_RES_TO_CREATE):
|
||||
all_resources_created.append(str(i) in resources)
|
||||
|
||||
@@ -164,7 +164,7 @@ def temporary_helper_function():
|
||||
return 1
|
||||
|
||||
# There should be no errors yet.
|
||||
assert len(ray.error_info()) == 0
|
||||
assert len(ray.errors()) == 0
|
||||
|
||||
# Create an actor.
|
||||
foo = Foo.remote()
|
||||
@@ -376,8 +376,9 @@ def test_actor_scope_or_intentionally_killed_message(ray_start_regular):
|
||||
a = Actor.remote()
|
||||
a.__ray_terminate__.remote()
|
||||
time.sleep(1)
|
||||
assert len(ray.error_info()) == 0, (
|
||||
"Should not have propogated an error - {}".format(ray.error_info()))
|
||||
assert len(
|
||||
ray.errors()) == 0, ("Should not have propogated an error - {}".format(
|
||||
ray.errors()))
|
||||
|
||||
|
||||
@pytest.mark.skip("This test does not work yet.")
|
||||
@@ -653,7 +654,7 @@ def test_warning_for_dead_node(ray_start_cluster_2_nodes):
|
||||
cluster = ray_start_cluster_2_nodes
|
||||
cluster.wait_for_nodes()
|
||||
|
||||
client_ids = {item["ClientID"] for item in ray.global_state.client_table()}
|
||||
client_ids = {item["ClientID"] for item in ray.nodes()}
|
||||
|
||||
# Try to make sure that the monitor has received at least one heartbeat
|
||||
# from the node.
|
||||
|
||||
@@ -18,8 +18,8 @@ import ray
|
||||
reason="Timeout package not installed; skipping test that may hang.")
|
||||
@pytest.mark.timeout(10)
|
||||
def test_replenish_resources(ray_start_regular):
|
||||
cluster_resources = ray.global_state.cluster_resources()
|
||||
available_resources = ray.global_state.available_resources()
|
||||
cluster_resources = ray.cluster_resources()
|
||||
available_resources = ray.available_resources()
|
||||
assert cluster_resources == available_resources
|
||||
|
||||
@ray.remote
|
||||
@@ -30,7 +30,7 @@ def test_replenish_resources(ray_start_regular):
|
||||
resources_reset = False
|
||||
|
||||
while not resources_reset:
|
||||
available_resources = ray.global_state.available_resources()
|
||||
available_resources = ray.available_resources()
|
||||
resources_reset = (cluster_resources == available_resources)
|
||||
assert resources_reset
|
||||
|
||||
@@ -40,7 +40,7 @@ def test_replenish_resources(ray_start_regular):
|
||||
reason="Timeout package not installed; skipping test that may hang.")
|
||||
@pytest.mark.timeout(10)
|
||||
def test_uses_resources(ray_start_regular):
|
||||
cluster_resources = ray.global_state.cluster_resources()
|
||||
cluster_resources = ray.cluster_resources()
|
||||
|
||||
@ray.remote
|
||||
def cpu_task():
|
||||
@@ -50,7 +50,7 @@ def test_uses_resources(ray_start_regular):
|
||||
resource_used = False
|
||||
|
||||
while not resource_used:
|
||||
available_resources = ray.global_state.available_resources()
|
||||
available_resources = ray.available_resources()
|
||||
resource_used = available_resources.get(
|
||||
"CPU", 0) == cluster_resources.get("CPU", 0) - 1
|
||||
|
||||
@@ -64,17 +64,17 @@ def test_uses_resources(ray_start_regular):
|
||||
def test_add_remove_cluster_resources(ray_start_cluster_head):
|
||||
"""Tests that Global State API is consistent with actual cluster."""
|
||||
cluster = ray_start_cluster_head
|
||||
assert ray.global_state.cluster_resources()["CPU"] == 1
|
||||
assert ray.cluster_resources()["CPU"] == 1
|
||||
nodes = []
|
||||
nodes += [cluster.add_node(num_cpus=1)]
|
||||
cluster.wait_for_nodes()
|
||||
assert ray.global_state.cluster_resources()["CPU"] == 2
|
||||
assert ray.cluster_resources()["CPU"] == 2
|
||||
|
||||
cluster.remove_node(nodes.pop())
|
||||
cluster.wait_for_nodes()
|
||||
assert ray.global_state.cluster_resources()["CPU"] == 1
|
||||
assert ray.cluster_resources()["CPU"] == 1
|
||||
|
||||
for i in range(5):
|
||||
nodes += [cluster.add_node(num_cpus=1)]
|
||||
cluster.wait_for_nodes()
|
||||
assert ray.global_state.cluster_resources()["CPU"] == 6
|
||||
assert ray.cluster_resources()["CPU"] == 6
|
||||
|
||||
@@ -30,17 +30,16 @@ def _test_cleanup_on_driver_exit(num_redis_shards):
|
||||
time.sleep(2)
|
||||
|
||||
def StateSummary():
|
||||
obj_tbl_len = len(ray.global_state.object_table())
|
||||
task_tbl_len = len(ray.global_state.task_table())
|
||||
func_tbl_len = len(ray.global_state.function_table())
|
||||
return obj_tbl_len, task_tbl_len, func_tbl_len
|
||||
obj_tbl_len = len(ray.objects())
|
||||
task_tbl_len = len(ray.tasks())
|
||||
return obj_tbl_len, task_tbl_len
|
||||
|
||||
def Driver(success):
|
||||
success.value = True
|
||||
# Start driver.
|
||||
ray.init(redis_address=redis_address)
|
||||
summary_start = StateSummary()
|
||||
if (0, 1) != summary_start[:2]:
|
||||
if (0, 1) != summary_start:
|
||||
success.value = False
|
||||
|
||||
# Two new objects.
|
||||
@@ -54,7 +53,7 @@ def _test_cleanup_on_driver_exit(num_redis_shards):
|
||||
|
||||
# 1 new function.
|
||||
attempts = 0
|
||||
while (2, 1, summary_start[2]) != StateSummary():
|
||||
while (2, 1) != StateSummary():
|
||||
time.sleep(0.1)
|
||||
attempts += 1
|
||||
if attempts == max_attempts_before_failing:
|
||||
@@ -63,7 +62,7 @@ def _test_cleanup_on_driver_exit(num_redis_shards):
|
||||
|
||||
ray.get(f.remote())
|
||||
attempts = 0
|
||||
while (4, 2, summary_start[2] + 1) != StateSummary():
|
||||
while (4, 2) != StateSummary():
|
||||
time.sleep(0.1)
|
||||
attempts += 1
|
||||
if attempts == max_attempts_before_failing:
|
||||
@@ -83,12 +82,12 @@ def _test_cleanup_on_driver_exit(num_redis_shards):
|
||||
# Check that objects, tasks, and functions are cleaned up.
|
||||
ray.init(redis_address=redis_address)
|
||||
attempts = 0
|
||||
while (0, 1) != StateSummary()[:2]:
|
||||
while (0, 1) != StateSummary():
|
||||
time.sleep(0.1)
|
||||
attempts += 1
|
||||
if attempts == max_attempts_before_failing:
|
||||
break
|
||||
assert (0, 1) == StateSummary()[:2]
|
||||
assert (0, 1) == StateSummary()
|
||||
|
||||
ray.shutdown()
|
||||
subprocess.Popen(["ray", "stop"]).wait()
|
||||
|
||||
@@ -19,7 +19,7 @@ def test_error_isolation(call_ray_start):
|
||||
ray.init(redis_address=redis_address)
|
||||
|
||||
# There shouldn't be any errors yet.
|
||||
assert len(ray.error_info()) == 0
|
||||
assert len(ray.errors()) == 0
|
||||
|
||||
error_string1 = "error_string1"
|
||||
error_string2 = "error_string2"
|
||||
@@ -33,13 +33,13 @@ def test_error_isolation(call_ray_start):
|
||||
ray.get(f.remote())
|
||||
|
||||
# Wait for the error to appear in Redis.
|
||||
while len(ray.error_info()) != 1:
|
||||
while len(ray.errors()) != 1:
|
||||
time.sleep(0.1)
|
||||
print("Waiting for error to appear.")
|
||||
|
||||
# Make sure we got the error.
|
||||
assert len(ray.error_info()) == 1
|
||||
assert error_string1 in ray.error_info()[0]["message"]
|
||||
assert len(ray.errors()) == 1
|
||||
assert error_string1 in ray.errors()[0]["message"]
|
||||
|
||||
# Start another driver and make sure that it does not receive this
|
||||
# error. Make the other driver throw an error, and make sure it
|
||||
@@ -51,7 +51,7 @@ import time
|
||||
ray.init(redis_address="{}")
|
||||
|
||||
time.sleep(1)
|
||||
assert len(ray.error_info()) == 0
|
||||
assert len(ray.errors()) == 0
|
||||
|
||||
@ray.remote
|
||||
def f():
|
||||
@@ -62,12 +62,12 @@ try:
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
while len(ray.error_info()) != 1:
|
||||
print(len(ray.error_info()))
|
||||
while len(ray.errors()) != 1:
|
||||
print(len(ray.errors()))
|
||||
time.sleep(0.1)
|
||||
assert len(ray.error_info()) == 1
|
||||
assert len(ray.errors()) == 1
|
||||
|
||||
assert "{}" in ray.error_info()[0]["message"]
|
||||
assert "{}" in ray.errors()[0]["message"]
|
||||
|
||||
print("success")
|
||||
""".format(redis_address, error_string2, error_string2)
|
||||
@@ -78,8 +78,8 @@ print("success")
|
||||
|
||||
# Make sure that the other error message doesn't show up for this
|
||||
# driver.
|
||||
assert len(ray.error_info()) == 1
|
||||
assert error_string1 in ray.error_info()[0]["message"]
|
||||
assert len(ray.errors()) == 1
|
||||
assert error_string1 in ray.errors()[0]["message"]
|
||||
|
||||
|
||||
def test_remote_function_isolation(call_ray_start):
|
||||
|
||||
@@ -52,10 +52,10 @@ def test_internal_config(ray_start_cluster_head):
|
||||
|
||||
cluster.remove_node(worker)
|
||||
time.sleep(1)
|
||||
assert ray.global_state.cluster_resources()["CPU"] == 2
|
||||
assert ray.cluster_resources()["CPU"] == 2
|
||||
|
||||
time.sleep(2)
|
||||
assert ray.global_state.cluster_resources()["CPU"] == 1
|
||||
assert ray.cluster_resources()["CPU"] == 1
|
||||
|
||||
|
||||
def test_wait_for_nodes(ray_start_cluster_head):
|
||||
@@ -70,12 +70,12 @@ def test_wait_for_nodes(ray_start_cluster_head):
|
||||
[cluster.remove_node(w) for w in workers]
|
||||
cluster.wait_for_nodes()
|
||||
|
||||
assert ray.global_state.cluster_resources()["CPU"] == 1
|
||||
assert ray.cluster_resources()["CPU"] == 1
|
||||
worker2 = cluster.add_node()
|
||||
cluster.wait_for_nodes()
|
||||
cluster.remove_node(worker2)
|
||||
cluster.wait_for_nodes()
|
||||
assert ray.global_state.cluster_resources()["CPU"] == 1
|
||||
assert ray.cluster_resources()["CPU"] == 1
|
||||
|
||||
|
||||
def test_worker_plasma_store_failure(ray_start_cluster_head):
|
||||
|
||||
@@ -80,7 +80,7 @@ def test_object_broadcast(ray_start_cluster_with_resource):
|
||||
|
||||
# Wait for profiling information to be pushed to the profile table.
|
||||
time.sleep(1)
|
||||
transfer_events = ray.global_state.chrome_tracing_object_transfer_dump()
|
||||
transfer_events = ray.object_transfer_timeline()
|
||||
|
||||
# Make sure that each object was transferred a reasonable number of times.
|
||||
for x_id in object_ids:
|
||||
@@ -160,7 +160,7 @@ def test_actor_broadcast(ray_start_cluster_with_resource):
|
||||
|
||||
# Wait for profiling information to be pushed to the profile table.
|
||||
time.sleep(1)
|
||||
transfer_events = ray.global_state.chrome_tracing_object_transfer_dump()
|
||||
transfer_events = ray.object_transfer_timeline()
|
||||
|
||||
# Make sure that each object was transferred a reasonable number of times.
|
||||
for x_id in object_ids:
|
||||
|
||||
@@ -393,7 +393,7 @@ def wait_for_errors(error_check):
|
||||
errors = []
|
||||
time_left = 100
|
||||
while time_left > 0:
|
||||
errors = ray.error_info()
|
||||
errors = ray.errors()
|
||||
if error_check(errors):
|
||||
break
|
||||
time_left -= 1
|
||||
|
||||
@@ -84,7 +84,7 @@ def run_string_as_driver_nonblocking(driver_script):
|
||||
|
||||
|
||||
def relevant_errors(error_type):
|
||||
return [info for info in ray.error_info() if info["type"] == error_type]
|
||||
return [info for info in ray.errors() if info["type"] == error_type]
|
||||
|
||||
|
||||
def wait_for_errors(error_type, num_errors, timeout=10):
|
||||
|
||||
Reference in New Issue
Block a user