Move global state API out of global_state object. (#4857)

This commit is contained in:
Robert Nishihara
2019-05-26 11:27:53 -07:00
committed by Philipp Moritz
parent ea8d7b4dc0
commit 6703519144
29 changed files with 387 additions and 403 deletions
+2 -2
View File
@@ -141,7 +141,7 @@ class Cluster(object):
start_time = time.time()
while time.time() - start_time < timeout:
clients = ray.experimental.state.parse_client_table(redis_client)
clients = ray.state._parse_client_table(redis_client)
object_store_socket_names = [
client["ObjectStoreSocketName"] for client in clients
]
@@ -174,7 +174,7 @@ class Cluster(object):
start_time = time.time()
while time.time() - start_time < timeout:
clients = ray.experimental.state.parse_client_table(redis_client)
clients = ray.state._parse_client_table(redis_client)
live_clients = [
client for client in clients
if client["EntryType"] == EntryType.INSERTION
+2 -2
View File
@@ -2439,7 +2439,7 @@ def test_checkpointing_save_exception(ray_start_regular,
assert ray.get(actor.was_resumed_from_checkpoint.remote()) is False
# Check that checkpointing errors were pushed to the driver.
errors = ray.error_info()
errors = ray.errors()
assert len(errors) > 0
for error in errors:
# An error for the actor process dying may also get pushed.
@@ -2483,7 +2483,7 @@ def test_checkpointing_load_exception(ray_start_regular,
assert ray.get(actor.was_resumed_from_checkpoint.remote()) is False
# Check that checkpointing errors were pushed to the driver.
errors = ray.error_info()
errors = ray.errors()
assert len(errors) > 0
for error in errors:
# An error for the actor process dying may also get pushed.
+28 -45
View File
@@ -935,7 +935,7 @@ def test_many_fractional_resources(shutdown_only):
stop_time = time.time() + 10
correct_available_resources = False
while time.time() < stop_time:
if ray.global_state.available_resources() == {
if ray.available_resources() == {
"CPU": 2.0,
"GPU": 2.0,
"Custom": 2.0,
@@ -1176,7 +1176,7 @@ def test_profiling_api(ray_start_2_cpus):
if time.time() - start_time > timeout_seconds:
raise Exception("Timed out while waiting for information in "
"profile table.")
profile_data = ray.global_state.chrome_tracing_dump()
profile_data = ray.timeline()
event_types = {event["cat"] for event in profile_data}
expected_types = [
"worker_idle",
@@ -1252,7 +1252,7 @@ def test_object_transfer_dump(ray_start_cluster):
# The profiling information only flushes once every second.
time.sleep(1.1)
transfer_dump = ray.global_state.chrome_tracing_object_transfer_dump()
transfer_dump = ray.object_transfer_timeline()
# Make sure the transfer dump can be serialized with JSON.
json.loads(json.dumps(transfer_dump))
assert len(transfer_dump) >= num_nodes**2
@@ -1559,12 +1559,12 @@ def test_free_objects_multi_node(ray_start_cluster):
# Case3: These cases test the deleting creating tasks for the object.
(a, b, c) = run_one_test(actors, False, False)
task_table = ray.global_state.task_table()
task_table = ray.tasks()
for obj in [a, b, c]:
assert ray._raylet.compute_task_id(obj).hex() in task_table
(a, b, c) = run_one_test(actors, False, True)
task_table = ray.global_state.task_table()
task_table = ray.tasks()
for obj in [a, b, c]:
assert ray._raylet.compute_task_id(obj).hex() not in task_table
@@ -2026,7 +2026,7 @@ def test_multiple_raylets(ray_start_cluster):
results.append(run_on_0_2.remote())
return names, results
client_table = ray.global_state.client_table()
client_table = ray.nodes()
store_names = []
store_names += [
client["ObjectStoreSocketName"] for client in client_table
@@ -2214,13 +2214,13 @@ def test_zero_capacity_deletion_semantics(shutdown_only):
ray.init(num_cpus=2, num_gpus=1, resources={"test_resource": 1})
def test():
resources = ray.global_state.available_resources()
resources = ray.available_resources()
MAX_RETRY_ATTEMPTS = 5
retry_count = 0
while resources and retry_count < MAX_RETRY_ATTEMPTS:
time.sleep(0.1)
resources = ray.global_state.available_resources()
resources = ray.available_resources()
retry_count += 1
if retry_count >= MAX_RETRY_ATTEMPTS:
@@ -2394,7 +2394,7 @@ def test_load_balancing_with_dependencies(ray_start_cluster):
def wait_for_num_tasks(num_tasks, timeout=10):
start_time = time.time()
while time.time() - start_time < timeout:
if len(ray.global_state.task_table()) >= num_tasks:
if len(ray.tasks()) >= num_tasks:
return
time.sleep(0.1)
raise Exception("Timed out while waiting for global state.")
@@ -2403,7 +2403,7 @@ def wait_for_num_tasks(num_tasks, timeout=10):
def wait_for_num_objects(num_objects, timeout=10):
start_time = time.time()
while time.time() - start_time < timeout:
if len(ray.global_state.object_table()) >= num_objects:
if len(ray.objects()) >= num_objects:
return
time.sleep(0.1)
raise Exception("Timed out while waiting for global state.")
@@ -2414,31 +2414,27 @@ def wait_for_num_objects(num_objects, timeout=10):
reason="New GCS API doesn't have a Python API yet.")
def test_global_state_api(shutdown_only):
with pytest.raises(Exception):
ray.global_state.object_table()
ray.objects()
with pytest.raises(Exception):
ray.global_state.task_table()
ray.tasks()
with pytest.raises(Exception):
ray.global_state.client_table()
with pytest.raises(Exception):
ray.global_state.function_table()
ray.nodes()
ray.init(num_cpus=5, num_gpus=3, resources={"CustomResource": 1})
resources = {"CPU": 5, "GPU": 3, "CustomResource": 1}
assert ray.global_state.cluster_resources() == resources
assert ray.cluster_resources() == resources
assert ray.global_state.object_table() == {}
assert ray.objects() == {}
driver_id = ray.experimental.state.binary_to_hex(
ray.worker.global_worker.worker_id)
driver_id = ray.utils.binary_to_hex(ray.worker.global_worker.worker_id)
driver_task_id = ray.worker.global_worker.current_task_id.hex()
# One task is put in the task table which corresponds to this driver.
wait_for_num_tasks(1)
task_table = ray.global_state.task_table()
task_table = ray.tasks()
assert len(task_table) == 1
assert driver_task_id == list(task_table.keys())[0]
task_spec = task_table[driver_task_id]["TaskSpec"]
@@ -2451,7 +2447,7 @@ def test_global_state_api(shutdown_only):
assert task_spec["FunctionID"] == nil_id_hex
assert task_spec["ReturnObjectIDs"] == []
client_table = ray.global_state.client_table()
client_table = ray.nodes()
node_ip_address = ray.worker.global_worker.node_ip_address
assert len(client_table) == 1
@@ -2466,24 +2462,19 @@ def test_global_state_api(shutdown_only):
# Wait for one additional task to complete.
wait_for_num_tasks(1 + 1)
task_table = ray.global_state.task_table()
task_table = ray.tasks()
assert len(task_table) == 1 + 1
task_id_set = set(task_table.keys())
task_id_set.remove(driver_task_id)
task_id = list(task_id_set)[0]
function_table = ray.global_state.function_table()
task_spec = task_table[task_id]["TaskSpec"]
assert task_spec["ActorID"] == nil_id_hex
assert task_spec["Args"] == [1, "hi", x_id]
assert task_spec["DriverID"] == driver_id
assert task_spec["ReturnObjectIDs"] == [result_id]
function_table_entry = function_table[task_spec["FunctionID"]]
assert function_table_entry["Name"] == "ray.tests.test_basic.f"
assert function_table_entry["DriverID"] == driver_id
assert function_table_entry["Module"] == "ray.tests.test_basic"
assert task_table[task_id] == ray.global_state.task_table(task_id)
assert task_table[task_id] == ray.tasks(task_id)
# Wait for two objects, one for the x_id and one for result_id.
wait_for_num_objects(2)
@@ -2492,7 +2483,7 @@ def test_global_state_api(shutdown_only):
timeout = 10
start_time = time.time()
while time.time() - start_time < timeout:
object_table = ray.global_state.object_table()
object_table = ray.objects()
tables_ready = (object_table[x_id]["ManagerIDs"] is not None and
object_table[result_id]["ManagerIDs"] is not None)
if tables_ready:
@@ -2501,11 +2492,11 @@ def test_global_state_api(shutdown_only):
raise Exception("Timed out while waiting for object table to "
"update.")
object_table = ray.global_state.object_table()
object_table = ray.objects()
assert len(object_table) == 2
assert object_table[x_id] == ray.global_state.object_table(x_id)
object_table_entry = ray.global_state.object_table(result_id)
assert object_table[x_id] == ray.objects(x_id)
object_table_entry = ray.objects(result_id)
assert object_table[result_id] == object_table_entry
@@ -2611,14 +2602,6 @@ def test_workers(shutdown_only):
while len(worker_ids) != num_workers:
worker_ids = set(ray.get([f.remote() for _ in range(10)]))
worker_info = ray.global_state.workers()
assert len(worker_info) >= num_workers
for worker_id, info in worker_info.items():
assert "node_ip_address" in info
assert "plasma_store_socket" in info
assert "stderr_file" in info
assert "stdout_file" in info
def test_specific_driver_id():
dummy_driver_id = ray.DriverID(b"00112233445566778899")
@@ -2816,7 +2799,7 @@ def test_socket_dir_not_existing(shutdown_only):
def test_raylet_is_robust_to_random_messages(ray_start_regular):
node_manager_address = None
node_manager_port = None
for client in ray.global_state.client_table():
for client in ray.nodes():
if "NodeManagerAddress" in client:
node_manager_address = client["NodeManagerAddress"]
node_manager_port = client["NodeManagerPort"]
@@ -2908,7 +2891,7 @@ def test_shutdown_disconnect_global_state():
ray.shutdown()
with pytest.raises(Exception) as e:
ray.global_state.object_table()
ray.objects()
assert str(e.value).endswith("ray.init has been called.")
@@ -2922,8 +2905,8 @@ def test_redis_lru_with_set(ray_start_object_store_memory):
removed = False
start_time = time.time()
while time.time() < start_time + 10:
if ray.global_state.redis_clients[0].delete(b"OBJECT" +
x_id.binary()) == 1:
if ray.state.state.redis_clients[0].delete(b"OBJECT" +
x_id.binary()) == 1:
removed = True
break
assert removed
+29 -44
View File
@@ -23,8 +23,8 @@ def test_dynamic_res_creation(ray_start_regular):
ray.get(set_res.remote(res_name, res_capacity))
available_res = ray.global_state.available_resources()
cluster_res = ray.global_state.cluster_resources()
available_res = ray.available_resources()
cluster_res = ray.cluster_resources()
assert available_res[res_name] == res_capacity
assert cluster_res[res_name] == res_capacity
@@ -43,8 +43,8 @@ def test_dynamic_res_deletion(shutdown_only):
ray.get(delete_res.remote(res_name))
available_res = ray.global_state.available_resources()
cluster_res = ray.global_state.cluster_resources()
available_res = ray.available_resources()
cluster_res = ray.cluster_resources()
assert res_name not in available_res
assert res_name not in cluster_res
@@ -69,7 +69,7 @@ def test_dynamic_res_infeasible_rescheduling(ray_start_regular):
oid = remote_task.remote() # This is infeasible
ray.get(set_res.remote(res_name, res_capacity)) # Now should be feasible
available_res = ray.global_state.available_resources()
available_res = ray.available_resources()
assert available_res[res_name] == res_capacity
successful, unsuccessful = ray.wait([oid], timeout=1)
@@ -88,7 +88,7 @@ def test_dynamic_res_updation_clientid(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
target_clientid = ray.global_state.client_table()[1]["ClientID"]
target_clientid = ray.nodes()[1]["ClientID"]
@ray.remote
def set_res(resource_name, resource_capacity, client_id):
@@ -102,7 +102,7 @@ def test_dynamic_res_updation_clientid(ray_start_cluster):
new_capacity = res_capacity + 1
ray.get(set_res.remote(res_name, new_capacity, target_clientid))
target_client = next(client for client in ray.global_state.client_table()
target_client = next(client for client in ray.nodes()
if client["ClientID"] == target_clientid)
resources = target_client["Resources"]
@@ -122,7 +122,7 @@ def test_dynamic_res_creation_clientid(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
target_clientid = ray.global_state.client_table()[1]["ClientID"]
target_clientid = ray.nodes()[1]["ClientID"]
@ray.remote
def set_res(resource_name, resource_capacity, res_client_id):
@@ -130,7 +130,7 @@ def test_dynamic_res_creation_clientid(ray_start_cluster):
resource_name, resource_capacity, client_id=res_client_id)
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
target_client = next(client for client in ray.global_state.client_table()
target_client = next(client for client in ray.nodes()
if client["ClientID"] == target_clientid)
resources = target_client["Resources"]
@@ -152,9 +152,7 @@ def test_dynamic_res_creation_clientid_multiple(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
target_clientids = [
client["ClientID"] for client in ray.global_state.client_table()
]
target_clientids = [client["ClientID"] for client in ray.nodes()]
@ray.remote
def set_res(resource_name, resource_capacity, res_client_id):
@@ -172,9 +170,8 @@ def test_dynamic_res_creation_clientid_multiple(ray_start_cluster):
while time.time() - start_time < TIMEOUT and not success:
resources_created = []
for cid in target_clientids:
target_client = next(client
for client in ray.global_state.client_table()
if client["ClientID"] == cid)
target_client = next(
client for client in ray.nodes() if client["ClientID"] == cid)
resources = target_client["Resources"]
resources_created.append(resources[res_name] == res_capacity)
success = all(resources_created)
@@ -196,7 +193,7 @@ def test_dynamic_res_deletion_clientid(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
target_clientid = ray.global_state.client_table()[1]["ClientID"]
target_clientid = ray.nodes()[1]["ClientID"]
# Launch the delete task
@ray.remote
@@ -206,10 +203,10 @@ def test_dynamic_res_deletion_clientid(ray_start_cluster):
ray.get(delete_res.remote(res_name, target_clientid))
target_client = next(client for client in ray.global_state.client_table()
target_client = next(client for client in ray.nodes()
if client["ClientID"] == target_clientid)
resources = target_client["Resources"]
print(ray.global_state.cluster_resources())
print(ray.cluster_resources())
assert res_name not in resources
@@ -228,9 +225,7 @@ def test_dynamic_res_creation_scheduler_consistency(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
clientids = [
client["ClientID"] for client in ray.global_state.client_table()
]
clientids = [client["ClientID"] for client in ray.nodes()]
@ray.remote
def set_res(resource_name, resource_capacity, res_client_id):
@@ -267,9 +262,7 @@ def test_dynamic_res_deletion_scheduler_consistency(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
clientids = [
client["ClientID"] for client in ray.global_state.client_table()
]
clientids = [client["ClientID"] for client in ray.nodes()]
@ray.remote
def delete_res(resource_name, res_client_id):
@@ -284,7 +277,7 @@ def test_dynamic_res_deletion_scheduler_consistency(ray_start_cluster):
# Create the resource on node1
target_clientid = clientids[1]
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
assert ray.global_state.cluster_resources()[res_name] == res_capacity
assert ray.cluster_resources()[res_name] == res_capacity
# Delete the resource
ray.get(delete_res.remote(res_name, target_clientid))
@@ -322,9 +315,7 @@ def test_dynamic_res_concurrent_res_increment(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
clientids = [
client["ClientID"] for client in ray.global_state.client_table()
]
clientids = [client["ClientID"] for client in ray.nodes()]
target_clientid = clientids[1]
@ray.remote
@@ -334,7 +325,7 @@ def test_dynamic_res_concurrent_res_increment(ray_start_cluster):
# Create the resource on node 1
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
assert ray.global_state.cluster_resources()[res_name] == res_capacity
assert ray.cluster_resources()[res_name] == res_capacity
# Task to hold the resource till the driver signals to finish
@ray.remote
@@ -376,7 +367,7 @@ def test_dynamic_res_concurrent_res_increment(ray_start_cluster):
}) # This should be infeasible
successful, unsuccessful = ray.wait([task_3], timeout=TIMEOUT_DURATION)
assert unsuccessful # The task did not complete because it's infeasible
assert ray.global_state.available_resources()[res_name] == updated_capacity
assert ray.available_resources()[res_name] == updated_capacity
def test_dynamic_res_concurrent_res_decrement(ray_start_cluster):
@@ -403,9 +394,7 @@ def test_dynamic_res_concurrent_res_decrement(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
clientids = [
client["ClientID"] for client in ray.global_state.client_table()
]
clientids = [client["ClientID"] for client in ray.nodes()]
target_clientid = clientids[1]
@ray.remote
@@ -415,7 +404,7 @@ def test_dynamic_res_concurrent_res_decrement(ray_start_cluster):
# Create the resource on node 1
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
assert ray.global_state.cluster_resources()[res_name] == res_capacity
assert ray.cluster_resources()[res_name] == res_capacity
# Task to hold the resource till the driver signals to finish
@ray.remote
@@ -457,7 +446,7 @@ def test_dynamic_res_concurrent_res_decrement(ray_start_cluster):
}) # This should be infeasible
successful, unsuccessful = ray.wait([task_3], timeout=TIMEOUT_DURATION)
assert unsuccessful # The task did not complete because it's infeasible
assert ray.global_state.available_resources()[res_name] == updated_capacity
assert ray.available_resources()[res_name] == updated_capacity
def test_dynamic_res_concurrent_res_delete(ray_start_cluster):
@@ -482,9 +471,7 @@ def test_dynamic_res_concurrent_res_delete(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
clientids = [
client["ClientID"] for client in ray.global_state.client_table()
]
clientids = [client["ClientID"] for client in ray.nodes()]
target_clientid = clientids[1]
@ray.remote
@@ -499,7 +486,7 @@ def test_dynamic_res_concurrent_res_delete(ray_start_cluster):
# Create the resource on node 1
ray.get(set_res.remote(res_name, res_capacity, target_clientid))
assert ray.global_state.cluster_resources()[res_name] == res_capacity
assert ray.cluster_resources()[res_name] == res_capacity
# Task to hold the resource till the driver signals to finish
@ray.remote
@@ -534,7 +521,7 @@ def test_dynamic_res_concurrent_res_delete(ray_start_cluster):
args=[], resources={res_name: 1}) # This should be infeasible
successful, unsuccessful = ray.wait([task_2], timeout=TIMEOUT_DURATION)
assert unsuccessful # The task did not complete because it's infeasible
assert res_name not in ray.global_state.available_resources()
assert res_name not in ray.available_resources()
def test_dynamic_res_creation_stress(ray_start_cluster):
@@ -553,9 +540,7 @@ def test_dynamic_res_creation_stress(ray_start_cluster):
ray.init(redis_address=cluster.redis_address)
clientids = [
client["ClientID"] for client in ray.global_state.client_table()
]
clientids = [client["ClientID"] for client in ray.nodes()]
target_clientid = clientids[1]
@ray.remote
@@ -578,7 +563,7 @@ def test_dynamic_res_creation_stress(ray_start_cluster):
start_time = time.time()
while time.time() - start_time < TIMEOUT and not success:
resources = ray.global_state.cluster_resources()
resources = ray.cluster_resources()
all_resources_created = []
for i in range(0, NUM_RES_TO_CREATE):
all_resources_created.append(str(i) in resources)
+5 -4
View File
@@ -164,7 +164,7 @@ def temporary_helper_function():
return 1
# There should be no errors yet.
assert len(ray.error_info()) == 0
assert len(ray.errors()) == 0
# Create an actor.
foo = Foo.remote()
@@ -376,8 +376,9 @@ def test_actor_scope_or_intentionally_killed_message(ray_start_regular):
a = Actor.remote()
a.__ray_terminate__.remote()
time.sleep(1)
assert len(ray.error_info()) == 0, (
"Should not have propogated an error - {}".format(ray.error_info()))
assert len(
ray.errors()) == 0, ("Should not have propogated an error - {}".format(
ray.errors()))
@pytest.mark.skip("This test does not work yet.")
@@ -653,7 +654,7 @@ def test_warning_for_dead_node(ray_start_cluster_2_nodes):
cluster = ray_start_cluster_2_nodes
cluster.wait_for_nodes()
client_ids = {item["ClientID"] for item in ray.global_state.client_table()}
client_ids = {item["ClientID"] for item in ray.nodes()}
# Try to make sure that the monitor has received at least one heartbeat
# from the node.
+9 -9
View File
@@ -18,8 +18,8 @@ import ray
reason="Timeout package not installed; skipping test that may hang.")
@pytest.mark.timeout(10)
def test_replenish_resources(ray_start_regular):
cluster_resources = ray.global_state.cluster_resources()
available_resources = ray.global_state.available_resources()
cluster_resources = ray.cluster_resources()
available_resources = ray.available_resources()
assert cluster_resources == available_resources
@ray.remote
@@ -30,7 +30,7 @@ def test_replenish_resources(ray_start_regular):
resources_reset = False
while not resources_reset:
available_resources = ray.global_state.available_resources()
available_resources = ray.available_resources()
resources_reset = (cluster_resources == available_resources)
assert resources_reset
@@ -40,7 +40,7 @@ def test_replenish_resources(ray_start_regular):
reason="Timeout package not installed; skipping test that may hang.")
@pytest.mark.timeout(10)
def test_uses_resources(ray_start_regular):
cluster_resources = ray.global_state.cluster_resources()
cluster_resources = ray.cluster_resources()
@ray.remote
def cpu_task():
@@ -50,7 +50,7 @@ def test_uses_resources(ray_start_regular):
resource_used = False
while not resource_used:
available_resources = ray.global_state.available_resources()
available_resources = ray.available_resources()
resource_used = available_resources.get(
"CPU", 0) == cluster_resources.get("CPU", 0) - 1
@@ -64,17 +64,17 @@ def test_uses_resources(ray_start_regular):
def test_add_remove_cluster_resources(ray_start_cluster_head):
"""Tests that Global State API is consistent with actual cluster."""
cluster = ray_start_cluster_head
assert ray.global_state.cluster_resources()["CPU"] == 1
assert ray.cluster_resources()["CPU"] == 1
nodes = []
nodes += [cluster.add_node(num_cpus=1)]
cluster.wait_for_nodes()
assert ray.global_state.cluster_resources()["CPU"] == 2
assert ray.cluster_resources()["CPU"] == 2
cluster.remove_node(nodes.pop())
cluster.wait_for_nodes()
assert ray.global_state.cluster_resources()["CPU"] == 1
assert ray.cluster_resources()["CPU"] == 1
for i in range(5):
nodes += [cluster.add_node(num_cpus=1)]
cluster.wait_for_nodes()
assert ray.global_state.cluster_resources()["CPU"] == 6
assert ray.cluster_resources()["CPU"] == 6
+8 -9
View File
@@ -30,17 +30,16 @@ def _test_cleanup_on_driver_exit(num_redis_shards):
time.sleep(2)
def StateSummary():
obj_tbl_len = len(ray.global_state.object_table())
task_tbl_len = len(ray.global_state.task_table())
func_tbl_len = len(ray.global_state.function_table())
return obj_tbl_len, task_tbl_len, func_tbl_len
obj_tbl_len = len(ray.objects())
task_tbl_len = len(ray.tasks())
return obj_tbl_len, task_tbl_len
def Driver(success):
success.value = True
# Start driver.
ray.init(redis_address=redis_address)
summary_start = StateSummary()
if (0, 1) != summary_start[:2]:
if (0, 1) != summary_start:
success.value = False
# Two new objects.
@@ -54,7 +53,7 @@ def _test_cleanup_on_driver_exit(num_redis_shards):
# 1 new function.
attempts = 0
while (2, 1, summary_start[2]) != StateSummary():
while (2, 1) != StateSummary():
time.sleep(0.1)
attempts += 1
if attempts == max_attempts_before_failing:
@@ -63,7 +62,7 @@ def _test_cleanup_on_driver_exit(num_redis_shards):
ray.get(f.remote())
attempts = 0
while (4, 2, summary_start[2] + 1) != StateSummary():
while (4, 2) != StateSummary():
time.sleep(0.1)
attempts += 1
if attempts == max_attempts_before_failing:
@@ -83,12 +82,12 @@ def _test_cleanup_on_driver_exit(num_redis_shards):
# Check that objects, tasks, and functions are cleaned up.
ray.init(redis_address=redis_address)
attempts = 0
while (0, 1) != StateSummary()[:2]:
while (0, 1) != StateSummary():
time.sleep(0.1)
attempts += 1
if attempts == max_attempts_before_failing:
break
assert (0, 1) == StateSummary()[:2]
assert (0, 1) == StateSummary()
ray.shutdown()
subprocess.Popen(["ray", "stop"]).wait()
+11 -11
View File
@@ -19,7 +19,7 @@ def test_error_isolation(call_ray_start):
ray.init(redis_address=redis_address)
# There shouldn't be any errors yet.
assert len(ray.error_info()) == 0
assert len(ray.errors()) == 0
error_string1 = "error_string1"
error_string2 = "error_string2"
@@ -33,13 +33,13 @@ def test_error_isolation(call_ray_start):
ray.get(f.remote())
# Wait for the error to appear in Redis.
while len(ray.error_info()) != 1:
while len(ray.errors()) != 1:
time.sleep(0.1)
print("Waiting for error to appear.")
# Make sure we got the error.
assert len(ray.error_info()) == 1
assert error_string1 in ray.error_info()[0]["message"]
assert len(ray.errors()) == 1
assert error_string1 in ray.errors()[0]["message"]
# Start another driver and make sure that it does not receive this
# error. Make the other driver throw an error, and make sure it
@@ -51,7 +51,7 @@ import time
ray.init(redis_address="{}")
time.sleep(1)
assert len(ray.error_info()) == 0
assert len(ray.errors()) == 0
@ray.remote
def f():
@@ -62,12 +62,12 @@ try:
except Exception as e:
pass
while len(ray.error_info()) != 1:
print(len(ray.error_info()))
while len(ray.errors()) != 1:
print(len(ray.errors()))
time.sleep(0.1)
assert len(ray.error_info()) == 1
assert len(ray.errors()) == 1
assert "{}" in ray.error_info()[0]["message"]
assert "{}" in ray.errors()[0]["message"]
print("success")
""".format(redis_address, error_string2, error_string2)
@@ -78,8 +78,8 @@ print("success")
# Make sure that the other error message doesn't show up for this
# driver.
assert len(ray.error_info()) == 1
assert error_string1 in ray.error_info()[0]["message"]
assert len(ray.errors()) == 1
assert error_string1 in ray.errors()[0]["message"]
def test_remote_function_isolation(call_ray_start):
+4 -4
View File
@@ -52,10 +52,10 @@ def test_internal_config(ray_start_cluster_head):
cluster.remove_node(worker)
time.sleep(1)
assert ray.global_state.cluster_resources()["CPU"] == 2
assert ray.cluster_resources()["CPU"] == 2
time.sleep(2)
assert ray.global_state.cluster_resources()["CPU"] == 1
assert ray.cluster_resources()["CPU"] == 1
def test_wait_for_nodes(ray_start_cluster_head):
@@ -70,12 +70,12 @@ def test_wait_for_nodes(ray_start_cluster_head):
[cluster.remove_node(w) for w in workers]
cluster.wait_for_nodes()
assert ray.global_state.cluster_resources()["CPU"] == 1
assert ray.cluster_resources()["CPU"] == 1
worker2 = cluster.add_node()
cluster.wait_for_nodes()
cluster.remove_node(worker2)
cluster.wait_for_nodes()
assert ray.global_state.cluster_resources()["CPU"] == 1
assert ray.cluster_resources()["CPU"] == 1
def test_worker_plasma_store_failure(ray_start_cluster_head):
+2 -2
View File
@@ -80,7 +80,7 @@ def test_object_broadcast(ray_start_cluster_with_resource):
# Wait for profiling information to be pushed to the profile table.
time.sleep(1)
transfer_events = ray.global_state.chrome_tracing_object_transfer_dump()
transfer_events = ray.object_transfer_timeline()
# Make sure that each object was transferred a reasonable number of times.
for x_id in object_ids:
@@ -160,7 +160,7 @@ def test_actor_broadcast(ray_start_cluster_with_resource):
# Wait for profiling information to be pushed to the profile table.
time.sleep(1)
transfer_events = ray.global_state.chrome_tracing_object_transfer_dump()
transfer_events = ray.object_transfer_timeline()
# Make sure that each object was transferred a reasonable number of times.
for x_id in object_ids:
+1 -1
View File
@@ -393,7 +393,7 @@ def wait_for_errors(error_check):
errors = []
time_left = 100
while time_left > 0:
errors = ray.error_info()
errors = ray.errors()
if error_check(errors):
break
time_left -= 1
+1 -1
View File
@@ -84,7 +84,7 @@ def run_string_as_driver_nonblocking(driver_script):
def relevant_errors(error_type):
return [info for info in ray.error_info() if info["type"] == error_type]
return [info for info in ray.errors() if info["type"] == error_type]
def wait_for_errors(error_type, num_errors, timeout=10):