wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-11 11:24:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove num_local_schedulers argument from ray.worker._init. (#3704)

Browse Source 
* Remove num_local_schedulers argument from ray.worker._init.

* Fix

* Fix tests.
This commit is contained in:
Robert Nishihara
2019-01-07 12:44:49 -08:00
committed by Philipp Moritz
parent e78562b2e8
commit c9d70f0dda
18 changed files with 388 additions and 513 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/actor_test.py
+117 -136
View File
@@ -13,7 +13,6 @@ import sys
import time
import ray
from ray.parameter import RayParams
import ray.ray_constants as ray_constants
import ray.test.test_utils
import ray.test.cluster_utils
@@ -40,9 +39,32 @@ def shutdown_only():
ray.shutdown()
@pytest.fixture()
def ray_start_cluster():
cluster = ray.test.cluster_utils.Cluster()
yield cluster
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
@pytest.fixture()
def two_node_cluster():
cluster = ray.test.cluster_utils.Cluster()
for _ in range(2):
cluster.add_node(num_cpus=1)
ray.init(redis_address=cluster.redis_address)
yield cluster
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
@pytest.fixture
def head_node_cluster(request):
timeout = getattr(request, 'param', 200)
timeout = getattr(request, "param", 200)
cluster = ray.test.cluster_utils.Cluster(
initialize_head=True,
connect=True,
@@ -741,13 +763,12 @@ def test_actors_on_nodes_with_no_cpus(ray_start_regular):
assert ready_ids == []
def test_actor_load_balancing(shutdown_only):
num_local_schedulers = 3
ray_params = RayParams(
start_ray_local=True,
num_cpus=1,
num_local_schedulers=num_local_schedulers)
ray.worker._init(ray_params)
def test_actor_load_balancing(ray_start_cluster):
cluster = ray_start_cluster
num_nodes = 3
for i in range(num_nodes):
cluster.add_node(num_cpus=1)
ray.init(redis_address=cluster.redis_address)
@ray.remote
class Actor1(object):
@@ -770,7 +791,7 @@ def test_actor_load_balancing(shutdown_only):
names = set(locations)
counts = [locations.count(name) for name in names]
print("Counts are {}.".format(counts))
if (len(names) == num_local_schedulers
if (len(names) == num_nodes
and all(count >= minimum_count for count in counts)):
break
attempts += 1
@@ -787,15 +808,14 @@ def test_actor_load_balancing(shutdown_only):
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Failing with new GCS API on Linux.")
def test_actor_gpus(shutdown_only):
num_local_schedulers = 3
num_gpus_per_scheduler = 4
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=num_local_schedulers,
num_cpus=(num_local_schedulers * [10 * num_gpus_per_scheduler]),
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
ray.worker._init(ray_params)
def test_actor_gpus(ray_start_cluster):
cluster = ray_start_cluster
num_nodes = 3
num_gpus_per_raylet = 4
for i in range(num_nodes):
cluster.add_node(
num_cpus=10 * num_gpus_per_raylet, num_gpus=num_gpus_per_raylet)
ray.init(redis_address=cluster.redis_address)
@ray.remote(num_gpus=1)
class Actor1(object):
@@ -808,18 +828,15 @@ def test_actor_gpus(shutdown_only):
tuple(self.gpu_ids))
# Create one actor per GPU.
actors = [
Actor1.remote()
for _ in range(num_local_schedulers * num_gpus_per_scheduler)
]
actors = [Actor1.remote() for _ in range(num_nodes * num_gpus_per_raylet)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get(
[actor.get_location_and_ids.remote() for actor in actors])
node_names = {location for location, gpu_id in locations_and_ids}
assert len(node_names) == num_local_schedulers
assert len(node_names) == num_nodes
location_actor_combinations = []
for node_name in node_names:
for gpu_id in range(num_gpus_per_scheduler):
for gpu_id in range(num_gpus_per_raylet):
location_actor_combinations.append((node_name, (gpu_id, )))
assert set(locations_and_ids) == set(location_actor_combinations)
@@ -830,15 +847,14 @@ def test_actor_gpus(shutdown_only):
assert ready_ids == []
def test_actor_multiple_gpus(shutdown_only):
num_local_schedulers = 3
num_gpus_per_scheduler = 5
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=num_local_schedulers,
num_cpus=(num_local_schedulers * [10 * num_gpus_per_scheduler]),
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
ray.worker._init(ray_params)
def test_actor_multiple_gpus(ray_start_cluster):
cluster = ray_start_cluster
num_nodes = 3
num_gpus_per_raylet = 5
for i in range(num_nodes):
cluster.add_node(
num_cpus=10 * num_gpus_per_raylet, num_gpus=num_gpus_per_raylet)
ray.init(redis_address=cluster.redis_address)
@ray.remote(num_gpus=2)
class Actor1(object):
@@ -851,12 +867,12 @@ def test_actor_multiple_gpus(shutdown_only):
tuple(self.gpu_ids))
# Create some actors.
actors1 = [Actor1.remote() for _ in range(num_local_schedulers * 2)]
actors1 = [Actor1.remote() for _ in range(num_nodes * 2)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get(
[actor.get_location_and_ids.remote() for actor in actors1])
node_names = {location for location, gpu_id in locations_and_ids}
assert len(node_names) == num_local_schedulers
assert len(node_names) == num_nodes
# Keep track of which GPU IDs are being used for each location.
gpus_in_use = {node_name: [] for node_name in node_names}
@@ -882,7 +898,7 @@ def test_actor_multiple_gpus(shutdown_only):
tuple(self.gpu_ids))
# Create some actors.
actors2 = [Actor2.remote() for _ in range(num_local_schedulers)]
actors2 = [Actor2.remote() for _ in range(num_nodes)]
# Make sure that no two actors are assigned to the same GPU.
locations_and_ids = ray.get(
[actor.get_location_and_ids.remote() for actor in actors2])
@@ -901,15 +917,14 @@ def test_actor_multiple_gpus(shutdown_only):
assert ready_ids == []
def test_actor_different_numbers_of_gpus(shutdown_only):
def test_actor_different_numbers_of_gpus(ray_start_cluster):
# Test that we can create actors on two nodes that have different
# numbers of GPUs.
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=3,
num_cpus=[10, 10, 10],
num_gpus=[0, 5, 10])
ray.worker._init(ray_params)
cluster = ray_start_cluster
cluster.add_node(num_cpus=10, num_gpus=0)
cluster.add_node(num_cpus=10, num_gpus=5)
cluster.add_node(num_cpus=10, num_gpus=10)
ray.init(redis_address=cluster.redis_address)
@ray.remote(num_gpus=1)
class Actor1(object):
@@ -942,19 +957,18 @@ def test_actor_different_numbers_of_gpus(shutdown_only):
assert ready_ids == []
def test_actor_multiple_gpus_from_multiple_tasks(shutdown_only):
num_local_schedulers = 5
num_gpus_per_scheduler = 5
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=num_local_schedulers,
redirect_output=True,
num_cpus=(num_local_schedulers * [10 * num_gpus_per_scheduler]),
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]),
_internal_config=json.dumps({
"num_heartbeats_timeout": 1000
}))
ray.worker._init(ray_params)
def test_actor_multiple_gpus_from_multiple_tasks(ray_start_cluster):
cluster = ray_start_cluster
num_nodes = 5
num_gpus_per_raylet = 5
for i in range(num_nodes):
cluster.add_node(
num_cpus=10 * num_gpus_per_raylet,
num_gpus=num_gpus_per_raylet,
_internal_config=json.dumps({
"num_heartbeats_timeout": 1000
}))
ray.init(redis_address=cluster.redis_address)
@ray.remote
def create_actors(i, n):
@@ -987,8 +1001,7 @@ def test_actor_multiple_gpus_from_multiple_tasks(shutdown_only):
return locations
all_locations = ray.get([
create_actors.remote(i, num_gpus_per_scheduler)
for i in range(num_local_schedulers)
create_actors.remote(i, num_gpus_per_raylet) for i in range(num_nodes)
])
# Make sure that no two actors are assigned to the same GPU.
@@ -996,7 +1009,7 @@ def test_actor_multiple_gpus_from_multiple_tasks(shutdown_only):
location
for locations in all_locations for location, gpu_id in locations
}
assert len(node_names) == num_local_schedulers
assert len(node_names) == num_nodes
# Keep track of which GPU IDs are being used for each location.
gpus_in_use = {node_name: [] for node_name in node_names}
@@ -1004,7 +1017,7 @@ def test_actor_multiple_gpus_from_multiple_tasks(shutdown_only):
for location, gpu_ids in locations:
gpus_in_use[location].extend(gpu_ids)
for node_name in node_names:
assert len(set(gpus_in_use[node_name])) == num_gpus_per_scheduler
assert len(set(gpus_in_use[node_name])) == num_gpus_per_raylet
@ray.remote(num_gpus=1)
class Actor(object):
@@ -1023,15 +1036,14 @@ def test_actor_multiple_gpus_from_multiple_tasks(shutdown_only):
@pytest.mark.skipif(
sys.version_info < (3, 0), reason="This test requires Python 3.")
def test_actors_and_tasks_with_gpus(shutdown_only):
num_local_schedulers = 3
num_gpus_per_scheduler = 6
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=num_local_schedulers,
num_cpus=num_gpus_per_scheduler,
num_gpus=(num_local_schedulers * [num_gpus_per_scheduler]))
ray.worker._init(ray_params)
def test_actors_and_tasks_with_gpus(ray_start_cluster):
cluster = ray_start_cluster
num_nodes = 3
num_gpus_per_raylet = 6
for i in range(num_nodes):
cluster.add_node(
num_cpus=num_gpus_per_raylet, num_gpus=num_gpus_per_raylet)
ray.init(redis_address=cluster.redis_address)
def check_intervals_non_overlapping(list_of_intervals):
for i in range(len(list_of_intervals)):
@@ -1056,7 +1068,7 @@ def test_actors_and_tasks_with_gpus(shutdown_only):
t2 = time.monotonic()
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 1
assert gpu_ids[0] in range(num_gpus_per_scheduler)
assert gpu_ids[0] in range(num_gpus_per_raylet)
return (ray.worker.global_worker.plasma_client.store_socket_name,
tuple(gpu_ids), [t1, t2])
@@ -1067,8 +1079,8 @@ def test_actors_and_tasks_with_gpus(shutdown_only):
t2 = time.monotonic()
gpu_ids = ray.get_gpu_ids()
assert len(gpu_ids) == 2
assert gpu_ids[0] in range(num_gpus_per_scheduler)
assert gpu_ids[1] in range(num_gpus_per_scheduler)
assert gpu_ids[0] in range(num_gpus_per_raylet)
assert gpu_ids[1] in range(num_gpus_per_raylet)
return (ray.worker.global_worker.plasma_client.store_socket_name,
tuple(gpu_ids), [t1, t2])
@@ -1077,7 +1089,7 @@ def test_actors_and_tasks_with_gpus(shutdown_only):
def __init__(self):
self.gpu_ids = ray.get_gpu_ids()
assert len(self.gpu_ids) == 1
assert self.gpu_ids[0] in range(num_gpus_per_scheduler)
assert self.gpu_ids[0] in range(num_gpus_per_raylet)
def get_location_and_ids(self):
assert ray.get_gpu_ids() == self.gpu_ids
@@ -1086,16 +1098,10 @@ def test_actors_and_tasks_with_gpus(shutdown_only):
def locations_to_intervals_for_many_tasks():
# Launch a bunch of GPU tasks.
locations_ids_and_intervals = ray.get([
f1.remote()
for _ in range(5 * num_local_schedulers * num_gpus_per_scheduler)
] + [
f2.remote()
for _ in range(5 * num_local_schedulers * num_gpus_per_scheduler)
] + [
f1.remote()
for _ in range(5 * num_local_schedulers * num_gpus_per_scheduler)
])
locations_ids_and_intervals = ray.get(
[f1.remote() for _ in range(5 * num_nodes * num_gpus_per_raylet)] +
[f2.remote() for _ in range(5 * num_nodes * num_gpus_per_raylet)] +
[f1.remote() for _ in range(5 * num_nodes * num_gpus_per_raylet)])
locations_to_intervals = collections.defaultdict(lambda: [])
for location, gpu_ids, interval in locations_ids_and_intervals:
@@ -1106,8 +1112,7 @@ def test_actors_and_tasks_with_gpus(shutdown_only):
# Run a bunch of GPU tasks.
locations_to_intervals = locations_to_intervals_for_many_tasks()
# Make sure that all GPUs were used.
assert (len(locations_to_intervals) == num_local_schedulers *
num_gpus_per_scheduler)
assert (len(locations_to_intervals) == num_nodes * num_gpus_per_raylet)
# For each GPU, verify that the set of tasks that used this specific
# GPU did not overlap in time.
for locations in locations_to_intervals:
@@ -1124,8 +1129,7 @@ def test_actors_and_tasks_with_gpus(shutdown_only):
# Run a bunch of GPU tasks.
locations_to_intervals = locations_to_intervals_for_many_tasks()
# Make sure that all but one of the GPUs were used.
assert (len(locations_to_intervals) ==
num_local_schedulers * num_gpus_per_scheduler - 1)
assert (len(locations_to_intervals) == num_nodes * num_gpus_per_raylet - 1)
# For each GPU, verify that the set of tasks that used this specific
# GPU did not overlap in time.
for locations in locations_to_intervals:
@@ -1141,8 +1145,8 @@ def test_actors_and_tasks_with_gpus(shutdown_only):
# Run a bunch of GPU tasks.
locations_to_intervals = locations_to_intervals_for_many_tasks()
# Make sure that all but 11 of the GPUs were used.
assert (len(locations_to_intervals) ==
num_local_schedulers * num_gpus_per_scheduler - 1 - 3)
assert (
len(locations_to_intervals) == num_nodes * num_gpus_per_raylet - 1 - 3)
# For each GPU, verify that the set of tasks that used this specific
# GPU did not overlap in time.
for locations in locations_to_intervals:
@@ -1154,8 +1158,7 @@ def test_actors_and_tasks_with_gpus(shutdown_only):
# Create more actors to fill up all the GPUs.
more_actors = [
Actor1.remote()
for _ in range(num_local_schedulers * num_gpus_per_scheduler - 1 - 3)
Actor1.remote() for _ in range(num_nodes * num_gpus_per_raylet - 1 - 3)
]
# Wait for the actors to finish being created.
ray.get([actor.get_location_and_ids.remote() for actor in more_actors])
@@ -1356,10 +1359,8 @@ def test_actor_init_fails(head_node_cluster):
def test_reconstruction_suppression(head_node_cluster):
num_local_schedulers = 10
worker_nodes = [
head_node_cluster.add_node() for _ in range(num_local_schedulers)
]
num_nodes = 10
worker_nodes = [head_node_cluster.add_node() for _ in range(num_nodes)]
@ray.remote(max_reconstructions=1)
class Counter(object):
@@ -1394,13 +1395,6 @@ def test_reconstruction_suppression(head_node_cluster):
def setup_counter_actor(test_checkpoint=False,
save_exception=False,
resume_exception=False):
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=2,
num_cpus=1,
redirect_output=True)
ray.worker._init(ray_params)
# Only set the checkpoint interval if we're testing with checkpointing.
checkpoint_interval = -1
if test_checkpoint:
@@ -1461,7 +1455,7 @@ def setup_counter_actor(test_checkpoint=False,
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
def test_checkpointing(shutdown_only):
def test_checkpointing(two_node_cluster):
actor, ids = setup_counter_actor(test_checkpoint=True)
# Wait for the last task to finish running.
ray.get(ids[-1])
@@ -1489,7 +1483,7 @@ def test_checkpointing(shutdown_only):
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
def test_remote_checkpoint(shutdown_only):
def test_remote_checkpoint(two_node_cluster):
actor, ids = setup_counter_actor(test_checkpoint=True)
# Do a remote checkpoint call and wait for it to finish.
@@ -1518,7 +1512,7 @@ def test_remote_checkpoint(shutdown_only):
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
def test_lost_checkpoint(shutdown_only):
def test_lost_checkpoint(two_node_cluster):
actor, ids = setup_counter_actor(test_checkpoint=True)
# Wait for the first fraction of tasks to finish running.
ray.get(ids[len(ids) // 10])
@@ -1547,7 +1541,7 @@ def test_lost_checkpoint(shutdown_only):
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
def test_checkpoint_exception(shutdown_only):
def test_checkpoint_exception(two_node_cluster):
actor, ids = setup_counter_actor(test_checkpoint=True, save_exception=True)
# Wait for the last task to finish running.
ray.get(ids[-1])
@@ -1578,7 +1572,7 @@ def test_checkpoint_exception(shutdown_only):
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
def test_checkpoint_resume_exception(shutdown_only):
def test_checkpoint_resume_exception(two_node_cluster):
actor, ids = setup_counter_actor(
test_checkpoint=True, resume_exception=True)
# Wait for the last task to finish running.
@@ -1608,7 +1602,7 @@ def test_checkpoint_resume_exception(shutdown_only):
@pytest.mark.skip("Fork/join consistency not yet implemented.")
def test_distributed_handle(self):
def test_distributed_handle(two_node_cluster):
counter, ids = setup_counter_actor(test_checkpoint=False)
@ray.remote
@@ -1648,7 +1642,7 @@ def test_distributed_handle(self):
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
def test_remote_checkpoint_distributed_handle(shutdown_only):
def test_remote_checkpoint_distributed_handle(two_node_cluster):
counter, ids = setup_counter_actor(test_checkpoint=True)
@ray.remote
@@ -1691,7 +1685,7 @@ def test_remote_checkpoint_distributed_handle(shutdown_only):
@pytest.mark.skip("Fork/join consistency not yet implemented.")
def test_checkpoint_distributed_handle(shutdown_only):
def test_checkpoint_distributed_handle(two_node_cluster):
counter, ids = setup_counter_actor(test_checkpoint=True)
@ray.remote
@@ -1729,13 +1723,6 @@ def test_checkpoint_distributed_handle(shutdown_only):
def _test_nondeterministic_reconstruction(num_forks, num_items_per_fork,
num_forks_to_wait):
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=2,
num_cpus=1,
redirect_output=True)
ray.worker._init(ray_params)
# Make a shared queue.
@ray.remote
class Queue(object):
@@ -1806,14 +1793,14 @@ def _test_nondeterministic_reconstruction(num_forks, num_items_per_fork,
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Currently doesn't work with the new GCS.")
def test_nondeterministic_reconstruction(shutdown_only):
def test_nondeterministic_reconstruction(two_node_cluster):
_test_nondeterministic_reconstruction(10, 100, 10)
@pytest.mark.skip("Nondeterministic reconstruction currently not supported "
"when there are concurrent forks that didn't finish "
"initial execution.")
def test_nondeterministic_reconstruction_concurrent_forks(shutdown_only):
def test_nondeterministic_reconstruction_concurrent_forks(two_node_cluster):
_test_nondeterministic_reconstruction(10, 100, 1)
@@ -2027,17 +2014,11 @@ def test_lifetime_and_transient_resources(ray_start_regular):
assert len(ready_ids) == 1
def test_custom_label_placement(shutdown_only):
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=2,
num_cpus=2,
resources=[{
"CustomResource1": 2
}, {
"CustomResource2": 2
}])
ray.worker._init(ray_params)
def test_custom_label_placement(ray_start_cluster):
cluster = ray_start_cluster
cluster.add_node(num_cpus=2, resources={"CustomResource1": 2})
cluster.add_node(num_cpus=2, resources={"CustomResource2": 2})
ray.init(redis_address=cluster.redis_address)
@ray.remote(resources={"CustomResource1": 1})
class ResourceActor1(object):
@@ -2263,22 +2244,22 @@ def test_actor_reconstruction_on_node_failure(head_node_cluster):
# this test. Because if this value is too small, suprious task reconstruction
# may happen and cause the test fauilure. If the value is too large, this test
# could be very slow. We can remove this once we support dynamic timeout.
@pytest.mark.parametrize('head_node_cluster', [1000], indirect=True)
@pytest.mark.parametrize("head_node_cluster", [1000], indirect=True)
def test_multiple_actor_reconstruction(head_node_cluster):
# This test can be made more stressful by increasing the numbers below.
# The total number of actors created will be
# num_actors_at_a_time * num_local_schedulers.
num_local_schedulers = 5
# num_actors_at_a_time * num_nodes.
num_nodes = 5
num_actors_at_a_time = 3
num_function_calls_at_a_time = 10
worker_nodes = [
head_node_cluster.add_node(
resources={"CPU": 3},
num_cpus=3,
_internal_config=json.dumps({
"initial_reconstruction_timeout_milliseconds": 200,
"num_heartbeats_timeout": 10,
})) for _ in range(num_local_schedulers)
})) for _ in range(num_nodes)
]
@ray.remote(max_reconstructions=ray.ray_constants.INFINITE_RECONSTRUCTION)
test/array_test.py
+7 -4
View File
@@ -10,7 +10,7 @@ import sys
import ray
import ray.experimental.array.remote as ra
import ray.experimental.array.distributed as da
from ray.parameter import RayParams
import ray.test.cluster_utils
if sys.version_info >= (3, 0):
from importlib import reload
@@ -75,12 +75,15 @@ def ray_start_two_nodes():
]:
reload(module)
# Start the Ray processes.
ray_params = RayParams(
start_ray_local=True, num_local_schedulers=2, num_cpus=[10, 10])
ray.worker._init(ray_params)
cluster = ray.test.cluster_utils.Cluster()
for _ in range(2):
cluster.add_node(num_cpus=10)
ray.init(redis_address=cluster.redis_address)
yield None
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
def test_distributed_array_methods(ray_start_two_nodes):
test/component_failures_test.py
+41 -51
View File
@@ -12,22 +12,10 @@ import numpy as np
import pytest
import ray
from ray.parameter import RayParams
from ray.test.cluster_utils import Cluster
from ray.test.test_utils import run_string_as_driver_nonblocking
@pytest.fixture
def ray_start_workers_separate():
# Start the Ray processes.
ray_params = RayParams(
num_cpus=1, start_ray_local=True, redirect_output=True)
ray.worker._init(ray_params)
yield None
# The code after the yield will run as teardown code.
ray.shutdown()
@pytest.fixture
def shutdown_only():
yield None
@@ -38,21 +26,22 @@ def shutdown_only():
@pytest.fixture
def ray_start_cluster():
node_args = {
"resources": dict(CPU=8),
"num_cpus": 8,
"_internal_config": json.dumps({
"initial_reconstruction_timeout_milliseconds": 1000,
"num_heartbeats_timeout": 10
})
}
# Start with 4 worker nodes and 8 cores each.
g = Cluster(initialize_head=True, connect=True, head_node_args=node_args)
cluster = Cluster(
initialize_head=True, connect=True, head_node_args=node_args)
workers = []
for _ in range(4):
workers.append(g.add_node(**node_args))
g.wait_for_nodes()
yield g
workers.append(cluster.add_node(**node_args))
cluster.wait_for_nodes()
yield cluster
ray.shutdown()
g.shutdown()
cluster.shutdown()
# This test checks that when a worker dies in the middle of a get, the plasma
@@ -235,23 +224,22 @@ ray.wait([ray.ObjectID(ray.utils.hex_to_binary("{}"))])
@pytest.fixture(params=[(1, 4), (4, 4)])
def ray_start_workers_separate_multinode(request):
num_local_schedulers = request.param[0]
num_nodes = request.param[0]
num_initial_workers = request.param[1]
# Start the Ray processes.
ray_params = RayParams(
num_local_schedulers=num_local_schedulers,
start_ray_local=True,
num_cpus=[num_initial_workers] * num_local_schedulers,
redirect_output=True)
ray.worker._init(ray_params)
yield num_local_schedulers, num_initial_workers
cluster = Cluster()
for _ in range(num_nodes):
cluster.add_node(num_cpus=num_initial_workers)
ray.init(redis_address=cluster.redis_address)
yield num_nodes, num_initial_workers
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
def test_worker_failed(ray_start_workers_separate_multinode):
num_local_schedulers, num_initial_workers = (
ray_start_workers_separate_multinode)
num_nodes, num_initial_workers = (ray_start_workers_separate_multinode)
@ray.remote
def f(x):
@@ -260,9 +248,7 @@ def test_worker_failed(ray_start_workers_separate_multinode):
# Submit more tasks than there are workers so that all workers and
# cores are utilized.
object_ids = [
f.remote(i) for i in range(num_initial_workers * num_local_schedulers)
]
object_ids = [f.remote(i) for i in range(num_initial_workers * num_nodes)]
object_ids += [f.remote(object_id) for object_id in object_ids]
# Allow the tasks some time to begin executing.
time.sleep(0.1)
@@ -276,22 +262,30 @@ def test_worker_failed(ray_start_workers_separate_multinode):
ray.get(object_ids)
@pytest.fixture
def ray_initialize_cluster():
# Start with 4 workers and 4 cores.
num_nodes = 4
num_workers_per_scheduler = 8
cluster = Cluster()
for _ in range(num_nodes):
cluster.add_node(
num_cpus=num_workers_per_scheduler,
_internal_config=json.dumps({
"initial_reconstruction_timeout_milliseconds": 1000,
"num_heartbeats_timeout": 10,
}))
ray.init(redis_address=cluster.redis_address)
yield None
ray.shutdown()
cluster.shutdown()
def _test_component_failed(component_type):
"""Kill a component on all worker nodes and check workload succeeds."""
# Start with 4 workers and 4 cores.
num_local_schedulers = 4
num_workers_per_scheduler = 8
ray_params = RayParams(
num_local_schedulers=num_local_schedulers,
start_ray_local=True,
num_cpus=[num_workers_per_scheduler] * num_local_schedulers,
redirect_output=True,
_internal_config=json.dumps({
"initial_reconstruction_timeout_milliseconds": 1000,
"num_heartbeats_timeout": 10,
}))
ray.worker._init(ray_params)
# Submit many tasks with many dependencies.
@ray.remote
def f(x):
@@ -346,20 +340,18 @@ def check_components_alive(component_type, check_component_alive):
assert not component.poll() is None
def test_raylet_failed():
def test_raylet_failed(ray_initialize_cluster):
# Kill all local schedulers on worker nodes.
_test_component_failed(ray.services.PROCESS_TYPE_RAYLET)
# The plasma stores should still be alive on the worker nodes.
check_components_alive(ray.services.PROCESS_TYPE_PLASMA_STORE, True)
ray.shutdown()
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
def test_plasma_store_failed():
def test_plasma_store_failed(ray_initialize_cluster):
# Kill all plasma stores on worker nodes.
_test_component_failed(ray.services.PROCESS_TYPE_PLASMA_STORE)
@@ -367,8 +359,6 @@ def test_plasma_store_failed():
check_components_alive(ray.services.PROCESS_TYPE_PLASMA_STORE, False)
check_components_alive(ray.services.PROCESS_TYPE_RAYLET, False)
ray.shutdown()
def test_actor_creation_node_failure(ray_start_cluster):
# TODO(swang): Refactor test_raylet_failed, etc to reuse the below code.
test/failure_test.py
+16 -15
View File
@@ -11,8 +11,8 @@ import tempfile
import threading
import time
from ray.parameter import RayParams
import ray.ray_constants as ray_constants
import ray.test.cluster_utils
from ray.utils import _random_string
import pytest
@@ -620,25 +620,26 @@ def test_warning_for_too_many_nested_tasks(shutdown_only):
@pytest.fixture
def ray_start_two_nodes():
# Start the Ray processes.
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=2,
num_cpus=0,
_internal_config=json.dumps({
"num_heartbeats_timeout": 40
}))
ray.worker._init(ray_params)
yield None
cluster = ray.test.cluster_utils.Cluster()
for _ in range(2):
cluster.add_node(
num_cpus=0,
_internal_config=json.dumps({
"num_heartbeats_timeout": 40
}))
ray.init(redis_address=cluster.redis_address)
yield cluster
# The code after the yield will run as teardown code.
ray.shutdown()
cluster.shutdown()
# Note that this test will take at least 10 seconds because it must wait for
# the monitor to detect enough missed heartbeats.
def test_warning_for_dead_node(ray_start_two_nodes):
# Wait for the raylet to appear in the client table.
while len(ray.global_state.client_table()) < 2:
time.sleep(0.1)
cluster = ray_start_two_nodes
cluster.wait_for_nodes(2)
client_ids = {item["ClientID"] for item in ray.global_state.client_table()}
@@ -647,8 +648,8 @@ def test_warning_for_dead_node(ray_start_two_nodes):
time.sleep(0.5)
# Kill both raylets.
ray.services.all_processes[ray.services.PROCESS_TYPE_RAYLET][1].kill()
ray.services.all_processes[ray.services.PROCESS_TYPE_RAYLET][0].kill()
cluster.list_all_nodes()[1].kill_raylet()
cluster.list_all_nodes()[0].kill_raylet()
# Check that we get warning messages for both raylets.
wait_for_errors(ray_constants.REMOVED_NODE_ERROR, 2, timeout=40)
test/multi_node_test_2.py
+2 -2
View File
@@ -20,7 +20,7 @@ def start_connected_cluster():
initialize_head=True,
connect=True,
head_node_args={
"resources": dict(CPU=1),
"num_cpus": 1,
"_internal_config": json.dumps({
"num_heartbeats_timeout": 10
})
@@ -38,7 +38,7 @@ def start_connected_longer_cluster():
initialize_head=True,
connect=True,
head_node_args={
"resources": dict(CPU=1),
"num_cpus": 1,
"_internal_config": json.dumps({
"num_heartbeats_timeout": 20
})
test/object_manager_test.py
+1 -1
View File
@@ -216,7 +216,7 @@ def test_object_transfer_retry(ray_start_empty_cluster):
"object_manager_repeated_push_delay_ms": repeated_push_delay * 1000
})
cluster.add_node(_internal_config=config)
cluster.add_node(resources={"GPU": 1}, _internal_config=config)
cluster.add_node(num_gpus=1, _internal_config=config)
ray.init(redis_address=cluster.redis_address)
@ray.remote(num_gpus=1)
test/runtest.py
+66 -83
View File
@@ -18,7 +18,6 @@ import numpy as np
import pytest
import ray
from ray.parameter import RayParams
import ray.ray_constants as ray_constants
import ray.test.cluster_utils
import ray.test.test_utils
@@ -304,22 +303,6 @@ def test_putting_object_that_closes_over_object_id(ray_start):
ray.put(f)
def test_python_workers(shutdown_only):
# Test the codepath for starting workers from the Python script,
# instead of the local scheduler. This codepath is for debugging
# purposes only.
num_workers = 4
ray_params = RayParams(num_cpus=num_workers, start_ray_local=True)
ray.worker._init(ray_params)
@ray.remote
def f(x):
return x
values = ray.get([f.remote(1) for i in range(num_workers * 2)])
assert values == [1] * (num_workers * 2)
def test_put_get(shutdown_only):
ray.init(num_cpus=0)
@@ -1074,7 +1057,6 @@ def test_object_transfer_dump(ray_start_cluster):
num_nodes = 3
for i in range(num_nodes):
cluster.add_node(resources={str(i): 1}, object_store_memory=10**9)
ray.init(redis_address=cluster.redis_address)
@ray.remote
@@ -1249,7 +1231,7 @@ def test_multithreading(shutdown_only):
ray.get(a.join.remote())
def test_free_objects_multi_node(shutdown_only):
def test_free_objects_multi_node(ray_start_cluster):
# This test will do following:
# 1. Create 3 raylets that each hold an actor.
# 2. Each actor creates an object which is the deletion target.
@@ -1261,20 +1243,14 @@ def test_free_objects_multi_node(shutdown_only):
# tasks, so the flushing operations may be executed in different
# workers and the plasma client holding the deletion target
# may not be flushed.
cluster = ray_start_cluster
config = json.dumps({"object_manager_repeated_push_delay_ms": 1000})
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=3,
num_cpus=[1, 1, 1],
resources=[{
"Custom0": 1
}, {
"Custom1": 1
}, {
"Custom2": 1
}],
_internal_config=config)
ray.worker._init(ray_params)
for i in range(3):
cluster.add_node(
num_cpus=1,
resources={"Custom{}".format(i): 1},
_internal_config=config)
ray.init(redis_address=cluster.redis_address)
@ray.remote(resources={"Custom0": 1})
class ActorOnNode0(object):
@@ -1718,10 +1694,11 @@ def test_zero_cpus(shutdown_only):
ray.get(f.remote())
def test_zero_cpus_actor(shutdown_only):
ray_params = RayParams(
start_ray_local=True, num_local_schedulers=2, num_cpus=[0, 2])
ray.worker._init(ray_params)
def test_zero_cpus_actor(ray_start_cluster):
cluster = ray_start_cluster
cluster.add_node(num_cpus=0)
cluster.add_node(num_cpus=2)
ray.init(redis_address=cluster.redis_address)
local_plasma = ray.worker.global_worker.plasma_client.store_socket_name
@@ -1786,16 +1763,16 @@ def test_fractional_resources(shutdown_only):
Foo2._remote([], {}, resources={"Custom": 1.5})
def test_multiple_local_schedulers(shutdown_only):
def test_multiple_local_schedulers(ray_start_cluster):
# This test will define a bunch of tasks that can only be assigned to
# specific local schedulers, and we will check that they are assigned
# to the correct local schedulers.
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=3,
num_cpus=[11, 5, 10],
num_gpus=[0, 5, 1])
address_info = ray.worker._init(ray_params)
cluster = ray_start_cluster
cluster.add_node(num_cpus=11, num_gpus=0)
cluster.add_node(num_cpus=5, num_gpus=5)
cluster.add_node(num_cpus=10, num_gpus=1)
ray.init(redis_address=cluster.redis_address)
cluster.wait_for_nodes(3)
# Define a bunch of remote functions that all return the socket name of
# the plasma store. Since there is a one-to-one correspondence between
@@ -1857,7 +1834,21 @@ def test_multiple_local_schedulers(shutdown_only):
results.append(run_on_0_2.remote())
return names, results
store_names = address_info["object_store_addresses"]
client_table = ray.global_state.client_table()
store_names = []
store_names += [
client["ObjectStoreSocketName"] for client in client_table
if client["Resources"]["GPU"] == 0
]
store_names += [
client["ObjectStoreSocketName"] for client in client_table
if client["Resources"]["GPU"] == 5
]
store_names += [
client["ObjectStoreSocketName"] for client in client_table
if client["Resources"]["GPU"] == 1
]
assert len(store_names) == 3
def validate_names_and_results(names, results):
for name, result in zip(names, ray.get(results)):
@@ -1898,17 +1889,11 @@ def test_multiple_local_schedulers(shutdown_only):
validate_names_and_results(names, results)
def test_custom_resources(shutdown_only):
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=2,
num_cpus=[3, 3],
resources=[{
"CustomResource": 0
}, {
"CustomResource": 1
}])
ray.worker._init(ray_params)
def test_custom_resources(ray_start_cluster):
cluster = ray_start_cluster
cluster.add_node(num_cpus=3, resources={"CustomResource": 0})
cluster.add_node(num_cpus=3, resources={"CustomResource": 1})
ray.init(redis_address=cluster.redis_address)
@ray.remote
def f():
@@ -1940,19 +1925,19 @@ def test_custom_resources(shutdown_only):
ray.get([h.remote() for _ in range(5)])
def test_two_custom_resources(shutdown_only):
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=2,
num_cpus=[3, 3],
resources=[{
def test_two_custom_resources(ray_start_cluster):
cluster = ray_start_cluster
cluster.add_node(
num_cpus=3, resources={
"CustomResource1": 1,
"CustomResource2": 2
}, {
})
cluster.add_node(
num_cpus=3, resources={
"CustomResource1": 3,
"CustomResource2": 4
}])
ray.worker._init(ray_params)
})
ray.init(redis_address=cluster.redis_address)
@ray.remote(resources={"CustomResource1": 1})
def f():
@@ -2131,7 +2116,7 @@ def test_max_call_tasks(shutdown_only):
def attempt_to_load_balance(remote_function,
args,
total_tasks,
num_local_schedulers,
num_nodes,
minimum_count,
num_attempts=100):
attempts = 0
@@ -2141,43 +2126,41 @@ def attempt_to_load_balance(remote_function,
names = set(locations)
counts = [locations.count(name) for name in names]
logger.info("Counts are {}.".format(counts))
if (len(names) == num_local_schedulers
if (len(names) == num_nodes
and all(count >= minimum_count for count in counts)):
break
attempts += 1
assert attempts < num_attempts
def test_load_balancing(shutdown_only):
def test_load_balancing(ray_start_cluster):
# This test ensures that tasks are being assigned to all local
# schedulers in a roughly equal manner.
num_local_schedulers = 3
cluster = ray_start_cluster
num_nodes = 3
num_cpus = 7
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=num_local_schedulers,
num_cpus=num_cpus)
ray.worker._init(ray_params)
for _ in range(num_nodes):
cluster.add_node(num_cpus=num_cpus)
ray.init(redis_address=cluster.redis_address)
@ray.remote
def f():
time.sleep(0.01)
return ray.worker.global_worker.plasma_client.store_socket_name
attempt_to_load_balance(f, [], 100, num_local_schedulers, 10)
attempt_to_load_balance(f, [], 1000, num_local_schedulers, 100)
attempt_to_load_balance(f, [], 100, num_nodes, 10)
attempt_to_load_balance(f, [], 1000, num_nodes, 100)
def test_load_balancing_with_dependencies(shutdown_only):
def test_load_balancing_with_dependencies(ray_start_cluster):
# This test ensures that tasks are being assigned to all local
# schedulers in a roughly equal manner even when the tasks have
# dependencies.
num_local_schedulers = 3
ray_params = RayParams(
start_ray_local=True,
num_local_schedulers=num_local_schedulers,
num_cpus=1)
ray.worker._init(ray_params)
cluster = ray_start_cluster
num_nodes = 3
for _ in range(num_nodes):
cluster.add_node(num_cpus=1)
ray.init(redis_address=cluster.redis_address)
@ray.remote
def f(x):
@@ -2189,7 +2172,7 @@ def test_load_balancing_with_dependencies(shutdown_only):
# schedulers.
x = ray.put(np.zeros(1000000))
attempt_to_load_balance(f, [x], 100, num_local_schedulers, 25)
attempt_to_load_balance(f, [x], 100, num_nodes, 25)
def wait_for_num_tasks(num_tasks, timeout=10):
test/stress_tests.py
+2 -6
View File
@@ -41,9 +41,7 @@ def ray_start_combination(request):
cluster = Cluster(
initialize_head=True,
head_node_args={
"resources": {
"CPU": 10
},
"num_cpus": 10,
"redis_max_memory": 10**7
})
for i in range(num_nodes - 1):
@@ -200,9 +198,7 @@ def ray_start_reconstruction(request):
cluster = Cluster(
initialize_head=True,
head_node_args={
"resources": {
"CPU": 1
},
"num_cpus": 1,
"object_store_memory": plasma_store_memory // num_nodes,
"redis_max_memory": 10**7,
"redirect_output": True,
test/tempfile_test.py
+8 -8
View File
@@ -70,8 +70,8 @@ def test_raylet_tempfiles():
assert top_levels == {"ray_ui.ipynb", "sockets", "logs"}
log_files = set(os.listdir(tempfile_services.get_logs_dir_path()))
assert log_files == {
"log_monitor.out", "log_monitor.err", "plasma_store_0.out",
"plasma_store_0.err", "webui.out", "webui.err", "monitor.out",
"log_monitor.out", "log_monitor.err", "plasma_store.out",
"plasma_store.err", "webui.out", "webui.err", "monitor.out",
"monitor.err", "redis-shard_0.out", "redis-shard_0.err", "redis.out",
"redis.err"
} # without raylet logs
@@ -84,10 +84,10 @@ def test_raylet_tempfiles():
assert top_levels == {"ray_ui.ipynb", "sockets", "logs"}
log_files = set(os.listdir(tempfile_services.get_logs_dir_path()))
assert log_files == {
"log_monitor.out", "log_monitor.err", "plasma_store_0.out",
"plasma_store_0.err", "webui.out", "webui.err", "monitor.out",
"log_monitor.out", "log_monitor.err", "plasma_store.out",
"plasma_store.err", "webui.out", "webui.err", "monitor.out",
"monitor.err", "redis-shard_0.out", "redis-shard_0.err", "redis.out",
"redis.err", "raylet_0.out", "raylet_0.err"
"redis.err", "raylet.out", "raylet.err"
} # with raylet logs
socket_files = set(os.listdir(tempfile_services.get_sockets_dir_path()))
assert socket_files == {"plasma_store", "raylet"}
@@ -99,10 +99,10 @@ def test_raylet_tempfiles():
time.sleep(3) # wait workers to start
log_files = set(os.listdir(tempfile_services.get_logs_dir_path()))
assert log_files.issuperset({
"log_monitor.out", "log_monitor.err", "plasma_store_0.out",
"plasma_store_0.err", "webui.out", "webui.err", "monitor.out",
"log_monitor.out", "log_monitor.err", "plasma_store.out",
"plasma_store.err", "webui.out", "webui.err", "monitor.out",
"monitor.err", "redis-shard_0.out", "redis-shard_0.err", "redis.out",
"redis.err", "raylet_0.out", "raylet_0.err"
"redis.err", "raylet.out", "raylet.err"
}) # with raylet logs
# Check numbers of worker log file.