wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-14 12:40:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove legacy Ray code. (#3121)

Browse Source 
* Remove legacy Ray code.

* Fix cmake and simplify monitor.

* Fix linting

* Updates

* Fix

* Implement some methods.

* Remove more plasma manager references.

* Fix

* Linting

* Fix

* Fix

* Make sure class IDs are strings.

* Some path fixes

* Fix

* Path fixes and update arrow

* Fixes.

* linting

* Fixes

* Java fixes

* Some java fixes

* TaskLanguage -> Language

* Minor

* Fix python test and remove unused method signature.

* Fix java tests

* Fix jenkins tests

* Remove commented out code.
This commit is contained in:
Robert Nishihara
2018-10-26 13:36:58 -07:00
committed by Philipp Moritz
parent 055daf17a0
commit 658c14282c
289 changed files with 2460 additions and 40708 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/actor_test.py
+11 -36
View File
@@ -1231,9 +1231,6 @@ def test_blocking_actor_task(shutdown_only):
assert remaining_ids == [x_id]
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
def test_exception_raised_when_actor_node_dies(shutdown_only):
ray.worker._init(start_ray_local=True, num_local_schedulers=2, num_cpus=1)
@@ -1278,9 +1275,7 @@ def test_exception_raised_when_actor_node_dies(shutdown_only):
process.wait()
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
@@ -1328,9 +1323,7 @@ def test_local_scheduler_dying(shutdown_only):
assert results == list(range(1, 1 + len(results)))
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
@@ -1465,9 +1458,7 @@ def setup_counter_actor(test_checkpoint=False,
return actor, ids
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
@@ -1495,9 +1486,7 @@ def test_checkpointing(shutdown_only):
assert num_inc_calls < x
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
@@ -1526,9 +1515,7 @@ def test_remote_checkpoint(shutdown_only):
assert x == 101
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
@@ -1557,9 +1544,7 @@ def test_lost_checkpoint(shutdown_only):
assert 5 < num_inc_calls
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
@@ -1590,9 +1575,7 @@ def test_checkpoint_exception(shutdown_only):
assert error["type"] == ray_constants.CHECKPOINT_PUSH_ERROR
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
@@ -1662,9 +1645,7 @@ def test_distributed_handle(self):
assert x == count + 1
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
@@ -1821,9 +1802,7 @@ def _test_nondeterministic_reconstruction(num_forks, num_items_per_fork,
assert queue == reconstructed_queue[:len(queue)]
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Currently doesn't work with the new GCS.")
@@ -1859,9 +1838,7 @@ def setup_queue_actor():
ray.shutdown()
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
def test_fork(setup_queue_actor):
queue = setup_queue_actor
@@ -1878,9 +1855,7 @@ def test_fork(setup_queue_actor):
assert filtered_items == list(range(1))
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
def test_fork_consistency(setup_queue_actor):
queue = setup_queue_actor
test/component_failures_test.py
+33 -214
View File
@@ -10,7 +10,6 @@ import pytest
import ray
from ray.test.test_utils import run_string_as_driver_nonblocking
import pyarrow as pa
@pytest.fixture
@@ -35,13 +34,10 @@ def shutdown_only():
# This test checks that when a worker dies in the middle of a get, the plasma
# store and raylet will not die.
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Not working with new GCS API.")
def test_dying_worker_get_raylet(shutdown_only):
def test_dying_worker_get(shutdown_only):
# Start the Ray processes.
ray.init(num_cpus=2)
@@ -88,9 +84,6 @@ def test_dying_worker_get_raylet(shutdown_only):
# This test checks that when a driver dies in the middle of a get, the plasma
# store and raylet will not die.
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Not working with new GCS API.")
@@ -132,51 +125,12 @@ ray.get(ray.ObjectID(ray.utils.hex_to_binary("{}")))
assert ray.services.all_processes_alive()
# This test checks that when a worker dies in the middle of a get, the
# plasma store and manager will not die.
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Not working with new GCS API.")
def test_dying_worker_get(ray_start_workers_separate):
obj_id = 20 * b"a"
@ray.remote
def f():
ray.worker.global_worker.plasma_client.get(ray.ObjectID(obj_id))
# Have the worker wait in a get call.
f.remote()
# Kill the worker.
time.sleep(1)
(ray.services.all_processes[ray.services.PROCESS_TYPE_WORKER][0]
.terminate())
time.sleep(0.1)
# Seal the object so the store attempts to notify the worker that the
# get has been fulfilled.
ray.worker.global_worker.plasma_client.create(
pa.plasma.ObjectID(obj_id), 100)
ray.worker.global_worker.plasma_client.seal(pa.plasma.ObjectID(obj_id))
time.sleep(0.1)
# Make sure that nothing has died.
assert ray.services.all_processes_alive(
exclude=[ray.services.PROCESS_TYPE_WORKER])
# This test checks that when a worker dies in the middle of a wait, the plasma
# store and raylet will not die.
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Not working with new GCS API.")
def test_dying_worker_wait_raylet(shutdown_only):
def test_dying_worker_wait(shutdown_only):
ray.init(num_cpus=2)
@ray.remote
@@ -215,9 +169,6 @@ def test_dying_worker_wait_raylet(shutdown_only):
# This test checks that when a driver dies in the middle of a wait, the plasma
# store and raylet will not die.
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Not working with new GCS API.")
@@ -259,42 +210,6 @@ ray.wait([ray.ObjectID(ray.utils.hex_to_binary("{}"))])
assert ray.services.all_processes_alive()
# This test checks that when a worker dies in the middle of a wait, the
# plasma store and manager will not die.
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Not working with new GCS API.")
def test_dying_worker_wait(ray_start_workers_separate):
obj_id = 20 * b"a"
@ray.remote
def f():
ray.worker.global_worker.plasma_client.wait([ray.ObjectID(obj_id)])
# Have the worker wait in a get call.
f.remote()
# Kill the worker.
time.sleep(1)
(ray.services.all_processes[ray.services.PROCESS_TYPE_WORKER][0]
.terminate())
time.sleep(0.1)
# Seal the object so the store attempts to notify the worker that the
# get has been fulfilled.
ray.worker.global_worker.plasma_client.create(
pa.plasma.ObjectID(obj_id), 100)
ray.worker.global_worker.plasma_client.seal(pa.plasma.ObjectID(obj_id))
time.sleep(0.1)
# Make sure that nothing has died.
assert ray.services.all_processes_alive(
exclude=[ray.services.PROCESS_TYPE_WORKER])
@pytest.fixture(params=[(1, 4), (4, 4)])
def ray_start_workers_separate_multinode(request):
num_local_schedulers = request.param[0]
@@ -341,9 +256,6 @@ def test_worker_failed(ray_start_workers_separate_multinode):
def _test_component_failed(component_type):
"""Kill a component on all worker nodes and check workload succeeds."""
# Raylet is able to pass a harder failure test than legacy ray.
use_raylet = os.environ.get("RAY_USE_XRAY") != "0"
# Start with 4 workers and 4 cores.
num_local_schedulers = 4
num_workers_per_scheduler = 8
@@ -354,85 +266,42 @@ def _test_component_failed(component_type):
num_cpus=[num_workers_per_scheduler] * num_local_schedulers,
redirect_output=True)
if use_raylet:
# Submit many tasks with many dependencies.
@ray.remote
def f(x):
return x
# Submit many tasks with many dependencies.
@ray.remote
def f(x):
return x
@ray.remote
def g(*xs):
return 1
@ray.remote
def g(*xs):
return 1
# Kill the component on all nodes except the head node as the tasks
# execute. Do this in a loop while submitting tasks between each
# component failure.
# NOTE(swang): Legacy ray hangs on this test if the plasma manager
# is killed.
time.sleep(0.1)
components = ray.services.all_processes[component_type]
for process in components[1:]:
# Submit a round of tasks with many dependencies.
x = 1
for _ in range(1000):
x = f.remote(x)
# Kill the component on all nodes except the head node as the tasks
# execute. Do this in a loop while submitting tasks between each
# component failure.
time.sleep(0.1)
components = ray.services.all_processes[component_type]
for process in components[1:]:
# Submit a round of tasks with many dependencies.
x = 1
for _ in range(1000):
x = f.remote(x)
xs = [g.remote(1)]
for _ in range(100):
xs.append(g.remote(*xs))
xs.append(g.remote(1))
xs = [g.remote(1)]
for _ in range(100):
xs.append(g.remote(*xs))
xs.append(g.remote(1))
# Kill a component on one of the nodes.
process.terminate()
time.sleep(1)
process.kill()
process.wait()
assert not process.poll() is None
# Make sure that we can still get the objects after the
# executing tasks died.
ray.get(x)
ray.get(xs)
else:
@ray.remote
def f(x, j):
time.sleep(0.2)
return x
# Submit more tasks than there are workers so that all workers and
# cores are utilized.
object_ids = [
f.remote(i, 0)
for i in range(num_workers_per_scheduler * num_local_schedulers)
]
object_ids += [f.remote(object_id, 1) for object_id in object_ids]
object_ids += [f.remote(object_id, 2) for object_id in object_ids]
# Kill the component on all nodes except the head node as the tasks
# execute.
time.sleep(0.1)
components = ray.services.all_processes[component_type]
for process in components[1:]:
process.terminate()
# while the local_scheduler is fetching object_ids,
# and would trigger `fetch_object_timeout_handler`,
# which leads to find the plasma manager or plasma
# store socket is broken, so local_scheduler failed.
# Kill a component on one of the nodes.
process.terminate()
time.sleep(1)
process.kill()
process.wait()
assert not process.poll() is None
for process in components[1:]:
process.kill()
process.wait()
assert not process.poll() is None
# Make sure that we can still get the objects after the executing
# tasks died.
results = ray.get(object_ids)
expected_results = 4 * list(
range(num_workers_per_scheduler * num_local_schedulers))
assert results == expected_results
# Make sure that we can still get the objects after the
# executing tasks died.
ray.get(x)
ray.get(xs)
def check_components_alive(component_type, check_component_alive):
@@ -451,58 +320,16 @@ def check_components_alive(component_type, check_component_alive):
assert not component.poll() is None
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only makes sense with xray.")
def test_raylet_failed():
# Kill all local schedulers on worker nodes.
_test_component_failed(ray.services.PROCESS_TYPE_RAYLET)
# The plasma stores and plasma managers should still be alive on the
# worker nodes.
# The plasma stores should still be alive on the worker nodes.
check_components_alive(ray.services.PROCESS_TYPE_PLASMA_STORE, True)
ray.shutdown()
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not make sense with xray.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
def test_local_scheduler_failed():
# Kill all local schedulers on worker nodes.
_test_component_failed(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER)
# The plasma stores and plasma managers should still be alive on the
# worker nodes.
check_components_alive(ray.services.PROCESS_TYPE_PLASMA_STORE, True)
check_components_alive(ray.services.PROCESS_TYPE_PLASMA_MANAGER, True)
check_components_alive(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER, False)
ray.shutdown()
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not make sense with xray.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
def test_plasma_manager_failed():
# Kill all plasma managers on worker nodes.
_test_component_failed(ray.services.PROCESS_TYPE_PLASMA_MANAGER)
# The plasma stores should still be alive (but unreachable) on the
# worker nodes.
check_components_alive(ray.services.PROCESS_TYPE_PLASMA_STORE, True)
check_components_alive(ray.services.PROCESS_TYPE_PLASMA_MANAGER, False)
check_components_alive(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER, False)
ray.shutdown()
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Hanging with new GCS API.")
@@ -512,8 +339,6 @@ def test_plasma_store_failed():
# No processes should be left alive on the worker nodes.
check_components_alive(ray.services.PROCESS_TYPE_PLASMA_STORE, False)
check_components_alive(ray.services.PROCESS_TYPE_PLASMA_MANAGER, False)
check_components_alive(ray.services.PROCESS_TYPE_LOCAL_SCHEDULER, False)
check_components_alive(ray.services.PROCESS_TYPE_RAYLET, False)
ray.shutdown()
@@ -526,9 +351,6 @@ def test_driver_lives_sequential():
ray.worker.init()
all_processes = ray.services.all_processes
processes = (all_processes[ray.services.PROCESS_TYPE_PLASMA_STORE] +
all_processes[ray.services.PROCESS_TYPE_PLASMA_MANAGER] +
all_processes[ray.services.PROCESS_TYPE_LOCAL_SCHEDULER] +
all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER] +
all_processes[ray.services.PROCESS_TYPE_RAYLET])
# Kill all the components sequentially.
@@ -549,9 +371,6 @@ def test_driver_lives_parallel():
ray.worker.init()
all_processes = ray.services.all_processes
processes = (all_processes[ray.services.PROCESS_TYPE_PLASMA_STORE] +
all_processes[ray.services.PROCESS_TYPE_PLASMA_MANAGER] +
all_processes[ray.services.PROCESS_TYPE_LOCAL_SCHEDULER] +
all_processes[ray.services.PROCESS_TYPE_GLOBAL_SCHEDULER] +
all_processes[ray.services.PROCESS_TYPE_RAYLET])
# Kill all the components in parallel.
test/failure_test.py
+2 -18
View File
@@ -394,9 +394,7 @@ def ray_start_object_store_memory():
ray.shutdown()
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
def test_put_error1(ray_start_object_store_memory):
num_objects = 3
object_size = 4 * 10**5
@@ -438,9 +436,7 @@ def test_put_error1(ray_start_object_store_memory):
wait_for_errors(ray_constants.PUT_RECONSTRUCTION_PUSH_ERROR, 1)
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
def test_put_error2(ray_start_object_store_memory):
# This is the same as the previous test, but it calls ray.put directly.
num_objects = 3
@@ -494,9 +490,6 @@ def test_version_mismatch(shutdown_only):
ray.__version__ = ray_version
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
def test_warning_monitor_died(shutdown_only):
ray.init(num_cpus=0)
@@ -538,9 +531,6 @@ def test_export_large_objects(ray_start_regular):
wait_for_errors(ray_constants.PICKLING_LARGE_OBJECT_PUSH_ERROR, 2)
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
def test_warning_for_infeasible_tasks(ray_start_regular):
# Check that we get warning messages for infeasible tasks.
@@ -561,9 +551,6 @@ def test_warning_for_infeasible_tasks(ray_start_regular):
wait_for_errors(ray_constants.INFEASIBLE_TASK_ERROR, 2)
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "1",
reason="This test only works with xray.")
def test_warning_for_infeasible_zero_cpu_actor(shutdown_only):
# Check that we cannot place an actor on a 0 CPU machine and that we get an
# infeasibility warning (even though the actor creation task itself
@@ -591,9 +578,6 @@ def ray_start_two_nodes():
# Note that this test will take at least 10 seconds because it must wait for
# the monitor to detect enough missed heartbeats.
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
def test_warning_for_dead_node(ray_start_two_nodes):
# Wait for the raylet to appear in the client table.
while len(ray.global_state.client_table()) < 2:
test/jenkins_tests/multi_node_docker_test.py
+5 -18
View File
@@ -80,12 +80,11 @@ class DockerRunner(object):
head node.
"""
def __init__(self, use_raylet):
def __init__(self):
"""Initialize the DockerRunner."""
self.head_container_id = None
self.worker_container_ids = []
self.head_container_ip = None
self.use_raylet = use_raylet
def _get_container_id(self, stdout_data):
"""Parse the docker container ID from stdout_data.
@@ -149,8 +148,6 @@ class DockerRunner(object):
"--num-cpus={}".format(num_cpus), "--num-gpus={}".format(num_gpus),
"--no-ui"
])
if self.use_raylet:
command.append("--use-raylet")
print("Starting head node with command:{}".format(command))
proc = subprocess.Popen(
@@ -177,8 +174,6 @@ class DockerRunner(object):
"--redis-address={:s}:6379".format(self.head_container_ip),
"--num-cpus={}".format(num_cpus), "--num-gpus={}".format(num_gpus)
])
if self.use_raylet:
command.append("--use-raylet")
print("Starting worker node with command:{}".format(command))
proc = subprocess.Popen(
command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -325,19 +320,13 @@ class DockerRunner(object):
# Start the different drivers.
driver_processes = []
if self.use_raylet:
use_raylet_env = 1
else:
use_raylet_env = 0
for i in range(len(driver_locations)):
# Get the container ID to run the ith driver in.
container_id = all_container_ids[driver_locations[i]]
command = [
"docker", "exec", container_id, "/bin/bash",
"-c", ("RAY_REDIS_ADDRESS={}:6379 RAY_DRIVER_INDEX={} "
"RAY_USE_XRAY={} python {}".format(
self.head_container_ip, i, use_raylet_env,
test_script))
"docker", "exec", container_id, "/bin/bash", "-c",
("RAY_REDIS_ADDRESS={}:6379 RAY_DRIVER_INDEX={} "
"python {}".format(self.head_container_ip, i, test_script))
]
print("Starting driver with command {}.".format(test_script))
# Start the driver.
@@ -404,8 +393,6 @@ if __name__ == "__main__":
"--development-mode",
action="store_true",
help="use local copies of the test scripts")
parser.add_argument(
"--use-raylet", action="store_true", help="use raylet mode in Docker")
args = parser.parse_args()
# Parse the number of CPUs and GPUs to use for each worker.
@@ -419,7 +406,7 @@ if __name__ == "__main__":
driver_locations = (None if args.driver_locations is None else
[int(i) for i in args.driver_locations.split(",")])
d = DockerRunner(args.use_raylet)
d = DockerRunner()
d.start_ray(
docker_image=args.docker_image,
mem_size=args.mem_size,
test/jenkins_tests/run_multi_node_tests.sh
+2 -7
View File
@@ -323,16 +323,14 @@ docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
python /ray/python/ray/experimental/sgd/test_sgd.py --num-iters=2
# No Xray for PyTorch
docker run -e RAY_USE_XRAY=0 --rm --shm-size=10G --memory=10G $DOCKER_SHA \
docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
python /ray/python/ray/rllib/train.py \
--env PongDeterministic-v4 \
--run A3C \
--stop '{"training_iteration": 2}' \
--config '{"num_workers": 2, "use_pytorch": true, "model": {"use_lstm": false, "grayscale": true, "zero_mean": false, "dim": 84, "channel_major": true}, "preprocessor_pref": "rllib"}'
# No Xray for PyTorch
docker run -e RAY_USE_XRAY=0 --rm --shm-size=10G --memory=10G $DOCKER_SHA \
docker run --rm --shm-size=10G --memory=10G $DOCKER_SHA \
python /ray/python/ray/rllib/train.py \
--env CartPole-v1 \
--run A3C \
@@ -343,7 +341,6 @@ python3 $ROOT_DIR/multi_node_docker_test.py \
--docker-image=$DOCKER_SHA \
--num-nodes=5 \
--num-redis-shards=10 \
--use-raylet \
--test-script=/ray/test/jenkins_tests/multi_node_tests/test_0.py
python3 $ROOT_DIR/multi_node_docker_test.py \
@@ -361,7 +358,6 @@ python3 $ROOT_DIR/multi_node_docker_test.py \
--num-redis-shards=2 \
--num-gpus=0,0,5,6,50 \
--num-drivers=100 \
--use-raylet \
--test-script=/ray/test/jenkins_tests/multi_node_tests/many_drivers_test.py
python3 $ROOT_DIR/multi_node_docker_test.py \
@@ -369,5 +365,4 @@ python3 $ROOT_DIR/multi_node_docker_test.py \
--num-nodes=1 \
--mem-size=60G \
--shm-size=60G \
--use-raylet \
--test-script=/ray/test/jenkins_tests/multi_node_tests/large_memory_test.py
test/multi_node_test.py
-3
View File
@@ -201,9 +201,6 @@ def ray_start_head_with_resources():
subprocess.Popen(["ray", "stop"]).wait()
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
def test_drivers_release_resources(ray_start_head_with_resources):
redis_address = ray_start_head_with_resources
test/multi_node_test_2.py
-10
View File
@@ -3,7 +3,6 @@ from __future__ import division
from __future__ import print_function
import logging
import os
import pytest
import ray
@@ -23,9 +22,6 @@ def start_connected_cluster():
g.shutdown()
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "1",
reason="This test only works with xray.")
def test_cluster():
"""Basic test for adding and removing nodes in cluster."""
g = Cluster(initialize_head=False)
@@ -38,9 +34,6 @@ def test_cluster():
assert not any(node.any_processes_alive() for node in g.list_all_nodes())
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "1",
reason="This test only works with xray.")
def test_wait_for_nodes(start_connected_cluster):
"""Unit test for `Cluster.wait_for_nodes`.
@@ -58,9 +51,6 @@ def test_wait_for_nodes(start_connected_cluster):
cluster.wait_for_nodes()
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "1",
reason="This test only works with xray.")
def test_worker_plasma_store_failure(start_connected_cluster):
cluster = start_connected_cluster
worker = cluster.add_node()
test/runtest.py
+23 -215
View File
@@ -292,7 +292,7 @@ def test_putting_object_that_closes_over_object_id(ray_start):
f
f = Foo()
with pytest.raises(ray.local_scheduler.common_error):
with pytest.raises(ray.raylet.common_error):
ray.put(f)
@@ -989,57 +989,6 @@ def test_running_function_on_all_workers(shutdown_only):
assert "fake_directory" not in ray.get(get_path2.remote())
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray (nor is it intended to).")
def test_logging_api(shutdown_only):
ray.init(num_cpus=1)
def events():
# This is a hack for getting the event log. It is not part of the
# API.
keys = ray.worker.global_worker.redis_client.keys("event_log:*")
res = []
for key in keys:
res.extend(
ray.worker.global_worker.redis_client.zrange(key, 0, -1))
return res
def wait_for_num_events(num_events, timeout=10):
start_time = time.time()
while time.time() - start_time < timeout:
if len(events()) >= num_events:
return
time.sleep(0.1)
print("Timing out of wait.")
@ray.remote
def test_log_span():
with ray.profile("event_type2", extra_data={"key": "val"}):
pass
# Make sure that we can call ray.log_span in a remote function.
ray.get(test_log_span.remote())
# Wait for the events to appear in the event log.
wait_for_num_events(1)
assert len(events()) == 1
@ray.remote
def test_log_span_exception():
with ray.log_span("event_type2", extra_data={"key": "val"}):
raise Exception("This failed.")
# Make sure that logging a span works if an exception is thrown.
test_log_span_exception.remote()
# Wait for the events to appear in the event log.
wait_for_num_events(2)
assert len(events()) == 2
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
def test_profiling_api(shutdown_only):
ray.init(num_cpus=2)
@@ -1197,9 +1146,6 @@ def test_multithreading(shutdown_only):
ray.get(test_multi_threading_in_worker.remote())
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
def test_free_objects_multi_node(shutdown_only):
ray.worker._init(
start_ray_local=True,
@@ -1212,8 +1158,7 @@ def test_free_objects_multi_node(shutdown_only):
"Custom1": 1
}, {
"Custom2": 1
}],
use_raylet=True)
}])
@ray.remote(resources={"Custom0": 1})
def run_on_0():
@@ -1638,9 +1583,6 @@ def test_gpu_ids(shutdown_only):
ray.get(a1.test.remote())
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
def test_zero_cpus(shutdown_only):
ray.init(num_cpus=0)
@@ -1668,9 +1610,6 @@ def test_zero_cpus_actor(shutdown_only):
assert ray.get(a.method.remote()) != local_plasma
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") == "0",
reason="This test only works with xray.")
def test_fractional_resources(shutdown_only):
ray.init(num_cpus=6, num_gpus=3, resources={"Custom": 1})
@@ -1793,10 +1732,7 @@ def test_multiple_local_schedulers(shutdown_only):
results.append(run_on_0_2.remote())
return names, results
store_names = [
object_store_address.name
for object_store_address in address_info["object_store_addresses"]
]
store_names = address_info["object_store_addresses"]
def validate_names_and_results(names, results):
for name, result in zip(names, ray.get(results)):
@@ -2043,8 +1979,7 @@ def test_blocking_tasks(shutdown_only):
object_ids = [f.remote(i, j) for j in range(2)]
return ray.wait(object_ids, num_returns=len(object_ids))
if os.environ.get("RAY_USE_XRAY") != "0":
ray.get([h.remote(i) for i in range(4)])
ray.get([h.remote(i) for i in range(4)])
@ray.remote
def _sleep(i):
@@ -2198,14 +2133,8 @@ def test_global_state_api(shutdown_only):
task_table = ray.global_state.task_table()
assert len(task_table) == 1
assert driver_task_id == list(task_table.keys())[0]
if not ray.worker.global_worker.use_raylet:
assert (task_table[driver_task_id]["State"] ==
ray.experimental.state.TASK_STATUS_RUNNING)
if not ray.worker.global_worker.use_raylet:
task_spec = task_table[driver_task_id]["TaskSpec"]
else:
assert len(task_table[driver_task_id]) == 1
task_spec = task_table[driver_task_id][0]["TaskSpec"]
assert len(task_table[driver_task_id]) == 1
task_spec = task_table[driver_task_id][0]["TaskSpec"]
assert task_spec["TaskID"] == driver_task_id
assert task_spec["ActorID"] == ray_constants.ID_SIZE * "ff"
@@ -2217,15 +2146,8 @@ def test_global_state_api(shutdown_only):
client_table = ray.global_state.client_table()
node_ip_address = ray.worker.global_worker.node_ip_address
if not ray.worker.global_worker.use_raylet:
assert len(client_table[node_ip_address]) == 3
manager_client = [
c for c in client_table[node_ip_address]
if c["ClientType"] == "plasma_manager"
][0]
else:
assert len(client_table) == 1
assert client_table[0]["NodeManagerAddress"] == node_ip_address
assert len(client_table) == 1
assert client_table[0]["NodeManagerAddress"] == node_ip_address
@ray.remote
def f(*xs):
@@ -2235,25 +2157,15 @@ def test_global_state_api(shutdown_only):
result_id = f.remote(1, "hi", x_id)
# Wait for one additional task to complete.
start_time = time.time()
while time.time() - start_time < 10:
wait_for_num_tasks(1 + 1)
task_table = ray.global_state.task_table()
assert len(task_table) == 1 + 1
task_id_set = set(task_table.keys())
task_id_set.remove(driver_task_id)
task_id = list(task_id_set)[0]
if ray.worker.global_worker.use_raylet:
break
if (task_table[task_id]["State"] ==
ray.experimental.state.TASK_STATUS_DONE):
break
time.sleep(0.1)
wait_for_num_tasks(1 + 1)
task_table = ray.global_state.task_table()
assert len(task_table) == 1 + 1
task_id_set = set(task_table.keys())
task_id_set.remove(driver_task_id)
task_id = list(task_id_set)[0]
function_table = ray.global_state.function_table()
if not ray.worker.global_worker.use_raylet:
task_spec = task_table[task_id]["TaskSpec"]
else:
task_spec = task_table[task_id][0]["TaskSpec"]
task_spec = task_table[task_id][0]["TaskSpec"]
assert task_spec["ActorID"] == ray_constants.ID_SIZE * "ff"
assert task_spec["Args"] == [1, "hi", x_id]
assert task_spec["DriverID"] == driver_id
@@ -2281,31 +2193,16 @@ def test_global_state_api(shutdown_only):
raise Exception("Timed out while waiting for object table to "
"update.")
# Wait for the object table to be updated.
if not ray.worker.global_worker.use_raylet:
wait_for_object_table()
object_table = ray.global_state.object_table()
assert len(object_table) == 2
if not ray.worker.global_worker.use_raylet:
db_client_id = manager_client["DBClientID"]
assert object_table[x_id]["IsPut"] is True
assert object_table[x_id]["TaskID"] == driver_task_id
assert object_table[x_id]["ManagerIDs"] == [db_client_id]
assert len(object_table[x_id]) == 1
assert object_table[x_id][0]["IsEviction"] is False
assert object_table[x_id][0]["NumEvictions"] == 0
assert object_table[result_id]["IsPut"] is False
assert object_table[result_id]["TaskID"] == task_id
assert object_table[result_id]["ManagerIDs"] == [db_client_id]
else:
assert len(object_table[x_id]) == 1
assert object_table[x_id][0]["IsEviction"] is False
assert object_table[x_id][0]["NumEvictions"] == 0
assert len(object_table[result_id]) == 1
assert object_table[result_id][0]["IsEviction"] is False
assert object_table[result_id][0]["NumEvictions"] == 0
assert len(object_table[result_id]) == 1
assert object_table[result_id][0]["IsEviction"] is False
assert object_table[result_id][0]["NumEvictions"] == 0
assert object_table[x_id] == ray.global_state.object_table(x_id)
object_table_entry = ray.global_state.object_table(result_id)
@@ -2346,45 +2243,6 @@ def test_log_file_api(shutdown_only):
assert found_message is True
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="New GCS API doesn't have a Python API yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray (nor is it intended to).")
def test_task_profile_api(shutdown_only):
ray.init(num_cpus=1, redirect_output=True)
@ray.remote
def f():
return 1
num_calls = 5
[f.remote() for _ in range(num_calls)]
# Make sure the event log has the correct number of events.
start_time = time.time()
while time.time() - start_time < 10:
profiles = ray.global_state.task_profiles(
100, start=0, end=time.time())
limited_profiles = ray.global_state.task_profiles(
1, start=0, end=time.time())
if len(profiles) == num_calls and len(limited_profiles) == 1:
break
time.sleep(0.1)
assert len(profiles) == num_calls
assert len(limited_profiles) == 1
# Make sure that each entry is properly formatted.
for task_id, data in profiles.items():
assert "execute_start" in data
assert "execute_end" in data
assert "get_arguments_start" in data
assert "get_arguments_end" in data
assert "store_outputs_start" in data
assert "store_outputs_end" in data
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="New GCS API doesn't have a Python API yet.")
@@ -2409,62 +2267,15 @@ def test_workers(shutdown_only):
for worker_id, info in worker_info.items():
assert "node_ip_address" in info
assert "local_scheduler_socket" in info
assert "plasma_manager_socket" in info
assert "plasma_store_socket" in info
assert "stderr_file" in info
assert "stdout_file" in info
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="New GCS API doesn't have a Python API yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
def test_dump_trace_file(shutdown_only):
ray.init(num_cpus=1, redirect_output=True)
@ray.remote
def f(*xs):
return 1
@ray.remote
class Foo(object):
def __init__(self):
pass
def method(self):
pass
# We use a number of test objects because objects that are not JSON
# serializable caused problems in the past.
test_objects = [
0, 0.5, "hi", b"hi",
ray.put(0),
np.zeros(3), [0], (0, ), {
0: 0
}, True, False, None
]
ray.get([f.remote(obj) for obj in test_objects])
actors = [Foo.remote() for _ in range(5)]
ray.get([actor.method.remote() for actor in actors])
ray.get([actor.method.remote() for actor in actors])
path = os.path.join("/tmp/ray_test_trace")
task_info = ray.global_state.task_profiles(100, start=0, end=time.time())
ray.global_state.dump_catapult_trace(path, task_info)
# TODO(rkn): This test is not perfect because it does not verify that
# the visualization actually renders (e.g., the context of the dumped
# trace could be malformed).
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="New GCS API doesn't have a Python API yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
def test_flush_api(shutdown_only):
ray.init(num_cpus=1)
@@ -2553,9 +2364,6 @@ def test_initialized_local_mode(shutdown_only_with_initialization_check):
assert ray.is_initialized()
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "1",
reason="This test only works with xray.")
def test_wait_reconstruction(shutdown_only):
ray.init(num_cpus=1, object_store_memory=10**8)
test/stress_tests.py
+3 -12
View File
@@ -162,9 +162,7 @@ def ray_start_reconstruction(request):
# Start the Redis global state store.
node_ip_address = "127.0.0.1"
use_raylet = os.environ.get("RAY_USE_XRAY") != "0"
redis_address, redis_shards = ray.services.start_redis(
node_ip_address, use_raylet=use_raylet)
redis_address, redis_shards = ray.services.start_redis(node_ip_address)
redis_ip_address = ray.services.get_ip_address(redis_address)
redis_port = ray.services.get_port(redis_address)
time.sleep(0.1)
@@ -176,18 +174,13 @@ def ray_start_reconstruction(request):
for i in range(num_local_schedulers):
store_stdout_file, store_stderr_file = (
ray.tempfile_services.new_plasma_store_log_file(i, True))
manager_stdout_file, manager_stderr_file = (
ray.tempfile_services.new_plasma_manager_log_file(i, True))
plasma_addresses.append(
ray.services.start_plasma_store(
node_ip_address,
redis_address,
objstore_memory=objstore_memory,
store_stdout_file=store_stdout_file,
store_stderr_file=store_stderr_file,
manager_stdout_file=manager_stdout_file,
manager_stderr_file=manager_stderr_file,
use_raylet=use_raylet))
store_stderr_file=store_stderr_file))
# Start the rest of the services in the Ray cluster.
address_info = {
@@ -401,9 +394,7 @@ def wait_for_errors(error_check):
return errors
@pytest.mark.skipif(
os.environ.get("RAY_USE_XRAY") != "0",
reason="This test does not work with xray yet.")
@pytest.mark.skip("This test does not work yet.")
@pytest.mark.skipif(
os.environ.get("RAY_USE_NEW_GCS") == "on",
reason="Failing with new GCS API on Linux.")
test/tempfile_test.py
+7 -12
View File
@@ -10,7 +10,6 @@ def test_conn_cluster():
# plasma_store_socket_name
with pytest.raises(Exception) as exc_info:
ray.init(
use_raylet=True,
redis_address="127.0.0.1:6379",
plasma_store_socket_name="/tmp/this_should_fail")
assert exc_info.value.args[0] == (
@@ -20,7 +19,6 @@ def test_conn_cluster():
# raylet_socket_name
with pytest.raises(Exception) as exc_info:
ray.init(
use_raylet=True,
redis_address="127.0.0.1:6379",
raylet_socket_name="/tmp/this_should_fail")
assert exc_info.value.args[0] == (
@@ -30,16 +28,14 @@ def test_conn_cluster():
# temp_dir
with pytest.raises(Exception) as exc_info:
ray.init(
use_raylet=True,
redis_address="127.0.0.1:6379",
temp_dir="/tmp/this_should_fail")
redis_address="127.0.0.1:6379", temp_dir="/tmp/this_should_fail")
assert exc_info.value.args[0] == (
"When connecting to an existing cluster, "
"temp_dir must not be provided.")
def test_tempdir():
ray.init(use_raylet=True, temp_dir="/tmp/i_am_a_temp_dir")
ray.init(temp_dir="/tmp/i_am_a_temp_dir")
assert os.path.exists(
"/tmp/i_am_a_temp_dir"), "Specified temp dir not found."
ray.shutdown()
@@ -47,7 +43,7 @@ def test_tempdir():
def test_raylet_socket_name():
ray.init(use_raylet=True, raylet_socket_name="/tmp/i_am_a_temp_socket")
ray.init(raylet_socket_name="/tmp/i_am_a_temp_socket")
assert os.path.exists(
"/tmp/i_am_a_temp_socket"), "Specified socket path not found."
ray.shutdown()
@@ -58,8 +54,7 @@ def test_raylet_socket_name():
def test_temp_plasma_store_socket():
ray.init(
use_raylet=True, plasma_store_socket_name="/tmp/i_am_a_temp_socket")
ray.init(plasma_store_socket_name="/tmp/i_am_a_temp_socket")
assert os.path.exists(
"/tmp/i_am_a_temp_socket"), "Specified socket path not found."
ray.shutdown()
@@ -70,7 +65,7 @@ def test_temp_plasma_store_socket():
def test_raylet_tempfiles():
ray.init(use_raylet=True, redirect_worker_output=False)
ray.init(redirect_worker_output=False)
top_levels = set(os.listdir(tempfile_services.get_temp_root()))
assert top_levels == {"ray_ui.ipynb", "sockets", "logs"}
log_files = set(os.listdir(tempfile_services.get_logs_dir_path()))
@@ -84,7 +79,7 @@ def test_raylet_tempfiles():
assert socket_files == {"plasma_store", "raylet"}
ray.shutdown()
ray.init(use_raylet=True, redirect_worker_output=True, num_workers=0)
ray.init(redirect_worker_output=True, num_workers=0)
top_levels = set(os.listdir(tempfile_services.get_temp_root()))
assert top_levels == {"ray_ui.ipynb", "sockets", "logs"}
log_files = set(os.listdir(tempfile_services.get_logs_dir_path()))
@@ -98,7 +93,7 @@ def test_raylet_tempfiles():
assert socket_files == {"plasma_store", "raylet"}
ray.shutdown()
ray.init(use_raylet=True, redirect_worker_output=True, num_workers=2)
ray.init(redirect_worker_output=True, num_workers=2)
top_levels = set(os.listdir(tempfile_services.get_temp_root()))
assert top_levels == {"ray_ui.ipynb", "sockets", "logs"}
time.sleep(3) # wait workers to start
test/xray_test.py
+1 -1
View File
@@ -11,7 +11,7 @@ test_values = [1, 1.0, "test", b"test", (0, 1), [0, 1], {0: 1}]
@pytest.fixture
def ray_start():
# Start the Ray processes.
ray.init(num_cpus=1, use_raylet=True)
ray.init(num_cpus=1)
yield None
# The code after the yield will run as teardown code.
ray.shutdown()