mirror of
https://github.com/wassname/ray.git
synced 2026-08-13 12:30:18 +08:00
Simplify put test and move it to failure tests. (#788)
This commit is contained in:
Robert Nishihara
committed by
Philipp Moritz
Philipp Moritz
parent
c394a65ffc
commit
37dafa4d14
@@ -2,6 +2,7 @@ from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
|
||||
import numpy as np
|
||||
import os
|
||||
import ray
|
||||
import sys
|
||||
@@ -277,5 +278,100 @@ class WorkerDeath(unittest.TestCase):
|
||||
ray.worker.cleanup()
|
||||
|
||||
|
||||
class PutErrorTest(unittest.TestCase):
|
||||
|
||||
def testPutError1(self):
|
||||
store_size = 10 ** 6
|
||||
ray.worker._init(start_ray_local=True, driver_mode=ray.SILENT_MODE,
|
||||
object_store_memory=store_size)
|
||||
|
||||
num_objects = 3
|
||||
object_size = 4 * 10 ** 5
|
||||
|
||||
# Define a task with a single dependency, a numpy array, that returns
|
||||
# another array.
|
||||
@ray.remote
|
||||
def single_dependency(i, arg):
|
||||
arg = np.copy(arg)
|
||||
arg[0] = i
|
||||
return arg
|
||||
|
||||
@ray.remote
|
||||
def put_arg_task():
|
||||
# Launch num_objects instances of the remote task, each dependent
|
||||
# on the one before it. The result of the first task should get
|
||||
# evicted.
|
||||
args = []
|
||||
arg = single_dependency.remote(0, np.zeros(object_size,
|
||||
dtype=np.uint8))
|
||||
for i in range(num_objects):
|
||||
arg = single_dependency.remote(i, arg)
|
||||
args.append(arg)
|
||||
|
||||
# Get the last value to force all tasks to finish.
|
||||
value = ray.get(args[-1])
|
||||
assert value[0] == i
|
||||
|
||||
# Get the first value (which should have been evicted) to force
|
||||
# reconstruction. Currently, since we're not able to reconstruct
|
||||
# `ray.put` objects that were evicted and whose originating tasks
|
||||
# are still running, this for-loop should hang and push an error to
|
||||
# the driver.
|
||||
ray.get(args[0])
|
||||
|
||||
put_arg_task.remote()
|
||||
|
||||
# Make sure we receive the correct error message.
|
||||
wait_for_errors(b"put_reconstruction", 1)
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
def testPutError2(self):
|
||||
# This is the same as the previous test, but it calls ray.put directly.
|
||||
store_size = 10 ** 6
|
||||
ray.worker._init(start_ray_local=True, driver_mode=ray.SILENT_MODE,
|
||||
object_store_memory=store_size)
|
||||
|
||||
num_objects = 3
|
||||
object_size = 4 * 10 ** 5
|
||||
|
||||
# Define a task with a single dependency, a numpy array, that returns
|
||||
# another array.
|
||||
@ray.remote
|
||||
def single_dependency(i, arg):
|
||||
arg = np.copy(arg)
|
||||
arg[0] = i
|
||||
return arg
|
||||
|
||||
@ray.remote
|
||||
def put_task():
|
||||
# Launch num_objects instances of the remote task, each dependent
|
||||
# on the one before it. The result of the first task should get
|
||||
# evicted.
|
||||
args = []
|
||||
arg = ray.put(np.zeros(object_size, dtype=np.uint8))
|
||||
for i in range(num_objects):
|
||||
arg = single_dependency.remote(i, arg)
|
||||
args.append(arg)
|
||||
|
||||
# Get the last value to force all tasks to finish.
|
||||
value = ray.get(args[-1])
|
||||
assert value[0] == i
|
||||
|
||||
# Get the first value (which should have been evicted) to force
|
||||
# reconstruction. Currently, since we're not able to reconstruct
|
||||
# `ray.put` objects that were evicted and whose originating tasks
|
||||
# are still running, this for-loop should hang and push an error to
|
||||
# the driver.
|
||||
ray.get(args[0])
|
||||
|
||||
put_task.remote()
|
||||
|
||||
# Make sure we receive the correct error message.
|
||||
wait_for_errors(b"put_reconstruction", 1)
|
||||
|
||||
ray.worker.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
||||
@@ -425,97 +425,6 @@ class ReconstructionTests(unittest.TestCase):
|
||||
self.assertTrue(all(error[b"data"] == b"__main__.foo"
|
||||
for error in errors))
|
||||
|
||||
def testPutErrors(self):
|
||||
# Define the size of one task's return argument so that the combined
|
||||
# sum of all objects' sizes is at least twice the plasma stores'
|
||||
# combined allotted memory.
|
||||
num_objects = 1000
|
||||
size = self.plasma_store_memory * 2 // (num_objects * 8)
|
||||
|
||||
# Define a task with a single dependency, a numpy array, that returns
|
||||
# another array.
|
||||
@ray.remote
|
||||
def single_dependency(i, arg):
|
||||
arg = np.copy(arg)
|
||||
arg[0] = i
|
||||
return arg
|
||||
|
||||
# Define a root task that calls `ray.put` to put an argument in the
|
||||
# object store.
|
||||
@ray.remote
|
||||
def put_arg_task(size):
|
||||
# Launch num_objects instances of the remote task, each dependent
|
||||
# on the one before it. The first instance of the task takes a
|
||||
# numpy array as an argument, which is put into the object store.
|
||||
args = []
|
||||
arg = single_dependency.remote(0, np.zeros(size))
|
||||
for i in range(num_objects):
|
||||
arg = single_dependency.remote(i, arg)
|
||||
args.append(arg)
|
||||
|
||||
# Get each value to force each task to finish. After some number of
|
||||
# gets, old values should be evicted.
|
||||
for i in range(num_objects):
|
||||
value = ray.get(args[i])
|
||||
self.assertEqual(value[0], i)
|
||||
# Get each value again to force reconstruction. Currently, since
|
||||
# we're not able to reconstruct `ray.put` objects that were evicted
|
||||
# and whose originating tasks are still running, this for-loop
|
||||
# should hang on its first iteration and push an error to the
|
||||
# driver.
|
||||
for i in range(num_objects):
|
||||
value = ray.get(args[i])
|
||||
self.assertEqual(value[0], i)
|
||||
|
||||
# Define a root task that calls `ray.put` directly.
|
||||
@ray.remote
|
||||
def put_task(size):
|
||||
# Launch num_objects instances of the remote task, each dependent
|
||||
# on the one before it. The first instance of the task takes an
|
||||
# object ID returned by ray.put.
|
||||
args = []
|
||||
arg = ray.put(np.zeros(size))
|
||||
for i in range(num_objects):
|
||||
arg = single_dependency.remote(i, arg)
|
||||
args.append(arg)
|
||||
|
||||
# Get each value to force each task to finish. After some number of
|
||||
# gets, old values should be evicted.
|
||||
for i in range(num_objects):
|
||||
value = ray.get(args[i])
|
||||
self.assertEqual(value[0], i)
|
||||
# Get each value again to force reconstruction. Currently, since
|
||||
# we're not able to reconstruct `ray.put` objects that were evicted
|
||||
# and whose originating tasks are still running, this for-loop
|
||||
# should hang on its first iteration and push an error to the
|
||||
# driver.
|
||||
for i in range(num_objects):
|
||||
value = ray.get(args[i])
|
||||
self.assertEqual(value[0], i)
|
||||
|
||||
put_arg_task.remote(size)
|
||||
|
||||
def error_check(errors):
|
||||
return len(errors) > 1
|
||||
errors = self.wait_for_errors(error_check)
|
||||
# Make sure all the errors have the correct type.
|
||||
self.assertTrue(all(error[b"type"] == b"put_reconstruction"
|
||||
for error in errors))
|
||||
self.assertTrue(all(error[b"data"] == b"__main__.put_arg_task"
|
||||
for error in errors))
|
||||
|
||||
put_task.remote(size)
|
||||
|
||||
def error_check(errors):
|
||||
return any(error[b"data"] == b"__main__.put_task"
|
||||
for error in errors)
|
||||
errors = self.wait_for_errors(error_check)
|
||||
# Make sure all the errors have the correct type.
|
||||
self.assertTrue(all(error[b"type"] == b"put_reconstruction"
|
||||
for error in errors))
|
||||
self.assertTrue(any(error[b"data"] == b"__main__.put_task"
|
||||
for error in errors))
|
||||
|
||||
def testDriverPutErrors(self):
|
||||
# Define the size of one task's return argument so that the combined
|
||||
# sum of all objects' sizes is at least twice the plasma stores'
|
||||
|
||||
Reference in New Issue
Block a user