wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-09 11:32:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Propagate error messages from functions that run on all workers. (#410)

Browse Source
This commit is contained in:
Robert Nishihara
2016-09-06 10:06:43 -07:00
committed by Philipp Moritz
parent 327d7ff689
commit d5cb3ac090
9 changed files with 55 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/failure_test.py
+18
View File
@@ -124,5 +124,23 @@ class TaskStatusTest(unittest.TestCase):
ray.worker.cleanup()
def testFailedFunctionToRun(self):
ray.init(start_ray_local=True, num_workers=2, driver_mode=ray.SILENT_MODE)
def f(worker):
if ray.worker.global_worker.mode == ray.WORKER_MODE:
raise Exception("Function to run failed.")
ray.worker.global_worker.run_function_on_all_workers(f)
for _ in range(100): # Retry if we need to wait longer.
if len(ray.task_info()["failed_function_to_runs"]) >= 2:
break
time.sleep(0.1)
# Check that the error message is in the task info.
self.assertEqual(len(ray.task_info()["failed_function_to_runs"]), 2)
self.assertTrue("Function to run failed." in ray.task_info()["failed_function_to_runs"][0]["error_message"])
self.assertTrue("Function to run failed." in ray.task_info()["failed_function_to_runs"][1]["error_message"])
ray.worker.cleanup()
if __name__ == "__main__":
unittest.main(verbosity=2)