Propagate error messages from functions that run on all workers. (#410)

test/failure_test.py

@@ -124,5 +124,23 @@ class TaskStatusTest(unittest.TestCase):
 
     ray.worker.cleanup()
 
+  def testFailedFunctionToRun(self):
+    ray.init(start_ray_local=True, num_workers=2, driver_mode=ray.SILENT_MODE)
+
+    def f(worker):
+      if ray.worker.global_worker.mode == ray.WORKER_MODE:
+        raise Exception("Function to run failed.")
+    ray.worker.global_worker.run_function_on_all_workers(f)
+    for _ in range(100): # Retry if we need to wait longer.
+      if len(ray.task_info()["failed_function_to_runs"]) >= 2:
+        break
+      time.sleep(0.1)
+    # Check that the error message is in the task info.
+    self.assertEqual(len(ray.task_info()["failed_function_to_runs"]), 2)
+    self.assertTrue("Function to run failed." in ray.task_info()["failed_function_to_runs"][0]["error_message"])
+    self.assertTrue("Function to run failed." in ray.task_info()["failed_function_to_runs"][1]["error_message"])
+
+    ray.worker.cleanup()
+
 if __name__ == "__main__":
   unittest.main(verbosity=2)

test/runtest.py

@@ -358,14 +358,14 @@ class APITest(unittest.TestCase):
   def testRunningFunctionOnAllWorkers(self):
     ray.init(start_ray_local=True, num_workers=1)
 
-    def f():
+    def f(worker):
       sys.path.append("fake_directory")
     ray.worker.global_worker.run_function_on_all_workers(f)
     @ray.remote
     def get_path():
       return sys.path
     self.assertEqual("fake_directory", ray.get(get_path.remote())[-1])
-    def f():
+    def f(worker):
       sys.path.pop(-1)
     ray.worker.global_worker.run_function_on_all_workers(f)
     self.assertTrue("fake_directory" not in ray.get(get_path.remote()))