Propagate backend error to worker (#4039)

test/actor_test.py

@@ -1405,7 +1405,7 @@ def test_exception_raised_when_actor_node_dies(head_node_cluster):
         # Submit some new actor tasks.
         x_ids = [actor.inc.remote() for _ in range(5)]
         for x_id in x_ids:
-            with pytest.raises(ray.worker.RayTaskError):
+            with pytest.raises(ray.exceptions.RayActorError):
                 # There is some small chance that ray.get will actually
                 # succeed (if the object is transferred before the raylet
                 # dies).
@@ -2128,7 +2128,7 @@ def test_actor_eviction(shutdown_only):
         try:
             ray.get(obj)
             num_success += 1
-        except ray.worker.RayTaskError:
+        except ray.exceptions.UnreconstructableError:
             num_evicted += 1
     # Some objects should have been evicted, and some should still be in the
     # object store.
@@ -2173,7 +2173,7 @@ def test_actor_reconstruction(ray_start_regular):
     pid = ray.get(actor.get_pid.remote())
     os.kill(pid, signal.SIGKILL)
     # The actor has exceeded max reconstructions, and this task should fail.
-    with pytest.raises(ray.worker.RayTaskError):
+    with pytest.raises(ray.exceptions.RayActorError):
         ray.get(actor.increase.remote())
 
     # Create another actor.
@@ -2181,7 +2181,7 @@ def test_actor_reconstruction(ray_start_regular):
     # Intentionlly exit the actor
     actor.__ray_terminate__.remote()
     # Check that the actor won't be reconstructed.
-    with pytest.raises(ray.worker.RayTaskError):
+    with pytest.raises(ray.exceptions.RayActorError):
         ray.get(actor.increase.remote())
 
 
@@ -2241,7 +2241,7 @@ def test_actor_reconstruction_on_node_failure(head_node_cluster):
     object_store_socket = ray.get(actor.get_object_store_socket.remote())
     kill_node(object_store_socket)
     # The actor has exceeded max reconstructions, and this task should fail.
-    with pytest.raises(ray.worker.RayTaskError):
+    with pytest.raises(ray.exceptions.RayActorError):
         ray.get(actor.increase.remote())
 
 

test/component_failures_test.py

@@ -279,7 +279,7 @@ def test_worker_failed(ray_start_workers_separate_multinode):
     for object_id in object_ids:
         try:
             ray.get(object_id)
-        except ray.worker.RayTaskError:
+        except (ray.exceptions.RayTaskError, ray.exceptions.RayWorkerError):
             pass
 
 
@@ -424,7 +424,7 @@ def test_actor_creation_node_failure(ray_start_cluster):
             for i, out in enumerate(children_out):
                 try:
                     ray.get(out)
-                except ray.worker.RayTaskError:
+                except ray.exceptions.RayActorError:
                     children[i] = Child.remote(death_probability)
         # Remove a node. Any actor creation tasks that were forwarded to this
         # node must be reconstructed.

test/failure_test.py

@@ -319,7 +319,8 @@ def test_worker_dying(ray_start_regular):
     def f():
         eval("exit()")
 
-    f.remote()
+    with pytest.raises(ray.exceptions.RayWorkerError):
+        ray.get(f.remote())
 
     wait_for_errors(ray_constants.WORKER_DIED_PUSH_ERROR, 1)
 
@@ -340,9 +341,9 @@ def test_actor_worker_dying(ray_start_regular):
 
     a = Actor.remote()
     [obj], _ = ray.wait([a.kill.remote()], timeout=5.0)
-    with pytest.raises(Exception):
+    with pytest.raises(ray.exceptions.RayActorError):
         ray.get(obj)
-    with pytest.raises(Exception):
+    with pytest.raises(ray.exceptions.RayTaskError):
         ray.get(consume.remote(obj))
     wait_for_errors(ray_constants.WORKER_DIED_PUSH_ERROR, 1)
 

test/runtest.py

@@ -2621,7 +2621,7 @@ def test_inline_objects(shutdown_only):
             value = ray.get(inline_object)
             assert value == "inline"
             inlined += 1
-        except ray.worker.RayTaskError:
+        except ray.exceptions.UnreconstructableError:
             pass
     # Make sure some objects were inlined. Some of them may not get inlined
     # because we evict the object soon after creating it.
@@ -2638,7 +2638,7 @@ def test_inline_objects(shutdown_only):
             ray.worker.global_worker.plasma_client.delete([plasma_id])
         # Objects created by an actor that were evicted and larger than the
         # maximum inline object size cannot be retrieved or reconstructed.
-        with pytest.raises(ray.worker.RayTaskError):
+        with pytest.raises(ray.exceptions.UnreconstructableError):
             ray.get(non_inline_object) == 10000 * [1]
 
 

test/test_signal.py

@@ -106,7 +106,7 @@ def test_task_crash(ray_start):
     try:
         ray.get(object_id)
     except Exception as e:
-        assert type(e) == ray.worker.RayTaskError
+        assert type(e) == ray.exceptions.RayTaskError
     finally:
         result_list = signal.receive([object_id], timeout=5)
         assert len(result_list) == 1
@@ -142,7 +142,7 @@ def test_actor_crash(ray_start):
     try:
         ray.get(a.crash.remote())
     except Exception as e:
-        assert type(e) == ray.worker.RayTaskError
+        assert type(e) == ray.exceptions.RayTaskError
     finally:
         result_list = signal.receive([a], timeout=5)
         assert len(result_list) == 1
@@ -184,7 +184,7 @@ def test_actor_crash_init2(ray_start):
     try:
         ray.get(a.method.remote())
     except Exception as e:
-        assert type(e) == ray.worker.RayTaskError
+        assert type(e) == ray.exceptions.RayTaskError
     finally:
         result_list = receive_all_signals([a], timeout=5)
         assert len(result_list) == 2