Re-raise actor initialization errors on method invocation (#2843)

If an actor constructor fails, save that error and re-raise it on any subsequent attempts to interact with the actor. Related to https://github.com/ray-project/ray/issues/282 and https://github.com/ray-project/ray/issues/1093.
This commit is contained in:
Eric Liang
2018-09-10 10:51:19 -07:00
committed by Robert Nishihara
parent 8414e413a2
commit 611259b2c7
4 changed files with 58 additions and 13 deletions
+17
View File
@@ -233,6 +233,7 @@ class Worker(object):
self.cached_remote_functions_and_actors = []
self.cached_functions_to_run = []
self.fetch_and_register_actor = None
self.actor_init_error = None
self.make_actor = None
self.actors = {}
self.actor_task_counter = 0
@@ -254,6 +255,17 @@ class Worker(object):
# Identity of the driver that this worker is processing.
self.task_driver_id = None
def mark_actor_init_failed(self, error):
"""Called to mark this actor as failed during initialization."""
self.actor_init_error = error
def reraise_actor_init_error(self):
"""Raises any previous actor initialization error."""
if self.actor_init_error is not None:
raise self.actor_init_error
def get_serialization_context(self, driver_id):
"""Get the SerializationContext of the driver that this worker is processing.
@@ -907,6 +919,8 @@ class Worker(object):
# Get task arguments from the object store.
try:
if function_name != "__ray_terminate__":
self.reraise_actor_init_error()
with profiling.profile("task:deserialize_arguments", worker=self):
arguments = self._get_arguments_for_execution(
function_name, args)
@@ -973,6 +987,9 @@ class Worker(object):
"function_id": function_id.id(),
"function_name": function_name
})
# Mark the actor init as failed
if self.actor_id != NIL_ACTOR_ID and function_name == "__init__":
self.mark_actor_init_failed(error)
def _become_actor(self, task):
"""Turn this worker into an actor.