Re-raise actor initialization errors on method invocation (#2843)

If an actor constructor fails, save that error and re-raise it on any subsequent attempts to interact with the actor. Related to https://github.com/ray-project/ray/issues/282 and https://github.com/ray-project/ray/issues/1093.

python/ray/tune/trainable.py

@@ -57,7 +57,6 @@ class Trainable(object):
                 object. If unspecified, a default logger is created.
         """
 
-        self._initialize_ok = False
         self._experiment_id = uuid.uuid4().hex
         self.config = config or {}
 
@@ -80,7 +79,6 @@ class Trainable(object):
         self._iterations_since_restore = 0
         self._restored = False
         self._setup()
-        self._initialize_ok = True
         self._local_ip = ray.services.get_node_ip_address()
 
     @classmethod
@@ -138,10 +136,6 @@ class Trainable(object):
             A dict that describes training progress.
         """
 
-        if not self._initialize_ok:
-            raise ValueError(
-                "Trainable initialization failed, see previous errors")
-
         start = time.time()
         result = self._train()
         result = result.copy()
@@ -282,9 +276,8 @@ class Trainable(object):
     def stop(self):
         """Releases all resources used by this trainable."""
 
-        if self._initialize_ok:
-            self._result_logger.close()
-            self._stop()
+        self._result_logger.close()
+        self._stop()
 
     def _train(self):
         """Subclasses should override this to implement train().

python/ray/worker.py

@@ -233,6 +233,7 @@ class Worker(object):
         self.cached_remote_functions_and_actors = []
         self.cached_functions_to_run = []
         self.fetch_and_register_actor = None
+        self.actor_init_error = None
         self.make_actor = None
         self.actors = {}
         self.actor_task_counter = 0
@@ -254,6 +255,17 @@ class Worker(object):
         # Identity of the driver that this worker is processing.
         self.task_driver_id = None
 
+    def mark_actor_init_failed(self, error):
+        """Called to mark this actor as failed during initialization."""
+
+        self.actor_init_error = error
+
+    def reraise_actor_init_error(self):
+        """Raises any previous actor initialization error."""
+
+        if self.actor_init_error is not None:
+            raise self.actor_init_error
+
     def get_serialization_context(self, driver_id):
         """Get the SerializationContext of the driver that this worker is processing.
 
@@ -907,6 +919,8 @@ class Worker(object):
 
         # Get task arguments from the object store.
         try:
+            if function_name != "__ray_terminate__":
+                self.reraise_actor_init_error()
             with profiling.profile("task:deserialize_arguments", worker=self):
                 arguments = self._get_arguments_for_execution(
                     function_name, args)
@@ -973,6 +987,9 @@ class Worker(object):
                 "function_id": function_id.id(),
                 "function_name": function_name
             })
+        # Mark the actor init as failed
+        if self.actor_id != NIL_ACTOR_ID and function_name == "__init__":
+            self.mark_actor_init_failed(error)
 
     def _become_actor(self, task):
         """Turn this worker into an actor.