Propagate backend error to worker (#4039)

src/ray/gcs/format/gcs.fbs

@@ -347,3 +347,23 @@ table ActorCheckpointIdData {
   // A list of the timestamps for each of the above `checkpoint_ids`.
   timestamps: [long];
 }
+
+// This enum type is used as object's metadata to indicate the object's creating
+// task has failed because of a certain error.
+// TODO(hchen): We may want to make these errors more specific. E.g., we may want
+// to distinguish between intentional and expected actor failures, and between
+// worker process failure and node failure.
+enum ErrorType:int {
+  // Indicates that a task failed because the worker died unexpectedly while executing it.
+  WORKER_DIED = 1,
+  // Indicates that a task failed because the actor died unexpectedly before finishing it.
+  ACTOR_DIED = 2,
+  // Indicates that an object is lost and cannot be reconstructed.
+  // Note, this currently only happens to actor objects. When the actor's state is already
+  // after the object's creating task, the actor cannot re-run the task.
+  // TODO(hchen): we may want to reuse this error type for more cases. E.g.,
+  // 1) A object that was put by the driver.
+  // 2) The object's creating task is already cleaned up from GCS (this currently
+  //    crashes raylet).
+  OBJECT_UNRECONSTRUCTABLE = 3,
+}

src/ray/raylet/node_manager.cc

@@ -574,7 +574,7 @@ void NodeManager::HandleActorStateTransition(const ActorID &actor_id,
     auto tasks_to_remove = local_queues_.GetTaskIdsForActor(actor_id);
     auto removed_tasks = local_queues_.RemoveTasks(tasks_to_remove);
     for (auto const &task : removed_tasks) {
-      TreatTaskAsFailed(task);
+      TreatTaskAsFailed(task, ErrorType::ACTOR_DIED);
     }
   } else {
     RAY_CHECK(actor_registration.GetState() == ActorState::RECONSTRUCTING);
@@ -858,7 +858,7 @@ void NodeManager::ProcessDisconnectClientMessage(
       // `HandleDisconnectedActor`.
       if (actor_id.is_nil()) {
         const Task &task = local_queues_.RemoveTask(task_id);
-        TreatTaskAsFailed(task);
+        TreatTaskAsFailed(task, ErrorType::WORKER_DIED);
       }
 
       if (!intentional_disconnect) {
@@ -1214,9 +1214,10 @@ bool NodeManager::CheckDependencyManagerInvariant() const {
   return true;
 }
 
-void NodeManager::TreatTaskAsFailed(const Task &task) {
+void NodeManager::TreatTaskAsFailed(const Task &task, const ErrorType &error_type) {
   const TaskSpecification &spec = task.GetTaskSpecification();
-  RAY_LOG(DEBUG) << "Treating task " << spec.TaskId() << " as failed.";
+  RAY_LOG(DEBUG) << "Treating task " << spec.TaskId() << " as failed because of error "
+                 << EnumNameErrorType(error_type) << ".";
   // If this was an actor creation task that tried to resume from a checkpoint,
   // then erase it here since the task did not finish.
   if (spec.IsActorCreationTask()) {
@@ -1231,20 +1232,22 @@ void NodeManager::TreatTaskAsFailed(const Task &task) {
     // information about the TaskSpecification implementation.
     num_returns -= 1;
   }
+  const std::string meta = std::to_string(static_cast<int>(error_type));
   for (int64_t i = 0; i < num_returns; i++) {
-    const ObjectID object_id = spec.ReturnId(i);
-
-    std::shared_ptr<Buffer> data;
-    // TODO(ekl): this writes an invalid arrow object, which is sufficient to
-    // signal that the worker failed, but it would be nice to return more
-    // detailed failure metadata in the future.
-    arrow::Status status =
-        store_client_.Create(object_id.to_plasma_id(), 1, NULL, 0, &data);
-    if (!status.IsPlasmaObjectExists()) {
-      // TODO(rkn): We probably don't want this checks. E.g., if the object
-      // store is full, we don't want to kill the raylet.
-      RAY_ARROW_CHECK_OK(status);
-      RAY_ARROW_CHECK_OK(store_client_.Seal(object_id.to_plasma_id()));
+    const auto object_id = spec.ReturnId(i).to_plasma_id();
+    arrow::Status status = store_client_.CreateAndSeal(object_id, "", meta);
+    if (!status.ok() && !status.IsPlasmaObjectExists()) {
+      // If we failed to save the error code, log a warning and push an error message
+      // to the driver.
+      std::ostringstream stream;
+      stream << "An plasma error (" << status.ToString() << ") occurred while saving"
+             << " error code to object " << object_id << ". Anyone who's getting this"
+             << " object may hang forever.";
+      std::string error_message = stream.str();
+      RAY_LOG(WARNING) << error_message;
+      RAY_CHECK_OK(gcs_client_->error_table().PushErrorToDriver(
+          task.GetTaskSpecification().DriverId(), "task", error_message,
+          current_time_ms()));
     }
   }
   // A task failing is equivalent to assigning and finishing the task, so clean
@@ -1297,7 +1300,7 @@ void NodeManager::TreatTaskAsFailedIfLost(const Task &task) {
               // The object does not exist on any nodes but has been created
               // before, so the object has been lost. Mark the task as failed to
               // prevent any tasks that depend on this object from hanging.
-              TreatTaskAsFailed(task);
+              TreatTaskAsFailed(task, ErrorType::OBJECT_UNRECONSTRUCTABLE);
               *task_marked_as_failed = true;
             }
           }
@@ -1343,7 +1346,7 @@ void NodeManager::SubmitTask(const Task &task, const Lineage &uncommitted_lineag
       if (actor_entry->second.GetState() == ActorState::DEAD) {
         // If this actor is dead, either because the actor process is dead
         // or because its residing node is dead, treat this task as failed.
-        TreatTaskAsFailed(task);
+        TreatTaskAsFailed(task, ErrorType::ACTOR_DIED);
       } else {
         // If this actor is alive, check whether this actor is local.
         auto node_manager_id = actor_entry->second.GetNodeManagerId();

src/ray/raylet/node_manager.h

@@ -157,8 +157,9 @@ class NodeManager {
   /// the local queue.
   ///
   /// \param task The task to fail.
+  /// \param error_type The type of the error that caused this task to fail.
   /// \return Void.
-  void TreatTaskAsFailed(const Task &task);
+  void TreatTaskAsFailed(const Task &task, const ErrorType &error_type);
   /// This is similar to TreatTaskAsFailed, but it will only mark the task as
   /// failed if at least one of the task's return values is lost. A return
   /// value is lost if it has been created before, but no longer exists on any