wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-09 10:37:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix: do not treat actor task as failed if the actor will be reconstructed (#3736)

Browse Source
This commit is contained in:
Hao Chen
2019-01-24 15:28:44 +08:00
committed by Robert Nishihara
parent 04ec47cbd4
commit bfcf254e52
3 changed files with 90 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ray/raylet/node_manager.cc
+37 -28
View File
@@ -809,33 +809,49 @@ void NodeManager::ProcessDisconnectClientMessage(
// If the client has any blocked tasks, mark them as unblocked. In
// particular, we are no longer waiting for their dependencies.
if (worker) {
while (!worker->GetBlockedTaskIds().empty()) {
// NOTE(swang): HandleTaskUnblocked will modify the worker, so it is
// not safe to pass in the iterator directly.
const TaskID task_id = *worker->GetBlockedTaskIds().begin();
HandleTaskUnblocked(client, task_id);
if (is_worker && worker->IsDead()) {
// Don't need to unblock the client if it's a worker and is already dead.
// Because in this case, its task is already cleaned up.
RAY_LOG(DEBUG) << "Skip unblocking worker because it's already dead.";
} else {
while (!worker->GetBlockedTaskIds().empty()) {
// NOTE(swang): HandleTaskUnblocked will modify the worker, so it is
// not safe to pass in the iterator directly.
const TaskID task_id = *worker->GetBlockedTaskIds().begin();
HandleTaskUnblocked(client, task_id);
}
}
}
// Remove the dead client from the pool and stop listening for messages.
if (is_worker) {
// The client is a worker. Handle the case where the worker is killed
// while executing a task. Clean up the assigned task's resources, push
// an error to the driver.
// (See design_docs/task_states.rst for the state transition diagram.)
const TaskID &task_id = worker->GetAssignedTaskId();
if (!task_id.is_nil() && !worker->IsDead()) {
// If the worker was killed intentionally, e.g., when the driver that created
// the task that this worker is currently executing exits, the task for this
// worker has already been removed from queue, so the following are skipped.
const Task &task = local_queues_.RemoveTask(task_id);
// Handle the task failure in order to raise an exception in the
// application.
TreatTaskAsFailed(task);
// The client is a worker.
if (worker->IsDead()) {
// If the worker was killed by us because the driver exited,
// treat it as intentionally disconnected.
intentional_disconnect = true;
}
const JobID &job_id = worker->GetAssignedDriverId();
const ActorID &actor_id = worker->GetActorId();
if (!actor_id.is_nil()) {
// If the worker was an actor, update actor state, reconstruct the actor if needed,
// and clean up actor's tasks if the actor is permanently dead.
HandleDisconnectedActor(actor_id, true, intentional_disconnect);
}
const TaskID &task_id = worker->GetAssignedTaskId();
// If the worker was running a task, clean up the task and push an error to
// the driver, unless the worker is already dead.
if (!task_id.is_nil() && !worker->IsDead()) {
// If the worker was an actor, the task was already cleaned up in
// `HandleDisconnectedActor`.
if (actor_id.is_nil()) {
const Task &task = local_queues_.RemoveTask(task_id);
TreatTaskAsFailed(task);
}
if (!intentional_disconnect) {
// Push the error to driver.
const JobID &job_id = worker->GetAssignedDriverId();
// TODO(rkn): Define this constant somewhere else.
std::string type = "worker_died";
std::ostringstream error_message;
@@ -846,16 +862,9 @@ void NodeManager::ProcessDisconnectClientMessage(
}
}
// Remove the dead client from the pool and stop listening for messages.
worker_pool_.DisconnectWorker(worker);
// If the worker was an actor, add it to the list of dead actors.
const ActorID &actor_id = worker->GetActorId();
if (!actor_id.is_nil()) {
RAY_LOG(DEBUG) << "The actor with ID " << actor_id << " died on "
<< gcs_client_->client_table().GetLocalClientId();
HandleDisconnectedActor(actor_id, /*was_local=*/true, intentional_disconnect);
}
const ClientID &client_id = gcs_client_->client_table().GetLocalClientId();
// Return the resources that were being used by this worker.