wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-30 12:30:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Actor checkpointing with object lineage reconstruction (#1004)

Browse Source 
* Worker reports error in previous task, actor task counter is incremented after task is successful

* Refactor actor task execution

- Return new task counter in GetTaskRequest
- Update worker state for actor tasks inside of the actor method
  executor

* Manually invoked checkpoint method

* Scheduling for actor checkpoint methods

* Fix python bugs in checkpointing

* Return task success from worker to local scheduler instead of actor counter

* Kill local schedulers halfway through actor execution instead of waiting for all tasks to execute once

* Remove redundant actor tasks during dispatch, reconstruct missing dependencies for actor tasks

* Make executor for temporary actor methods

* doc

* Set default argument for whether the previous task was a success

* Refactor actor method call

* Simplify checkpoint task submission

* lint

* fix philipp's comments

* Add missing line

* Make actor reconstruction tests run faster

* Unimportant whitespace.

* Unimportant whitespace.

* Update checkpoint method signature

* Documentation and handle exceptions during checkpoint save/resume

* Rename get_task message field to actor_checkpoint_failed

* Fix bug.

* Remove debugging check, redirect test output
This commit is contained in:
Stephanie Wang
2017-10-12 09:53:32 -07:00
committed by Robert Nishihara
parent b585001881
commit 3764f2f2e1
14 changed files with 608 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/local_scheduler/local_scheduler_algorithm.cc
+92 -44
View File
@@ -53,6 +53,10 @@ typedef struct {
* restrict the submission of tasks on actors to the process that created the
* actor. */
int64_t task_counter;
/** The index of the task assigned to this actor. Set to -1 if no task is
* currently assigned. If the actor process reports back success for the
* assigned task execution, task_counter should be set to this value. */
int64_t assigned_task_counter;
/** A queue of tasks to be executed on this actor. The tasks will be sorted by
* the order of their actor counters. */
std::list<TaskQueueEntry> *task_queue;
@@ -234,6 +238,7 @@ void create_actor(SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker) {
LocalActorInfo entry;
entry.task_counter = 0;
entry.assigned_task_counter = -1;
entry.task_queue = new std::list<TaskQueueEntry>();
entry.worker = worker;
entry.worker_available = false;
@@ -309,39 +314,56 @@ bool dispatch_actor_task(LocalSchedulerState *state,
/* There should be some queued tasks for this actor. */
CHECK(!entry.task_queue->empty());
TaskQueueEntry first_task = entry.task_queue->front();
int64_t next_task_counter = TaskSpec_actor_counter(first_task.spec);
if (next_task_counter != entry.task_counter) {
/* We cannot execute the next task on this actor without violating the
* in-order execution guarantee for actor tasks. */
CHECK(next_task_counter > entry.task_counter);
return false;
}
/* If the worker is not available, we cannot assign a task to it. */
if (!entry.worker_available) {
return false;
}
/* Find the first task that either matches the task counter or that is a
* checkpoint method. Remove any tasks that we have already executed past
* (e.g., by executing a more recent checkpoint method). */
auto task = entry.task_queue->begin();
int64_t next_task_counter = TaskSpec_actor_counter(task->spec);
while (next_task_counter != entry.task_counter) {
if (next_task_counter < entry.task_counter) {
/* A task that we have already executed past. Remove it. */
task = entry.task_queue->erase(task);
/* If there are no more tasks in the queue, wait. */
if (task == entry.task_queue->end()) {
algorithm_state->actors_with_pending_tasks.erase(actor_id);
return false;
}
/* Move on to the next task. */
next_task_counter = TaskSpec_actor_counter(task->spec);
} else if (TaskSpec_actor_is_checkpoint_method(task->spec)) {
/* A later task that is a checkpoint method. Checkpoint methods can
* always be executed. */
break;
} else {
/* A later task that is not a checkpoint. Wait for the preceding tasks to
* execute. */
return false;
}
}
/* If there are not enough resources available, we cannot assign the task. */
CHECK(0 ==
TaskSpec_get_required_resource(first_task.spec, ResourceIndex_GPU));
CHECK(0 == TaskSpec_get_required_resource(task->spec, ResourceIndex_GPU));
if (!check_dynamic_resources(
state,
TaskSpec_get_required_resource(first_task.spec, ResourceIndex_CPU), 0,
TaskSpec_get_required_resource(first_task.spec,
ResourceIndex_CustomResource))) {
state, TaskSpec_get_required_resource(task->spec, ResourceIndex_CPU),
0, TaskSpec_get_required_resource(task->spec,
ResourceIndex_CustomResource))) {
return false;
}
/* Assign the first task in the task queue to the worker and mark the worker
* as unavailable. */
entry.task_counter += 1;
assign_task_to_worker(state, first_task.spec, first_task.task_spec_size,
entry.worker);
assign_task_to_worker(state, task->spec, task->task_spec_size, entry.worker);
entry.assigned_task_counter = next_task_counter;
entry.worker_available = false;
/* Free the task queue entry. */
TaskQueueEntry_free(&first_task);
TaskQueueEntry_free(&(*task));
/* Remove the task from the actor's task queue. */
entry.task_queue->pop_front();
entry.task_queue->erase(task);
/* If there are no more tasks in the queue, then indicate that the actor has
* no tasks. */
@@ -414,12 +436,11 @@ void add_task_to_actor_queue(LocalSchedulerState *state,
* guaranteeing in-order execution of the tasks on the actor). TODO(rkn): This
* check will fail if the fault-tolerance mechanism resubmits a task on an
* actor. */
bool task_is_redundant = false;
if (task_counter < entry.task_counter) {
LOG_INFO(
"A task that has already been executed has been resubmitted, so we "
"are ignoring it. This should only happen during reconstruction.");
task_is_redundant = true;
return;
}
/* Create a new task queue entry. */
@@ -437,32 +458,51 @@ void add_task_to_actor_queue(LocalSchedulerState *state,
if (it != entry.task_queue->end() &&
task_counter == TaskSpec_actor_counter(it->spec)) {
LOG_INFO(
"A task that has already been executed has been resubmitted, so we "
"are ignoring it. This should only happen during reconstruction.");
task_is_redundant = true;
"A task was resubmitted, so we are ignoring it. This should only "
"happen during reconstruction.");
return;
}
if (!task_is_redundant) {
entry.task_queue->insert(it, elt);
/* The task has a counter that has not been executed or submitted before. Add
* it to the actor queue. */
entry.task_queue->insert(it, elt);
/* Update the task table. */
if (state->db != NULL) {
Task *task = Task_alloc(spec, task_spec_size, TASK_STATUS_QUEUED,
get_db_client_id(state->db));
if (from_global_scheduler) {
/* If the task is from the global scheduler, it's already been added to
* the task table, so just update the entry. */
task_table_update(state->db, task, NULL, NULL, NULL);
} else {
/* Otherwise, this is the first time the task has been seen in the
* system (unless it's a resubmission of a previous task), so add the
* entry. */
task_table_add_task(state->db, task, NULL, NULL, NULL);
/* Update the task table. */
if (state->db != NULL) {
Task *task = Task_alloc(spec, task_spec_size, TASK_STATUS_QUEUED,
get_db_client_id(state->db));
if (from_global_scheduler) {
/* If the task is from the global scheduler, it's already been added to
* the task table, so just update the entry. */
task_table_update(state->db, task, NULL, NULL, NULL);
} else {
/* Otherwise, this is the first time the task has been seen in the
* system (unless it's a resubmission of a previous task), so add the
* entry. */
task_table_add_task(state->db, task, NULL, NULL, NULL);
}
}
/* Record the fact that this actor has a task waiting to execute. */
algorithm_state->actors_with_pending_tasks.insert(actor_id);
/* Register a missing dependency on the preceding task. TODO(swang): Unify
* with `fetch_missing_dependencies` for non-actor tasks. */
if (entry.task_counter != task_counter) {
int64_t num_args = TaskSpec_num_args(spec);
/* The last argument represents dependency on a preceding task. If it is by
* reference, then it is an explicit dependency. */
if (TaskSpec_arg_by_ref(spec, num_args - 1)) {
ObjectID dummy_object_id = TaskSpec_arg_id(spec, num_args - 1);
if (algorithm_state->local_objects.count(dummy_object_id) == 0) {
ObjectEntry entry;
/* TODO(swang): Objects in `remote_objects` will get fetched from
* remote plasma managers. Do not fetch actor dummy objects. Otherwise,
* if the plasma manager associated with the dead local scheduler is
* still alive, reconstruction will never complete. */
state->algorithm_state->remote_objects[dummy_object_id] = entry;
}
}
/* Record the fact that this actor has a task waiting to execute. */
algorithm_state->actors_with_pending_tasks.insert(actor_id);
}
}
@@ -1202,7 +1242,8 @@ void handle_actor_worker_disconnect(LocalSchedulerState *state,
void handle_actor_worker_available(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
LocalSchedulerClient *worker) {
LocalSchedulerClient *worker,
bool actor_checkpoint_failed) {
ActorID actor_id = worker->actor_id;
CHECK(!ActorID_equal(actor_id, NIL_ACTOR_ID));
/* Get the actor info for this worker. */
@@ -1212,6 +1253,13 @@ void handle_actor_worker_available(LocalSchedulerState *state,
CHECK(worker == entry.worker);
CHECK(!entry.worker_available);
/* If the assigned task was not a checkpoint task, or if it was but it
* loaded the checkpoint successfully, then we update the actor's counter
* to the assigned counter. */
if (!actor_checkpoint_failed) {
entry.task_counter = entry.assigned_task_counter + 1;
}
entry.assigned_task_counter = -1;
entry.worker_available = true;
/* Assign new tasks if possible. */
dispatch_all_tasks(state, algorithm_state);