wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-11 11:24:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Actor fault tolerance using object lineage reconstruction (#902)

Browse Source 
* Revert Python actor reconstruction

* Actor reconstruction using object lineage

* Add dummy arguments and return values for actor tasks

* Pin dummy outputs for actor tasks

* Skip checkpointing test for now

* TODOs

* minor edits

* Generate dummy object dependencies in Python, not C

* Fix linting.

* Move actor counter and dummy objects inside of the actor handle

* Refactor Worker._process_task, suppress exception propagation for
sequential actor tasks
This commit is contained in:
Stephanie Wang
2017-09-10 19:29:28 -07:00
committed by Robert Nishihara
parent d8aa826e63
commit 99c8b1f38c
7 changed files with 207 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/local_scheduler/local_scheduler.cc
+15 -13
View File
@@ -633,24 +633,25 @@ void reconstruct_task_update_callback(Task *task,
* Suppress the reconstruction request. */
return;
}
/* Otherwise, the test-and-set succeeded, so resubmit the task for execution
* to ensure that reconstruction will happen. */
LocalSchedulerState *state = (LocalSchedulerState *) user_context;
TaskSpec *spec = Task_task_spec(task);
/* If the task is an actor task, then we currently do not reconstruct it.
* TODO(rkn): Handle this better. */
if (!ActorID_equal(TaskSpec_actor_id(spec), NIL_ACTOR_ID)) {
LOG_WARN("We are not resubmitting this task because it is an actor task.");
} else {
/* Resubmit the task. */
handle_task_submitted(state, state->algorithm_state, spec,
if (ActorID_equal(TaskSpec_actor_id(spec), NIL_ACTOR_ID)) {
handle_task_submitted(state, state->algorithm_state, Task_task_spec(task),
Task_task_spec_size(task));
/* Recursively reconstruct the task's inputs, if necessary. */
for (int64_t i = 0; i < TaskSpec_num_args(spec); ++i) {
if (TaskSpec_arg_by_ref(spec, i)) {
ObjectID arg_id = TaskSpec_arg_id(spec, i);
reconstruct_object(state, arg_id);
}
} else {
handle_actor_task_submitted(state, state->algorithm_state,
Task_task_spec(task),
Task_task_spec_size(task));
}
/* Recursively reconstruct the task's inputs, if necessary. */
for (int64_t i = 0; i < TaskSpec_num_args(spec); ++i) {
if (TaskSpec_arg_by_ref(spec, i)) {
ObjectID arg_id = TaskSpec_arg_id(spec, i);
reconstruct_object(state, arg_id);
}
}
}
@@ -1178,6 +1179,7 @@ void handle_actor_creation_callback(ActorID actor_id,
/* TODO(rkn): We should kill the actor here if it is still around. Also,
* if it hasn't registered yet, we should keep track of its PID so we can
* kill it anyway. */
/* TODO(swang): Evict actor dummy objects as part of actor cleanup. */
}
}
src/local_scheduler/local_scheduler_algorithm.cc
+3
View File
@@ -498,6 +498,9 @@ void fetch_missing_dependency(LocalSchedulerState *state,
* Fetch a queued task's missing object dependencies. The fetch requests will
* be retried every kLocalSchedulerFetchTimeoutMilliseconds until all
* objects are available locally.
* TODO(swang): For actor task dummy objects, we should still request
* reconstruction for missing dependencies, but we should not request transfer
* from other nodes.
*
* @param state The scheduler state.
* @param algorithm_state The scheduling algorithm state.