wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-26 13:37:24 +08:00
Code Issues Packages Projects Releases Wiki Activity

Allow remote functions to specify max executions and kill worker once limit is reached. (#660)

Browse Source 
* implement restarting workers after certain number of task executions

* Clean up python code.

* Don't start new worker when an actor disconnects.

* Move wait_for_pid_to_exit to test_utils.py.

* Add test.

* Fix linting errors.

* Fix linting.

* Fix typo.
This commit is contained in:
Philipp Moritz
2017-06-13 00:34:58 -07:00
committed by Robert Nishihara
parent 4374ad1453
commit 54925996ca
9 changed files with 214 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/local_scheduler/local_scheduler.cc
+35 -26
View File
@@ -532,6 +532,33 @@ void assign_task_to_worker(LocalSchedulerState *state,
}
}
void finish_task(LocalSchedulerState *state, LocalSchedulerClient *worker) {
if (worker->task_in_progress != NULL) {
TaskSpec *spec = Task_task_spec(worker->task_in_progress);
/* Return dynamic resources back for the task in progress. TODO(rkn): We
* are currently ignoring resource bookkeeping for actor methods. */
if (ActorID_equal(worker->actor_id, NIL_ACTOR_ID)) {
CHECK(worker->cpus_in_use ==
TaskSpec_get_required_resource(spec, ResourceIndex_CPU));
CHECK(worker->gpus_in_use.size() ==
TaskSpec_get_required_resource(spec, ResourceIndex_GPU));
release_resources(state, worker, worker->cpus_in_use,
worker->gpus_in_use.size());
}
/* If we're connected to Redis, update tables. */
if (state->db != NULL) {
/* Update control state tables. */
Task_set_state(worker->task_in_progress, TASK_STATUS_DONE);
task_table_update(state->db, worker->task_in_progress, NULL, NULL, NULL);
/* The call to task_table_update takes ownership of the
* task_in_progress, so we set the pointer to NULL so it is not used. */
} else {
Task_free(worker->task_in_progress);
}
worker->task_in_progress = NULL;
}
}
void process_plasma_notification(event_loop *loop,
int client_sock,
void *context,
@@ -874,8 +901,14 @@ void process_message(event_loop *loop,
case MessageType_TaskDone: {
} break;
case MessageType_DisconnectClient: {
finish_task(state, worker);
CHECK(!worker->disconnected);
worker->disconnected = true;
/* If the disconnected worker was not an actor, start a new worker to make
* sure there are enough workers in the pool. */
if (ActorID_equal(worker->actor_id, NIL_ACTOR_ID)) {
start_worker(state, NIL_ACTOR_ID);
}
} break;
case MessageType_EventLogMessage: {
/* Parse the message. */
@@ -894,32 +927,8 @@ void process_message(event_loop *loop,
send_client_register_reply(state, worker);
} break;
case MessageType_GetTask: {
/* If this worker reports a completed task: account for resources. */
if (worker->task_in_progress != NULL) {
TaskSpec *spec = Task_task_spec(worker->task_in_progress);
/* Return dynamic resources back for the task in progress. TODO(rkn): We
* are currently ignoring resource bookkeeping for actor methods. */
if (ActorID_equal(worker->actor_id, NIL_ACTOR_ID)) {
CHECK(worker->cpus_in_use ==
TaskSpec_get_required_resource(spec, ResourceIndex_CPU));
CHECK(worker->gpus_in_use.size() ==
TaskSpec_get_required_resource(spec, ResourceIndex_GPU));
release_resources(state, worker, worker->cpus_in_use,
worker->gpus_in_use.size());
}
/* If we're connected to Redis, update tables. */
if (state->db != NULL) {
/* Update control state tables. */
Task_set_state(worker->task_in_progress, TASK_STATUS_DONE);
task_table_update(state->db, worker->task_in_progress, NULL, NULL,
NULL);
/* The call to task_table_update takes ownership of the
* task_in_progress, so we set the pointer to NULL so it is not used. */
} else {
Task_free(worker->task_in_progress);
}
worker->task_in_progress = NULL;
}
/* If this worker reports a completed task, account for resources. */
finish_task(state, worker);
/* Let the scheduling algorithm process the fact that there is an available
* worker. */
if (ActorID_equal(worker->actor_id, NIL_ACTOR_ID)) {
src/local_scheduler/local_scheduler.h
+10
View File
@@ -48,6 +48,16 @@ void assign_task_to_worker(LocalSchedulerState *state,
int64_t task_spec_size,
LocalSchedulerClient *worker);
/*
* This function is called whenever a task has finished on one of the workers.
* It updates the resource accounting and the global state store.
*
* @param state The local scheduler state.
* @param worker The worker that finished the task.
* @return Void.
*/
void finish_task(LocalSchedulerState *state, LocalSchedulerClient *worker);
/**
* This is the callback that is used to process a notification from the Plasma
* store that an object has been sealed.