wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-10 12:38:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Treat actor creation like a regular task. (#1668)

Browse Source 
* Treat actor creation like a regular task.

* Small cleanups.

* Change semantics of actor resource handling.

* Bug fix.

* Minor linting

* Bug fix

* Fix jenkins test.

* Fix actor tests

* Some cleanups

* Bug fix

* Fix bug.

* Remove cached actor tasks when a driver is removed.

* Add more info to taskspec in global state API.

* Fix cyclic import bug in tune.

* Fix

* Fix linting.

* Fix linting.

* Don't schedule any tasks (especially actor creaiton tasks) on local schedulers with 0 CPUs.

* Bug fix.

* Add test for 0 CPU case

* Fix linting

* Address comments.

* Fix typos and add comment.

* Add assertion and fix test.
This commit is contained in:
Robert Nishihara
2018-03-16 11:18:07 -07:00
committed by Stephanie Wang
parent 3c080f4baa
commit 96913be939
36 changed files with 901 additions and 798 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/format/common.fbs
+13
View File
@@ -29,6 +29,10 @@ table TaskInfo {
parent_task_id: string;
// A count of the number of tasks submitted by the parent task before this one.
parent_counter: int;
// The ID of the actor to create if this is an actor creation task.
actor_creation_id: string;
// The dummy object ID of the actor creation task if this is an actor method.
actor_creation_dummy_object_id: string;
// Actor ID of the task. This is the actor that this task is executed on
// or NIL_ACTOR_ID if the task is just a normal task.
actor_id: string;
@@ -162,3 +166,12 @@ table DriverTableMessage {
// The driver ID of the driver that died.
driver_id: string;
}
table ActorCreationNotification {
// The ID of the actor that was created.
actor_id: string;
// The ID of the driver that created the actor.
driver_id: string;
// The ID of the local scheduler that created the actor.
local_scheduler_id: string;
}
src/common/lib/python/common_extension.cc
+34 -7
View File
@@ -272,9 +272,9 @@ static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
/* ID of the driver that this task originates from. */
UniqueID driver_id;
/* ID of the actor this task should run on. */
UniqueID actor_id = UniqueID::nil();
UniqueID actor_id = ActorID::nil();
/* ID of the actor handle used to submit this task. */
UniqueID actor_handle_id = UniqueID::nil();
UniqueID actor_handle_id = ActorHandleID::nil();
/* How many tasks have been launched on the actor so far? */
int actor_counter = 0;
/* True if this is an actor checkpoint task and false otherwise. */
@@ -289,15 +289,21 @@ static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
TaskID parent_task_id;
/* The number of tasks that the parent task has called prior to this one. */
int parent_counter;
// The actor creation ID.
ActorID actor_creation_id = ActorID::nil();
// The dummy object for the actor creation task (if this is an actor method).
ObjectID actor_creation_dummy_object_id = ObjectID::nil();
/* Arguments of the task that are execution-dependent. These must be
* PyObjectIDs). */
PyObject *execution_arguments = NULL;
/* Dictionary of resource requirements for this task. */
PyObject *resource_map = NULL;
if (!PyArg_ParseTuple(args, "O&O&OiO&i|O&O&iOOO", &PyObjectToUniqueID,
if (!PyArg_ParseTuple(args, "O&O&OiO&i|O&O&O&O&iOOO", &PyObjectToUniqueID,
&driver_id, &PyObjectToUniqueID, &function_id,
&arguments, &num_returns, &PyObjectToUniqueID,
&parent_task_id, &parent_counter, &PyObjectToUniqueID,
&actor_creation_id, &PyObjectToUniqueID,
&actor_creation_dummy_object_id, &PyObjectToUniqueID,
&actor_id, &PyObjectToUniqueID, &actor_handle_id,
&actor_counter, &is_actor_checkpoint_method_object,
&execution_arguments, &resource_map)) {
@@ -312,10 +318,11 @@ static int PyTask_init(PyTask *self, PyObject *args, PyObject *kwds) {
Py_ssize_t size = PyList_Size(arguments);
/* Construct the task specification. */
TaskSpec_start_construct(g_task_builder, driver_id, parent_task_id,
parent_counter, actor_id, actor_handle_id,
actor_counter, is_actor_checkpoint_method,
function_id, num_returns);
TaskSpec_start_construct(
g_task_builder, driver_id, parent_task_id, parent_counter,
actor_creation_id, actor_creation_dummy_object_id, actor_id,
actor_handle_id, actor_counter, is_actor_checkpoint_method, function_id,
num_returns);
/* Add the task arguments. */
for (Py_ssize_t i = 0; i < size; ++i) {
PyObject *arg = PyList_GetItem(arguments, i);
@@ -463,6 +470,21 @@ static PyObject *PyTask_arguments(PyObject *self) {
return arg_list;
}
static PyObject *PyTask_actor_creation_id(PyObject *self) {
ActorID actor_creation_id =
TaskSpec_actor_creation_id(((PyTask *) self)->spec);
return PyObjectID_make(actor_creation_id);
}
static PyObject *PyTask_actor_creation_dummy_object_id(PyObject *self) {
ActorID actor_creation_dummy_object_id = ActorID::nil();
if (TaskSpec_is_actor_task(((PyTask *) self)->spec)) {
actor_creation_dummy_object_id =
TaskSpec_actor_creation_dummy_object_id(((PyTask *) self)->spec);
}
return PyObjectID_make(actor_creation_dummy_object_id);
}
static PyObject *PyTask_required_resources(PyObject *self) {
TaskSpec *task = ((PyTask *) self)->spec;
PyObject *required_resources = PyDict_New();
@@ -520,6 +542,11 @@ static PyMethodDef PyTask_methods[] = {
"Return the task ID for this task."},
{"arguments", (PyCFunction) PyTask_arguments, METH_NOARGS,
"Return the arguments for the task."},
{"actor_creation_id", (PyCFunction) PyTask_actor_creation_id, METH_NOARGS,
"Return the actor creation ID for the task."},
{"actor_creation_dummy_object_id",
(PyCFunction) PyTask_actor_creation_dummy_object_id, METH_NOARGS,
"Return the actor creation dummy object ID for the task."},
{"required_resources", (PyCFunction) PyTask_required_resources, METH_NOARGS,
"Return the resource vector of the task."},
{"returns", (PyCFunction) PyTask_returns, METH_NOARGS,
src/common/state/actor_notification_table.cc
+25
View File
@@ -1,6 +1,31 @@
#include "actor_notification_table.h"
#include "common_protocol.h"
#include "redis.h"
void publish_actor_creation_notification(DBHandle *db_handle,
const ActorID &actor_id,
const WorkerID &driver_id,
const DBClientID &local_scheduler_id) {
// Create a flatbuffer object to serialize and publish.
flatbuffers::FlatBufferBuilder fbb;
// Create the flatbuffers message.
auto message = CreateActorCreationNotification(
fbb, to_flatbuf(fbb, actor_id), to_flatbuf(fbb, driver_id),
to_flatbuf(fbb, local_scheduler_id));
fbb.Finish(message);
ActorCreationNotificationData *data =
(ActorCreationNotificationData *) malloc(
sizeof(ActorCreationNotificationData) + fbb.GetSize());
data->size = fbb.GetSize();
memcpy(&data->flatbuffer_data[0], fbb.GetBufferPointer(), fbb.GetSize());
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(data), NULL, NULL,
redis_publish_actor_creation_notification, NULL);
}
void actor_notification_table_subscribe(
DBHandle *db_handle,
actor_notification_table_subscribe_callback subscribe_callback,
src/common/state/actor_notification_table.h
+25 -4
View File
@@ -11,12 +11,33 @@
/* Callback for subscribing to the local scheduler table. */
typedef void (*actor_notification_table_subscribe_callback)(
ActorID actor_id,
WorkerID driver_id,
DBClientID local_scheduler_id,
bool reconstruct,
const ActorID &actor_id,
const WorkerID &driver_id,
const DBClientID &local_scheduler_id,
void *user_context);
/// Publish an actor creation notification. This is published by a local
/// scheduler once it creates an actor.
///
/// \param db_handle Database handle.
/// \param actor_id The ID of the actor that was created.
/// \param driver_id The ID of the driver that created the actor.
/// \param local_scheduler_id The ID of the local scheduler that created the
/// actor.
/// \return Void.
void publish_actor_creation_notification(DBHandle *db_handle,
const ActorID &actor_id,
const WorkerID &driver_id,
const DBClientID &local_scheduler_id);
/// Data that is needed to publish an actor creation notification.
typedef struct {
/// The size of the flatbuffer object.
int64_t size;
/// The information to be sent.
uint8_t flatbuffer_data[0];
} ActorCreationNotificationData;
/**
* Register a callback to process actor notification events.
*
src/common/state/local_scheduler_table.cc
+2 -1
View File
@@ -1,5 +1,6 @@
#include "common_protocol.h"
#include "local_scheduler_table.h"
#include "common_protocol.h"
#include "redis.h"
void local_scheduler_table_subscribe(
src/common/state/redis.cc
+46 -32
View File
@@ -1033,7 +1033,8 @@ void redis_task_table_test_and_update_callback(redisAsyncContext *c,
* delayed when added to the task table if they are submitted to a local
* scheduler before it receives the notification that maps the actor to a
* local scheduler. */
RAY_LOG(ERROR) << "No task found during task_table_test_and_update";
RAY_LOG(ERROR) << "No task found during task_table_test_and_update for "
<< "task with ID " << callback_data->id;
return;
}
/* Determine whether the update happened. */
@@ -1541,6 +1542,40 @@ void redis_plasma_manager_send_heartbeat(TableCallbackData *callback_data) {
destroy_timer_callback(db->loop, callback_data);
}
void redis_publish_actor_creation_notification_callback(redisAsyncContext *c,
void *r,
void *privdata) {
REDIS_CALLBACK_HEADER(db, callback_data, r);
redisReply *reply = (redisReply *) r;
RAY_CHECK(reply->type == REDIS_REPLY_INTEGER);
RAY_LOG(DEBUG) << reply->integer << " subscribers received this publish.";
// At the very least, the local scheduler that publishes this message should
// also receive it.
RAY_CHECK(reply->integer >= 1);
RAY_CHECK(callback_data->done_callback == NULL);
// Clean up the timer and callback.
destroy_timer_callback(db->loop, callback_data);
}
void redis_publish_actor_creation_notification(
TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
ActorCreationNotificationData *data =
(ActorCreationNotificationData *) callback_data->data->Get();
int status = redisAsyncCommand(
db->context, redis_publish_actor_creation_notification_callback,
(void *) callback_data->timer_id, "PUBLISH actor_notifications %b",
&data->flatbuffer_data[0], data->size);
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context,
"error in redis_publish_actor_creation_notification");
}
}
void redis_actor_notification_table_subscribe_callback(redisAsyncContext *c,
void *r,
void *privdata) {
@@ -1554,43 +1589,22 @@ void redis_actor_notification_table_subscribe_callback(redisAsyncContext *c,
<< message_type->str;
if (strcmp(message_type->str, "message") == 0) {
/* Handle an actor notification message. Parse the payload and call the
* subscribe callback. */
// Handle an actor notification message. Parse the payload and call the
// subscribe callback.
redisReply *payload = reply->element[2];
ActorNotificationTableSubscribeData *data =
(ActorNotificationTableSubscribeData *) callback_data->data->Get();
/* The payload should be the concatenation of three IDs. */
ActorID actor_id;
WorkerID driver_id;
DBClientID local_scheduler_id;
bool reconstruct;
RAY_CHECK(sizeof(actor_id) + sizeof(driver_id) +
sizeof(local_scheduler_id) + 1 ==
payload->len);
char *current_ptr = payload->str;
/* Parse the actor ID. */
memcpy(&actor_id, current_ptr, sizeof(actor_id));
current_ptr += sizeof(actor_id);
/* Parse the driver ID. */
memcpy(&driver_id, current_ptr, sizeof(driver_id));
current_ptr += sizeof(driver_id);
/* Parse the local scheduler ID. */
memcpy(&local_scheduler_id, current_ptr, sizeof(local_scheduler_id));
current_ptr += sizeof(local_scheduler_id);
/* Parse the reconstruct bit. */
if (*current_ptr == '1') {
reconstruct = true;
} else if (*current_ptr == '0') {
reconstruct = false;
} else {
reconstruct = false; // We set this value to avoid a compiler warning.
RAY_LOG(FATAL) << "This code should be unreachable.";
}
current_ptr += 1;
auto message =
flatbuffers::GetRoot<ActorCreationNotification>(payload->str);
ActorID actor_id = from_flatbuf(*message->actor_id());
WorkerID driver_id = from_flatbuf(*message->driver_id());
DBClientID local_scheduler_id =
from_flatbuf(*message->local_scheduler_id());
if (data->subscribe_callback) {
data->subscribe_callback(actor_id, driver_id, local_scheduler_id,
reconstruct, data->subscribe_context);
data->subscribe_context);
}
} else if (strcmp(message_type->str, "subscribe") == 0) {
/* The reply for the initial SUBSCRIBE command. */
src/common/state/redis.h
+8
View File
@@ -332,6 +332,14 @@ void redis_plasma_manager_send_heartbeat(TableCallbackData *callback_data);
*/
void redis_actor_table_mark_removed(DBHandle *db, ActorID actor_id);
/// Publish an actor creation notification.
///
/// \param callback_data Data structure containing redis connection and timeout
/// information.
/// \return Void.
void redis_publish_actor_creation_notification(
TableCallbackData *callback_data);
/**
* Subscribe to updates about newly created actors.
*
src/common/task.cc
+36 -4
View File
@@ -37,8 +37,10 @@ class TaskBuilder {
void Start(UniqueID driver_id,
TaskID parent_task_id,
int64_t parent_counter,
ActorID actor_creation_id,
ObjectID actor_creation_dummy_object_id,
ActorID actor_id,
ActorID actor_handle_id,
ActorHandleID actor_handle_id,
int64_t actor_counter,
bool is_actor_checkpoint_method,
FunctionID function_id,
@@ -46,6 +48,8 @@ class TaskBuilder {
driver_id_ = driver_id;
parent_task_id_ = parent_task_id;
parent_counter_ = parent_counter;
actor_creation_id_ = actor_creation_id;
actor_creation_dummy_object_id_ = actor_creation_dummy_object_id;
actor_id_ = actor_id;
actor_handle_id_ = actor_handle_id;
actor_counter_ = actor_counter;
@@ -58,6 +62,9 @@ class TaskBuilder {
sha256_update(&ctx, (BYTE *) &driver_id, sizeof(driver_id));
sha256_update(&ctx, (BYTE *) &parent_task_id, sizeof(parent_task_id));
sha256_update(&ctx, (BYTE *) &parent_counter, sizeof(parent_counter));
sha256_update(&ctx, (BYTE *) &actor_creation_id, sizeof(actor_creation_id));
sha256_update(&ctx, (BYTE *) &actor_creation_dummy_object_id,
sizeof(actor_creation_dummy_object_id));
sha256_update(&ctx, (BYTE *) &actor_id, sizeof(actor_id));
sha256_update(&ctx, (BYTE *) &actor_counter, sizeof(actor_counter));
sha256_update(&ctx, (BYTE *) &is_actor_checkpoint_method,
@@ -103,6 +110,8 @@ class TaskBuilder {
auto message = CreateTaskInfo(
fbb, to_flatbuf(fbb, driver_id_), to_flatbuf(fbb, task_id),
to_flatbuf(fbb, parent_task_id_), parent_counter_,
to_flatbuf(fbb, actor_creation_id_),
to_flatbuf(fbb, actor_creation_dummy_object_id_),
to_flatbuf(fbb, actor_id_), to_flatbuf(fbb, actor_handle_id_),
actor_counter_, is_actor_checkpoint_method_,
to_flatbuf(fbb, function_id_), arguments, fbb.CreateVector(returns),
@@ -127,6 +136,8 @@ class TaskBuilder {
UniqueID driver_id_;
TaskID parent_task_id_;
int64_t parent_counter_;
ActorID actor_creation_id_;
ObjectID actor_creation_dummy_object_id_;
ActorID actor_id_;
ActorID actor_handle_id_;
int64_t actor_counter_;
@@ -170,15 +181,18 @@ void TaskSpec_start_construct(TaskBuilder *builder,
UniqueID driver_id,
TaskID parent_task_id,
int64_t parent_counter,
ActorID actor_creation_id,
ObjectID actor_creation_dummy_object_id,
ActorID actor_id,
ActorID actor_handle_id,
int64_t actor_counter,
bool is_actor_checkpoint_method,
FunctionID function_id,
int64_t num_returns) {
builder->Start(driver_id, parent_task_id, parent_counter, actor_id,
actor_handle_id, actor_counter, is_actor_checkpoint_method,
function_id, num_returns);
builder->Start(driver_id, parent_task_id, parent_counter, actor_creation_id,
actor_creation_dummy_object_id, actor_id, actor_handle_id,
actor_counter, is_actor_checkpoint_method, function_id,
num_returns);
}
TaskSpec *TaskSpec_finish_construct(TaskBuilder *builder, int64_t *size) {
@@ -233,6 +247,24 @@ bool TaskSpec_is_actor_task(TaskSpec *spec) {
return !TaskSpec_actor_id(spec).is_nil();
}
ActorID TaskSpec_actor_creation_id(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->actor_creation_id());
}
ObjectID TaskSpec_actor_creation_dummy_object_id(TaskSpec *spec) {
RAY_CHECK(spec);
// The task must be an actor method.
RAY_CHECK(TaskSpec_is_actor_task(spec));
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
return from_flatbuf(*message->actor_creation_dummy_object_id());
}
bool TaskSpec_is_actor_creation_task(TaskSpec *spec) {
return !TaskSpec_actor_creation_id(spec).is_nil();
}
int64_t TaskSpec_actor_counter(TaskSpec *spec) {
RAY_CHECK(spec);
auto message = flatbuffers::GetRoot<TaskInfo>(spec);
src/common/task.h
+33 -5
View File
@@ -190,6 +190,9 @@ void free_task_builder(TaskBuilder *builder);
* @param parent_task_id The task ID of the task that submitted this task.
* @param parent_counter A counter indicating how many tasks were submitted by
* the parent task prior to this one.
* @param actor_creation_id The actor creation ID of this task.
* @param actor_creation_dummy_object_id The dummy object for the corresponding
* actor creation task, assuming this is an actor method.
* @param actor_id The ID of the actor that this task is for. If it is not an
* actor task, then this if NIL_ACTOR_ID.
* @param actor_handle_id The ID of the actor handle that this task was
@@ -210,8 +213,10 @@ void TaskSpec_start_construct(TaskBuilder *B,
UniqueID driver_id,
TaskID parent_task_id,
int64_t parent_counter,
UniqueID actor_id,
UniqueID actor_handle_id,
ActorID actor_creation_id,
ObjectID actor_creation_dummy_object_id,
ActorID actor_id,
ActorHandleID actor_handle_id,
int64_t actor_counter,
bool is_actor_checkpoint_method,
FunctionID function_id,
@@ -241,7 +246,7 @@ FunctionID TaskSpec_function(TaskSpec *spec);
* @param spec The task_spec in question.
* @return The actor ID of the actor the task is part of.
*/
UniqueID TaskSpec_actor_id(TaskSpec *spec);
ActorID TaskSpec_actor_id(TaskSpec *spec);
/**
* Return the actor handle ID of the task.
@@ -249,7 +254,7 @@ UniqueID TaskSpec_actor_id(TaskSpec *spec);
* @param spec The task_spec in question.
* @return The ID of the actor handle that the task was submitted through.
*/
UniqueID TaskSpec_actor_handle_id(TaskSpec *spec);
ActorID TaskSpec_actor_handle_id(TaskSpec *spec);
/**
* Return whether this task is for an actor.
@@ -259,6 +264,26 @@ UniqueID TaskSpec_actor_handle_id(TaskSpec *spec);
*/
bool TaskSpec_is_actor_task(TaskSpec *spec);
/// Return whether this task is an actor creation task or not.
///
/// \param spec The task_spec in question.
/// \return True if this task is an actor creation task and false otherwise.
bool TaskSpec_is_actor_creation_task(TaskSpec *spec);
/// Return the actor creation ID of the task. The task must be an actor creation
/// task.
///
/// \param spec The task_spec in question.
/// \return The actor creation ID if this is an actor creation task.
ActorID TaskSpec_actor_creation_id(TaskSpec *spec);
/// Return the actor creation dummy object ID of the task. The task must be an
/// actor task.
///
/// \param spec The task_spec in question.
/// \return The actor creation dummy object ID corresponding to this actor task.
ObjectID TaskSpec_actor_creation_dummy_object_id(TaskSpec *spec);
/**
* Return the actor counter of the task. This starts at 0 and increments by 1
* every time a new task is submitted to run on the actor.
@@ -508,7 +533,10 @@ typedef enum {
/** The task was not able to finish. */
TASK_STATUS_LOST = 32,
/** The task will be submitted for reexecution. */
TASK_STATUS_RECONSTRUCTING = 64
TASK_STATUS_RECONSTRUCTING = 64,
/** An actor task is cached at a local scheduler and is waiting for the
* corresponding actor to be created. */
TASK_STATUS_ACTOR_CACHED = 128
} scheduling_state;
/** A task is an execution of a task specification. It has a state of execution
src/common/test/example_task.h
+2 -2
View File
@@ -14,8 +14,8 @@ static inline TaskExecutionSpec example_task_execution_spec_with_args(
TaskID parent_task_id = TaskID::from_random();
FunctionID func_id = FunctionID::from_random();
TaskSpec_start_construct(g_task_builder, UniqueID::nil(), parent_task_id, 0,
ActorID::nil(), ActorID::nil(), 0, false, func_id,
num_returns);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, num_returns);
for (int64_t i = 0; i < num_args; ++i) {
ObjectID arg_id;
if (arg_ids == NULL) {
src/common/test/task_tests.cc
+19 -18
View File
@@ -16,8 +16,8 @@ TEST task_test(void) {
FunctionID func_id = FunctionID::from_random();
TaskBuilder *builder = make_task_builder();
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ActorID::nil(), 0, false, func_id,
2);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 2);
UniqueID arg1 = UniqueID::from_random();
TaskSpec_args_add_ref(builder, &arg1, 1);
@@ -56,8 +56,8 @@ TEST deterministic_ids_test(void) {
/* Construct a first task. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ActorID::nil(), 0, false, func_id,
3);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size1;
@@ -65,8 +65,8 @@ TEST deterministic_ids_test(void) {
/* Construct a second identical task. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ActorID::nil(), 0, false, func_id,
3);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size2;
@@ -86,8 +86,8 @@ TEST deterministic_ids_test(void) {
/* Construct a task with a different parent task ID. */
TaskSpec_start_construct(builder, DriverID::nil(), TaskID::from_random(), 0,
ActorID::nil(), ActorID::nil(), 0, false, func_id,
3);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size3;
@@ -95,8 +95,8 @@ TEST deterministic_ids_test(void) {
/* Construct a task with a different parent counter. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 1,
ActorID::nil(), ActorID::nil(), 0, false, func_id,
3);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size4;
@@ -104,8 +104,9 @@ TEST deterministic_ids_test(void) {
/* Construct a task with a different function ID. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ActorID::nil(), 0, false,
FunctionID::from_random(), 3);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, FunctionID::from_random(),
3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, arg2, 11);
int64_t size5;
@@ -113,8 +114,8 @@ TEST deterministic_ids_test(void) {
/* Construct a task with a different object ID argument. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ActorID::nil(), 0, false, func_id,
3);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
ObjectID object_id = ObjectID::from_random();
TaskSpec_args_add_ref(builder, &object_id, 1);
TaskSpec_args_add_val(builder, arg2, 11);
@@ -123,8 +124,8 @@ TEST deterministic_ids_test(void) {
/* Construct a task with a different value argument. */
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ActorID::nil(), 0, false, func_id,
3);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 3);
TaskSpec_args_add_ref(builder, &arg1, 1);
TaskSpec_args_add_val(builder, (uint8_t *) "hello_world", 11);
int64_t size7;
@@ -168,8 +169,8 @@ TEST send_task(void) {
TaskID parent_task_id = TaskID::from_random();
FunctionID func_id = FunctionID::from_random();
TaskSpec_start_construct(builder, DriverID::nil(), parent_task_id, 0,
ActorID::nil(), ActorID::nil(), 0, false, func_id,
2);
ActorID::nil(), ObjectID::nil(), ActorID::nil(),
ActorID::nil(), 0, false, func_id, 2);
ObjectID object_id = ObjectID::from_random();
TaskSpec_args_add_ref(builder, &object_id, 1);
TaskSpec_args_add_val(builder, (uint8_t *) "Hello", 5);
src/global_scheduler/global_scheduler_algorithm.cc
+8
View File
@@ -24,6 +24,14 @@ void GlobalSchedulerPolicyState_free(GlobalSchedulerPolicyState *policy_state) {
*/
bool constraints_satisfied_hard(const LocalScheduler *scheduler,
const TaskSpec *spec) {
if (scheduler->info.static_resources.count("CPU") == 1 &&
scheduler->info.static_resources.at("CPU") == 0) {
// Don't give tasks to local schedulers that have 0 CPUs. This can be an
// issue for actor creation tasks that require 0 CPUs (but the subsequent
// actor methods require some CPUs).
return false;
}
for (auto const &resource_pair : TaskSpec_get_required_resources(spec)) {
std::string resource_name = resource_pair.first;
double resource_quantity = resource_pair.second;
src/local_scheduler/format/local_scheduler.fbs
-9
View File
@@ -76,17 +76,8 @@ table RegisterClientRequest {
is_worker: bool;
// The ID of the worker or driver.
client_id: string;
// The ID of the actor. This is NIL_ACTOR_ID if the worker is not an actor.
actor_id: string;
// The process ID of this worker.
worker_pid: long;
// The number of GPUs required by this actor.
num_gpus: long;
}
table RegisterClientReply {
// The IDs of the GPUs that are reserved for this worker.
gpu_ids: [int];
}
table DisconnectClient {
src/local_scheduler/local_scheduler.cc
+63 -137
View File
@@ -226,19 +226,7 @@ void LocalSchedulerState_free(LocalSchedulerState *state) {
event_loop_destroy(loop);
}
/**
* Start a new worker as a child process.
*
* @param state The state of the local scheduler.
* @return Void.
*/
void start_worker(LocalSchedulerState *state,
ActorID actor_id,
bool reconstruct) {
/* Non-actors can't be started in reconstruct mode. */
if (actor_id.is_nil()) {
RAY_CHECK(!reconstruct);
}
void start_worker(LocalSchedulerState *state) {
/* We can't start a worker if we don't have the path to the worker script. */
if (state->config.start_worker_command == NULL) {
RAY_LOG(DEBUG) << "No valid command to start worker provided. Cannot start "
@@ -261,18 +249,6 @@ void start_worker(LocalSchedulerState *state,
command_vector.push_back(state->config.start_worker_command[i]);
}
/* Pass in the worker's actor ID. */
const char *actor_id_string = "--actor-id";
std::string id_string = actor_id.hex();
command_vector.push_back(actor_id_string);
command_vector.push_back(id_string.c_str());
/* Add a flag for reconstructing the actor if necessary. */
const char *reconstruct_string = "--reconstruct";
if (reconstruct) {
command_vector.push_back(reconstruct_string);
}
/* Add a NULL pointer to the end. */
command_vector.push_back(NULL);
@@ -419,7 +395,7 @@ LocalSchedulerState *LocalSchedulerState_init(
/* Start the initial set of workers. */
for (int i = 0; i < num_workers; ++i) {
start_worker(state, ActorID::nil(), false);
start_worker(state);
}
/* Initialize the time at which the previous heartbeat was sent. */
@@ -489,9 +465,6 @@ void acquire_resources(
RAY_CHECK(state->dynamic_resources[resource_name] >= resource_quantity);
}
state->dynamic_resources[resource_name] -= resource_quantity;
if (resource_name == std::string("CPU")) {
RAY_CHECK(worker->resources_in_use[resource_name] == 0);
}
worker->resources_in_use[resource_name] += resource_quantity;
}
@@ -520,9 +493,6 @@ void release_resources(
}
// Do bookkeeping for general resources types.
if (resource_name == std::string("CPU")) {
RAY_CHECK(resource_quantity == worker->resources_in_use[resource_name]);
}
state->dynamic_resources[resource_name] += resource_quantity;
worker->resources_in_use[resource_name] -= resource_quantity;
}
@@ -599,10 +569,44 @@ void assign_task_to_worker(LocalSchedulerState *state,
void finish_task(LocalSchedulerState *state, LocalSchedulerClient *worker) {
if (worker->task_in_progress != NULL) {
TaskSpec *spec = Task_task_execution_spec(worker->task_in_progress)->Spec();
/* Return dynamic resources back for the task in progress. */
RAY_CHECK(worker->resources_in_use["CPU"] ==
TaskSpec_get_required_resource(spec, "CPU"));
if (worker->actor_id.is_nil()) {
// Return dynamic resources back for the task in progress.
if (TaskSpec_is_actor_creation_task(spec)) {
// Resources required by the actor creation task are acquired for the
// actor's lifetime, so don't return anything here. TODO(rkn): Should the
// actor creation task require 1 CPU in addition to any resources acquired
// for the lifetime of the actor? If not, then the local scheduler may
// schedule an arbitrary number of actor creation tasks concurrently (if
// they don't acquire any resources for their entire lifetime). In
// practice this will usually be rate-limited by the rate at which we can
// create new workers.
ActorID actor_creation_id = TaskSpec_actor_creation_id(spec);
WorkerID driver_id = TaskSpec_driver_id(spec);
// The driver must be alive because if the driver had been removed, then
// this worker would have been killed (because it was executing a task for
// the driver).
RAY_CHECK(is_driver_alive(state, driver_id));
// Update the worker struct with this actor ID.
RAY_CHECK(worker->actor_id.is_nil());
worker->actor_id = actor_creation_id;
// Extract the initial execution dependency from the actor creation task.
RAY_CHECK(TaskSpec_num_returns(spec) == 1);
ObjectID initial_execution_dependency = TaskSpec_return(spec, 0);
// Let the scheduling algorithm process the presence of this new worker.
handle_convert_worker_to_actor(state, state->algorithm_state,
actor_creation_id,
initial_execution_dependency, worker);
// Publish the actor creation notification. The corresponding callback
// handle_actor_creation_callback will update state->actor_mapping.
publish_actor_creation_notification(
state->db, actor_creation_id, driver_id, get_db_client_id(state->db));
} else if (worker->actor_id.is_nil()) {
// Return dynamic resources back for the task in progress.
RAY_CHECK(worker->resources_in_use["CPU"] ==
TaskSpec_get_required_resource(spec, "CPU"));
// Return GPU resources.
RAY_CHECK(worker->gpus_in_use.size() ==
TaskSpec_get_required_resource(spec, "GPU"));
release_resources(state, worker, worker->resources_in_use);
@@ -610,9 +614,7 @@ void finish_task(LocalSchedulerState *state, LocalSchedulerClient *worker) {
// Actor tasks should only specify CPU requirements.
RAY_CHECK(0 == TaskSpec_get_required_resource(spec, "GPU"));
std::unordered_map<std::string, double> cpu_resources;
cpu_resources["CPU"] = worker->resources_in_use["CPU"];
std::unordered_map<std::string, double> resources_to_release =
worker->resources_in_use;
cpu_resources["CPU"] = TaskSpec_get_required_resource(spec, "CPU");
release_resources(state, worker, cpu_resources);
}
/* If we're connected to Redis, update tables. */
@@ -902,29 +904,6 @@ void reconstruct_object(LocalSchedulerState *state,
reconstruct_object_lookup_callback, (void *) state);
}
void send_client_register_reply(LocalSchedulerState *state,
LocalSchedulerClient *worker) {
flatbuffers::FlatBufferBuilder fbb;
auto message =
CreateRegisterClientReply(fbb, fbb.CreateVector(worker->gpus_in_use));
fbb.Finish(message);
/* Send the message to the client. */
if (write_message(worker->sock, MessageType_RegisterClientReply,
fbb.GetSize(), fbb.GetBufferPointer()) < 0) {
if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
/* Something went wrong, so kill the worker. */
kill_worker(state, worker, false, false);
RAY_LOG(WARNING) << "Failed to give send register client reply to worker "
<< "on fd " << worker->sock
<< ". The client may have hung up.";
} else {
RAY_LOG(FATAL) << "Failed to send register client reply to client on fd "
<< worker->sock;
}
}
}
void handle_client_register(LocalSchedulerState *state,
LocalSchedulerClient *worker,
const RegisterClientRequest *message) {
@@ -940,40 +919,6 @@ void handle_client_register(LocalSchedulerState *state,
/* Update the actor mapping with the actor ID of the worker (if an actor is
* running on the worker). */
worker->pid = message->worker_pid();
ActorID actor_id = from_flatbuf(*message->actor_id());
if (!actor_id.is_nil()) {
/* Make sure that the local scheduler is aware that it is responsible for
* this actor. */
RAY_CHECK(state->actor_mapping.count(actor_id) == 1);
RAY_CHECK(state->actor_mapping[actor_id].local_scheduler_id ==
get_db_client_id(state->db));
/* Update the worker struct with this actor ID. */
RAY_CHECK(worker->actor_id.is_nil());
worker->actor_id = actor_id;
/* Let the scheduling algorithm process the presence of this new
* worker. */
handle_actor_worker_connect(state, state->algorithm_state, actor_id,
worker);
/* If there are enough GPUs available, allocate them and reply to the
* actor. */
double num_gpus_required = (double) message->num_gpus();
std::unordered_map<std::string, double> gpu_resources;
gpu_resources["GPU"] = num_gpus_required;
if (check_dynamic_resources(state, gpu_resources)) {
acquire_resources(state, worker, gpu_resources);
} else {
/* TODO(rkn): This means that an actor wants to register but that there
* aren't enough GPUs for it. We should queue this request, and reply to
* the actor when GPUs become available. */
RAY_LOG(WARNING) << "Attempting to create an actor but there aren't "
<< "enough available GPUs. We'll start the worker "
<< "anyway without any GPUs, but this is incorrect "
<< "behavior.";
}
}
/* Register worker process id with the scheduler. */
/* Determine if this worker is one of our child processes. */
RAY_LOG(DEBUG) << "PID is " << worker->pid;
@@ -987,15 +932,6 @@ void handle_client_register(LocalSchedulerState *state,
state->child_pids.erase(it);
RAY_LOG(DEBUG) << "Found matching child pid " << worker->pid;
}
/* If the worker is an actor that corresponds to a driver that has been
* removed, then kill the worker. */
if (!actor_id.is_nil()) {
WorkerID driver_id = state->actor_mapping[actor_id].driver_id;
if (state->removed_drivers.count(driver_id) == 1) {
kill_worker(state, worker, false, false);
}
}
} else {
/* Register the driver. Currently we don't do anything here. */
}
@@ -1164,7 +1100,7 @@ void process_message(event_loop *loop,
/* If the disconnected worker was not an actor, start a new worker to make
* sure there are enough workers in the pool. */
if (worker->actor_id.is_nil()) {
start_worker(state, ActorID::nil(), false);
start_worker(state);
}
} break;
case MessageType_EventLogMessage: {
@@ -1180,7 +1116,6 @@ void process_message(event_loop *loop,
case MessageType_RegisterClientRequest: {
auto message = flatbuffers::GetRoot<RegisterClientRequest>(input);
handle_client_register(state, worker, message);
send_client_register_reply(state, worker);
} break;
case MessageType_GetTask: {
/* If this worker reports a completed task, account for resources. */
@@ -1360,14 +1295,12 @@ void handle_task_scheduled_callback(Task *original_task,
* @param actor_id The ID of the actor being created.
* @param local_scheduler_id The ID of the local scheduler that is responsible
* for creating the actor.
* @param reconstruct True if the actor should be started in "reconstruct" mode.
* @param context The context for this callback.
* @return Void.
*/
void handle_actor_creation_callback(ActorID actor_id,
WorkerID driver_id,
DBClientID local_scheduler_id,
bool reconstruct,
void handle_actor_creation_callback(const ActorID &actor_id,
const WorkerID &driver_id,
const DBClientID &local_scheduler_id,
void *context) {
LocalSchedulerState *state = (LocalSchedulerState *) context;
@@ -1376,26 +1309,19 @@ void handle_actor_creation_callback(ActorID actor_id,
return;
}
if (!reconstruct) {
/* Make sure the actor entry is not already present in the actor map table.
* TODO(rkn): We will need to remove this check to handle the case where the
* corresponding publish is retried and the case in which a task that
* creates an actor is resubmitted due to fault tolerance. */
RAY_CHECK(state->actor_mapping.count(actor_id) == 0);
} else {
/* In this case, the actor already exists. Check that the driver hasn't
* changed but that the local scheduler has. */
// TODO(rkn): If we do not have perfect task suppression and it is possible
// for a task to be executed simultaneously on two nodes, then we will need to
// detect and handle that case.
if (state->actor_mapping.count(actor_id) != 0) {
// This actor already exists.
auto it = state->actor_mapping.find(actor_id);
RAY_CHECK(it != state->actor_mapping.end());
RAY_CHECK(it->second.driver_id == driver_id);
RAY_CHECK(!(it->second.local_scheduler_id == local_scheduler_id));
/* If the actor was previously assigned to this local scheduler, kill the
* actor. */
if (it->second.local_scheduler_id == get_db_client_id(state->db)) {
/* TODO(rkn): We should kill the actor here if it is still around. Also,
* if it hasn't registered yet, we should keep track of its PID so we can
* kill it anyway. */
/* TODO(swang): Evict actor dummy objects as part of actor cleanup. */
// TODO(rkn): The actor was previously assigned to this local scheduler.
// We should kill the actor here if it is still around. Also, if it hasn't
// registered yet, we should keep track of its PID so we can kill it
// anyway.
// TODO(swang): Evict actor dummy objects as part of actor cleanup.
}
}
@@ -1407,15 +1333,9 @@ void handle_actor_creation_callback(ActorID actor_id,
entry.driver_id = driver_id;
state->actor_mapping[actor_id] = entry;
/* If this local scheduler is responsible for the actor, then start a new
* worker for the actor. */
if (local_scheduler_id == get_db_client_id(state->db)) {
start_worker(state, actor_id, reconstruct);
}
/* Let the scheduling algorithm process the fact that a new actor has been
* created. */
handle_actor_creation_notification(state, state->algorithm_state, actor_id,
reconstruct);
handle_actor_creation_notification(state, state->algorithm_state, actor_id);
}
int heartbeat_handler(event_loop *loop, timer_id id, void *context) {
@@ -1515,6 +1435,12 @@ void start_server(
loop, RayConfig::instance()
.local_scheduler_reconstruction_timeout_milliseconds(),
reconstruct_object_timeout_handler, g_state);
// Create a timer for rerunning actor creation tasks for actor tasks that are
// cached locally.
event_loop_add_timer(
loop, RayConfig::instance()
.local_scheduler_reconstruction_timeout_milliseconds(),
rerun_actor_creation_tasks_timeout_handler, g_state);
/* Run event loop. */
event_loop_run(loop);
}
src/local_scheduler/local_scheduler.h
+1 -7
View File
@@ -104,15 +104,9 @@ void kill_worker(LocalSchedulerState *state,
* scheduler.
*
* @param state The local scheduler state.
* @param actor_id The ID of the actor for this worker. If this worker is not an
* actor, then NIL_ACTOR_ID should be used.
* @param reconstruct True if the worker is an actor and is being started in
* reconstruct mode.
* @param Void.
*/
void start_worker(LocalSchedulerState *state,
ActorID actor_id,
bool reconstruct);
void start_worker(LocalSchedulerState *state);
/**
* Check if a certain quantity of dynamic resources are available. If num_cpus
src/local_scheduler/local_scheduler_algorithm.cc
+135 -41
View File
@@ -6,6 +6,7 @@
#include "state/task_table.h"
#include "state/actor_notification_table.h"
#include "state/db_client_table.h"
#include "state/local_scheduler_table.h"
#include "state/object_table.h"
#include "local_scheduler_shared.h"
@@ -205,6 +206,8 @@ void provide_scheduler_info(LocalSchedulerState *state,
*
* @param algorithm_state The state of the scheduling algorithm.
* @param actor_id The actor ID of the actor being created.
* @param initial_execution_dependency The dummy object ID of the actor
* creation task.
* @param worker The worker struct for the worker that is running this actor.
* If the worker struct has not been created yet (meaning that the worker
* that is running this actor has not registered with the local scheduler
@@ -213,14 +216,15 @@ void provide_scheduler_info(LocalSchedulerState *state,
* @return Void.
*/
void create_actor(SchedulingAlgorithmState *algorithm_state,
ActorID actor_id,
const ActorID &actor_id,
const ObjectID &initial_execution_dependency,
LocalSchedulerClient *worker) {
LocalActorInfo entry;
entry.task_counters[ActorHandleID::nil()] = 0;
entry.frontier_dependencies[ActorHandleID::nil()] = ObjectID::nil();
/* The actor has not yet executed any tasks, so there are no execution
* dependencies for the next task to be scheduled. */
entry.execution_dependency = ObjectID::nil();
entry.execution_dependency = initial_execution_dependency;
entry.task_queue = new std::list<TaskExecutionSpec>();
entry.worker = worker;
entry.worker_available = false;
@@ -315,11 +319,7 @@ bool dispatch_actor_task(LocalSchedulerState *state,
* deterministic reconstruction ordering for tasks whose updates are
* reflected in the task table. */
std::vector<ObjectID> ordered_execution_dependencies;
/* Only overwrite execution dependencies for tasks that have a
* submission-time dependency (meaning it is not the initial task). */
if (!entry.execution_dependency.is_nil()) {
ordered_execution_dependencies.push_back(entry.execution_dependency);
}
ordered_execution_dependencies.push_back(entry.execution_dependency);
task->SetExecutionDependencies(ordered_execution_dependencies);
/* Assign the first task in the task queue to the worker and mark the worker
@@ -342,19 +342,21 @@ bool dispatch_actor_task(LocalSchedulerState *state,
return true;
}
void handle_actor_worker_connect(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
ActorID actor_id,
LocalSchedulerClient *worker) {
void handle_convert_worker_to_actor(
LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
const ActorID &actor_id,
const ObjectID &initial_execution_dependency,
LocalSchedulerClient *worker) {
if (algorithm_state->local_actor_infos.count(actor_id) == 0) {
create_actor(algorithm_state, actor_id, worker);
create_actor(algorithm_state, actor_id, initial_execution_dependency,
worker);
} else {
/* In this case, the LocalActorInfo struct was already been created by the
* first call to add_task_to_actor_queue. However, the worker field was not
* filled out, so fill out the correct worker field now. */
algorithm_state->local_actor_infos[actor_id].worker = worker;
}
dispatch_actor_task(state, algorithm_state, actor_id);
}
/**
@@ -420,14 +422,6 @@ void insert_actor_task_queue(LocalSchedulerState *state,
return;
}
/* Handle the case in which there is no LocalActorInfo struct yet. */
if (algorithm_state->local_actor_infos.count(actor_id) == 0) {
/* Create the actor struct with a NULL worker because the worker struct has
* not been created yet. The correct worker struct will be inserted when the
* actor worker connects to the local scheduler. */
create_actor(algorithm_state, actor_id, NULL);
RAY_CHECK(algorithm_state->local_actor_infos.count(actor_id) == 1);
}
LocalActorInfo &entry =
algorithm_state->local_actor_infos.find(actor_id)->second;
if (entry.task_counters.count(task_handle_id) == 0) {
@@ -799,6 +793,40 @@ int reconstruct_object_timeout_handler(event_loop *loop,
.local_scheduler_reconstruction_timeout_milliseconds();
}
int rerun_actor_creation_tasks_timeout_handler(event_loop *loop,
timer_id id,
void *context) {
int64_t start_time = current_time_ms();
LocalSchedulerState *state = (LocalSchedulerState *) context;
// Create a set of the dummy object IDs for the actor creation tasks to
// reconstruct.
std::unordered_set<ObjectID, UniqueIDHasher> actor_dummy_objects;
for (auto const &execution_spec :
state->algorithm_state->cached_submitted_actor_tasks) {
ObjectID actor_creation_dummy_object_id =
TaskSpec_actor_creation_dummy_object_id(execution_spec.Spec());
actor_dummy_objects.insert(actor_creation_dummy_object_id);
}
// Issue reconstruct calls.
for (auto const &object_id : actor_dummy_objects) {
reconstruct_object(state, object_id);
}
// Print a warning if this method took too long.
int64_t end_time = current_time_ms();
if (end_time - start_time >
RayConfig::instance().max_time_for_handler_milliseconds()) {
RAY_LOG(WARNING) << "reconstruct_object_timeout_handler took "
<< end_time - start_time << " milliseconds.";
}
return RayConfig::instance()
.local_scheduler_reconstruction_timeout_milliseconds();
}
/**
* Return true if there are still some resources available and false otherwise.
*
@@ -855,7 +883,7 @@ void dispatch_tasks(LocalSchedulerState *state,
if (state->child_pids.size() == 0) {
/* If there are no workers, including those pending PID registration,
* then we must start a new one to replenish the worker pool. */
start_worker(state, ActorID::nil(), false);
start_worker(state);
}
return;
}
@@ -904,10 +932,9 @@ void dispatch_all_tasks(LocalSchedulerState *state,
/* Attempt to dispatch actor tasks. */
auto it = algorithm_state->actors_with_pending_tasks.begin();
while (it != algorithm_state->actors_with_pending_tasks.end()) {
/* Terminate early if there are no more resources available. */
if (!resources_available(state)) {
break;
}
// We cannot short-circuit and exit here if there are no resources
// available because actor methods may require 0 CPUs.
/* We increment the iterator ahead of time because the call to
* dispatch_actor_task may invalidate the current iterator. */
ActorID actor_id = *it;
@@ -1078,18 +1105,46 @@ void give_task_to_local_scheduler_retry(UniqueID id,
RAY_CHECK(TaskSpec_is_actor_task(spec));
ActorID actor_id = TaskSpec_actor_id(spec);
RAY_CHECK(state->actor_mapping.count(actor_id) == 1);
if (state->actor_mapping[actor_id].local_scheduler_id ==
get_db_client_id(state->db)) {
/* The task is now scheduled to us. Call the callback directly. */
handle_task_scheduled(state, state->algorithm_state, *execution_spec);
} else {
/* The task is scheduled to a remote local scheduler. Try to hand it to
* them again. */
if (state->actor_mapping.count(actor_id) == 0) {
// Process the actor task submission again. This will cache the task
// locally until a new actor creation notification is broadcast. We will
// attempt to reissue the actor creation tasks for all cached actor tasks
// in rerun_actor_creation_tasks_timeout_handler.
handle_actor_task_submitted(state, state->algorithm_state, *execution_spec);
return;
}
DBClientID remote_local_scheduler_id =
state->actor_mapping[actor_id].local_scheduler_id;
// TODO(rkn): db_client_table_cache_get is a blocking call, is this a
// performance issue?
DBClient remote_local_scheduler =
db_client_table_cache_get(state->db, remote_local_scheduler_id);
// Check if the local scheduler that we're assigning this task to is still
// alive.
if (remote_local_scheduler.is_alive) {
// The local scheduler is still alive, which means that perhaps it hasn't
// subscribed to the appropriate channel yet, so retrying should suffice.
// This should be rare.
give_task_to_local_scheduler(
state, state->algorithm_state, *execution_spec,
state->actor_mapping[actor_id].local_scheduler_id);
} else {
// The local scheduler is dead, so we will need to recreate the actor by
// invoking reconstruction.
RAY_LOG(INFO) << "Local scheduler " << remote_local_scheduler_id
<< " that was running actor " << actor_id << " died.";
RAY_CHECK(state->actor_mapping.count(actor_id) == 1);
// Update the actor mapping.
state->actor_mapping.erase(actor_id);
// Process the actor task submission again. This will cache the task
// locally until a new actor creation notification is broadcast. We will
// attempt to reissue the actor creation tasks for all cached actor tasks
// in rerun_actor_creation_tasks_timeout_handler.
handle_actor_task_submitted(state, state->algorithm_state, *execution_spec);
}
}
@@ -1188,6 +1243,12 @@ bool resource_constraints_satisfied(LocalSchedulerState *state,
return false;
}
}
if (TaskSpec_is_actor_creation_task(spec) &&
state->static_resources["CPU"] != 0) {
return false;
}
return true;
}
@@ -1199,10 +1260,10 @@ void handle_task_submitted(LocalSchedulerState *state,
* resource is currently unavailable, then consider queueing task locally and
* recheck dynamic next time. */
/* If this task's constraints are satisfied, dependencies are available
* locally, and there is an available worker, then enqueue the task in the
* dispatch queue and trigger task dispatch. Otherwise, pass the task along to
* the global scheduler if there is one. */
// If this task's constraints are satisfied, dependencies are available
// locally, and there is an available worker, then enqueue the task in the
// dispatch queue and trigger task dispatch. Otherwise, pass the task along to
// the global scheduler if there is one.
if (resource_constraints_satisfied(state, spec) &&
(algorithm_state->available_workers.size() > 0) &&
can_run(algorithm_state, execution_spec)) {
@@ -1224,6 +1285,11 @@ void handle_actor_task_submitted(LocalSchedulerState *state,
ActorID actor_id = TaskSpec_actor_id(task_spec);
if (state->actor_mapping.count(actor_id) == 0) {
// Create a copy of the task to write to the task table.
Task *task = Task_alloc(
task_spec, execution_spec.SpecSize(), TASK_STATUS_ACTOR_CACHED,
get_db_client_id(state->db), execution_spec.ExecutionDependencies());
/* Add this task to a queue of tasks that have been submitted but the local
* scheduler doesn't know which actor is responsible for them. These tasks
* will be resubmitted (internally by the local scheduler) whenever a new
@@ -1232,6 +1298,18 @@ void handle_actor_task_submitted(LocalSchedulerState *state,
TaskExecutionSpec task_entry = TaskExecutionSpec(&execution_spec);
algorithm_state->cached_submitted_actor_tasks.push_back(
std::move(task_entry));
#if !RAY_USE_NEW_GCS
// Even if the task can't be assigned to a worker yet, we should still write
// it to the task table. TODO(rkn): There's no need to do this more than
// once, and we could run into problems if we have very large numbers of
// tasks in this cache.
task_table_add_task(state->db, task, NULL, NULL, NULL);
#else
RAY_CHECK_OK(TaskTableAdd(&state->gcs_client, task));
Task_free(task);
#endif
return;
}
@@ -1255,8 +1333,7 @@ void handle_actor_task_submitted(LocalSchedulerState *state,
void handle_actor_creation_notification(
LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
ActorID actor_id,
bool reconstruct) {
ActorID actor_id) {
int num_cached_actor_tasks =
algorithm_state->cached_submitted_actor_tasks.size();
@@ -1281,7 +1358,12 @@ void handle_task_scheduled(LocalSchedulerState *state,
* the database. */
RAY_CHECK(state->db != NULL);
RAY_CHECK(state->config.global_scheduler_exists);
/* Push the task to the appropriate queue. */
// Currently, the global scheduler will never assign a task to a local
// scheduler that has 0 CPUs.
RAY_CHECK(state->static_resources["CPU"] != 0);
// Push the task to the appropriate queue.
queue_task_locally(state, algorithm_state, execution_spec, true);
dispatch_tasks(state, algorithm_state);
}
@@ -1652,6 +1734,18 @@ void handle_driver_removed(LocalSchedulerState *state,
}
}
// Remove this driver's tasks from the cached actor tasks. Note that this loop
// could be very slow if the vector of cached actor tasks is very long.
for (auto it = algorithm_state->cached_submitted_actor_tasks.begin();
it != algorithm_state->cached_submitted_actor_tasks.end();) {
TaskSpec *spec = (*it).Spec();
if (TaskSpec_driver_id(spec) == driver_id) {
it = algorithm_state->cached_submitted_actor_tasks.erase(it);
} else {
++it;
}
}
/* TODO(rkn): Should we clean up the actor data structures? */
}
src/local_scheduler/local_scheduler_algorithm.h
+23 -8
View File
@@ -76,14 +76,12 @@ void handle_actor_task_submitted(LocalSchedulerState *state,
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param actor_id The ID of the actor being created.
* @param reconstruct True if the actor is being created in "reconstruct" mode.
* @return Void.
*/
void handle_actor_creation_notification(
LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
ActorID actor_id,
bool reconstruct);
ActorID actor_id);
/**
* This function will be called when a task is assigned by the global scheduler
@@ -177,13 +175,17 @@ void handle_actor_worker_available(LocalSchedulerState *state,
* @param state The state of the local scheduler.
* @param algorithm_state State maintained by the scheduling algorithm.
* @param actor_id The ID of the actor running on the worker.
* @param worker The worker that was connected.
* @param initial_execution_dependency The dummy object ID of the actor
* creation task.
* @param worker The worker that was converted to an actor.
* @return Void.
*/
void handle_actor_worker_connect(LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
ActorID actor_id,
LocalSchedulerClient *worker);
void handle_convert_worker_to_actor(
LocalSchedulerState *state,
SchedulingAlgorithmState *algorithm_state,
const ActorID &actor_id,
const ObjectID &initial_execution_dependency,
LocalSchedulerClient *worker);
/**
* Handle the fact that a worker running an actor has disconnected.
@@ -292,6 +294,19 @@ int fetch_object_timeout_handler(event_loop *loop, timer_id id, void *context);
int reconstruct_object_timeout_handler(event_loop *loop,
timer_id id,
void *context);
/// This function initiates reconstruction for the actor creation tasks
/// corresponding to the actor tasks cached in the local scheduler.
///
/// \param loop The local scheduler's event loop.
/// \param id The ID of the timer that triggers this function.
/// \param context The function's context.
/// \return An integer representing the time interval in seconds before the
/// next invocation of the function.
int rerun_actor_creation_tasks_timeout_handler(event_loop *loop,
timer_id id,
void *context);
/**
* Check whether an object, including actor dummy objects, is locally
* available.
src/local_scheduler/local_scheduler_client.cc
+11 -37
View File
@@ -12,49 +12,22 @@
LocalSchedulerConnection *LocalSchedulerConnection_init(
const char *local_scheduler_socket,
UniqueID client_id,
ActorID actor_id,
bool is_worker,
int64_t num_gpus) {
bool is_worker) {
LocalSchedulerConnection *result = new LocalSchedulerConnection();
result->conn = connect_ipc_sock_retry(local_scheduler_socket, -1, -1);
result->actor_id = actor_id;
/* Register with the local scheduler.
* NOTE(swang): If the local scheduler exits and we are registered as a
* worker, we will get killed. */
flatbuffers::FlatBufferBuilder fbb;
auto message = CreateRegisterClientRequest(
fbb, is_worker, to_flatbuf(fbb, client_id),
to_flatbuf(fbb, result->actor_id), getpid(), num_gpus);
fbb, is_worker, to_flatbuf(fbb, client_id), getpid());
fbb.Finish(message);
/* Register the process ID with the local scheduler. */
int success = write_message(result->conn, MessageType_RegisterClientRequest,
fbb.GetSize(), fbb.GetBufferPointer());
RAY_CHECK(success == 0) << "Unable to register worker with local scheduler";
/* Wait for a confirmation from the local scheduler. */
int64_t type;
int64_t reply_size;
uint8_t *reply;
read_message(result->conn, &type, &reply_size, &reply);
if (type == DISCONNECT_CLIENT) {
RAY_LOG(DEBUG) << "Exiting because local scheduler closed connection.";
exit(1);
}
RAY_CHECK(type == MessageType_RegisterClientReply);
/* Parse the reply object. */
auto reply_message = flatbuffers::GetRoot<RegisterClientReply>(reply);
for (size_t i = 0; i < reply_message->gpu_ids()->size(); ++i) {
result->gpu_ids.push_back(reply_message->gpu_ids()->Get(i));
}
/* If the worker is not an actor, there should not be any GPU IDs here. */
if (ActorID_equal(result->actor_id, ActorID::nil())) {
RAY_CHECK(reply_message->gpu_ids()->size() == 0);
}
free(reply);
return result;
}
@@ -119,20 +92,21 @@ TaskSpec *local_scheduler_get_task(LocalSchedulerConnection *conn,
/* Parse the flatbuffer object. */
auto reply_message = flatbuffers::GetRoot<GetTaskReply>(reply);
/* Set the GPU IDs for this task. We only do this for non-actor tasks because
* for actors the GPUs are associated with the actor itself and not with the
* actor methods. */
if (ActorID_equal(conn->actor_id, ActorID::nil())) {
/* Create a copy of the task spec so we can free the reply. */
*task_size = reply_message->task_spec()->size();
TaskSpec *data = (TaskSpec *) reply_message->task_spec()->data();
TaskSpec *spec = TaskSpec_copy(data, *task_size);
// Set the GPU IDs for this task. We only do this for non-actor tasks because
// for actors the GPUs are associated with the actor itself and not with the
// actor methods. Note that this also processes GPUs for actor creation tasks.
if (!TaskSpec_is_actor_task(spec)) {
conn->gpu_ids.clear();
for (size_t i = 0; i < reply_message->gpu_ids()->size(); ++i) {
conn->gpu_ids.push_back(reply_message->gpu_ids()->Get(i));
}
}
/* Create a copy of the task spec so we can free the reply. */
*task_size = reply_message->task_spec()->size();
TaskSpec *data = (TaskSpec *) reply_message->task_spec()->data();
TaskSpec *spec = TaskSpec_copy(data, *task_size);
/* Free the original message from the local scheduler. */
free(reply);
/* Return the copy of the task spec and pass ownership to the caller. */
src/local_scheduler/local_scheduler_client.h
+1 -10
View File
@@ -8,9 +8,6 @@ struct LocalSchedulerConnection {
/** File descriptor of the Unix domain socket that connects to local
* scheduler. */
int conn;
/** The actor ID of this client. If this client is not an actor, then this
* should be NIL_ACTOR_ID. */
ActorID actor_id;
/** The IDs of the GPUs that this client can use. */
std::vector<int> gpu_ids;
};
@@ -20,20 +17,14 @@ struct LocalSchedulerConnection {
*
* @param local_scheduler_socket The name of the socket to use to connect to the
* local scheduler.
* @param actor_id The ID of the actor running on this worker. If no actor is
* running on this actor, this should be NIL_ACTOR_ID.
* @param is_worker Whether this client is a worker. If it is a worker, an
* additional message will be sent to register as one.
* @param num_gpus The number of GPUs required by this worker. This is only
* used if the worker is an actor.
* @return The connection information.
*/
LocalSchedulerConnection *LocalSchedulerConnection_init(
const char *local_scheduler_socket,
UniqueID worker_id,
ActorID actor_id,
bool is_worker,
int64_t num_gpus);
bool is_worker);
/**
* Disconnect from the local scheduler.
src/local_scheduler/local_scheduler_extension.cc
+3 -7
View File
@@ -19,19 +19,15 @@ static int PyLocalSchedulerClient_init(PyLocalSchedulerClient *self,
PyObject *kwds) {
char *socket_name;
UniqueID client_id;
ActorID actor_id;
PyObject *is_worker;
int num_gpus;
if (!PyArg_ParseTuple(args, "sO&O&Oi", &socket_name, PyStringToUniqueID,
&client_id, PyStringToUniqueID, &actor_id, &is_worker,
&num_gpus)) {
if (!PyArg_ParseTuple(args, "sO&O", &socket_name, PyStringToUniqueID,
&client_id, &is_worker)) {
self->local_scheduler_connection = NULL;
return -1;
}
/* Connect to the local scheduler. */
self->local_scheduler_connection = LocalSchedulerConnection_init(
socket_name, client_id, actor_id, (bool) PyObject_IsTrue(is_worker),
num_gpus);
socket_name, client_id, (bool) PyObject_IsTrue(is_worker));
return 0;
}
src/local_scheduler/test/local_scheduler_tests.cc
+3 -4
View File
@@ -124,9 +124,8 @@ LocalSchedulerMock *LocalSchedulerMock_init(int num_workers,
std::thread(register_clients, num_mock_workers, mock);
for (int i = 0; i < num_mock_workers; ++i) {
mock->conns[i] =
LocalSchedulerConnection_init(local_scheduler_socket_name.c_str(),
WorkerID::nil(), ActorID::nil(), true, 0);
mock->conns[i] = LocalSchedulerConnection_init(
local_scheduler_socket_name.c_str(), WorkerID::nil(), true);
}
background_thread.join();
@@ -666,7 +665,7 @@ TEST start_kill_workers_test(void) {
static_cast<size_t>(num_workers - 1));
/* Start a worker after the local scheduler has been initialized. */
start_worker(local_scheduler->local_scheduler_state, ActorID::nil(), false);
start_worker(local_scheduler->local_scheduler_state);
/* Accept the workers as clients to the plasma manager. */
int new_worker_fd = accept_client(local_scheduler->plasma_manager_fd);
/* The new worker should register its process ID. */
src/ray/raylet/task_spec.cc
+2 -1
View File
@@ -104,7 +104,8 @@ TaskSpecification::TaskSpecification(
// Serialize the TaskSpecification.
auto spec = CreateTaskInfo(
fbb, to_flatbuf(fbb, driver_id), to_flatbuf(fbb, task_id),
to_flatbuf(fbb, parent_task_id), parent_counter, to_flatbuf(fbb, WorkerID::nil()),
to_flatbuf(fbb, parent_task_id), parent_counter, to_flatbuf(fbb, ActorID::nil()),
to_flatbuf(fbb, ActorID::nil()), to_flatbuf(fbb, WorkerID::nil()),
to_flatbuf(fbb, ActorHandleID::nil()), 0, false, to_flatbuf(fbb, function_id),
fbb.CreateVector(arguments), fbb.CreateVector(returns),
map_to_flatbuf(fbb, required_resources));