wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-05 13:21:03 +08:00
Code Issues Packages Projects Releases Wiki Activity

Push an error to the driver when the workload hangs on ray.put reconstruction (#382)

Browse Source 
* Fix worker blocked bug

* tmp

* Push an error to the driver on ray.put for non-driver tasks

* Fix result table tests

* Fix test, logging

* Address comments

* Fix suppression bug

* Fix redis module test

* Edit error message

* Get values in chunks during reconstruction

* Test case for driver ray.put errors

* Error for evicting ray.put objects from the driver

* Fix tests

* Reduce verbosity

* Documentation
This commit is contained in:
Stephanie Wang
2017-03-21 00:16:48 -07:00
committed by Robert Nishihara
parent 4618fd45b1
commit 083e7a28ad
21 changed files with 528 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/format/common.fbs
+12
View File
@@ -90,6 +90,9 @@ table TaskReply {
local_scheduler_id: string;
// A string of bytes representing the task specification.
task_spec: string;
// A boolean representing whether the update was successful. This field
// should only be used for test-and-set operations.
updated: bool;
}
root_type TaskReply;
@@ -127,3 +130,12 @@ table LocalSchedulerInfoMessage {
}
root_type LocalSchedulerInfoMessage;
table ResultTableReply {
// The task ID of the task that created the object.
task_id: string;
// Whether the task created the object through a ray.put.
is_put: bool;
}
root_type ResultTableReply;
src/common/io.cc
+9 -2
View File
@@ -116,6 +116,10 @@ int connect_ipc_sock_retry(const char *socket_pathname,
if (fd >= 0) {
break;
}
if (num_attempts == 0) {
LOG_ERROR("Connection to socket failed for pathname %s.",
socket_pathname);
}
/* Sleep for timeout milliseconds. */
usleep(timeout * 1000);
}
@@ -147,7 +151,7 @@ int connect_ipc_sock(const char *socket_pathname) {
if (connect(socket_fd, (struct sockaddr *) &socket_address,
sizeof(socket_address)) != 0) {
LOG_ERROR("Connection to socket failed for pathname %s.", socket_pathname);
close(socket_fd);
return -1;
}
@@ -173,6 +177,10 @@ int connect_inet_sock_retry(const char *ip_addr,
if (fd >= 0) {
break;
}
if (num_attempts == 0) {
LOG_ERROR("Connection to socket failed for address %s:%d.", ip_addr,
port);
}
/* Sleep for timeout milliseconds. */
usleep(timeout * 1000);
}
@@ -203,7 +211,6 @@ int connect_inet_sock(const char *ip_addr, int port) {
addr.sin_port = htons(port);
if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) != 0) {
LOG_ERROR("Connection to socket failed for address %s:%d.", ip_addr, port);
close(fd);
return -1;
}
src/common/lib/python/common_extension.cc
-11
View File
@@ -543,14 +543,3 @@ PyObject *check_simple_value(PyObject *self, PyObject *args) {
}
Py_RETURN_FALSE;
}
PyObject *compute_put_id(PyObject *self, PyObject *args) {
int put_index;
TaskID task_id;
if (!PyArg_ParseTuple(args, "O&i", &PyObjectToUniqueID, &task_id,
&put_index)) {
return NULL;
}
ObjectID put_id = task_compute_put_id(task_id, put_index);
return PyObjectID_make(put_id);
}
src/common/redis_module/ray_redis_module.cc
+63 -28
View File
@@ -690,27 +690,40 @@ int ObjectInfoSubscribe_RedisCommand(RedisModuleCtx *ctx,
*
* This is called from a client with the command:
*
* RAY.RESULT_TABLE_ADD <object id> <task id>
* RAY.RESULT_TABLE_ADD <object id> <task id> <is_put>
*
* @param object_id A string representing the object ID.
* @param task_id A string representing the task ID of the task that produced
* the object.
* @param is_put An integer that is 1 if the object was created through ray.put
* and 0 if created by return value.
* @return OK if the operation was successful.
*/
int ResultTableAdd_RedisCommand(RedisModuleCtx *ctx,
RedisModuleString **argv,
int argc) {
if (argc != 3) {
if (argc != 4) {
return RedisModule_WrongArity(ctx);
}
/* Set the task ID under field "task" in the object info table. */
RedisModuleString *object_id = argv[1];
RedisModuleString *task_id = argv[2];
RedisModuleString *is_put = argv[3];
/* Check to make sure the is_put field was a 0 or a 1. */
long long is_put_integer;
if ((RedisModule_StringToLongLong(is_put, &is_put_integer) !=
REDISMODULE_OK) ||
(is_put_integer != 0 && is_put_integer != 1)) {
return RedisModule_ReplyWithError(
ctx, "The is_put field must be either a 0 or a 1.");
}
RedisModuleKey *key;
key = OpenPrefixedKey(ctx, OBJECT_INFO_PREFIX, object_id, REDISMODULE_WRITE);
RedisModule_HashSet(key, REDISMODULE_HASH_CFIELDS, "task", task_id, NULL);
RedisModule_HashSet(key, REDISMODULE_HASH_CFIELDS, "task", task_id, "is_put",
is_put, NULL);
/* Clean up. */
RedisModule_CloseKey(key);
@@ -723,13 +736,19 @@ int ResultTableAdd_RedisCommand(RedisModuleCtx *ctx,
* Reply with information about a task ID. This is used by
* RAY.RESULT_TABLE_LOOKUP and RAY.TASK_TABLE_GET.
*
* @param ctx The Redis context.
* @param task_id The task ID of the task to reply about.
* @param updated A boolean representing whether the task was updated during
* this operation. This field is only used for
* RAY.TASK_TABLE_TEST_AND_UPDATE operations.
* @return NIL if the task ID is not in the task table. An error if the task ID
* is in the task table but the appropriate fields are not there, and
* an array of the task scheduling state, the local scheduler ID, and
* the task spec for the task otherwise.
*/
int ReplyWithTask(RedisModuleCtx *ctx, RedisModuleString *task_id) {
int ReplyWithTask(RedisModuleCtx *ctx,
RedisModuleString *task_id,
bool updated) {
RedisModuleKey *key =
OpenPrefixedKey(ctx, TASK_PREFIX, task_id, REDISMODULE_READ);
@@ -762,7 +781,7 @@ int ReplyWithTask(RedisModuleCtx *ctx, RedisModuleString *task_id) {
auto message =
CreateTaskReply(fbb, RedisStringToFlatbuf(fbb, task_id), state_integer,
RedisStringToFlatbuf(fbb, local_scheduler_id),
RedisStringToFlatbuf(fbb, task_spec));
RedisStringToFlatbuf(fbb, task_spec), updated);
fbb.Finish(message);
RedisModuleString *reply = RedisModule_CreateString(
@@ -790,10 +809,8 @@ int ReplyWithTask(RedisModuleCtx *ctx, RedisModuleString *task_id) {
* RAY.RESULT_TABLE_LOOKUP <object id>
*
* @param object_id A string representing the object ID.
* @return NIL if the object ID is not in the result table or if the
* corresponding task ID is not in the task table. Otherwise, this
* returns an array of the scheduling state, the local scheduler ID, and
* the task spec for the task corresponding to this object ID.
* @return NIL if the object ID is not in the result table. Otherwise, this
* returns a ResultTableReply flatbuffer.
*/
int ResultTableLookup_RedisCommand(RedisModuleCtx *ctx,
RedisModuleString **argv,
@@ -814,13 +831,36 @@ int ResultTableLookup_RedisCommand(RedisModuleCtx *ctx,
}
RedisModuleString *task_id;
RedisModule_HashGet(key, REDISMODULE_HASH_CFIELDS, "task", &task_id, NULL);
RedisModuleString *is_put;
RedisModule_HashGet(key, REDISMODULE_HASH_CFIELDS, "task", &task_id, "is_put",
&is_put, NULL);
RedisModule_CloseKey(key);
if (task_id == NULL) {
if (task_id == NULL || is_put == NULL) {
return RedisModule_ReplyWithNull(ctx);
}
RedisModule_ReplyWithString(ctx, task_id);
/* Check to make sure the is_put field was a 0 or a 1. */
long long is_put_integer;
if (RedisModule_StringToLongLong(is_put, &is_put_integer) != REDISMODULE_OK ||
(is_put_integer != 0 && is_put_integer != 1)) {
RedisModule_FreeString(ctx, is_put);
RedisModule_FreeString(ctx, task_id);
return RedisModule_ReplyWithError(
ctx, "The is_put field must be either a 0 or a 1.");
}
/* Make and return the flatbuffer reply. */
flatbuffers::FlatBufferBuilder fbb;
auto message = CreateResultTableReply(fbb, RedisStringToFlatbuf(fbb, task_id),
bool(is_put_integer));
fbb.Finish(message);
RedisModuleString *reply = RedisModule_CreateString(
ctx, (const char *) fbb.GetBufferPointer(), fbb.GetSize());
RedisModule_ReplyWithString(ctx, reply);
/* Clean up. */
RedisModule_FreeString(ctx, reply);
RedisModule_FreeString(ctx, is_put);
RedisModule_FreeString(ctx, task_id);
return REDISMODULE_OK;
@@ -971,12 +1011,8 @@ int TaskTableUpdate_RedisCommand(RedisModuleCtx *ctx,
* instance) to update the task entry with.
* @param ray_client_id A string that is the ray client ID of the associated
* local scheduler, if any, to update the task entry with.
* @return If the current scheduling state does not match the test bitmask,
* returns nil. Else, returns the same as RAY.TASK_TABLE_GET: an array
* of strings representing the updated task fields in the following
* order: 1) (integer) scheduling state 2) (string) associated local
* scheduler ID, if any 3) (string) the task specification, which can be
* cast to a task_spec.
* @return Returns the task entry as a TaskReply. The reply will reflect the
* update, if it happened.
*/
int TaskTableTestAndUpdate_RedisCommand(RedisModuleCtx *ctx,
RedisModuleString **argv,
@@ -1015,20 +1051,19 @@ int TaskTableTestAndUpdate_RedisCommand(RedisModuleCtx *ctx,
return RedisModule_ReplyWithError(
ctx, "Invalid test value for scheduling state");
}
if ((current_state_integer & test_state_bitmask) == 0) {
/* The current value does not match the test bitmask, so do not perform the
* update. */
RedisModule_CloseKey(key);
return RedisModule_ReplyWithNull(ctx);
bool updated = false;
if (current_state_integer & test_state_bitmask) {
/* The test passed, so perform the update. */
RedisModule_HashSet(key, REDISMODULE_HASH_CFIELDS, "state", state,
"local_scheduler_id", argv[4], NULL);
updated = true;
}
/* The test passed, so perform the update. */
RedisModule_HashSet(key, REDISMODULE_HASH_CFIELDS, "state", state,
"local_scheduler_id", argv[4], NULL);
/* Clean up. */
RedisModule_CloseKey(key);
/* Construct a reply by getting the task from the task ID. */
return ReplyWithTask(ctx, argv[1]);
return ReplyWithTask(ctx, argv[1], updated);
}
/**
@@ -1052,7 +1087,7 @@ int TaskTableGet_RedisCommand(RedisModuleCtx *ctx,
}
/* Construct a reply by getting the task from the task ID. */
return ReplyWithTask(ctx, argv[1]);
return ReplyWithTask(ctx, argv[1], false);
}
extern "C" {
src/common/state/error_table.h
+7 -2
View File
@@ -18,14 +18,19 @@ typedef enum {
/** An object was added with a different hash from the existing
* one. */
OBJECT_HASH_MISMATCH_ERROR_INDEX = 0,
/** An object that was created through a ray.put is lost. */
PUT_RECONSTRUCTION_ERROR_INDEX,
/** The total number of error types. */
MAX_ERROR_INDEX
} error_index;
/** Information about the error to be displayed to the user. */
static const char *error_types[] = {"object_hash_mismatch"};
static const char *error_types[] = {"object_hash_mismatch",
"put_reconstruction"};
static const char *error_messages[] = {
"A nondeterministic task was reexecuted."};
"A nondeterministic task was reexecuted.",
"An object created by ray.put was evicted and could not be reconstructed. "
"The driver may need to be restarted."};
/**
* Push an error to the given Python driver.
src/common/state/object_table.cc
+7 -4
View File
@@ -104,13 +104,16 @@ void object_info_subscribe(DBHandle *db_handle,
void result_table_add(DBHandle *db_handle,
ObjectID object_id,
TaskID task_id_arg,
TaskID task_id,
bool is_put,
RetryInfo *retry,
result_table_done_callback done_callback,
void *user_context) {
TaskID *task_id_copy = (TaskID *) malloc(sizeof(TaskID));
memcpy(task_id_copy, task_id_arg.id, sizeof(*task_id_copy));
init_table_callback(db_handle, object_id, __func__, task_id_copy, retry,
ResultTableAddInfo *info =
(ResultTableAddInfo *) malloc(sizeof(ResultTableAddInfo));
info->task_id = task_id;
info->is_put = is_put;
init_table_callback(db_handle, object_id, __func__, info, retry,
(table_done_callback) done_callback,
redis_result_table_add, user_context);
}
src/common/state/object_table.h
+13
View File
@@ -224,6 +224,15 @@ typedef struct {
typedef void (*result_table_done_callback)(ObjectID object_id,
void *user_context);
/** Information about a result table entry to add. */
typedef struct {
/** The task ID of the task that created the requested object. */
TaskID task_id;
/** True if the object was created through a put, and false if created by
* return value. */
bool is_put;
} ResultTableAddInfo;
/**
* Add information about a new object to the object table. This
* is immutable information like the ID of the task that
@@ -232,6 +241,8 @@ typedef void (*result_table_done_callback)(ObjectID object_id,
* @param db_handle Handle to object_table database.
* @param object_id ID of the object to add.
* @param task_id ID of the task that creates this object.
* @param is_put A boolean that is true if the object was created through a
* ray.put, and false if the object was created by return value.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Context passed by the caller.
@@ -240,6 +251,7 @@ typedef void (*result_table_done_callback)(ObjectID object_id,
void result_table_add(DBHandle *db_handle,
ObjectID object_id,
TaskID task_id,
bool is_put,
RetryInfo *retry,
result_table_done_callback done_callback,
void *user_context);
@@ -247,6 +259,7 @@ void result_table_add(DBHandle *db_handle,
/** Callback called when the result table lookup completes. */
typedef void (*result_table_lookup_callback)(ObjectID object_id,
TaskID task_id,
bool is_put,
void *user_context);
/**
src/common/state/redis.cc
+18 -9
View File
@@ -342,12 +342,14 @@ void redis_result_table_add(TableCallbackData *callback_data) {
CHECK(callback_data);
DBHandle *db = callback_data->db_handle;
ObjectID id = callback_data->id;
TaskID *result_task_id = (TaskID *) callback_data->data;
ResultTableAddInfo *info = (ResultTableAddInfo *) callback_data->data;
int is_put = info->is_put ? 1 : 0;
/* Add the result entry to the result table. */
int status = redisAsyncCommand(
db->context, redis_result_table_add_callback,
(void *) callback_data->timer_id, "RAY.RESULT_TABLE_ADD %b %b", id.id,
sizeof(id.id), result_task_id->id, sizeof(result_task_id->id));
(void *) callback_data->timer_id, "RAY.RESULT_TABLE_ADD %b %b %d", id.id,
sizeof(id.id), info->task_id.id, sizeof(info->task_id.id), is_put);
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "Error in result table add");
}
@@ -386,16 +388,19 @@ void redis_result_table_lookup_callback(redisAsyncContext *c,
reply->type);
/* Parse the task from the reply. */
TaskID result_id = NIL_TASK_ID;
bool is_put = false;
if (reply->type == REDIS_REPLY_STRING) {
CHECK(reply->len == sizeof(result_id));
memcpy(&result_id, reply->str, reply->len);
auto message = flatbuffers::GetRoot<ResultTableReply>(reply->str);
result_id = from_flatbuf(message->task_id());
is_put = message->is_put();
}
/* Call the done callback if there is one. */
result_table_lookup_callback done_callback =
(result_table_lookup_callback) callback_data->done_callback;
if (done_callback != NULL) {
done_callback(callback_data->id, result_id, callback_data->user_context);
done_callback(callback_data->id, result_id, is_put,
callback_data->user_context);
}
/* Clean up timer and callback. */
destroy_timer_callback(db->loop, callback_data);
@@ -761,11 +766,15 @@ void redis_task_table_test_and_update_callback(redisAsyncContext *c,
redisReply *reply = (redisReply *) r;
/* Parse the task from the reply. */
Task *task = parse_and_construct_task_from_redis_reply(reply);
/* Determine whether the update happened. */
auto message = flatbuffers::GetRoot<TaskReply>(reply->str);
bool updated = message->updated();
/* Call the done callback if there is one. */
task_table_get_callback done_callback =
(task_table_get_callback) callback_data->done_callback;
task_table_test_and_update_callback done_callback =
(task_table_test_and_update_callback) callback_data->done_callback;
if (done_callback != NULL) {
done_callback(task, callback_data->user_context);
done_callback(task, callback_data->user_context, updated);
}
/* Free the task if it is not NULL. */
if (task != NULL) {
src/common/state/task_table.cc
+8 -7
View File
@@ -33,13 +33,14 @@ void task_table_update(DBHandle *db_handle,
redis_task_table_update, user_context);
}
void task_table_test_and_update(DBHandle *db_handle,
TaskID task_id,
int test_state_bitmask,
int update_state,
RetryInfo *retry,
task_table_get_callback done_callback,
void *user_context) {
void task_table_test_and_update(
DBHandle *db_handle,
TaskID task_id,
int test_state_bitmask,
int update_state,
RetryInfo *retry,
task_table_test_and_update_callback done_callback,
void *user_context) {
TaskTableTestAndUpdateData *update_data =
(TaskTableTestAndUpdateData *) malloc(sizeof(TaskTableTestAndUpdateData));
update_data->test_state_bitmask = test_state_bitmask;
src/common/state/task_table.h
+15 -7
View File
@@ -28,6 +28,13 @@ typedef void (*task_table_done_callback)(TaskID task_id, void *user_context);
* was not in the task table, then the task pointer will be NULL. */
typedef void (*task_table_get_callback)(Task *task, void *user_context);
/* Callback called when a task table test-and-update operation completes. If
* the task ID was not in the task table, then the task pointer will be NULL.
* If the update succeeded, the updated field will be set to true. */
typedef void (*task_table_test_and_update_callback)(Task *task,
void *user_context,
bool updated);
/**
* Get a task's entry from the task table.
*
@@ -107,13 +114,14 @@ void task_table_update(DBHandle *db_handle,
* fail_callback.
* @return Void.
*/
void task_table_test_and_update(DBHandle *db_handle,
TaskID task_id,
int test_state_bitmask,
int update_state,
RetryInfo *retry,
task_table_get_callback done_callback,
void *user_context);
void task_table_test_and_update(
DBHandle *db_handle,
TaskID task_id,
int test_state_bitmask,
int update_state,
RetryInfo *retry,
task_table_test_and_update_callback done_callback,
void *user_context);
/* Data that is needed to test and set the task's scheduling state. */
typedef struct {
src/common/test/object_table_tests.cc
+3 -1
View File
@@ -34,6 +34,7 @@ void new_object_fail_callback(UniqueID id,
void new_object_done_callback(ObjectID object_id,
TaskID task_id,
bool is_put,
void *user_context) {
new_object_succeeded = 1;
CHECK(ObjectID_equal(object_id, new_object_id));
@@ -60,7 +61,7 @@ void new_object_task_callback(TaskID task_id, void *user_context) {
.fail_callback = new_object_fail_callback,
};
DBHandle *db = (DBHandle *) user_context;
result_table_add(db, new_object_id, new_object_task_id, &retry,
result_table_add(db, new_object_id, new_object_task_id, false, &retry,
new_object_lookup_callback, (void *) db);
}
@@ -95,6 +96,7 @@ TEST new_object_test(void) {
void new_object_no_task_callback(ObjectID object_id,
TaskID task_id,
bool is_put,
void *user_context) {
new_object_succeeded = 1;
CHECK(IS_NIL_ID(task_id));