wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-11 05:51:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Second Part of Internal API Refactor (#1326)

Browse Source
This commit is contained in:
Philipp Moritz
2017-12-26 16:22:04 -08:00
committed by Robert Nishihara
parent 4bb5b6bd5b
commit 3d224c4edf
58 changed files with 537 additions and 677 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/state/actor_notification_table.cc
+1 -1
View File
@@ -12,7 +12,7 @@ void actor_notification_table_subscribe(
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, NIL_ID, __func__,
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(sub_data), retry, NULL,
redis_actor_notification_table_subscribe, NULL);
}
src/common/state/db_client_table.cc
+4 -3
View File
@@ -24,7 +24,7 @@ void db_client_table_subscribe(
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, NIL_ID, __func__,
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(sub_data), retry,
(table_done_callback) done_callback,
redis_db_client_table_subscribe, user_context);
@@ -71,7 +71,7 @@ void db_client_table_cache_init(DBHandle *db_handle) {
}
DBClient db_client_table_cache_get(DBHandle *db_handle, DBClientID client_id) {
CHECK(!DBClientID_is_nil(client_id));
CHECK(!client_id.is_nil());
return redis_cache_get_db_client(db_handle, client_id);
}
@@ -82,7 +82,8 @@ void plasma_manager_send_heartbeat(DBHandle *db_handle) {
RayConfig::instance().heartbeat_timeout_milliseconds();
heartbeat_retry.fail_callback = NULL;
init_table_callback(db_handle, NIL_ID, __func__, new CommonCallbackData(NULL),
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(NULL),
(RetryInfo *) &heartbeat_retry, NULL,
redis_plasma_manager_send_heartbeat, NULL);
}
src/common/state/driver_table.cc
+1 -1
View File
@@ -9,7 +9,7 @@ void driver_table_subscribe(DBHandle *db_handle,
(DriverTableSubscribeData *) malloc(sizeof(DriverTableSubscribeData));
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, NIL_ID, __func__,
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(sub_data), retry, NULL,
redis_driver_table_subscribe, NULL);
}
src/common/state/error_table.cc
+7 -6
View File
@@ -13,7 +13,7 @@ void push_error(DBHandle *db_handle,
DBClientID driver_id,
int error_index,
size_t data_length,
unsigned char *data) {
const unsigned char *data) {
CHECK(error_index >= 0 && error_index < MAX_ERROR_INDEX);
/* Allocate a struct to hold the error information. */
ErrorInfo *info = (ErrorInfo *) malloc(sizeof(ErrorInfo) + data_length);
@@ -22,10 +22,11 @@ void push_error(DBHandle *db_handle,
info->data_length = data_length;
memcpy(info->data, data, data_length);
/* Generate a random key to identify this error message. */
CHECK(sizeof(info->error_key) >= UNIQUE_ID_SIZE);
UniqueID error_key = globally_unique_id();
memcpy(info->error_key, error_key.id, sizeof(info->error_key));
CHECK(sizeof(info->error_key) >= sizeof(UniqueID));
UniqueID error_key = UniqueID::from_random();
memcpy(info->error_key, error_key.data(), sizeof(info->error_key));
init_table_callback(db_handle, NIL_ID, __func__, new CommonCallbackData(info),
NULL, NULL, redis_push_error, NULL);
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(info), NULL, NULL,
redis_push_error, NULL);
}
src/common/state/error_table.h
+1 -1
View File
@@ -48,6 +48,6 @@ void push_error(DBHandle *db_handle,
DBClientID driver_id,
int error_index,
size_t data_length,
unsigned char *data);
const unsigned char *data);
#endif
src/common/state/local_scheduler_table.cc
+4 -3
View File
@@ -13,7 +13,7 @@ void local_scheduler_table_subscribe(
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, NIL_ID, __func__,
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(sub_data), retry, NULL,
redis_local_scheduler_table_subscribe, NULL);
}
@@ -37,8 +37,9 @@ void local_scheduler_table_send_info(DBHandle *db_handle,
data->size = fbb.GetSize();
memcpy(&data->flatbuffer_data[0], fbb.GetBufferPointer(), fbb.GetSize());
init_table_callback(db_handle, NIL_ID, __func__, new CommonCallbackData(data),
retry, NULL, redis_local_scheduler_table_send_info, NULL);
init_table_callback(db_handle, UniqueID::nil(), __func__,
new CommonCallbackData(data), retry, NULL,
redis_local_scheduler_table_send_info, NULL);
}
void local_scheduler_table_disconnect(DBHandle *db_handle) {
src/common/state/object_table.cc
+3 -3
View File
@@ -67,7 +67,7 @@ void object_table_subscribe_to_notifications(
sub_data->subscribe_all = subscribe_all;
init_table_callback(
db_handle, NIL_OBJECT_ID, __func__, new CommonCallbackData(sub_data),
db_handle, ObjectID::nil(), __func__, new CommonCallbackData(sub_data),
retry, (table_done_callback) done_callback,
redis_object_table_subscribe_to_notifications, user_context);
}
@@ -85,7 +85,7 @@ void object_table_request_notifications(DBHandle *db_handle,
data->num_object_ids = num_object_ids;
memcpy(data->object_ids, object_ids, num_object_ids * sizeof(ObjectID));
init_table_callback(db_handle, NIL_OBJECT_ID, __func__,
init_table_callback(db_handle, ObjectID::nil(), __func__,
new CommonCallbackData(data), retry, NULL,
redis_object_table_request_notifications, NULL);
}
@@ -101,7 +101,7 @@ void object_info_subscribe(DBHandle *db_handle,
sub_data->subscribe_callback = subscribe_callback;
sub_data->subscribe_context = subscribe_context;
init_table_callback(db_handle, NIL_OBJECT_ID, __func__,
init_table_callback(db_handle, ObjectID::nil(), __func__,
new CommonCallbackData(sub_data), retry,
(table_done_callback) done_callback,
redis_object_info_subscribe, user_context);
src/common/state/redis.cc
+59 -57
View File
@@ -207,8 +207,8 @@ void db_connect_shard(const std::string &db_address,
argv[0] = "RAY.CONNECT";
argvlen[0] = strlen(argv[0]);
/* Set the client ID argument. */
argv[1] = (char *) client.id;
argvlen[1] = sizeof(client.id);
argv[1] = (char *) client.data();
argvlen[1] = sizeof(client);
/* Set the node IP address argument. */
argv[2] = node_ip_address;
argvlen[2] = strlen(node_ip_address);
@@ -265,7 +265,7 @@ DBHandle *db_connect(const std::string &db_primary_address,
}
/* Create a client ID for this client. */
DBClientID client = globally_unique_id();
DBClientID client = DBClientID::from_random();
DBHandle *db = new DBHandle();
@@ -325,7 +325,7 @@ void db_disconnect(DBHandle *db) {
* reconnect and get assigned a different client ID. */
redisReply *reply =
(redisReply *) redisCommand(db->sync_context, "RAY.DISCONNECT %b",
db->client.id, sizeof(db->client.id));
db->client.data(), sizeof(db->client));
CHECK(reply->type != REDIS_REPLY_ERROR);
CHECKM(strcmp(reply->str, "OK") == 0, "reply->str is %s", reply->str);
freeReplyObject(reply);
@@ -408,8 +408,8 @@ void redis_object_table_add(TableCallbackData *callback_data) {
int status = redisAsyncCommand(
context, redis_object_table_add_callback,
(void *) callback_data->timer_id, "RAY.OBJECT_TABLE_ADD %b %lld %b %b",
obj_id.id, sizeof(obj_id.id), (long long) object_size, digest,
(size_t) DIGEST_SIZE, db->client.id, sizeof(db->client.id));
obj_id.data(), sizeof(obj_id), (long long) object_size, digest,
(size_t) DIGEST_SIZE, db->client.data(), sizeof(db->client));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_object_table_add");
@@ -456,7 +456,7 @@ void redis_object_table_remove(TableCallbackData *callback_data) {
int status = redisAsyncCommand(
context, redis_object_table_remove_callback,
(void *) callback_data->timer_id, "RAY.OBJECT_TABLE_REMOVE %b %b",
obj_id.id, sizeof(obj_id.id), client_id->id, sizeof(client_id->id));
obj_id.data(), sizeof(obj_id), client_id->data(), sizeof(*client_id));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_object_table_remove");
@@ -473,8 +473,8 @@ void redis_object_table_lookup(TableCallbackData *callback_data) {
int status = redisAsyncCommand(context, redis_object_table_lookup_callback,
(void *) callback_data->timer_id,
"RAY.OBJECT_TABLE_LOOKUP %b", obj_id.id,
sizeof(obj_id.id));
"RAY.OBJECT_TABLE_LOOKUP %b", obj_id.data(),
sizeof(obj_id));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in object_table lookup");
}
@@ -508,10 +508,11 @@ void redis_result_table_add(TableCallbackData *callback_data) {
redisAsyncContext *context = get_redis_context(db, id);
/* Add the result entry to the result table. */
int status = redisAsyncCommand(
context, redis_result_table_add_callback,
(void *) callback_data->timer_id, "RAY.RESULT_TABLE_ADD %b %b %d", id.id,
sizeof(id.id), info->task_id.id, sizeof(info->task_id.id), is_put);
int status =
redisAsyncCommand(context, redis_result_table_add_callback,
(void *) callback_data->timer_id,
"RAY.RESULT_TABLE_ADD %b %b %d", id.data(), sizeof(id),
info->task_id.data(), sizeof(info->task_id), is_put);
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "Error in result table add");
}
@@ -554,7 +555,7 @@ void redis_result_table_lookup_callback(redisAsyncContext *c,
"Unexpected reply type %d in redis_result_table_lookup_callback",
reply->type);
/* Parse the task from the reply. */
TaskID result_id = NIL_TASK_ID;
TaskID result_id = TaskID::nil();
bool is_put = false;
if (reply->type == REDIS_REPLY_STRING) {
auto message = flatbuffers::GetRoot<ResultTableReply>(reply->str);
@@ -581,14 +582,14 @@ void redis_result_table_lookup(TableCallbackData *callback_data) {
int status =
redisAsyncCommand(context, redis_result_table_lookup_callback,
(void *) callback_data->timer_id,
"RAY.RESULT_TABLE_LOOKUP %b", id.id, sizeof(id.id));
"RAY.RESULT_TABLE_LOOKUP %b", id.data(), sizeof(id));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "Error in result table lookup");
}
}
DBClient redis_db_client_table_get(DBHandle *db,
unsigned char *client_id,
const unsigned char *client_id,
size_t client_id_len) {
redisReply *reply =
(redisReply *) redisCommand(db->sync_context, "HGETALL %s%b",
@@ -602,7 +603,7 @@ DBClient redis_db_client_table_get(DBHandle *db,
const char *key = reply->element[j]->str;
const char *value = reply->element[j + 1]->str;
if (strcmp(key, "ray_client_id") == 0) {
memcpy(db_client.id.id, value, sizeof(db_client.id));
memcpy(db_client.id.mutable_data(), value, sizeof(db_client.id));
num_fields++;
} else if (strcmp(key, "client_type") == 0) {
db_client.client_type = std::string(value);
@@ -637,8 +638,8 @@ void redis_cache_set_db_client(DBHandle *db, DBClient client) {
DBClient redis_cache_get_db_client(DBHandle *db, DBClientID db_client_id) {
auto it = db->db_client_cache.find(db_client_id);
if (it == db->db_client_cache.end()) {
DBClient db_client =
redis_db_client_table_get(db, db_client_id.id, sizeof(db_client_id.id));
DBClient db_client = redis_db_client_table_get(db, db_client_id.data(),
sizeof(db_client_id));
db->db_client_cache[db_client_id] = db_client;
it = db->db_client_cache.find(db_client_id);
}
@@ -672,7 +673,8 @@ void redis_object_table_lookup_callback(redisAsyncContext *c,
for (size_t j = 0; j < reply->elements; ++j) {
CHECK(reply->element[j]->type == REDIS_REPLY_STRING);
DBClientID manager_id;
memcpy(manager_id.id, reply->element[j]->str, sizeof(manager_id.id));
memcpy(manager_id.mutable_data(), reply->element[j]->str,
sizeof(manager_id));
manager_ids.push_back(manager_id);
}
@@ -743,7 +745,7 @@ void object_table_redis_subscribe_to_notifications_callback(
if (callback_data->done_callback != NULL) {
object_table_lookup_done_callback done_callback =
(object_table_lookup_done_callback) callback_data->done_callback;
done_callback(NIL_ID, false, std::vector<DBClientID>(),
done_callback(ray::UniqueID::nil(), false, std::vector<DBClientID>(),
callback_data->user_context);
}
/* If the initial SUBSCRIBE was successful, clean up the timer, but don't
@@ -783,7 +785,7 @@ void redis_object_table_subscribe_to_notifications(
db->subscribe_contexts[i],
object_table_redis_subscribe_to_notifications_callback,
(void *) callback_data->timer_id, "SUBSCRIBE %s%b",
object_channel_prefix, db->client.id, sizeof(db->client.id));
object_channel_prefix, db->client.data(), sizeof(db->client));
}
if ((status == REDIS_ERR) || db->subscribe_contexts[i]->err) {
@@ -827,11 +829,11 @@ void redis_object_table_request_notifications(
argv[0] = "RAY.OBJECT_TABLE_REQUEST_NOTIFICATIONS";
argvlen[0] = strlen(argv[0]);
/* Set the client ID argument. */
argv[1] = (char *) db->client.id;
argvlen[1] = sizeof(db->client.id);
argv[1] = (char *) db->client.data();
argvlen[1] = sizeof(db->client);
/* Set the object ID arguments. */
argv[2] = (char *) object_ids[i].id;
argvlen[2] = sizeof(object_ids[i].id);
argv[2] = (char *) object_ids[i].data();
argvlen[2] = sizeof(object_ids[i]);
int status = redisAsyncCommandArgv(
context, redis_object_table_request_notifications_callback,
@@ -881,8 +883,8 @@ void redis_task_table_get_task(TableCallbackData *callback_data) {
int status = redisAsyncCommand(context, redis_task_table_get_task_callback,
(void *) callback_data->timer_id,
"RAY.TASK_TABLE_GET %b", task_id.id,
sizeof(task_id.id));
"RAY.TASK_TABLE_GET %b", task_id.data(),
sizeof(task_id));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_task_table_get_task");
}
@@ -942,8 +944,8 @@ void redis_task_table_add_task(TableCallbackData *callback_data) {
int status = redisAsyncCommand(
context, redis_task_table_add_task_callback,
(void *) callback_data->timer_id, "RAY.TASK_TABLE_ADD %b %d %b %b %b",
task_id.id, sizeof(task_id.id), state, local_scheduler_id.id,
sizeof(local_scheduler_id.id), fbb.GetBufferPointer(),
task_id.data(), sizeof(task_id), state, local_scheduler_id.data(),
sizeof(local_scheduler_id), fbb.GetBufferPointer(),
(size_t) fbb.GetSize(), spec, execution_spec->SpecSize());
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_task_table_add_task");
@@ -1004,8 +1006,8 @@ void redis_task_table_update(TableCallbackData *callback_data) {
int status = redisAsyncCommand(
context, redis_task_table_update_callback,
(void *) callback_data->timer_id, "RAY.TASK_TABLE_UPDATE %b %d %b %b",
task_id.id, sizeof(task_id.id), state, local_scheduler_id.id,
sizeof(local_scheduler_id.id), fbb.GetBufferPointer(),
task_id.data(), sizeof(task_id), state, local_scheduler_id.data(),
sizeof(local_scheduler_id), fbb.GetBufferPointer(),
(size_t) fbb.GetSize());
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_task_table_update");
@@ -1055,24 +1057,24 @@ void redis_task_table_test_and_update(TableCallbackData *callback_data) {
int status;
/* If the test local scheduler ID is NIL, then ignore it. */
if (IS_NIL_ID(update_data->test_local_scheduler_id)) {
if (update_data->test_local_scheduler_id.is_nil()) {
status = redisAsyncCommand(
context, redis_task_table_test_and_update_callback,
(void *) callback_data->timer_id,
"RAY.TASK_TABLE_TEST_AND_UPDATE %b %d %d %b", task_id.id,
sizeof(task_id.id), update_data->test_state_bitmask,
update_data->update_state, update_data->local_scheduler_id.id,
sizeof(update_data->local_scheduler_id.id));
"RAY.TASK_TABLE_TEST_AND_UPDATE %b %d %d %b", task_id.data(),
sizeof(task_id), update_data->test_state_bitmask,
update_data->update_state, update_data->local_scheduler_id.data(),
sizeof(update_data->local_scheduler_id));
} else {
status = redisAsyncCommand(
context, redis_task_table_test_and_update_callback,
(void *) callback_data->timer_id,
"RAY.TASK_TABLE_TEST_AND_UPDATE %b %d %d %b %b", task_id.id,
sizeof(task_id.id), update_data->test_state_bitmask,
update_data->update_state, update_data->local_scheduler_id.id,
sizeof(update_data->local_scheduler_id.id),
update_data->test_local_scheduler_id.id,
sizeof(update_data->test_local_scheduler_id.id));
"RAY.TASK_TABLE_TEST_AND_UPDATE %b %d %d %b %b", task_id.data(),
sizeof(task_id), update_data->test_state_bitmask,
update_data->update_state, update_data->local_scheduler_id.data(),
sizeof(update_data->local_scheduler_id),
update_data->test_local_scheduler_id.data(),
sizeof(update_data->test_local_scheduler_id));
}
if ((status == REDIS_ERR) || context->err) {
@@ -1152,7 +1154,7 @@ void redis_task_table_subscribe(TableCallbackData *callback_data) {
const char *TASK_CHANNEL_PREFIX = "TT:";
for (auto subscribe_context : db->subscribe_contexts) {
int status;
if (IS_NIL_ID(data->local_scheduler_id)) {
if (data->local_scheduler_id.is_nil()) {
/* TODO(swang): Implement the state_filter by translating the bitmask into
* a Redis key-matching pattern. */
status = redisAsyncCommand(
@@ -1164,8 +1166,8 @@ void redis_task_table_subscribe(TableCallbackData *callback_data) {
status = redisAsyncCommand(
subscribe_context, redis_task_table_subscribe_callback,
(void *) callback_data->timer_id, "SUBSCRIBE %s%b:%d",
TASK_CHANNEL_PREFIX, (char *) local_scheduler_id.id,
sizeof(local_scheduler_id.id), data->state_filter);
TASK_CHANNEL_PREFIX, (char *) local_scheduler_id.data(),
sizeof(local_scheduler_id), data->state_filter);
}
if ((status == REDIS_ERR) || subscribe_context->err) {
LOG_REDIS_DEBUG(subscribe_context, "error in redis_task_table_subscribe");
@@ -1201,7 +1203,7 @@ void redis_db_client_table_remove(TableCallbackData *callback_data) {
int status =
redisAsyncCommand(db->context, redis_db_client_table_remove_callback,
(void *) callback_data->timer_id, "RAY.DISCONNECT %b",
callback_data->id.id, sizeof(callback_data->id.id));
callback_data->id.data(), sizeof(callback_data->id));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error in db_client_table_remove");
}
@@ -1512,7 +1514,7 @@ void redis_plasma_manager_send_heartbeat(TableCallbackData *callback_data) {
* memory for callback data each time. */
int status = redisAsyncCommand(
db->context, NULL, (void *) callback_data->timer_id,
"PUBLISH plasma_managers %b", db->client.id, sizeof(db->client.id));
"PUBLISH plasma_managers %b", db->client.data(), sizeof(db->client));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context,
"error in redis_plasma_manager_send_heartbeat");
@@ -1598,7 +1600,7 @@ void redis_actor_notification_table_subscribe(
void redis_actor_table_mark_removed(DBHandle *db, ActorID actor_id) {
int status =
redisAsyncCommand(db->context, NULL, NULL, "HSET Actor:%b removed \"1\"",
actor_id.id, sizeof(actor_id.id));
actor_id.data(), sizeof(actor_id));
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->context, "error in redis_actor_table_mark_removed");
}
@@ -1633,7 +1635,7 @@ void redis_object_info_subscribe_callback(redisAsyncContext *c,
ObjectInfoSubscribeData *data =
(ObjectInfoSubscribeData *) callback_data->data->Get();
ObjectID object_id;
memcpy(object_id.id, payload->str, sizeof(object_id.id));
memcpy(object_id.mutable_data(), payload->str, sizeof(object_id));
/* payload->str should have the format: "ObjectID:object_size_int" */
LOG_DEBUG("obj:info channel received message <%s>", payload->str);
if (data->subscribe_callback) {
@@ -1676,11 +1678,11 @@ void redis_push_error_hmset_callback(redisAsyncContext *c,
/* Add the error to this driver's list of errors. */
ErrorInfo *info = (ErrorInfo *) callback_data->data->Get();
int status = redisAsyncCommand(db->context, redis_push_error_rpush_callback,
(void *) callback_data->timer_id,
"RPUSH ErrorKeys Error:%b:%b",
info->driver_id.id, sizeof(info->driver_id.id),
info->error_key, sizeof(info->error_key));
int status = redisAsyncCommand(
db->context, redis_push_error_rpush_callback,
(void *) callback_data->timer_id, "RPUSH ErrorKeys Error:%b:%b",
info->driver_id.data(), sizeof(info->driver_id), info->error_key,
sizeof(info->error_key));
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->subscribe_context, "error in redis_push_error rpush");
}
@@ -1698,8 +1700,8 @@ void redis_push_error(TableCallbackData *callback_data) {
int status = redisAsyncCommand(
db->context, redis_push_error_hmset_callback,
(void *) callback_data->timer_id,
"HMSET Error:%b:%b type %s message %s data %b", info->driver_id.id,
sizeof(info->driver_id.id), info->error_key, sizeof(info->error_key),
"HMSET Error:%b:%b type %s message %s data %b", info->driver_id.data(),
sizeof(info->driver_id), info->error_key, sizeof(info->error_key),
error_type, error_message, info->data, info->data_length);
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->subscribe_context, "error in redis_push_error hmset");