wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-30 12:30:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

Shard Redis. (#539)

Browse Source 
* Implement sharding in the Ray core

* Single node Python modifications to do sharding

* Do the sharding in redis.cc

* Pipe num_redis_shards through start_ray.py and worker.py.

* Use multiple redis shards in multinode tests.

* first steps for sharding ray.global_state

* Fix problem in multinode docker test.

* fix runtest.py

* fix some tests

* fix redis shard startup

* fix redis sharding

* fix

* fix bug introduced by the map-iterator being consumed

* fix sharding bug

* shard event table

* update number of Redis clients to be 64K

* Fix object table tests by flushing shards in between unit tests

* Fix local scheduler tests

* Documentation

* Register shard locations in the primary shard

* Add plasma unit tests back to build

* lint

* lint and fix build

* Fix

* Address Robert's comments

* Refactor start_ray_processes to start Redis shard

* lint

* Fix global scheduler python tests

* Fix redis module test

* Fix plasma test

* Fix component failure test

* Fix local scheduler test

* Fix runtest.py

* Fix global scheduler test for python3

* Fix task_table_test_and_update bug, from actor task table submission race

* Fix jenkins tests.

* Retry Redis shard connections

* Fix test cases

* Convert database clients to DBClient struct

* Fix race condition when subscribing to db client table

* Remove unused lines, add APITest for sharded Ray

* Fix

* Fix memory leak

* Suppress ReconstructionTests output

* Suppress output for APITestSharded

* Reissue task table add/update commands if initial command does not publish to any subscribers.

* fix

* Fix linting.

* fix tests

* fix linting

* fix python test

* fix linting
This commit is contained in:
Stephanie Wang
2017-05-18 17:40:41 -07:00
committed by Philipp Moritz
parent 0a4304725f
commit ee08c8274b
39 changed files with 1336 additions and 651 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/common_protocol.h
+2
View File
@@ -5,6 +5,8 @@
#include "common.h"
#define DB_CLIENT_PREFIX "CL:"
/**
* Convert an object ID to a flatbuffer string.
*
src/common/lib/python/common_extension.cc
+10
View File
@@ -172,6 +172,14 @@ static PyObject *PyObjectID_richcompare(PyObjectID *self,
return result;
}
static PyObject *PyObjectID_redis_shard_hash(PyObjectID *self) {
/* NOTE: The hash function used here must match the one in get_redis_context
* in src/common/state/redis.cc. Changes to the hash function should only be
* made through UniqueIDHasher in src/common/common.h */
UniqueIDHasher hash;
return PyLong_FromSize_t(hash(self->object_id));
}
static long PyObjectID_hash(PyObjectID *self) {
PyObject *tuple = PyTuple_New(UNIQUE_ID_SIZE);
for (int i = 0; i < UNIQUE_ID_SIZE; ++i) {
@@ -201,6 +209,8 @@ static PyObject *PyObjectID___reduce__(PyObjectID *self) {
static PyMethodDef PyObjectID_methods[] = {
{"id", (PyCFunction) PyObjectID_id, METH_NOARGS,
"Return the hash associated with this ObjectID"},
{"redis_shard_hash", (PyCFunction) PyObjectID_redis_shard_hash, METH_NOARGS,
"Return the redis shard that this ObjectID is associated with"},
{"hex", (PyCFunction) PyObjectID_hex, METH_NOARGS,
"Return the object ID as a string in hex."},
{"__reduce__", (PyCFunction) PyObjectID___reduce__, METH_NOARGS,
src/common/logging.cc
+8 -5
View File
@@ -67,11 +67,14 @@ void RayLogger_log(RayLogger *logger,
if (logger->is_direct) {
DBHandle *db = (DBHandle *) logger->conn;
/* Fill in the client ID and send the message to Redis. */
int status = redisAsyncCommand(
db->context, NULL, NULL, utstring_body(formatted_message),
(char *) db->client.id, sizeof(db->client.id));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error while logging message to log table");
redisAsyncContext *context = get_redis_context(db, db->client);
int status =
redisAsyncCommand(context, NULL, NULL, utstring_body(formatted_message),
(char *) db->client.id, sizeof(db->client.id));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error while logging message to log table");
}
} else {
/* If we don't own a Redis connection, we leave our client
src/common/net.cc
+11
View File
@@ -1,5 +1,9 @@
#include "net.h"
#include <arpa/inet.h>
#include <sstream>
#include "common.h"
int parse_ip_addr_port(const char *ip_addr_port, char *ip_addr, int *port) {
@@ -11,3 +15,10 @@ int parse_ip_addr_port(const char *ip_addr_port, char *ip_addr, int *port) {
*port = atoi(port_str);
return 0;
}
/* Return true if the ip address is valid. */
bool valid_ip_address(const std::string &ip_address) {
struct sockaddr_in sa;
int result = inet_pton(AF_INET, ip_address.c_str(), &sa.sin_addr);
return result == 1;
}
src/common/redis_module/ray_redis_module.cc
+16 -1
View File
@@ -27,7 +27,6 @@
* TODO(pcm): Fill this out.
*/
#define DB_CLIENT_PREFIX "CL:"
#define OBJECT_INFO_PREFIX "OI:"
#define OBJECT_LOCATION_PREFIX "OL:"
#define OBJECT_NOTIFICATION_PREFIX "ON:"
@@ -929,6 +928,10 @@ int TaskTableWrite(RedisModuleCtx *ctx,
RedisModuleCallReply *reply =
RedisModule_Call(ctx, "PUBLISH", "ss", publish_topic, publish_message);
/* See how many clients received this publish. */
long long num_clients = RedisModule_CallReplyInteger(reply);
CHECKM(num_clients <= 1, "Published to %lld clients.", num_clients);
RedisModule_FreeString(ctx, publish_message);
RedisModule_FreeString(ctx, publish_topic);
if (existing_task_spec != NULL) {
@@ -938,6 +941,18 @@ int TaskTableWrite(RedisModuleCtx *ctx,
if (reply == NULL) {
return RedisModule_ReplyWithError(ctx, "PUBLISH unsuccessful");
}
if (num_clients == 0) {
LOG_WARN(
"No subscribers received this publish. This most likely means that "
"either the intended recipient has not subscribed yet or that the "
"pubsub connection to the intended recipient has been broken.");
/* This reply will be received by redis_task_table_update_callback or
* redis_task_table_add_task_callback in redis.cc, which will then reissue
* the command. */
return RedisModule_ReplyWithError(ctx,
"No subscribers received message.");
}
}
RedisModule_ReplyWithSimpleString(ctx, "OK");
src/common/state/db.h
+5 -2
View File
@@ -11,6 +11,9 @@ typedef struct DBHandle DBHandle;
*
* @param db_address The hostname to use to connect to the database.
* @param db_port The port to use to connect to the database.
* @param db_shards_addresses The list of database shard IP addresses.
* @param db_shards_ports The list of database shard ports, in the same order
* as db_shards_addresses.
* @param client_type The type of this client.
* @param node_ip_address The hostname of the client that is connecting.
* @param num_args The number of extra arguments that should be supplied. This
@@ -21,8 +24,8 @@ typedef struct DBHandle DBHandle;
* @return This returns a handle to the database, which must be freed with
* db_disconnect after use.
*/
DBHandle *db_connect(const char *db_address,
int db_port,
DBHandle *db_connect(const std::string &db_primary_address,
int db_primary_port,
const char *client_type,
const char *node_ip_address,
int num_args,
src/common/state/db_client_table.h
+15 -4
View File
@@ -29,11 +29,22 @@ void db_client_table_remove(DBHandle *db_handle,
* ==== Subscribing to the db client table ====
*/
/* An entry in the db client table. */
typedef struct {
/** The database client ID. */
DBClientID id;
/** The database client type. */
const char *client_type;
/** An optional auxiliary address for an associated database client on the
* same node. */
const char *aux_address;
/** Whether or not the database client exists. If this is false for an entry,
* then it will never again be true. */
bool is_insertion;
} DBClient;
/* Callback for subscribing to the db client table. */
typedef void (*db_client_table_subscribe_callback)(DBClientID db_client_id,
const char *client_type,
const char *aux_address,
bool is_insertion,
typedef void (*db_client_table_subscribe_callback)(DBClient *db_client,
void *user_context);
/**
src/common/state/redis.cc
+475 -162
View File
@@ -4,6 +4,7 @@
#include <stdbool.h>
#include <stdlib.h>
#include <unistd.h>
#include <vector>
extern "C" {
/* Including hiredis here is necessary on Windows for typedefs used in ae.h. */
@@ -26,6 +27,7 @@ extern "C" {
#include "event_loop.h"
#include "redis.h"
#include "io.h"
#include "net.h"
#include "format/common_generated.h"
@@ -77,52 +79,128 @@ extern int usleep(useconds_t usec);
do { \
} while (0)
DBHandle *db_connect(const char *db_address,
int db_port,
const char *client_type,
const char *node_ip_address,
int num_args,
const char **args) {
/* Check that the number of args is even. These args will be passed to the
* RAY.CONNECT Redis command, which takes arguments in pairs. */
if (num_args % 2 != 0) {
LOG_FATAL("The number of extra args must be divisible by two.");
}
redisAsyncContext *get_redis_context(DBHandle *db, UniqueID id) {
/* NOTE: The hash function used here must match the one in
* PyObjectID_redis_shard_hash in src/common/lib/python/common_extension.cc.
* Changes to the hash function should only be made through
* UniqueIDHasher in src/common/common.h */
UniqueIDHasher index;
return db->contexts[index(id) % db->contexts.size()];
}
DBHandle *db = (DBHandle *) malloc(sizeof(DBHandle));
/* Sync connection for initial handshake */
redisAsyncContext *get_redis_subscribe_context(DBHandle *db, UniqueID id) {
UniqueIDHasher index;
return db->subscribe_contexts[index(id) % db->subscribe_contexts.size()];
}
void get_redis_shards(redisContext *context,
std::vector<std::string> &db_shards_addresses,
std::vector<int> &db_shards_ports) {
/* Get the total number of Redis shards in the system. */
int num_attempts = 0;
redisReply *reply = NULL;
while (num_attempts < REDIS_DB_CONNECT_RETRIES) {
/* Try to read the number of Redis shards from the primary shard. If the
* entry is present, exit. */
reply = (redisReply *) redisCommand(context, "GET NumRedisShards");
if (reply->type != REDIS_REPLY_NIL) {
break;
}
/* Sleep for a little, and try again if the entry isn't there yet. */
freeReplyObject(reply);
usleep(REDIS_DB_CONNECT_WAIT_MS * 1000);
num_attempts++;
continue;
}
CHECKM(num_attempts < REDIS_DB_CONNECT_RETRIES,
"No entry found for NumRedisShards");
CHECKM(reply->type == REDIS_REPLY_STRING,
"Expected string, found Redis type %d for NumRedisShards",
reply->type);
int num_redis_shards = atoi(reply->str);
CHECKM(num_redis_shards >= 1, "Expected at least one Redis shard, found %d.",
num_redis_shards);
freeReplyObject(reply);
/* Get the addresses of all of the Redis shards. */
num_attempts = 0;
while (num_attempts < REDIS_DB_CONNECT_RETRIES) {
/* Try to read the Redis shard locations from the primary shard. If we find
* that all of them are present, exit. */
reply = (redisReply *) redisCommand(context, "LRANGE RedisShards 0 -1");
if (reply->elements == num_redis_shards) {
break;
}
/* Sleep for a little, and try again if not all Redis shard addresses have
* been added yet. */
freeReplyObject(reply);
usleep(REDIS_DB_CONNECT_WAIT_MS * 1000);
num_attempts++;
continue;
}
CHECKM(num_attempts < REDIS_DB_CONNECT_RETRIES,
"Expected %d Redis shard addresses, found %d", num_redis_shards,
(int) reply->elements);
/* Parse the Redis shard addresses. */
char db_shard_address[16];
int db_shard_port;
for (int i = 0; i < reply->elements; ++i) {
/* Parse the shard addresses and ports. */
CHECK(reply->element[i]->type == REDIS_REPLY_STRING);
CHECK(parse_ip_addr_port(reply->element[i]->str, db_shard_address,
&db_shard_port) == 0);
db_shards_addresses.push_back(std::string(db_shard_address));
db_shards_ports.push_back(db_shard_port);
}
freeReplyObject(reply);
}
void db_connect_shard(const std::string &db_address,
int db_port,
DBClientID client,
const char *client_type,
const char *node_ip_address,
int num_args,
const char **args,
DBHandle *db,
redisAsyncContext **context_out,
redisAsyncContext **subscribe_context_out,
redisContext **sync_context_out) {
/* Synchronous connection for initial handshake */
redisReply *reply;
int connection_attempts = 0;
redisContext *context = redisConnect(db_address, db_port);
while (context == NULL || context->err) {
redisContext *sync_context = redisConnect(db_address.c_str(), db_port);
while (sync_context == NULL || sync_context->err) {
if (connection_attempts >= REDIS_DB_CONNECT_RETRIES) {
break;
}
LOG_WARN("Failed to connect to Redis, retrying.");
/* Sleep for a little. */
usleep(REDIS_DB_CONNECT_WAIT_MS * 1000);
context = redisConnect(db_address, db_port);
sync_context = redisConnect(db_address.c_str(), db_port);
connection_attempts += 1;
}
CHECK_REDIS_CONNECT(redisContext, context,
CHECK_REDIS_CONNECT(redisContext, sync_context,
"could not establish synchronous connection to redis "
"%s:%d",
db_address, db_port);
db_address.c_str(), db_port);
/* Configure Redis to generate keyspace notifications for list events. This
* should only need to be done once (by whoever started Redis), but since
* Redis may be started in multiple places (e.g., for testing or when starting
* processes by hand), it is easier to do it multiple times. */
reply = (redisReply *) redisCommand(context,
reply = (redisReply *) redisCommand(sync_context,
"CONFIG SET notify-keyspace-events Kl");
CHECKM(reply != NULL, "db_connect failed on CONFIG SET");
freeReplyObject(reply);
/* Also configure Redis to not run in protected mode, so clients on other
* hosts can connect to it. */
reply = (redisReply *) redisCommand(context, "CONFIG SET protected-mode no");
reply =
(redisReply *) redisCommand(sync_context, "CONFIG SET protected-mode no");
CHECKM(reply != NULL, "db_connect failed on CONFIG SET");
freeReplyObject(reply);
/* Create a client ID for this client. */
DBClientID client = globally_unique_id();
/* Construct the argument arrays for RAY.CONNECT. */
int argc = num_args + 4;
@@ -133,7 +211,7 @@ DBHandle *db_connect(const char *db_address,
argvlen[0] = strlen(argv[0]);
/* Set the client ID argument. */
argv[1] = (char *) client.id;
argvlen[1] = sizeof(db->client.id);
argvlen[1] = sizeof(client.id);
/* Set the node IP address argument. */
argv[2] = node_ip_address;
argvlen[2] = strlen(node_ip_address);
@@ -152,7 +230,7 @@ DBHandle *db_connect(const char *db_address,
/* Register this client with Redis. RAY.CONNECT is a custom Redis command that
* we've defined. */
reply = (redisReply *) redisCommandArgv(context, argc, argv, argvlen);
reply = (redisReply *) redisCommandArgv(sync_context, argc, argv, argvlen);
CHECKM(reply != NULL, "db_connect failed on RAY.CONNECT");
CHECK(reply->type != REDIS_REPLY_ERROR);
CHECK(strcmp(reply->str, "OK") == 0);
@@ -160,25 +238,75 @@ DBHandle *db_connect(const char *db_address,
free(argv);
free(argvlen);
*sync_context_out = sync_context;
/* Establish connection for control data. */
redisAsyncContext *context = redisAsyncConnect(db_address.c_str(), db_port);
CHECK_REDIS_CONNECT(redisAsyncContext, context,
"could not establish asynchronous connection to redis "
"%s:%d",
db_address.c_str(), db_port);
context->data = (void *) db;
*context_out = context;
/* Establish async connection for subscription. */
redisAsyncContext *subscribe_context =
redisAsyncConnect(db_address.c_str(), db_port);
CHECK_REDIS_CONNECT(redisAsyncContext, subscribe_context,
"could not establish asynchronous subscription "
"connection to redis %s:%d",
db_address.c_str(), db_port);
subscribe_context->data = (void *) db;
*subscribe_context_out = subscribe_context;
}
DBHandle *db_connect(const std::string &db_primary_address,
int db_primary_port,
const char *client_type,
const char *node_ip_address,
int num_args,
const char **args) {
/* Check that the number of args is even. These args will be passed to the
* RAY.CONNECT Redis command, which takes arguments in pairs. */
if (num_args % 2 != 0) {
LOG_FATAL("The number of extra args must be divisible by two.");
}
/* Create a client ID for this client. */
DBClientID client = globally_unique_id();
DBHandle *db = new DBHandle();
db->client_type = strdup(client_type);
db->client = client;
db->db_client_cache = NULL;
db->sync_context = context;
/* Establish async connection */
db->context = redisAsyncConnect(db_address, db_port);
CHECK_REDIS_CONNECT(redisAsyncContext, db->context,
"could not establish asynchronous connection to redis "
"%s:%d",
db_address, db_port);
db->context->data = (void *) db;
/* Establish async connection for subscription */
db->sub_context = redisAsyncConnect(db_address, db_port);
CHECK_REDIS_CONNECT(redisAsyncContext, db->sub_context,
"could not establish asynchronous subscription "
"connection to redis %s:%d",
db_address, db_port);
db->sub_context->data = (void *) db;
redisAsyncContext *context;
redisAsyncContext *subscribe_context;
redisContext *sync_context;
/* Connect to the primary redis instance. */
db_connect_shard(db_primary_address, db_primary_port, client, client_type,
node_ip_address, num_args, args, db, &context,
&subscribe_context, &sync_context);
db->context = context;
db->subscribe_context = subscribe_context;
db->sync_context = sync_context;
/* Get the shard locations. */
std::vector<std::string> db_shards_addresses;
std::vector<int> db_shards_ports;
get_redis_shards(db->sync_context, db_shards_addresses, db_shards_ports);
CHECKM(db_shards_addresses.size() > 0, "No Redis shards found");
/* Connect to the shards. */
for (int i = 0; i < db_shards_addresses.size(); ++i) {
db_connect_shard(db_shards_addresses[i], db_shards_ports[i], client,
client_type, node_ip_address, num_args, args, db, &context,
&subscribe_context, &sync_context);
db->contexts.push_back(context);
db->subscribe_contexts.push_back(subscribe_context);
redisFree(sync_context);
}
return db;
}
@@ -193,10 +321,17 @@ void db_disconnect(DBHandle *db) {
CHECK(strcmp(reply->str, "OK") == 0);
freeReplyObject(reply);
/* Clean up the Redis connection state. */
/* Clean up the primary Redis connection state. */
redisFree(db->sync_context);
redisAsyncFree(db->context);
redisAsyncFree(db->sub_context);
redisAsyncFree(db->subscribe_context);
/* Clean up the Redis shards. */
CHECK(db->contexts.size() == db->subscribe_contexts.size());
for (int i = 0; i < db->contexts.size(); ++i) {
redisAsyncFree(db->contexts[i]);
redisAsyncFree(db->subscribe_contexts[i]);
}
/* Clean up memory. */
DBClientCacheEntry *e, *tmp;
@@ -206,21 +341,36 @@ void db_disconnect(DBHandle *db) {
free(e);
}
free(db->client_type);
free(db);
delete db;
}
void db_attach(DBHandle *db, event_loop *loop, bool reattach) {
db->loop = loop;
/* Attach primary redis instance to the event loop. */
int err = redisAeAttach(loop, db->context);
/* If the database is reattached in the tests, redis normally gives
* an error which we can safely ignore. */
if (!reattach) {
CHECKM(err == REDIS_OK, "failed to attach the event loop");
}
err = redisAeAttach(loop, db->sub_context);
err = redisAeAttach(loop, db->subscribe_context);
if (!reattach) {
CHECKM(err == REDIS_OK, "failed to attach the event loop");
}
/* Attach other redis shards to the event loop. */
CHECK(db->contexts.size() == db->subscribe_contexts.size());
for (int i = 0; i < db->contexts.size(); ++i) {
int err = redisAeAttach(loop, db->contexts[i]);
/* If the database is reattached in the tests, redis normally gives
* an error which we can safely ignore. */
if (!reattach) {
CHECKM(err == REDIS_OK, "failed to attach the event loop");
}
err = redisAeAttach(loop, db->subscribe_contexts[i]);
if (!reattach) {
CHECKM(err == REDIS_OK, "failed to attach the event loop");
}
}
}
/*
@@ -264,14 +414,16 @@ void redis_object_table_add(TableCallbackData *callback_data) {
int64_t object_size = info->object_size;
unsigned char *digest = info->digest;
redisAsyncContext *context = get_redis_context(db, obj_id);
int status = redisAsyncCommand(
db->context, redis_object_table_add_callback,
context, redis_object_table_add_callback,
(void *) callback_data->timer_id, "RAY.OBJECT_TABLE_ADD %b %ld %b %b",
obj_id.id, sizeof(obj_id.id), object_size, digest, (size_t) DIGEST_SIZE,
db->client.id, sizeof(db->client.id));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error in redis_object_table_add");
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_object_table_add");
}
}
@@ -309,13 +461,16 @@ void redis_object_table_remove(TableCallbackData *callback_data) {
if (client_id == NULL) {
client_id = &db->client;
}
redisAsyncContext *context = get_redis_context(db, obj_id);
int status = redisAsyncCommand(
db->context, redis_object_table_remove_callback,
context, redis_object_table_remove_callback,
(void *) callback_data->timer_id, "RAY.OBJECT_TABLE_REMOVE %b %b",
obj_id.id, sizeof(obj_id.id), client_id->id, sizeof(client_id->id));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error in redis_object_table_remove");
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_object_table_remove");
}
}
@@ -324,12 +479,15 @@ void redis_object_table_lookup(TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
ObjectID obj_id = callback_data->id;
int status = redisAsyncCommand(
db->context, redis_object_table_lookup_callback,
(void *) callback_data->timer_id, "RAY.OBJECT_TABLE_LOOKUP %b", obj_id.id,
sizeof(obj_id.id));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error in object_table lookup");
redisAsyncContext *context = get_redis_context(db, obj_id);
int status = redisAsyncCommand(context, redis_object_table_lookup_callback,
(void *) callback_data->timer_id,
"RAY.OBJECT_TABLE_LOOKUP %b", obj_id.id,
sizeof(obj_id.id));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in object_table lookup");
}
}
@@ -358,13 +516,15 @@ void redis_result_table_add(TableCallbackData *callback_data) {
ResultTableAddInfo *info = (ResultTableAddInfo *) callback_data->data;
int is_put = info->is_put ? 1 : 0;
redisAsyncContext *context = get_redis_context(db, id);
/* Add the result entry to the result table. */
int status = redisAsyncCommand(
db->context, redis_result_table_add_callback,
context, redis_result_table_add_callback,
(void *) callback_data->timer_id, "RAY.RESULT_TABLE_ADD %b %b %d", id.id,
sizeof(id.id), info->task_id.id, sizeof(info->task_id.id), is_put);
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "Error in result table add");
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "Error in result table add");
}
}
@@ -423,12 +583,13 @@ void redis_result_table_lookup(TableCallbackData *callback_data) {
CHECK(callback_data);
DBHandle *db = callback_data->db_handle;
ObjectID id = callback_data->id;
redisAsyncContext *context = get_redis_context(db, id);
int status =
redisAsyncCommand(db->context, redis_result_table_lookup_callback,
redisAsyncCommand(context, redis_result_table_lookup_callback,
(void *) callback_data->timer_id,
"RAY.RESULT_TABLE_LOOKUP %b", id.id, sizeof(id.id));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "Error in result table lookup");
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "Error in result table lookup");
}
}
@@ -586,28 +747,33 @@ void redis_object_table_subscribe_to_notifications(
* src/common/redismodule/ray_redis_module.cc. */
const char *object_channel_prefix = "OC:";
const char *object_channel_bcast = "BCAST";
int status = REDIS_OK;
/* Subscribe to notifications from the object table. This uses the client ID
* as the channel name so this channel is specific to this client. TODO(rkn):
* The channel name should probably be the client ID with some prefix. */
CHECKM(callback_data->data != NULL,
"Object table subscribe data passed as NULL.");
if (((ObjectTableSubscribeData *) (callback_data->data))->subscribe_all) {
/* Subscribe to the object broadcast channel. */
status = redisAsyncCommand(
db->sub_context, object_table_redis_subscribe_to_notifications_callback,
(void *) callback_data->timer_id, "SUBSCRIBE %s%s",
object_channel_prefix, object_channel_bcast);
} else {
status = redisAsyncCommand(
db->sub_context, object_table_redis_subscribe_to_notifications_callback,
(void *) callback_data->timer_id, "SUBSCRIBE %s%b",
object_channel_prefix, db->client.id, sizeof(db->client.id));
}
for (int i = 0; i < db->subscribe_contexts.size(); ++i) {
int status = REDIS_OK;
/* Subscribe to notifications from the object table. This uses the client ID
* as the channel name so this channel is specific to this client.
* TODO(rkn):
* The channel name should probably be the client ID with some prefix. */
CHECKM(callback_data->data != NULL,
"Object table subscribe data passed as NULL.");
if (((ObjectTableSubscribeData *) (callback_data->data))->subscribe_all) {
/* Subscribe to the object broadcast channel. */
status = redisAsyncCommand(
db->subscribe_contexts[i],
object_table_redis_subscribe_to_notifications_callback,
(void *) callback_data->timer_id, "SUBSCRIBE %s%s",
object_channel_prefix, object_channel_bcast);
} else {
status = redisAsyncCommand(
db->subscribe_contexts[i],
object_table_redis_subscribe_to_notifications_callback,
(void *) callback_data->timer_id, "SUBSCRIBE %s%b",
object_channel_prefix, db->client.id, sizeof(db->client.id));
}
if ((status == REDIS_ERR) || db->sub_context->err) {
LOG_REDIS_DEBUG(db->sub_context,
"error in redis_object_table_subscribe_to_notifications");
if ((status == REDIS_ERR) || db->subscribe_contexts[i]->err) {
LOG_REDIS_DEBUG(db->subscribe_contexts[i],
"error in redis_object_table_subscribe_to_notifications");
}
}
}
@@ -633,31 +799,33 @@ void redis_object_table_request_notifications(
int num_object_ids = request_data->num_object_ids;
ObjectID *object_ids = request_data->object_ids;
/* Create the arguments for the Redis command. */
int num_args = 1 + 1 + num_object_ids;
const char **argv = (const char **) malloc(sizeof(char *) * num_args);
size_t *argvlen = (size_t *) malloc(sizeof(size_t) * num_args);
/* Set the command name argument. */
argv[0] = "RAY.OBJECT_TABLE_REQUEST_NOTIFICATIONS";
argvlen[0] = strlen(argv[0]);
/* Set the client ID argument. */
argv[1] = (char *) db->client.id;
argvlen[1] = sizeof(db->client.id);
/* Set the object ID arguments. */
for (int i = 0; i < num_object_ids; ++i) {
argv[2 + i] = (char *) object_ids[i].id;
argvlen[2 + i] = sizeof(object_ids[i].id);
}
redisAsyncContext *context = get_redis_context(db, object_ids[i]);
int status = redisAsyncCommandArgv(
db->context, redis_object_table_request_notifications_callback,
(void *) callback_data->timer_id, num_args, argv, argvlen);
free(argv);
free(argvlen);
/* Create the arguments for the Redis command. */
int num_args = 1 + 1 + 1;
const char **argv = (const char **) malloc(sizeof(char *) * num_args);
size_t *argvlen = (size_t *) malloc(sizeof(size_t) * num_args);
/* Set the command name argument. */
argv[0] = "RAY.OBJECT_TABLE_REQUEST_NOTIFICATIONS";
argvlen[0] = strlen(argv[0]);
/* Set the client ID argument. */
argv[1] = (char *) db->client.id;
argvlen[1] = sizeof(db->client.id);
/* Set the object ID arguments. */
argv[2] = (char *) object_ids[i].id;
argvlen[2] = sizeof(object_ids[i].id);
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context,
"error in redis_object_table_subscribe_to_notifications");
int status = redisAsyncCommandArgv(
context, redis_object_table_request_notifications_callback,
(void *) callback_data->timer_id, num_args, argv, argvlen);
free(argv);
free(argvlen);
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context,
"error in redis_object_table_subscribe_to_notifications");
}
}
}
@@ -690,12 +858,14 @@ void redis_task_table_get_task(TableCallbackData *callback_data) {
CHECK(callback_data->data == NULL);
TaskID task_id = callback_data->id;
int status = redisAsyncCommand(
db->context, redis_task_table_get_task_callback,
(void *) callback_data->timer_id, "RAY.TASK_TABLE_GET %b", task_id.id,
sizeof(task_id.id));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error in redis_task_table_get_task");
redisAsyncContext *context = get_redis_context(db, task_id);
int status = redisAsyncCommand(context, redis_task_table_get_task_callback,
(void *) callback_data->timer_id,
"RAY.TASK_TABLE_GET %b", task_id.id,
sizeof(task_id.id));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_task_table_get_task");
}
}
@@ -706,6 +876,36 @@ void redis_task_table_add_task_callback(redisAsyncContext *c,
/* Do some minimal checking. */
redisReply *reply = (redisReply *) r;
/* If the publish which happens inside of the call to RAY.TASK_TABLE_ADD was
* not received by any subscribers, then reissue the command. TODO(rkn): This
* entire if block should be temporary. Once we address the problem where in
* which a global scheduler may publish a task to a local scheduler before the
* local scheduler has subscribed to the relevant channel, we shouldn't need
* this block any more. */
if (reply->type == REDIS_REPLY_ERROR &&
strcmp(reply->str, "No subscribers received message.") == 0) {
Task *task = (Task *) callback_data->data;
TaskID task_id = Task_task_id(task);
DBClientID local_scheduler_id = Task_local_scheduler(task);
redisAsyncContext *context = get_redis_context(db, task_id);
int state = Task_state(task);
TaskSpec *spec = Task_task_spec(task);
/* Reissue the command. */
CHECKM(task != NULL, "NULL task passed to redis_task_table_add_task.");
int status = redisAsyncCommand(
context, redis_task_table_add_task_callback,
(void *) callback_data->timer_id, "RAY.TASK_TABLE_ADD %b %d %b %b",
task_id.id, sizeof(task_id.id), state, local_scheduler_id.id,
sizeof(local_scheduler_id.id), spec, Task_task_spec_size(task));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_task_table_add_task");
}
/* Since we are reissuing the same command with the same callback data,
* return early to avoid freeing the callback data. */
return;
}
CHECKM(strcmp(reply->str, "OK") == 0, "reply->str is %s", reply->str);
/* Call the done callback if there is one. */
if (callback_data->done_callback != NULL) {
@@ -722,17 +922,18 @@ void redis_task_table_add_task(TableCallbackData *callback_data) {
Task *task = (Task *) callback_data->data;
TaskID task_id = Task_task_id(task);
DBClientID local_scheduler_id = Task_local_scheduler(task);
redisAsyncContext *context = get_redis_context(db, task_id);
int state = Task_state(task);
TaskSpec *spec = Task_task_spec(task);
CHECKM(task != NULL, "NULL task passed to redis_task_table_add_task.");
int status = redisAsyncCommand(
db->context, redis_task_table_add_task_callback,
context, redis_task_table_add_task_callback,
(void *) callback_data->timer_id, "RAY.TASK_TABLE_ADD %b %d %b %b",
task_id.id, sizeof(task_id.id), state, local_scheduler_id.id,
sizeof(local_scheduler_id.id), spec, Task_task_spec_size(task));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error in redis_task_table_add_task");
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_task_table_add_task");
}
}
@@ -743,6 +944,35 @@ void redis_task_table_update_callback(redisAsyncContext *c,
/* Do some minimal checking. */
redisReply *reply = (redisReply *) r;
/* If the publish which happens inside of the call to RAY.TASK_TABLE_UPDATE
* was not received by any subscribers, then reissue the command. TODO(rkn):
* This entire if block should be temporary. Once we address the problem where
* in which a global scheduler may publish a task to a local scheduler before
* the local scheduler has subscribed to the relevant channel, we shouldn't
* need this block any more. */
if (reply->type == REDIS_REPLY_ERROR &&
strcmp(reply->str, "No subscribers received message.") == 0) {
Task *task = (Task *) callback_data->data;
TaskID task_id = Task_task_id(task);
redisAsyncContext *context = get_redis_context(db, task_id);
DBClientID local_scheduler_id = Task_local_scheduler(task);
int state = Task_state(task);
/* Reissue the command. */
CHECKM(task != NULL, "NULL task passed to redis_task_table_update.");
int status = redisAsyncCommand(
context, redis_task_table_update_callback,
(void *) callback_data->timer_id, "RAY.TASK_TABLE_UPDATE %b %d %b",
task_id.id, sizeof(task_id.id), state, local_scheduler_id.id,
sizeof(local_scheduler_id.id));
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_task_table_update");
}
/* Since we are reissuing the same command with the same callback data,
* return early to avoid freeing the callback data. */
return;
}
CHECKM(strcmp(reply->str, "OK") == 0, "reply->str is %s", reply->str);
/* Call the done callback if there is one. */
if (callback_data->done_callback != NULL) {
@@ -758,17 +988,18 @@ void redis_task_table_update(TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
Task *task = (Task *) callback_data->data;
TaskID task_id = Task_task_id(task);
redisAsyncContext *context = get_redis_context(db, task_id);
DBClientID local_scheduler_id = Task_local_scheduler(task);
int state = Task_state(task);
CHECKM(task != NULL, "NULL task passed to redis_task_table_update.");
int status = redisAsyncCommand(
db->context, redis_task_table_update_callback,
context, redis_task_table_update_callback,
(void *) callback_data->timer_id, "RAY.TASK_TABLE_UPDATE %b %d %b",
task_id.id, sizeof(task_id.id), state, local_scheduler_id.id,
sizeof(local_scheduler_id.id));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error in redis_task_table_update");
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_task_table_update");
}
}
@@ -779,6 +1010,15 @@ void redis_task_table_test_and_update_callback(redisAsyncContext *c,
redisReply *reply = (redisReply *) r;
/* Parse the task from the reply. */
Task *task = parse_and_construct_task_from_redis_reply(reply);
if (task == NULL) {
/* A NULL task means that the task was not in the task table. NOTE(swang):
* For normal tasks, this is not expected behavior, but actor tasks may be
* delayed when added to the task table if they are submitted to a local
* scheduler before it receives the notification that maps the actor to a
* local scheduler. */
LOG_ERROR("No task found during task_table_test_and_update");
return;
}
/* Determine whether the update happened. */
auto message = flatbuffers::GetRoot<TaskReply>(reply->str);
bool updated = message->updated();
@@ -800,18 +1040,19 @@ void redis_task_table_test_and_update_callback(redisAsyncContext *c,
void redis_task_table_test_and_update(TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
TaskID task_id = callback_data->id;
redisAsyncContext *context = get_redis_context(db, task_id);
TaskTableTestAndUpdateData *update_data =
(TaskTableTestAndUpdateData *) callback_data->data;
int status = redisAsyncCommand(
db->context, redis_task_table_test_and_update_callback,
context, redis_task_table_test_and_update_callback,
(void *) callback_data->timer_id,
"RAY.TASK_TABLE_TEST_AND_UPDATE %b %d %d %b", task_id.id,
sizeof(task_id.id), update_data->test_state_bitmask,
update_data->update_state, update_data->local_scheduler_id.id,
sizeof(update_data->local_scheduler_id.id));
if ((status == REDIS_ERR) || db->context->err) {
LOG_REDIS_DEBUG(db->context, "error in redis_task_table_test_and_update");
if ((status == REDIS_ERR) || context->err) {
LOG_REDIS_DEBUG(context, "error in redis_task_table_test_and_update");
}
}
@@ -879,24 +1120,26 @@ void redis_task_table_subscribe(TableCallbackData *callback_data) {
/* TASK_CHANNEL_PREFIX is defined in ray_redis_module.cc and must be kept in
* sync with that file. */
const char *TASK_CHANNEL_PREFIX = "TT:";
int status;
if (IS_NIL_ID(data->local_scheduler_id)) {
/* TODO(swang): Implement the state_filter by translating the bitmask into
* a Redis key-matching pattern. */
status =
redisAsyncCommand(db->sub_context, redis_task_table_subscribe_callback,
(void *) callback_data->timer_id, "PSUBSCRIBE %s*:%d",
TASK_CHANNEL_PREFIX, data->state_filter);
} else {
DBClientID local_scheduler_id = data->local_scheduler_id;
status =
redisAsyncCommand(db->sub_context, redis_task_table_subscribe_callback,
(void *) callback_data->timer_id, "SUBSCRIBE %s%b:%d",
TASK_CHANNEL_PREFIX, (char *) local_scheduler_id.id,
sizeof(local_scheduler_id.id), data->state_filter);
}
if ((status == REDIS_ERR) || db->sub_context->err) {
LOG_REDIS_DEBUG(db->sub_context, "error in redis_task_table_subscribe");
for (auto subscribe_context : db->subscribe_contexts) {
int status;
if (IS_NIL_ID(data->local_scheduler_id)) {
/* TODO(swang): Implement the state_filter by translating the bitmask into
* a Redis key-matching pattern. */
status = redisAsyncCommand(
subscribe_context, redis_task_table_subscribe_callback,
(void *) callback_data->timer_id, "PSUBSCRIBE %s*:%d",
TASK_CHANNEL_PREFIX, data->state_filter);
} else {
DBClientID local_scheduler_id = data->local_scheduler_id;
status = redisAsyncCommand(
subscribe_context, redis_task_table_subscribe_callback,
(void *) callback_data->timer_id, "SUBSCRIBE %s%b:%d",
TASK_CHANNEL_PREFIX, (char *) local_scheduler_id.id,
sizeof(local_scheduler_id.id), data->state_filter);
}
if ((status == REDIS_ERR) || subscribe_context->err) {
LOG_REDIS_DEBUG(subscribe_context, "error in redis_task_table_subscribe");
}
}
}
@@ -934,6 +1177,55 @@ void redis_db_client_table_remove(TableCallbackData *callback_data) {
}
}
void redis_db_client_table_scan(DBHandle *db,
std::vector<DBClient> &db_clients) {
/* TODO(swang): Integrate this functionality with the Ray Redis module. To do
* this, we need the KEYS or SCAN command in Redis modules. */
/* Get all the database client keys. */
redisReply *reply = (redisReply *) redisCommand(db->sync_context, "KEYS %s*",
DB_CLIENT_PREFIX);
if (reply->type == REDIS_REPLY_NIL) {
return;
}
/* Get all the database client information. */
CHECK(reply->type == REDIS_REPLY_ARRAY);
for (int i = 0; i < reply->elements; ++i) {
redisReply *client_reply = (redisReply *) redisCommand(
db->sync_context, "HGETALL %b", reply->element[i]->str,
reply->element[i]->len);
CHECK(reply->type == REDIS_REPLY_ARRAY);
CHECK(reply->elements > 0);
DBClient db_client;
memset(&db_client, 0, sizeof(db_client));
int num_fields = 0;
/* Parse the fields into a DBClient. */
for (int j = 0; j < client_reply->elements; j = j + 2) {
const char *key = client_reply->element[j]->str;
const char *value = client_reply->element[j + 1]->str;
if (strcmp(key, "ray_client_id") == 0) {
memcpy(db_client.id.id, value, sizeof(db_client.id));
num_fields++;
} else if (strcmp(key, "client_type") == 0) {
db_client.client_type = strdup(value);
num_fields++;
} else if (strcmp(key, "aux_address") == 0) {
db_client.aux_address = strdup(value);
num_fields++;
} else if (strcmp(key, "deleted") == 0) {
bool is_deleted = atoi(value);
db_client.is_insertion = !is_deleted;
num_fields++;
}
}
freeReplyObject(client_reply);
/* The client ID, type, and whether it is deleted are all mandatory fields.
* Auxiliary address is optional. */
CHECK(num_fields >= 3);
db_clients.push_back(db_client);
}
freeReplyObject(reply);
}
void redis_db_client_table_subscribe_callback(redisAsyncContext *c,
void *r,
void *privdata) {
@@ -956,35 +1248,54 @@ void redis_db_client_table_subscribe_callback(redisAsyncContext *c,
/* Note that we do not destroy the callback data yet because the
* subscription callback needs this data. */
event_loop_remove_timer(db->loop, callback_data->timer_id);
/* Get the current db client table entries, in case we missed notifications
* before the initial subscription. This must be done before we process any
* notifications from the subscription channel, so that we don't readd an
* entry that has already been deleted. */
std::vector<DBClient> db_clients;
redis_db_client_table_scan(db, db_clients);
/* Call the subscription callback for all entries that we missed. */
DBClientTableSubscribeData *data =
(DBClientTableSubscribeData *) callback_data->data;
for (auto db_client : db_clients) {
data->subscribe_callback(&db_client, data->subscribe_context);
if (db_client.client_type != NULL) {
free((void *) db_client.client_type);
}
if (db_client.aux_address != NULL) {
free((void *) db_client.aux_address);
}
}
return;
}
/* Otherwise, parse the payload and call the callback. */
auto message =
flatbuffers::GetRoot<SubscribeToDBClientTableReply>(payload->str);
DBClientID client = from_flatbuf(message->db_client_id());
/* Parse the client type and auxiliary address from the response. If there is
* only client type, then the update was a delete. */
char *client_type = (char *) message->client_type()->data();
char *aux_address = (char *) message->aux_address()->data();
bool is_insertion = message->is_insertion();
DBClient db_client;
db_client.id = from_flatbuf(message->db_client_id());
db_client.client_type = (char *) message->client_type()->data();
db_client.aux_address = message->aux_address()->data();
db_client.is_insertion = message->is_insertion();
/* Call the subscription callback. */
DBClientTableSubscribeData *data =
(DBClientTableSubscribeData *) callback_data->data;
if (data->subscribe_callback) {
data->subscribe_callback(client, client_type, aux_address, is_insertion,
data->subscribe_context);
data->subscribe_callback(&db_client, data->subscribe_context);
}
}
void redis_db_client_table_subscribe(TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
int status = redisAsyncCommand(
db->sub_context, redis_db_client_table_subscribe_callback,
db->subscribe_context, redis_db_client_table_subscribe_callback,
(void *) callback_data->timer_id, "SUBSCRIBE db_clients");
if ((status == REDIS_ERR) || db->sub_context->err) {
LOG_REDIS_DEBUG(db->sub_context,
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->subscribe_context,
"error in db_client_table_register_callback");
}
}
@@ -1042,10 +1353,10 @@ void redis_local_scheduler_table_subscribe_callback(redisAsyncContext *c,
void redis_local_scheduler_table_subscribe(TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
int status = redisAsyncCommand(
db->sub_context, redis_local_scheduler_table_subscribe_callback,
db->subscribe_context, redis_local_scheduler_table_subscribe_callback,
(void *) callback_data->timer_id, "SUBSCRIBE local_schedulers");
if ((status == REDIS_ERR) || db->sub_context->err) {
LOG_REDIS_DEBUG(db->sub_context,
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->subscribe_context,
"error in redis_local_scheduler_table_subscribe");
}
}
@@ -1130,10 +1441,11 @@ void redis_driver_table_subscribe_callback(redisAsyncContext *c,
void redis_driver_table_subscribe(TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
int status = redisAsyncCommand(
db->sub_context, redis_driver_table_subscribe_callback,
db->subscribe_context, redis_driver_table_subscribe_callback,
(void *) callback_data->timer_id, "SUBSCRIBE driver_deaths");
if ((status == REDIS_ERR) || db->sub_context->err) {
LOG_REDIS_DEBUG(db->sub_context, "error in redis_driver_table_subscribe");
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->subscribe_context,
"error in redis_driver_table_subscribe");
}
}
@@ -1240,10 +1552,10 @@ void redis_actor_notification_table_subscribe(
TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
int status = redisAsyncCommand(
db->sub_context, redis_actor_notification_table_subscribe_callback,
db->subscribe_context, redis_actor_notification_table_subscribe_callback,
(void *) callback_data->timer_id, "SUBSCRIBE actor_notifications");
if ((status == REDIS_ERR) || db->sub_context->err) {
LOG_REDIS_DEBUG(db->sub_context,
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->subscribe_context,
"error in redis_actor_notification_table_subscribe");
}
}
@@ -1290,10 +1602,11 @@ void redis_object_info_subscribe_callback(redisAsyncContext *c,
void redis_object_info_subscribe(TableCallbackData *callback_data) {
DBHandle *db = callback_data->db_handle;
int status = redisAsyncCommand(
db->sub_context, redis_object_info_subscribe_callback,
db->subscribe_context, redis_object_info_subscribe_callback,
(void *) callback_data->timer_id, "PSUBSCRIBE obj:info");
if ((status == REDIS_ERR) || db->sub_context->err) {
LOG_REDIS_DEBUG(db->sub_context, "error in object_info_register_callback");
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->subscribe_context,
"error in object_info_register_callback");
}
}
@@ -1324,8 +1637,8 @@ void redis_push_error_hmset_callback(redisAsyncContext *c,
"RPUSH ErrorKeys Error:%b:%b",
info->driver_id.id, sizeof(info->driver_id.id),
info->error_key, sizeof(info->error_key));
if ((status == REDIS_ERR) || db->sub_context->err) {
LOG_REDIS_DEBUG(db->sub_context, "error in redis_push_error rpush");
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->subscribe_context, "error in redis_push_error rpush");
}
}
@@ -1344,8 +1657,8 @@ void redis_push_error(TableCallbackData *callback_data) {
"HMSET Error:%b:%b type %s message %s data %b", info->driver_id.id,
sizeof(info->driver_id.id), info->error_key, sizeof(info->error_key),
error_type, error_message, info->data, info->data_length);
if ((status == REDIS_ERR) || db->sub_context->err) {
LOG_REDIS_DEBUG(db->sub_context, "error in redis_push_error hmset");
if ((status == REDIS_ERR) || db->subscribe_context->err) {
LOG_REDIS_DEBUG(db->subscribe_context, "error in redis_push_error hmset");
}
}
src/common/state/redis.h
+51 -4
View File
@@ -34,11 +34,24 @@ struct DBHandle {
char *client_type;
/** Unique ID for this client. */
DBClientID client;
/** Redis context for all non-subscribe connections. */
/** Primary redis context for all non-subscribe connections. This is used for
* the database client table, heartbeats, and errors that should be pushed to
* the driver. */
redisAsyncContext *context;
/** Redis context for "subscribe" communication. Yes, we need a separate one
* for that, see https://github.com/redis/hiredis/issues/55. */
redisAsyncContext *sub_context;
/** Primary redis context for "subscribe" communication. A separate context
* is needed for this communication (see
* https://github.com/redis/hiredis/issues/55). This is used for the
* database client table, heartbeats, and errors that should be pushed to
* the driver. */
redisAsyncContext *subscribe_context;
/** Redis contexts for shards for all non-subscribe connections. All requests
* to the object table, task table, and event table should be directed here.
* The correct shard can be retrieved using get_redis_context below. */
std::vector<redisAsyncContext *> contexts;
/** Redis contexts for shards for "subscribe" communication. All requests
* to the object table, task table, and event table should be directed here.
* The correct shard can be retrieved using get_redis_context below. */
std::vector<redisAsyncContext *> subscribe_contexts;
/** The event loop this global state store connection is part of. */
event_loop *loop;
/** Index of the database connection in the event loop */
@@ -51,6 +64,40 @@ struct DBHandle {
redisContext *sync_context;
};
/**
* Get the Redis asynchronous context responsible for non-subscription
* communication for the given UniqueID.
*
* @param db The database handle.
* @param id The ID whose location we are querying for.
* @return The redisAsyncContext responsible for the given ID.
*/
redisAsyncContext *get_redis_context(DBHandle *db, UniqueID id);
/**
* Get the Redis asynchronous context responsible for subscription
* communication for the given UniqueID.
*
* @param db The database handle.
* @param id The ID whose location we are querying for.
* @return The redisAsyncContext responsible for the given ID.
*/
redisAsyncContext *get_redis_subscribe_context(DBHandle *db, UniqueID id);
/**
* Get a list of Redis shard IP addresses from the primary shard.
*
* @param context A Redis context connected to the primary shard.
* @param db_shards_addresses The IP addresses for the shards registered
* with the primary shard will be added to this vector.
* @param db_shards_ports The IP ports for the shards registered with the
* primary shard will be added to this vector, in the same order as
* db_shards_addresses.
*/
void get_redis_shards(redisContext *context,
std::vector<std::string> &db_shards_addresses,
std::vector<int> &db_shards_ports);
void redis_object_table_get_entry(redisAsyncContext *c,
void *r,
void *privdata);
src/common/state/task_table.h
+4 -3
View File
@@ -151,9 +151,10 @@ typedef void (*task_table_subscribe_callback)(Task *task, void *user_context);
* @param local_scheduler_id The db_client_id of the local scheduler whose
* events we want to listen to. If you want to subscribe to updates from
* all local schedulers, pass in NIL_ID.
* @param state_filter Flags for events we want to listen to. If you want
* to listen to all events, use state_filter = TASK_WAITING |
* TASK_SCHEDULED | TASK_RUNNING | TASK_DONE.
* @param state_filter Events we want to listen to. Can have values from the
* enum "scheduling_state" in task.h.
* TODO(pcm): Make it possible to combine these using flags like
* TASK_STATUS_WAITING | TASK_STATUS_SCHEDULED.
* @param retry Information about retrying the request to the database.
* @param done_callback Function to be called when database returns result.
* @param user_context Data that will be passed to done_callback and
src/common/test/db_tests.cc
+10 -9
View File
@@ -72,12 +72,12 @@ TEST object_table_lookup_test(void) {
event_loop *loop = event_loop_create();
/* This uses manager_port1. */
const char *db_connect_args1[] = {"address", "127.0.0.1:12345"};
DBHandle *db1 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
2, db_connect_args1);
DBHandle *db1 = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
manager_addr, 2, db_connect_args1);
/* This uses manager_port2. */
const char *db_connect_args2[] = {"address", "127.0.0.1:12346"};
DBHandle *db2 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
2, db_connect_args2);
DBHandle *db2 = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
manager_addr, 2, db_connect_args2);
db_attach(db1, loop, false);
db_attach(db2, loop, false);
UniqueID id = globally_unique_id();
@@ -148,8 +148,8 @@ void task_table_test_callback(Task *callback_task, void *user_data) {
TEST task_table_test(void) {
task_table_test_callback_called = 0;
event_loop *loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "local_scheduler", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "local_scheduler",
"127.0.0.1", 0, NULL);
db_attach(db, loop, false);
DBClientID local_scheduler_id = globally_unique_id();
int64_t task_spec_size;
@@ -184,8 +184,8 @@ void task_table_all_test_callback(Task *task, void *user_data) {
TEST task_table_all_test(void) {
event_loop *loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "local_scheduler", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "local_scheduler",
"127.0.0.1", 0, NULL);
db_attach(db, loop, false);
int64_t task_spec_size;
TaskSpec *spec = example_task_spec(1, 1, &task_spec_size);
@@ -222,7 +222,8 @@ TEST unique_client_id_test(void) {
DBClientID ids[num_conns];
DBHandle *db;
for (int i = 0; i < num_conns; ++i) {
db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
ids[i] = get_db_client_id(db);
db_disconnect(db);
}
src/common/test/object_table_tests.cc
+51 -57
View File
@@ -65,6 +65,15 @@ void new_object_task_callback(TaskID task_id, void *user_context) {
new_object_lookup_callback, (void *) db);
}
void task_table_subscribe_done(TaskID task_id, void *user_context) {
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = NULL,
};
DBHandle *db = (DBHandle *) user_context;
task_table_add_task(db, Task_copy(new_object_task), &retry,
new_object_task_callback, db);
}
TEST new_object_test(void) {
new_object_failed = 0;
new_object_succeeded = 0;
@@ -73,16 +82,16 @@ TEST new_object_test(void) {
new_object_task_spec = Task_task_spec(new_object_task);
new_object_task_id = TaskSpec_task_id(new_object_task_spec);
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = new_object_fail_callback,
};
task_table_add_task(db, Task_copy(new_object_task), &retry,
new_object_task_callback, db);
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, &retry,
task_table_subscribe_done, db);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
@@ -109,8 +118,8 @@ TEST new_object_no_task_test(void) {
new_object_id = globally_unique_id();
new_object_task_id = globally_unique_id();
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -151,8 +160,8 @@ void lookup_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST lookup_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = lookup_fail_callback,
@@ -161,6 +170,9 @@ TEST lookup_timeout_test(void) {
(void *) lookup_timeout_context);
/* Disconnect the database to see if the lookup times out. */
close(db->context->c.fd);
for (auto context : db->contexts) {
close(context->c.fd);
}
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
@@ -187,8 +199,8 @@ void add_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST add_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = add_fail_callback,
@@ -197,6 +209,9 @@ TEST add_timeout_test(void) {
add_done_callback, (void *) add_timeout_context);
/* Disconnect the database to see if the lookup times out. */
close(db->context->c.fd);
for (auto context : db->contexts) {
close(context->c.fd);
}
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
@@ -225,8 +240,8 @@ void subscribe_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST subscribe_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -236,7 +251,10 @@ TEST subscribe_timeout_test(void) {
object_table_subscribe_to_notifications(db, false, subscribe_done_callback,
NULL, &retry, NULL, NULL);
/* Disconnect the database to see if the lookup times out. */
close(db->sub_context->c.fd);
close(db->subscribe_context->c.fd);
for (auto subscribe_context : db->subscribe_contexts) {
close(subscribe_context->c.fd);
}
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
@@ -324,8 +342,8 @@ TEST add_lookup_test(void) {
lookup_retry_succeeded = 0;
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11235"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, true);
RetryInfo retry = {
.num_retries = 5,
@@ -385,8 +403,8 @@ void add_remove_callback(ObjectID object_id, bool success, void *user_context) {
TEST add_remove_lookup_test(void) {
g_loop = event_loop_create();
lookup_retry_succeeded = 0;
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, true);
RetryInfo retry = {
.num_retries = 5,
@@ -407,29 +425,6 @@ TEST add_remove_lookup_test(void) {
PASS();
}
/* === Test subscribe retry === */
const char *subscribe_retry_context = "subscribe_retry";
int subscribe_retry_succeeded = 0;
int64_t reconnect_sub_context_callback(event_loop *loop,
int64_t timer_id,
void *context) {
DBHandle *db = (DBHandle *) context;
/* Reconnect to redis. This is not reconnecting the pub/sub channel. */
redisAsyncFree(db->sub_context);
redisAsyncFree(db->context);
redisFree(db->sync_context);
db->sub_context = redisAsyncConnect("127.0.0.1", 6379);
db->sub_context->data = (void *) db;
db->context = redisAsyncConnect("127.0.0.1", 6379);
db->context->data = (void *) db;
db->sync_context = redisConnect("127.0.0.1", 6379);
/* Re-attach the database to the event loop (the file descriptor changed). */
db_attach(db, loop, true);
return EVENT_LOOP_TIMER_DONE;
}
/* ==== Test if late succeed is working correctly ==== */
/* === Test lookup late succeed === */
@@ -454,8 +449,8 @@ void lookup_late_done_callback(ObjectID object_id,
TEST lookup_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0,
@@ -498,8 +493,8 @@ void add_late_done_callback(ObjectID object_id,
TEST add_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0, .timeout = 0, .fail_callback = add_late_fail_callback,
@@ -543,8 +538,8 @@ void subscribe_late_done_callback(ObjectID object_id,
TEST subscribe_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0,
@@ -611,8 +606,8 @@ TEST subscribe_success_test(void) {
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11236"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, false);
subscribe_id = globally_unique_id();
@@ -680,8 +675,8 @@ TEST subscribe_object_present_test(void) {
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11236"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, false);
UniqueID id = globally_unique_id();
RetryInfo retry = {
@@ -732,8 +727,8 @@ void subscribe_object_not_present_object_available_callback(
TEST subscribe_object_not_present_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
UniqueID id = globally_unique_id();
RetryInfo retry = {
@@ -796,8 +791,8 @@ TEST subscribe_object_available_later_test(void) {
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11236"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, false);
UniqueID id = globally_unique_id();
RetryInfo retry = {
@@ -849,8 +844,8 @@ TEST subscribe_object_available_subscribe_all(void) {
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11236"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, false);
UniqueID id = globally_unique_id();
RetryInfo retry = {
@@ -904,7 +899,6 @@ SUITE(object_table_tests) {
RUN_REDIS_TEST(add_late_test);
RUN_REDIS_TEST(subscribe_late_test);
RUN_REDIS_TEST(subscribe_success_test);
RUN_REDIS_TEST(subscribe_object_present_test);
RUN_REDIS_TEST(subscribe_object_not_present_test);
RUN_REDIS_TEST(subscribe_object_available_later_test);
RUN_REDIS_TEST(subscribe_object_available_subscribe_all);
src/common/test/redis_tests.cc
+4 -4
View File
@@ -102,8 +102,8 @@ TEST async_redis_socket_test(void) {
utarray_push_back(connections, &socket_fd);
/* Start connection to Redis. */
DBHandle *db =
db_connect("127.0.0.1", 6379, "test_process", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "test_process",
"127.0.0.1", 0, NULL);
db_attach(db, loop, false);
/* Send a command to the Redis process. */
@@ -177,8 +177,8 @@ TEST logging_test(void) {
utarray_push_back(connections, &socket_fd);
/* Start connection to Redis. */
DBHandle *conn =
db_connect("127.0.0.1", 6379, "test_process", "127.0.0.1", 0, NULL);
DBHandle *conn = db_connect(std::string("127.0.0.1"), 6379, "test_process",
"127.0.0.1", 0, NULL);
db_attach(conn, loop, false);
/* Send a command to the Redis process. */
src/common/test/run_tests.sh
+9 -2
View File
@@ -5,8 +5,14 @@
# Cause the script to exit if a single command fails.
set -e
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
# Start the Redis shards.
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
sleep 1s
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
./src/common/common_tests
./src/common/db_tests
./src/common/io_tests
@@ -14,4 +20,5 @@ sleep 1s
./src/common/redis_tests
./src/common/task_table_tests
./src/common/object_table_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6379 shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
src/common/test/run_valgrind.sh
+9 -1
View File
@@ -5,8 +5,14 @@
# Cause the script to exit if a single command fails.
set -e
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
# Start the Redis shards.
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
sleep 1s
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
valgrind --leak-check=full --error-exitcode=1 ./src/common/common_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/db_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/io_tests
@@ -14,4 +20,6 @@ valgrind --leak-check=full --error-exitcode=1 ./src/common/task_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/redis_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/task_table_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/object_table_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
src/common/test/task_table_tests.cc
+47 -22
View File
@@ -40,8 +40,8 @@ void lookup_nil_success_callback(Task *task, void *context) {
TEST lookup_nil_test(void) {
lookup_nil_id = globally_unique_id();
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -96,14 +96,16 @@ void add_success_callback(TaskID task_id, void *context) {
TEST add_lookup_test(void) {
add_lookup_task = example_task(1, 1, TASK_STATUS_WAITING);
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 1000,
.fail_callback = add_lookup_fail_callback,
};
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, &retry,
NULL, NULL);
task_table_add_task(db, Task_copy(add_lookup_task), &retry,
add_success_callback, (void *) db);
/* Disconnect the database to see if the lookup times out. */
@@ -136,8 +138,8 @@ void subscribe_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST subscribe_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -148,7 +150,10 @@ TEST subscribe_timeout_test(void) {
subscribe_done_callback,
(void *) subscribe_timeout_context);
/* Disconnect the database to see if the subscribe times out. */
close(db->sub_context->c.fd);
close(db->subscribe_context->c.fd);
for (int i = 0; i < db->subscribe_contexts.size(); ++i) {
close(db->subscribe_contexts[i]->c.fd);
}
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
@@ -177,17 +182,22 @@ void publish_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST publish_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
Task *task = example_task(1, 1, TASK_STATUS_WAITING);
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = publish_fail_callback,
};
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, &retry,
NULL, NULL);
task_table_add_task(db, task, &retry, publish_done_callback,
(void *) publish_timeout_context);
/* Disconnect the database to see if the publish times out. */
close(db->context->c.fd);
for (int i = 0; i < db->contexts.size(); ++i) {
close(db->contexts[i]->c.fd);
}
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
@@ -204,9 +214,14 @@ int64_t reconnect_db_callback(event_loop *loop,
void *context) {
DBHandle *db = (DBHandle *) context;
/* Reconnect to redis. */
redisAsyncFree(db->sub_context);
db->sub_context = redisAsyncConnect("127.0.0.1", 6379);
db->sub_context->data = (void *) db;
redisAsyncFree(db->subscribe_context);
db->subscribe_context = redisAsyncConnect("127.0.0.1", 6379);
db->subscribe_context->data = (void *) db;
for (int i = 0; i < db->subscribe_contexts.size(); ++i) {
redisAsyncFree(db->subscribe_contexts[i]);
db->subscribe_contexts[i] = redisAsyncConnect("127.0.0.1", 6380 + i);
db->subscribe_contexts[i]->data = (void *) db;
}
/* Re-attach the database to the event loop (the file descriptor changed). */
db_attach(db, loop, true);
return EVENT_LOOP_TIMER_DONE;
@@ -239,8 +254,8 @@ void subscribe_retry_fail_callback(UniqueID id,
TEST subscribe_retry_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -251,7 +266,10 @@ TEST subscribe_retry_test(void) {
subscribe_retry_done_callback,
(void *) subscribe_retry_context);
/* Disconnect the database to see if the subscribe times out. */
close(db->sub_context->c.fd);
close(db->subscribe_context->c.fd);
for (int i = 0; i < db->subscribe_contexts.size(); ++i) {
close(db->subscribe_contexts[i]->c.fd);
}
/* Install handler for reconnecting the database. */
event_loop_add_timer(g_loop, 150,
(event_loop_timer_handler) reconnect_db_callback, db);
@@ -286,8 +304,8 @@ void publish_retry_fail_callback(UniqueID id,
TEST publish_retry_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
Task *task = example_task(1, 1, TASK_STATUS_WAITING);
RetryInfo retry = {
@@ -295,10 +313,15 @@ TEST publish_retry_test(void) {
.timeout = 100,
.fail_callback = publish_retry_fail_callback,
};
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, &retry,
NULL, NULL);
task_table_add_task(db, task, &retry, publish_retry_done_callback,
(void *) publish_retry_context);
/* Disconnect the database to see if the publish times out. */
close(db->sub_context->c.fd);
close(db->subscribe_context->c.fd);
for (int i = 0; i < db->subscribe_contexts.size(); ++i) {
close(db->subscribe_contexts[i]->c.fd);
}
/* Install handler for reconnecting the database. */
event_loop_add_timer(g_loop, 150,
(event_loop_timer_handler) reconnect_db_callback, db);
@@ -335,8 +358,8 @@ void subscribe_late_done_callback(TaskID task_id, void *user_context) {
TEST subscribe_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0,
@@ -380,8 +403,8 @@ void publish_late_done_callback(TaskID task_id, void *user_context) {
TEST publish_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
Task *task = example_task(1, 1, TASK_STATUS_WAITING);
RetryInfo retry = {
@@ -389,6 +412,8 @@ TEST publish_late_test(void) {
.timeout = 0,
.fail_callback = publish_late_fail_callback,
};
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, NULL, NULL,
NULL);
task_table_add_task(db, task, &retry, publish_late_done_callback,
(void *) publish_late_context);
/* Install handler for terminating the event loop. */
src/common/test/test_common.h
+22 -1
View File
@@ -2,11 +2,13 @@
#define TEST_COMMON_H
#include <unistd.h>
#include <vector>
#include "common.h"
#include "io.h"
#include "hiredis/hiredis.h"
#include "utstring.h"
#include "state/redis.h"
#ifndef _WIN32
/* This function is actually not declared in standard POSIX, so declare it. */
@@ -48,10 +50,29 @@ static inline int bind_inet_sock_retry(int *fd) {
}
/* Flush redis. */
static inline void flushall_redis() {
static inline void flushall_redis(void) {
/* Flush the primary shard. */
redisContext *context = redisConnect("127.0.0.1", 6379);
std::vector<std::string> db_shards_addresses;
std::vector<int> db_shards_ports;
get_redis_shards(context, db_shards_addresses, db_shards_ports);
freeReplyObject(redisCommand(context, "FLUSHALL"));
/* Readd the shard locations. */
freeReplyObject(redisCommand(context, "SET NumRedisShards %d",
db_shards_addresses.size()));
for (int i = 0; i < db_shards_addresses.size(); ++i) {
freeReplyObject(redisCommand(context, "RPUSH RedisShards %s:%d",
db_shards_addresses[i].c_str(),
db_shards_ports[i]));
}
redisFree(context);
/* Flush the remaining shards. */
for (int i = 0; i < db_shards_addresses.size(); ++i) {
context = redisConnect(db_shards_addresses[i].c_str(), db_shards_ports[i]);
freeReplyObject(redisCommand(context, "FLUSHALL"));
redisFree(context);
}
}
/* Cleanup method for running tests with the greatest library.
src/global_scheduler/global_scheduler.cc
+40 -30
View File
@@ -62,14 +62,14 @@ void assign_task_to_local_scheduler(GlobalSchedulerState *state,
GlobalSchedulerState *GlobalSchedulerState_init(event_loop *loop,
const char *node_ip_address,
const char *redis_addr,
int redis_port) {
const char *redis_primary_addr,
int redis_primary_port) {
GlobalSchedulerState *state =
(GlobalSchedulerState *) malloc(sizeof(GlobalSchedulerState));
/* Must initialize state to 0. Sets hashmap head(s) to NULL. */
memset(state, 0, sizeof(GlobalSchedulerState));
state->db = db_connect(redis_addr, redis_port, "global_scheduler",
node_ip_address, 0, NULL);
state->db = db_connect(std::string(redis_primary_addr), redis_primary_port,
"global_scheduler", node_ip_address, 0, NULL);
db_attach(state->db, loop, false);
utarray_new(state->local_schedulers, &local_scheduler_icd);
state->policy_state = GlobalSchedulerPolicyState_init();
@@ -253,26 +253,34 @@ void remove_local_scheduler(GlobalSchedulerState *state, int index) {
* @param aux_address: an ip:port pair for the plasma manager associated with
* this db client.
*/
void process_new_db_client(DBClientID db_client_id,
const char *client_type,
const char *aux_address,
bool is_insertion,
void *user_context) {
void process_new_db_client(DBClient *db_client, void *user_context) {
GlobalSchedulerState *state = (GlobalSchedulerState *) user_context;
char id_string[ID_STRING_SIZE];
LOG_DEBUG("db client table callback for db client = %s",
ObjectID_to_string(db_client_id, id_string, ID_STRING_SIZE));
ObjectID_to_string(db_client->id, id_string, ID_STRING_SIZE));
UNUSED(id_string);
if (strncmp(client_type, "local_scheduler", strlen("local_scheduler")) == 0) {
if (is_insertion) {
/* This is a notification for an insert. */
add_local_scheduler(state, db_client_id, aux_address);
if (strncmp(db_client->client_type, "local_scheduler",
strlen("local_scheduler")) == 0) {
if (db_client->is_insertion) {
/* This is a notification for an insert. We may receive duplicate
* notifications since we read the entire table before processing
* notifications. Filter out local schedulers that we already added. */
for (LocalScheduler *scheduler =
(LocalScheduler *) utarray_front(state->local_schedulers);
scheduler != NULL; scheduler = (LocalScheduler *) utarray_next(
state->local_schedulers, scheduler)) {
if (UNIQUE_ID_EQ(scheduler->id, db_client->id)) {
return;
}
}
add_local_scheduler(state, db_client->id, db_client->aux_address);
} else {
int i = 0;
for (; i < utarray_len(state->local_schedulers); ++i) {
LocalScheduler *active_worker =
(LocalScheduler *) utarray_eltptr(state->local_schedulers, i);
if (DBClientID_equal(active_worker->id, db_client_id)) {
if (DBClientID_equal(active_worker->id, db_client->id)) {
break;
}
}
@@ -418,11 +426,11 @@ int heartbeat_timeout_handler(event_loop *loop, timer_id id, void *context) {
}
void start_server(const char *node_ip_address,
const char *redis_addr,
int redis_port) {
const char *redis_primary_addr,
int redis_primary_port) {
event_loop *loop = event_loop_create();
g_state =
GlobalSchedulerState_init(loop, node_ip_address, redis_addr, redis_port);
g_state = GlobalSchedulerState_init(loop, node_ip_address, redis_primary_addr,
redis_primary_port);
/* TODO(rkn): subscribe to notifications from the object table. */
/* Subscribe to notifications about new local schedulers. TODO(rkn): this
* needs to also get all of the clients that registered with the database
@@ -458,15 +466,15 @@ void start_server(const char *node_ip_address,
int main(int argc, char *argv[]) {
signal(SIGTERM, signal_handler);
/* IP address and port of redis. */
char *redis_addr_port = NULL;
/* IP address and port of the primary redis instance. */
char *redis_primary_addr_port = NULL;
/* The IP address of the node that this global scheduler is running on. */
char *node_ip_address = NULL;
int c;
while ((c = getopt(argc, argv, "h:r:")) != -1) {
switch (c) {
case 'r':
redis_addr_port = optarg;
redis_primary_addr_port = optarg;
break;
case 'h':
node_ip_address = optarg;
@@ -476,16 +484,18 @@ int main(int argc, char *argv[]) {
exit(-1);
}
}
char redis_addr[16];
int redis_port;
if (!redis_addr_port ||
parse_ip_addr_port(redis_addr_port, redis_addr, &redis_port) == -1) {
LOG_ERROR(
"specify the redis address like 127.0.0.1:6379 with the -r switch");
exit(-1);
char redis_primary_addr[16];
int redis_primary_port;
if (!redis_primary_addr_port ||
parse_ip_addr_port(redis_primary_addr_port, redis_primary_addr,
&redis_primary_port) == -1) {
LOG_FATAL(
"specify the primary redis address like 127.0.0.1:6379 with the -r "
"switch");
}
if (!node_ip_address) {
LOG_FATAL("specify the node IP address with the -h switch");
}
start_server(node_ip_address, redis_addr, redis_port);
start_server(node_ip_address, redis_primary_addr, redis_primary_port);
}
src/local_scheduler/local_scheduler.cc
+22 -28
View File
@@ -16,6 +16,7 @@
#include "local_scheduler_shared.h"
#include "local_scheduler.h"
#include "local_scheduler_algorithm.h"
#include "net.h"
#include "state/actor_notification_table.h"
#include "state/db.h"
#include "state/driver_table.h"
@@ -296,8 +297,8 @@ const char **parse_command(const char *command) {
LocalSchedulerState *LocalSchedulerState_init(
const char *node_ip_address,
event_loop *loop,
const char *redis_addr,
int redis_port,
const char *redis_primary_addr,
int redis_primary_port,
const char *local_scheduler_socket_name,
const char *plasma_store_socket_name,
const char *plasma_manager_socket_name,
@@ -323,7 +324,7 @@ LocalSchedulerState *LocalSchedulerState_init(
state->loop = loop;
/* Connect to Redis if a Redis address is provided. */
if (redis_addr != NULL) {
if (redis_primary_addr != NULL) {
int num_args;
const char **db_connect_args = NULL;
/* Use UT_string to convert the resource value into a string. */
@@ -354,8 +355,9 @@ LocalSchedulerState *LocalSchedulerState_init(
db_connect_args[4] = "num_gpus";
db_connect_args[5] = utstring_body(num_gpus);
}
state->db = db_connect(redis_addr, redis_port, "local_scheduler",
node_ip_address, num_args, db_connect_args);
state->db = db_connect(std::string(redis_primary_addr), redis_primary_port,
"local_scheduler", node_ip_address, num_args,
db_connect_args);
utstring_free(num_cpus);
utstring_free(num_gpus);
free(db_connect_args);
@@ -548,8 +550,6 @@ void process_plasma_notification(event_loop *loop,
void reconstruct_task_update_callback(Task *task,
void *user_context,
bool updated) {
/* The task ID should be in the task table. */
CHECK(task != NULL);
if (!updated) {
/* The test-and-set of the task's scheduling state failed, so the task was
* either not finished yet, or it was already being reconstructed.
@@ -578,7 +578,6 @@ void reconstruct_task_update_callback(Task *task,
void reconstruct_put_task_update_callback(Task *task,
void *user_context,
bool updated) {
CHECK(task != NULL);
if (updated) {
/* The update to TASK_STATUS_RECONSTRUCTING succeeded, so continue with
* reconstruction as usual. */
@@ -1111,8 +1110,8 @@ int heartbeat_handler(event_loop *loop, timer_id id, void *context) {
void start_server(const char *node_ip_address,
const char *socket_name,
const char *redis_addr,
int redis_port,
const char *redis_primary_addr,
int redis_primary_port,
const char *plasma_store_socket_name,
const char *plasma_manager_socket_name,
const char *plasma_manager_address,
@@ -1126,8 +1125,8 @@ void start_server(const char *node_ip_address,
int fd = bind_ipc_sock(socket_name, true);
event_loop *loop = event_loop_create();
g_state = LocalSchedulerState_init(
node_ip_address, loop, redis_addr, redis_port, socket_name,
plasma_store_socket_name, plasma_manager_socket_name,
node_ip_address, loop, redis_primary_addr, redis_primary_port,
socket_name, plasma_store_socket_name, plasma_manager_socket_name,
plasma_manager_address, global_scheduler_exists, static_resource_conf,
start_worker_command, num_workers);
/* Register a callback for registering new clients. */
@@ -1173,8 +1172,8 @@ int main(int argc, char *argv[]) {
signal(SIGTERM, signal_handler);
/* Path of the listening socket of the local scheduler. */
char *scheduler_socket_name = NULL;
/* IP address and port of redis. */
char *redis_addr_port = NULL;
/* IP address and port of the primary redis instance. */
char *redis_primary_addr_port = NULL;
/* Socket name for the local Plasma store. */
char *plasma_store_socket_name = NULL;
/* Socket name for the local Plasma manager. */
@@ -1199,7 +1198,7 @@ int main(int argc, char *argv[]) {
scheduler_socket_name = optarg;
break;
case 'r':
redis_addr_port = optarg;
redis_primary_addr_port = optarg;
break;
case 'p':
plasma_store_socket_name = optarg;
@@ -1266,7 +1265,7 @@ int main(int argc, char *argv[]) {
char *redis_addr = NULL;
int redis_port = -1;
if (!redis_addr_port) {
if (!redis_primary_addr_port) {
/* Start the local scheduler without connecting to Redis. In this case, all
* submitted tasks will be queued and scheduled locally. */
if (plasma_manager_socket_name) {
@@ -1275,27 +1274,22 @@ int main(int argc, char *argv[]) {
"then a redis address must be provided with the -r switch");
}
} else {
char redis_addr_buffer[16] = {0};
char redis_port_str[6] = {0};
/* Parse the Redis address into an IP address and a port. */
int num_assigned = sscanf(redis_addr_port, "%15[0-9.]:%5[0-9]",
redis_addr_buffer, redis_port_str);
if (num_assigned != 2) {
char redis_primary_addr[16];
int redis_primary_port;
/* Parse the primary Redis address into an IP address and a port. */
if (parse_ip_addr_port(redis_primary_addr_port, redis_primary_addr,
&redis_primary_port) == -1) {
LOG_FATAL(
"if a redis address is provided with the -r switch, it should be "
"formatted like 127.0.0.1:6379");
}
redis_addr = redis_addr_buffer;
redis_port = strtol(redis_port_str, NULL, 10);
if (redis_port == 0) {
LOG_FATAL("Unable to parse port number from redis address %s",
redis_addr_port);
}
if (!plasma_manager_socket_name) {
LOG_FATAL(
"please specify socket for connecting to Plasma manager with -m "
"switch");
}
redis_addr = redis_primary_addr;
redis_port = redis_primary_port;
}
start_server(node_ip_address, scheduler_socket_name, redis_addr, redis_port,
src/local_scheduler/local_scheduler_algorithm.cc
+2 -1
View File
@@ -839,7 +839,8 @@ void handle_actor_task_submitted(LocalSchedulerState *state,
/* Add this task to a queue of tasks that have been submitted but the local
* scheduler doesn't know which actor is responsible for them. These tasks
* will be resubmitted (internally by the local scheduler) whenever a new
* actor notification arrives. */
* actor notification arrives. NOTE(swang): These tasks have not yet been
* added to the task table. */
utarray_push_back(algorithm_state->cached_submitted_actor_tasks, &spec);
utarray_push_back(algorithm_state->cached_submitted_actor_task_sizes,
&task_spec_size);
src/local_scheduler/test/local_scheduler_tests.cc
+21 -2
View File
@@ -18,6 +18,7 @@
#include "task.h"
#include "state/object_table.h"
#include "state/task_table.h"
#include "state/redis.h"
#include "local_scheduler_shared.h"
#include "local_scheduler.h"
@@ -182,7 +183,16 @@ TEST object_reconstruction_test(void) {
/* Add an empty object table entry for the object we want to reconstruct, to
* simulate it having been created and evicted. */
const char *client_id = "clientid";
/* Lookup the shard locations for the object table. */
std::vector<std::string> db_shards_addresses;
std::vector<int> db_shards_ports;
redisContext *context = redisConnect("127.0.0.1", 6379);
get_redis_shards(context, db_shards_addresses, db_shards_ports);
redisFree(context);
/* There should only be one shard, so we can safely add the empty object
* table entry to the first one. */
ASSERT(db_shards_addresses.size() == 1);
context = redisConnect(db_shards_addresses[0].c_str(), db_shards_ports[0]);
redisReply *reply = (redisReply *) redisCommand(
context, "RAY.OBJECT_TABLE_ADD %b %ld %b %s", return_id.id,
sizeof(return_id.id), 1, NIL_DIGEST, (size_t) DIGEST_SIZE, client_id);
@@ -273,7 +283,16 @@ TEST object_reconstruction_recursive_test(void) {
/* Add an empty object table entry for each object we want to reconstruct, to
* simulate their having been created and evicted. */
const char *client_id = "clientid";
/* Lookup the shard locations for the object table. */
std::vector<std::string> db_shards_addresses;
std::vector<int> db_shards_ports;
redisContext *context = redisConnect("127.0.0.1", 6379);
get_redis_shards(context, db_shards_addresses, db_shards_ports);
redisFree(context);
/* There should only be one shard, so we can safely add the empty object
* table entry to the first one. */
ASSERT(db_shards_addresses.size() == 1);
context = redisConnect(db_shards_addresses[0].c_str(), db_shards_ports[0]);
for (int i = 0; i < NUM_TASKS; ++i) {
ObjectID return_id = TaskSpec_return(specs[i], 0);
redisReply *reply = (redisReply *) redisCommand(
@@ -406,8 +425,8 @@ TEST object_reconstruction_suppression_test(void) {
} else {
/* Connect a plasma manager client so we can call object_table_add. */
const char *db_connect_args[] = {"address", "127.0.0.1:12346"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1",
2, db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, local_scheduler->loop, false);
/* Add the object to the object table. */
object_table_add(db, return_id, 1, (unsigned char *) NIL_DIGEST, NULL,
src/local_scheduler/test/run_tests.sh
+8 -1
View File
@@ -5,10 +5,17 @@
# Cause the script to exit if a single command fails.
set -e
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
# Start the Redis shards.
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
sleep 1s
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 &
sleep 0.5s
./src/local_scheduler/local_scheduler_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
killall plasma_store
src/local_scheduler/test/run_valgrind.sh
+8 -1
View File
@@ -5,10 +5,17 @@
# Cause the script to exit if a single command fails.
set -e
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
# Start the Redis shards.
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
sleep 1s
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 &
sleep 0.5s
valgrind --leak-check=full --show-leak-kinds=all --error-exitcode=1 ./src/local_scheduler/local_scheduler_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
killall plasma_store
src/plasma/plasma_manager.cc
+24 -22
View File
@@ -493,8 +493,8 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name,
const char *manager_socket_name,
const char *manager_addr,
int manager_port,
const char *db_addr,
int db_port) {
const char *redis_primary_addr,
int redis_primary_port) {
PlasmaManagerState *state =
(PlasmaManagerState *) malloc(sizeof(PlasmaManagerState));
state->loop = event_loop_create();
@@ -504,7 +504,7 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name,
state->fetch_requests = NULL;
state->object_wait_requests_local = NULL;
state->object_wait_requests_remote = NULL;
if (db_addr) {
if (redis_primary_addr) {
/* Get the manager port as a string. */
UT_string *manager_address_str;
utstring_new(manager_address_str);
@@ -519,8 +519,9 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name,
db_connect_args[3] = manager_socket_name;
db_connect_args[4] = "address";
db_connect_args[5] = utstring_body(manager_address_str);
state->db = db_connect(db_addr, db_port, "plasma_manager", manager_addr,
num_args, db_connect_args);
state->db =
db_connect(std::string(redis_primary_addr), redis_primary_port,
"plasma_manager", manager_addr, num_args, db_connect_args);
utstring_free(manager_address_str);
free(db_connect_args);
db_attach(state->db, state->loop, false);
@@ -1594,8 +1595,8 @@ void start_server(const char *store_socket_name,
const char *manager_socket_name,
const char *master_addr,
int port,
const char *db_addr,
int db_port) {
const char *redis_primary_addr,
int redis_primary_port) {
/* Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
* to a client that has already died, the manager could die. */
signal(SIGPIPE, SIG_IGN);
@@ -1610,9 +1611,9 @@ void start_server(const char *store_socket_name,
int local_sock = bind_ipc_sock(manager_socket_name, false);
CHECKM(local_sock >= 0, "Unable to bind local manager socket");
g_manager_state =
PlasmaManagerState_init(store_socket_name, manager_socket_name,
master_addr, port, db_addr, db_port);
g_manager_state = PlasmaManagerState_init(
store_socket_name, manager_socket_name, master_addr, port,
redis_primary_addr, redis_primary_port);
CHECK(g_manager_state);
CHECK(listen(remote_sock, 5) != -1);
@@ -1664,8 +1665,8 @@ int main(int argc, char *argv[]) {
char *master_addr = NULL;
/* Port number the manager should use. */
int port = -1;
/* IP address and port of state database. */
char *db_host = NULL;
/* IP address and port of the primary redis instance. */
char *redis_primary_addr_port = NULL;
int c;
while ((c = getopt(argc, argv, "s:m:h:p:r:")) != -1) {
switch (c) {
@@ -1682,7 +1683,7 @@ int main(int argc, char *argv[]) {
port = atoi(optarg);
break;
case 'r':
db_host = optarg;
redis_primary_addr_port = optarg;
break;
default:
LOG_FATAL("unknown option %c", c);
@@ -1708,15 +1709,16 @@ int main(int argc, char *argv[]) {
"please specify port the plasma manager shall listen to in the"
"format 12345 with -p switch");
}
char db_addr[16];
int db_port;
if (db_host) {
parse_ip_addr_port(db_host, db_addr, &db_port);
start_server(store_socket_name, manager_socket_name, master_addr, port,
db_addr, db_port);
} else {
start_server(store_socket_name, manager_socket_name, master_addr, port,
NULL, 0);
char redis_primary_addr[16];
int redis_primary_port;
if (!redis_primary_addr_port ||
parse_ip_addr_port(redis_primary_addr_port, redis_primary_addr,
&redis_primary_port) == -1) {
LOG_FATAL(
"specify the primary redis address like 127.0.0.1:6379 with the -r "
"switch");
}
start_server(store_socket_name, manager_socket_name, master_addr, port,
redis_primary_addr, redis_primary_port);
}
#endif
src/plasma/test/run_tests.sh
+15 -6
View File
@@ -9,11 +9,18 @@ sleep 1
killall plasma_store
./src/plasma/serialization_tests
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
redis_pid=$!
# Start the Redis shards.
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
redis_pid1=$!
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
redis_pid2=$!
sleep 1
# flush the redis server
./src/common/thirdparty/redis/src/redis-cli flushall &
# Flush the redis server
./src/common/thirdparty/redis/src/redis-cli flushall
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
sleep 1
./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 &
plasma1_pid=$!
@@ -31,5 +38,7 @@ kill $plasma4_pid
kill $plasma3_pid
kill $plasma2_pid
kill $plasma1_pid
kill $redis_pid
wait $redis_pid
kill $redis_pid1
wait $redis_pid1
kill $redis_pid2
wait $redis_pid2