wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-11 12:43:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Deploy Redis module and start using custom Redis commands. (#128)

Browse Source 
* Add RAY.CONNECT Redis command.

* Add RAY.GET_CLIENT_ADDRESS command.

* Build and clean Redis in common Makefile.

* Use custom Redis module in Ray and use custom CONNECT and GET_CLIENT_ADDRESS commands.

* Fixes.

* Remove mapping from redis client ID to ray db client ID.

* Fix.
This commit is contained in:
Robert Nishihara
2016-12-16 14:40:44 -08:00
parent 1c95840765
commit 58a873eb20
11 changed files with 168 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/Makefile
+9 -5
View File
@@ -2,7 +2,7 @@ CC = gcc
CFLAGS = -g -Wall -Wextra -Werror=implicit-function-declaration -Wno-typedef-redefinition -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -fPIC -I. -Ithirdparty -Ithirdparty/ae
BUILD = build
all: hiredis redis $(BUILD)/libcommon.a
all: hiredis redis redismodule $(BUILD)/libcommon.a
$(BUILD)/libcommon.a: event_loop.o common.o task.o io.o net.o state/redis.o state/table.o state/object_table.o state/task_table.o state/db_client_table.o thirdparty/ae/ae.o thirdparty/sha256.o
ar rcs $@ $^
@@ -31,6 +31,7 @@ $(BUILD)/redis_tests: hiredis test/redis_tests.c $(BUILD)/libcommon.a logging.h
clean:
rm -f *.o state/*.o test/*.o thirdparty/ae/*.o
rm -rf $(BUILD)/*
cd redis_module; make clean
redis:
cd thirdparty ; bash ./build-redis.sh
@@ -38,9 +39,12 @@ redis:
hiredis:
cd thirdparty/hiredis ; make
redismodule:
cd redis_module && make && cd ..
test: CFLAGS += -DRAY_COMMON_LOG_LEVEL=4
test: hiredis redis $(BUILD)/common_tests $(BUILD)/task_table_tests $(BUILD)/object_table_tests $(BUILD)/db_tests $(BUILD)/io_tests $(BUILD)/task_tests $(BUILD)/redis_tests FORCE
./thirdparty/redis/src/redis-server &
test: hiredis redis redismodule $(BUILD)/common_tests $(BUILD)/task_table_tests $(BUILD)/object_table_tests $(BUILD)/db_tests $(BUILD)/io_tests $(BUILD)/task_tests $(BUILD)/redis_tests FORCE
./thirdparty/redis/src/redis-server --loadmodule ./redis_module/ray_redis_module.so &
sleep 1s
./build/common_tests
./build/db_tests
@@ -50,10 +54,10 @@ test: hiredis redis $(BUILD)/common_tests $(BUILD)/task_table_tests $(BUILD)/obj
./build/task_table_tests
./build/object_table_tests
./thirdparty/redis/src/redis-cli shutdown
cd redis_module && make && sleep 1 && python runtest.py && cd ..
python ./redis_module/runtest.py
valgrind: test
./thirdparty/redis/src/redis-server &
./thirdparty/redis/src/redis-server --loadmodule redis_module/ray_redis_module.so &
sleep 1s
valgrind --leak-check=full --error-exitcode=1 ./build/common_tests
valgrind --leak-check=full --error-exitcode=1 ./build/db_tests
src/common/redis_module/ray_redis_module.c
+126
View File
@@ -19,6 +19,7 @@
* TODO(pcm): Fill this out.
*/
#define DB_CLIENT_PREFIX "CL:"
#define OBJECT_INFO_PREFIX "OI:"
#define OBJECT_LOCATION_PREFIX "OL:"
#define OBJECT_SUBSCRIBE_PREFIX "OS:"
@@ -41,6 +42,117 @@ RedisModuleKey *OpenPrefixedKey(RedisModuleCtx *ctx,
return key;
}
/**
* Register a client with Redis. This is called from a client with the command:
*
* RAY.CONNECT <client type> <address> <ray client id> <aux address>
*
* @param client_type The type of the client (e.g., plasma_manager).
* @param address The address of the client.
* @param ray_client_id The db client ID of the client.
* @param aux_address An auxiliary address. This is currently just used by the
* local scheduler to record the address of the plasma manager that it is
* connected to.
* @return OK if the operation was successful.
*/
int Connect_RedisCommand(RedisModuleCtx *ctx,
RedisModuleString **argv,
int argc) {
if (argc != 5) {
return RedisModule_WrongArity(ctx);
}
RedisModuleString *client_type = argv[1];
RedisModuleString *address = argv[2];
RedisModuleString *ray_client_id = argv[3];
RedisModuleString *aux_address = argv[4];
/* Add this client to the Ray db client table. */
RedisModuleKey *db_client_table_key =
OpenPrefixedKey(ctx, DB_CLIENT_PREFIX, ray_client_id, REDISMODULE_WRITE);
RedisModule_HashSet(db_client_table_key, REDISMODULE_HASH_CFIELDS,
"client_type", client_type, "address", address,
"aux_address", aux_address, NULL);
/* Clean up. */
RedisModule_CloseKey(db_client_table_key);
/* Construct strings to publish on the db client channel. */
RedisModuleString *channel_name =
RedisModule_CreateString(ctx, "db_clients", strlen("db_clients"));
RedisModuleString *client_info =
RedisModule_CreateStringFromString(ctx, ray_client_id);
RedisModule_StringAppendBuffer(ctx, client_info, ":", strlen(":"));
/* Append the client type. */
size_t client_type_size;
const char *client_type_str =
RedisModule_StringPtrLen(client_type, &client_type_size);
RedisModule_StringAppendBuffer(ctx, client_info, client_type_str,
client_type_size);
/* Append a space. */
RedisModule_StringAppendBuffer(ctx, client_info, " ", strlen(" "));
/* Append the aux address. */
size_t aux_address_size;
const char *aux_address_str =
RedisModule_StringPtrLen(aux_address, &aux_address_size);
RedisModule_StringAppendBuffer(ctx, client_info, aux_address_str,
aux_address_size);
/* Publish the client info on the db client channel. */
RedisModuleCallReply *reply;
reply = RedisModule_Call(ctx, "PUBLISH", "ss", channel_name, client_info);
RedisModule_FreeString(ctx, channel_name);
RedisModule_FreeString(ctx, client_info);
if (reply == NULL) {
return RedisModule_ReplyWithError(ctx, "PUBLISH unsuccessful");
}
RedisModule_ReplyWithSimpleString(ctx, "OK");
return REDISMODULE_OK;
}
/**
* Get the address of a client from its db client ID. This is called from a
* client with the command:
*
* RAY.GET_CLIENT_ADDRESS <ray client id>
*
* @param ray_client_id The db client ID of the client.
* @return The address of the client if the operation was successful.
*/
int GetClientAddress_RedisCommand(RedisModuleCtx *ctx,
RedisModuleString **argv,
int argc) {
if (argc != 2) {
return RedisModule_WrongArity(ctx);
}
RedisModuleString *ray_client_id = argv[1];
/* Get the request client address from the db client table. */
RedisModuleKey *db_client_table_key =
OpenPrefixedKey(ctx, DB_CLIENT_PREFIX, ray_client_id, REDISMODULE_READ);
if (db_client_table_key == NULL) {
/* There is no client with this ID. */
RedisModule_CloseKey(db_client_table_key);
return RedisModule_ReplyWithError(ctx, "invalid client ID");
}
RedisModuleString *address;
RedisModule_HashGet(db_client_table_key, REDISMODULE_HASH_CFIELDS, "address",
&address, NULL);
if (address == NULL) {
/* The key did not exist. This should not happen. */
RedisModule_CloseKey(db_client_table_key);
return RedisModule_ReplyWithError(
ctx, "Client does not have an address field. This shouldn't happen.");
}
RedisModule_ReplyWithString(ctx, address);
/* Cleanup. */
RedisModule_CloseKey(db_client_table_key);
RedisModule_FreeString(ctx, address);
return REDISMODULE_OK;
}
/**
* Lookup an entry in the object table.
*
@@ -81,6 +193,9 @@ int ObjectTableLookup_RedisCommand(RedisModuleCtx *ctx,
} while (RedisModule_ZsetRangeNext(key));
RedisModule_ReplySetArrayLength(ctx, num_results);
/* Clean up. */
RedisModule_CloseKey(key);
return REDISMODULE_OK;
}
@@ -285,6 +400,17 @@ int RedisModule_OnLoad(RedisModuleCtx *ctx,
return REDISMODULE_ERR;
}
if (RedisModule_CreateCommand(ctx, "ray.connect", Connect_RedisCommand,
"write", 0, 0, 0) == REDISMODULE_ERR) {
return REDISMODULE_ERR;
}
if (RedisModule_CreateCommand(ctx, "ray.get_client_address",
GetClientAddress_RedisCommand, "write", 0, 0,
0) == REDISMODULE_ERR) {
return REDISMODULE_ERR;
}
if (RedisModule_CreateCommand(ctx, "ray.object_table_lookup",
ObjectTableLookup_RedisCommand, "readonly", 0,
0, 0) == REDISMODULE_ERR) {
src/common/state/redis.c
+8 -37
View File
@@ -174,43 +174,14 @@ db_handle *db_connect_extended(const char *address,
freeReplyObject(reply);
/* Add new client using optimistic locking. */
db_client_id client = globally_unique_id();
while (true) {
reply = redisCommand(context, "WATCH %s", client_type);
freeReplyObject(reply);
reply = redisCommand(context, "HLEN %s", client_type);
freeReplyObject(reply);
reply = redisCommand(context, "MULTI");
freeReplyObject(reply);
reply = redisCommand(context,
"HMSET db_clients:%b client_type %s address %s:%d "
"db_client_id %b aux_address %s",
(char *) client.id, sizeof(client.id), client_type,
client_addr, client_port, (char *) client.id,
sizeof(client.id), aux_address);
CHECKM(reply != NULL, "db_connect failed on HMSET");
freeReplyObject(reply);
{
UT_string *tmpbuf;
utstring_new(tmpbuf);
utstring_printf(tmpbuf, "%s %s", client_type, aux_address);
reply =
redisCommand(context, "PUBLISH db_clients %b:%s", (char *) client.id,
sizeof(client.id), utstring_body(tmpbuf));
CHECKM(reply != NULL, "db_connect failed on PUBLISH");
freeReplyObject(reply);
utstring_free(tmpbuf);
}
reply = redisCommand(context, "EXEC");
CHECKM(reply != NULL, "db_connect failed on EXEC");
CHECK(reply);
if (reply->type != REDIS_REPLY_NIL) {
freeReplyObject(reply);
break;
}
freeReplyObject(reply);
}
/* Register this client with Redis. RAY.CONNECT is a custom Redis command that
* we've defined. */
reply = redisCommand(context, "RAY.CONNECT %s %s:%d %b %s", client_type,
client_addr, client_port, (char *) client.id,
sizeof(client.id), aux_address);
CHECKM(reply != NULL, "db_connect failed on RAY.CONNECT");
freeReplyObject(reply);
db->client_type = strdup(client_type);
db->client = client;
@@ -576,7 +547,7 @@ void redis_get_cached_db_client(db_handle *db,
if (!entry) {
/* This is a very rare case. It should happen at most once per db client. */
redisReply *reply =
redisCommand(db->sync_context, "HGET db_clients:%b address",
redisCommand(db->sync_context, "RAY.GET_CLIENT_ADDRESS %b",
(char *) db_client_id.id, sizeof(db_client_id.id));
CHECKM(reply->type == REDIS_REPLY_STRING, "REDIS reply type=%d",
reply->type);