wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-06 13:31:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

Shard Redis. (#539)

Browse Source 
* Implement sharding in the Ray core

* Single node Python modifications to do sharding

* Do the sharding in redis.cc

* Pipe num_redis_shards through start_ray.py and worker.py.

* Use multiple redis shards in multinode tests.

* first steps for sharding ray.global_state

* Fix problem in multinode docker test.

* fix runtest.py

* fix some tests

* fix redis shard startup

* fix redis sharding

* fix

* fix bug introduced by the map-iterator being consumed

* fix sharding bug

* shard event table

* update number of Redis clients to be 64K

* Fix object table tests by flushing shards in between unit tests

* Fix local scheduler tests

* Documentation

* Register shard locations in the primary shard

* Add plasma unit tests back to build

* lint

* lint and fix build

* Fix

* Address Robert's comments

* Refactor start_ray_processes to start Redis shard

* lint

* Fix global scheduler python tests

* Fix redis module test

* Fix plasma test

* Fix component failure test

* Fix local scheduler test

* Fix runtest.py

* Fix global scheduler test for python3

* Fix task_table_test_and_update bug, from actor task table submission race

* Fix jenkins tests.

* Retry Redis shard connections

* Fix test cases

* Convert database clients to DBClient struct

* Fix race condition when subscribing to db client table

* Remove unused lines, add APITest for sharded Ray

* Fix

* Fix memory leak

* Suppress ReconstructionTests output

* Suppress output for APITestSharded

* Reissue task table add/update commands if initial command does not publish to any subscribers.

* fix

* Fix linting.

* fix tests

* fix linting

* fix python test

* fix linting
This commit is contained in:
Stephanie Wang
2017-05-18 17:40:41 -07:00
committed by Philipp Moritz
parent 0a4304725f
commit ee08c8274b
39 changed files with 1336 additions and 651 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/test/db_tests.cc
+10 -9
View File
@@ -72,12 +72,12 @@ TEST object_table_lookup_test(void) {
event_loop *loop = event_loop_create();
/* This uses manager_port1. */
const char *db_connect_args1[] = {"address", "127.0.0.1:12345"};
DBHandle *db1 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
2, db_connect_args1);
DBHandle *db1 = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
manager_addr, 2, db_connect_args1);
/* This uses manager_port2. */
const char *db_connect_args2[] = {"address", "127.0.0.1:12346"};
DBHandle *db2 = db_connect("127.0.0.1", 6379, "plasma_manager", manager_addr,
2, db_connect_args2);
DBHandle *db2 = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
manager_addr, 2, db_connect_args2);
db_attach(db1, loop, false);
db_attach(db2, loop, false);
UniqueID id = globally_unique_id();
@@ -148,8 +148,8 @@ void task_table_test_callback(Task *callback_task, void *user_data) {
TEST task_table_test(void) {
task_table_test_callback_called = 0;
event_loop *loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "local_scheduler", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "local_scheduler",
"127.0.0.1", 0, NULL);
db_attach(db, loop, false);
DBClientID local_scheduler_id = globally_unique_id();
int64_t task_spec_size;
@@ -184,8 +184,8 @@ void task_table_all_test_callback(Task *task, void *user_data) {
TEST task_table_all_test(void) {
event_loop *loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "local_scheduler", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "local_scheduler",
"127.0.0.1", 0, NULL);
db_attach(db, loop, false);
int64_t task_spec_size;
TaskSpec *spec = example_task_spec(1, 1, &task_spec_size);
@@ -222,7 +222,8 @@ TEST unique_client_id_test(void) {
DBClientID ids[num_conns];
DBHandle *db;
for (int i = 0; i < num_conns; ++i) {
db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
ids[i] = get_db_client_id(db);
db_disconnect(db);
}
src/common/test/object_table_tests.cc
+51 -57
View File
@@ -65,6 +65,15 @@ void new_object_task_callback(TaskID task_id, void *user_context) {
new_object_lookup_callback, (void *) db);
}
void task_table_subscribe_done(TaskID task_id, void *user_context) {
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = NULL,
};
DBHandle *db = (DBHandle *) user_context;
task_table_add_task(db, Task_copy(new_object_task), &retry,
new_object_task_callback, db);
}
TEST new_object_test(void) {
new_object_failed = 0;
new_object_succeeded = 0;
@@ -73,16 +82,16 @@ TEST new_object_test(void) {
new_object_task_spec = Task_task_spec(new_object_task);
new_object_task_id = TaskSpec_task_id(new_object_task_spec);
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 100,
.fail_callback = new_object_fail_callback,
};
task_table_add_task(db, Task_copy(new_object_task), &retry,
new_object_task_callback, db);
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, &retry,
task_table_subscribe_done, db);
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
@@ -109,8 +118,8 @@ TEST new_object_no_task_test(void) {
new_object_id = globally_unique_id();
new_object_task_id = globally_unique_id();
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -151,8 +160,8 @@ void lookup_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST lookup_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = lookup_fail_callback,
@@ -161,6 +170,9 @@ TEST lookup_timeout_test(void) {
(void *) lookup_timeout_context);
/* Disconnect the database to see if the lookup times out. */
close(db->context->c.fd);
for (auto context : db->contexts) {
close(context->c.fd);
}
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
@@ -187,8 +199,8 @@ void add_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST add_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = add_fail_callback,
@@ -197,6 +209,9 @@ TEST add_timeout_test(void) {
add_done_callback, (void *) add_timeout_context);
/* Disconnect the database to see if the lookup times out. */
close(db->context->c.fd);
for (auto context : db->contexts) {
close(context->c.fd);
}
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
@@ -225,8 +240,8 @@ void subscribe_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST subscribe_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -236,7 +251,10 @@ TEST subscribe_timeout_test(void) {
object_table_subscribe_to_notifications(db, false, subscribe_done_callback,
NULL, &retry, NULL, NULL);
/* Disconnect the database to see if the lookup times out. */
close(db->sub_context->c.fd);
close(db->subscribe_context->c.fd);
for (auto subscribe_context : db->subscribe_contexts) {
close(subscribe_context->c.fd);
}
event_loop_run(g_loop);
db_disconnect(db);
destroy_outstanding_callbacks(g_loop);
@@ -324,8 +342,8 @@ TEST add_lookup_test(void) {
lookup_retry_succeeded = 0;
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11235"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, true);
RetryInfo retry = {
.num_retries = 5,
@@ -385,8 +403,8 @@ void add_remove_callback(ObjectID object_id, bool success, void *user_context) {
TEST add_remove_lookup_test(void) {
g_loop = event_loop_create();
lookup_retry_succeeded = 0;
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, true);
RetryInfo retry = {
.num_retries = 5,
@@ -407,29 +425,6 @@ TEST add_remove_lookup_test(void) {
PASS();
}
/* === Test subscribe retry === */
const char *subscribe_retry_context = "subscribe_retry";
int subscribe_retry_succeeded = 0;
int64_t reconnect_sub_context_callback(event_loop *loop,
int64_t timer_id,
void *context) {
DBHandle *db = (DBHandle *) context;
/* Reconnect to redis. This is not reconnecting the pub/sub channel. */
redisAsyncFree(db->sub_context);
redisAsyncFree(db->context);
redisFree(db->sync_context);
db->sub_context = redisAsyncConnect("127.0.0.1", 6379);
db->sub_context->data = (void *) db;
db->context = redisAsyncConnect("127.0.0.1", 6379);
db->context->data = (void *) db;
db->sync_context = redisConnect("127.0.0.1", 6379);
/* Re-attach the database to the event loop (the file descriptor changed). */
db_attach(db, loop, true);
return EVENT_LOOP_TIMER_DONE;
}
/* ==== Test if late succeed is working correctly ==== */
/* === Test lookup late succeed === */
@@ -454,8 +449,8 @@ void lookup_late_done_callback(ObjectID object_id,
TEST lookup_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0,
@@ -498,8 +493,8 @@ void add_late_done_callback(ObjectID object_id,
TEST add_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0, .timeout = 0, .fail_callback = add_late_fail_callback,
@@ -543,8 +538,8 @@ void subscribe_late_done_callback(ObjectID object_id,
TEST subscribe_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0,
@@ -611,8 +606,8 @@ TEST subscribe_success_test(void) {
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11236"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, false);
subscribe_id = globally_unique_id();
@@ -680,8 +675,8 @@ TEST subscribe_object_present_test(void) {
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11236"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, false);
UniqueID id = globally_unique_id();
RetryInfo retry = {
@@ -732,8 +727,8 @@ void subscribe_object_not_present_object_available_callback(
TEST subscribe_object_not_present_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
UniqueID id = globally_unique_id();
RetryInfo retry = {
@@ -796,8 +791,8 @@ TEST subscribe_object_available_later_test(void) {
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11236"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, false);
UniqueID id = globally_unique_id();
RetryInfo retry = {
@@ -849,8 +844,8 @@ TEST subscribe_object_available_subscribe_all(void) {
g_loop = event_loop_create();
/* Construct the arguments to db_connect. */
const char *db_connect_args[] = {"address", "127.0.0.1:11236"};
DBHandle *db = db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 2,
db_connect_args);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 2, db_connect_args);
db_attach(db, g_loop, false);
UniqueID id = globally_unique_id();
RetryInfo retry = {
@@ -904,7 +899,6 @@ SUITE(object_table_tests) {
RUN_REDIS_TEST(add_late_test);
RUN_REDIS_TEST(subscribe_late_test);
RUN_REDIS_TEST(subscribe_success_test);
RUN_REDIS_TEST(subscribe_object_present_test);
RUN_REDIS_TEST(subscribe_object_not_present_test);
RUN_REDIS_TEST(subscribe_object_available_later_test);
RUN_REDIS_TEST(subscribe_object_available_subscribe_all);
src/common/test/redis_tests.cc
+4 -4
View File
@@ -102,8 +102,8 @@ TEST async_redis_socket_test(void) {
utarray_push_back(connections, &socket_fd);
/* Start connection to Redis. */
DBHandle *db =
db_connect("127.0.0.1", 6379, "test_process", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "test_process",
"127.0.0.1", 0, NULL);
db_attach(db, loop, false);
/* Send a command to the Redis process. */
@@ -177,8 +177,8 @@ TEST logging_test(void) {
utarray_push_back(connections, &socket_fd);
/* Start connection to Redis. */
DBHandle *conn =
db_connect("127.0.0.1", 6379, "test_process", "127.0.0.1", 0, NULL);
DBHandle *conn = db_connect(std::string("127.0.0.1"), 6379, "test_process",
"127.0.0.1", 0, NULL);
db_attach(conn, loop, false);
/* Send a command to the Redis process. */
src/common/test/run_tests.sh
+9 -2
View File
@@ -5,8 +5,14 @@
# Cause the script to exit if a single command fails.
set -e
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
# Start the Redis shards.
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
sleep 1s
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
./src/common/common_tests
./src/common/db_tests
./src/common/io_tests
@@ -14,4 +20,5 @@ sleep 1s
./src/common/redis_tests
./src/common/task_table_tests
./src/common/object_table_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6379 shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
src/common/test/run_valgrind.sh
+9 -1
View File
@@ -5,8 +5,14 @@
# Cause the script to exit if a single command fails.
set -e
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so &
# Start the Redis shards.
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &
./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6380 &
sleep 1s
# Register the shard location with the primary shard.
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
valgrind --leak-check=full --error-exitcode=1 ./src/common/common_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/db_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/io_tests
@@ -14,4 +20,6 @@ valgrind --leak-check=full --error-exitcode=1 ./src/common/task_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/redis_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/task_table_tests
valgrind --leak-check=full --error-exitcode=1 ./src/common/object_table_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
src/common/test/task_table_tests.cc
+47 -22
View File
@@ -40,8 +40,8 @@ void lookup_nil_success_callback(Task *task, void *context) {
TEST lookup_nil_test(void) {
lookup_nil_id = globally_unique_id();
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -96,14 +96,16 @@ void add_success_callback(TaskID task_id, void *context) {
TEST add_lookup_test(void) {
add_lookup_task = example_task(1, 1, TASK_STATUS_WAITING);
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
.timeout = 1000,
.fail_callback = add_lookup_fail_callback,
};
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, &retry,
NULL, NULL);
task_table_add_task(db, Task_copy(add_lookup_task), &retry,
add_success_callback, (void *) db);
/* Disconnect the database to see if the lookup times out. */
@@ -136,8 +138,8 @@ void subscribe_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST subscribe_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -148,7 +150,10 @@ TEST subscribe_timeout_test(void) {
subscribe_done_callback,
(void *) subscribe_timeout_context);
/* Disconnect the database to see if the subscribe times out. */
close(db->sub_context->c.fd);
close(db->subscribe_context->c.fd);
for (int i = 0; i < db->subscribe_contexts.size(); ++i) {
close(db->subscribe_contexts[i]->c.fd);
}
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
@@ -177,17 +182,22 @@ void publish_fail_callback(UniqueID id, void *user_context, void *user_data) {
TEST publish_timeout_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
Task *task = example_task(1, 1, TASK_STATUS_WAITING);
RetryInfo retry = {
.num_retries = 5, .timeout = 100, .fail_callback = publish_fail_callback,
};
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, &retry,
NULL, NULL);
task_table_add_task(db, task, &retry, publish_done_callback,
(void *) publish_timeout_context);
/* Disconnect the database to see if the publish times out. */
close(db->context->c.fd);
for (int i = 0; i < db->contexts.size(); ++i) {
close(db->contexts[i]->c.fd);
}
aeProcessEvents(g_loop, AE_TIME_EVENTS);
event_loop_run(g_loop);
db_disconnect(db);
@@ -204,9 +214,14 @@ int64_t reconnect_db_callback(event_loop *loop,
void *context) {
DBHandle *db = (DBHandle *) context;
/* Reconnect to redis. */
redisAsyncFree(db->sub_context);
db->sub_context = redisAsyncConnect("127.0.0.1", 6379);
db->sub_context->data = (void *) db;
redisAsyncFree(db->subscribe_context);
db->subscribe_context = redisAsyncConnect("127.0.0.1", 6379);
db->subscribe_context->data = (void *) db;
for (int i = 0; i < db->subscribe_contexts.size(); ++i) {
redisAsyncFree(db->subscribe_contexts[i]);
db->subscribe_contexts[i] = redisAsyncConnect("127.0.0.1", 6380 + i);
db->subscribe_contexts[i]->data = (void *) db;
}
/* Re-attach the database to the event loop (the file descriptor changed). */
db_attach(db, loop, true);
return EVENT_LOOP_TIMER_DONE;
@@ -239,8 +254,8 @@ void subscribe_retry_fail_callback(UniqueID id,
TEST subscribe_retry_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 5,
@@ -251,7 +266,10 @@ TEST subscribe_retry_test(void) {
subscribe_retry_done_callback,
(void *) subscribe_retry_context);
/* Disconnect the database to see if the subscribe times out. */
close(db->sub_context->c.fd);
close(db->subscribe_context->c.fd);
for (int i = 0; i < db->subscribe_contexts.size(); ++i) {
close(db->subscribe_contexts[i]->c.fd);
}
/* Install handler for reconnecting the database. */
event_loop_add_timer(g_loop, 150,
(event_loop_timer_handler) reconnect_db_callback, db);
@@ -286,8 +304,8 @@ void publish_retry_fail_callback(UniqueID id,
TEST publish_retry_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
Task *task = example_task(1, 1, TASK_STATUS_WAITING);
RetryInfo retry = {
@@ -295,10 +313,15 @@ TEST publish_retry_test(void) {
.timeout = 100,
.fail_callback = publish_retry_fail_callback,
};
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, &retry,
NULL, NULL);
task_table_add_task(db, task, &retry, publish_retry_done_callback,
(void *) publish_retry_context);
/* Disconnect the database to see if the publish times out. */
close(db->sub_context->c.fd);
close(db->subscribe_context->c.fd);
for (int i = 0; i < db->subscribe_contexts.size(); ++i) {
close(db->subscribe_contexts[i]->c.fd);
}
/* Install handler for reconnecting the database. */
event_loop_add_timer(g_loop, 150,
(event_loop_timer_handler) reconnect_db_callback, db);
@@ -335,8 +358,8 @@ void subscribe_late_done_callback(TaskID task_id, void *user_context) {
TEST subscribe_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
RetryInfo retry = {
.num_retries = 0,
@@ -380,8 +403,8 @@ void publish_late_done_callback(TaskID task_id, void *user_context) {
TEST publish_late_test(void) {
g_loop = event_loop_create();
DBHandle *db =
db_connect("127.0.0.1", 6379, "plasma_manager", "127.0.0.1", 0, NULL);
DBHandle *db = db_connect(std::string("127.0.0.1"), 6379, "plasma_manager",
"127.0.0.1", 0, NULL);
db_attach(db, g_loop, false);
Task *task = example_task(1, 1, TASK_STATUS_WAITING);
RetryInfo retry = {
@@ -389,6 +412,8 @@ TEST publish_late_test(void) {
.timeout = 0,
.fail_callback = publish_late_fail_callback,
};
task_table_subscribe(db, NIL_ID, TASK_STATUS_WAITING, NULL, NULL, NULL, NULL,
NULL);
task_table_add_task(db, task, &retry, publish_late_done_callback,
(void *) publish_late_context);
/* Install handler for terminating the event loop. */
src/common/test/test_common.h
+22 -1
View File
@@ -2,11 +2,13 @@
#define TEST_COMMON_H
#include <unistd.h>
#include <vector>
#include "common.h"
#include "io.h"
#include "hiredis/hiredis.h"
#include "utstring.h"
#include "state/redis.h"
#ifndef _WIN32
/* This function is actually not declared in standard POSIX, so declare it. */
@@ -48,10 +50,29 @@ static inline int bind_inet_sock_retry(int *fd) {
}
/* Flush redis. */
static inline void flushall_redis() {
static inline void flushall_redis(void) {
/* Flush the primary shard. */
redisContext *context = redisConnect("127.0.0.1", 6379);
std::vector<std::string> db_shards_addresses;
std::vector<int> db_shards_ports;
get_redis_shards(context, db_shards_addresses, db_shards_ports);
freeReplyObject(redisCommand(context, "FLUSHALL"));
/* Readd the shard locations. */
freeReplyObject(redisCommand(context, "SET NumRedisShards %d",
db_shards_addresses.size()));
for (int i = 0; i < db_shards_addresses.size(); ++i) {
freeReplyObject(redisCommand(context, "RPUSH RedisShards %s:%d",
db_shards_addresses[i].c_str(),
db_shards_ports[i]));
}
redisFree(context);
/* Flush the remaining shards. */
for (int i = 0; i < db_shards_addresses.size(); ++i) {
context = redisConnect(db_shards_addresses[i].c_str(), db_shards_ports[i]);
freeReplyObject(redisCommand(context, "FLUSHALL"));
redisFree(context);
}
}
/* Cleanup method for running tests with the greatest library.