wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-17 11:25:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Stopped nodes can rejoin immediately (#428)

Browse Source 
* Ignore deleted clients when reading address info from Redis

* Remove self from db_client table when exiting cleanly

* Fix valgrind test

* Do not call plasma_perform_release when disconnecting
This commit is contained in:
Stephanie Wang
2017-04-05 23:50:38 -07:00
committed by Robert Nishihara
parent 4043769ba2
commit 93679df724
6 changed files with 93 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/state/redis.cc
+12
View File
@@ -183,9 +183,21 @@ DBHandle *db_connect(const char *db_address,
}
void db_disconnect(DBHandle *db) {
/* Notify others that this client is disconnecting from Redis. If a client of
* the same type on the same node wants to reconnect again, they must
* reconnect and get assigned a different client ID. */
redisReply *reply =
(redisReply *) redisCommand(db->sync_context, "RAY.DISCONNECT %b",
db->client.id, sizeof(db->client.id));
CHECK(strcmp(reply->str, "OK") == 0);
freeReplyObject(reply);
/* Clean up the Redis connection state. */
redisFree(db->sync_context);
redisAsyncFree(db->context);
redisAsyncFree(db->sub_context);
/* Clean up memory. */
DBClientCacheEntry *e, *tmp;
HASH_ITER(hh, db->db_client_cache, e, tmp) {
free(e->addr);
src/local_scheduler/local_scheduler.cc
+35 -30
View File
@@ -155,6 +155,37 @@ void LocalSchedulerState_free(LocalSchedulerState *state) {
* the possibility of orphan worker processes. */
signal(SIGTERM, SIG_DFL);
/* Kill any child processes that didn't register as a worker yet. */
pid_t *worker_pid;
for (worker_pid = (pid_t *) utarray_front(state->child_pids);
worker_pid != NULL;
worker_pid = (pid_t *) utarray_next(state->child_pids, worker_pid)) {
kill(*worker_pid, SIGKILL);
waitpid(*worker_pid, NULL, 0);
LOG_DEBUG("Killed pid %d", *worker_pid);
}
utarray_free(state->child_pids);
/* Kill any registered workers. */
/* TODO(swang): It's possible that the local scheduler will exit before all
* of its task table updates make it to redis. */
for (LocalSchedulerClient **worker =
(LocalSchedulerClient **) utarray_front(state->workers);
worker != NULL;
worker = (LocalSchedulerClient **) utarray_front(state->workers)) {
kill_worker(*worker, true);
}
/* Disconnect from plasma. */
plasma_disconnect(state->plasma_conn);
state->plasma_conn = NULL;
/* Disconnect from the database. */
if (state->db != NULL) {
db_disconnect(state->db);
state->db = NULL;
}
/* Free the command for starting new workers. */
if (state->config.start_worker_command != NULL) {
int i = 0;
@@ -168,39 +199,11 @@ void LocalSchedulerState_free(LocalSchedulerState *state) {
state->config.start_worker_command = NULL;
}
/* Kill any child processes that didn't register as a worker yet. */
pid_t *worker_pid;
for (worker_pid = (pid_t *) utarray_front(state->child_pids);
worker_pid != NULL;
worker_pid = (pid_t *) utarray_next(state->child_pids, worker_pid)) {
kill(*worker_pid, SIGKILL);
waitpid(*worker_pid, NULL, 0);
LOG_DEBUG("Killed pid %d", *worker_pid);
}
utarray_free(state->child_pids);
/* Free the list of workers and any tasks that are still in progress on those
* workers. */
/* TODO(swang): It's possible that the local scheduler will exit before all
* of its task table updates make it to redis. */
for (LocalSchedulerClient **worker =
(LocalSchedulerClient **) utarray_front(state->workers);
worker != NULL;
worker = (LocalSchedulerClient **) utarray_front(state->workers)) {
kill_worker(*worker, true);
}
utarray_free(state->workers);
state->workers = NULL;
/* Disconnect from the database. */
if (state->db != NULL) {
db_disconnect(state->db);
state->db = NULL;
}
/* Disconnect from plasma. */
plasma_disconnect(state->plasma_conn);
state->plasma_conn = NULL;
/* Free the mapping from the actor ID to the ID of the local scheduler
* responsible for that actor. */
actor_map_entry *current_actor_map_entry, *temp_actor_map_entry;
@@ -825,7 +828,7 @@ void new_client_connection(event_loop *loop,
/* We need this code so we can clean up when we get a SIGTERM signal. */
LocalSchedulerState *g_state;
LocalSchedulerState *g_state = NULL;
void signal_handler(int signal) {
LOG_DEBUG("Signal was %d", signal);
@@ -834,7 +837,9 @@ void signal_handler(int signal) {
* free the local scheduler state at most once. If another SIGTERM is
* caught during this call, there is the possibility of orphan worker
* processes. */
LocalSchedulerState_free(g_state);
if (g_state) {
LocalSchedulerState_free(g_state);
}
exit(0);
}
}
src/plasma/plasma_client.cc
+12 -12
View File
@@ -650,27 +650,27 @@ PlasmaConnection *plasma_connect(const char *store_socket_name,
}
void plasma_disconnect(PlasmaConnection *conn) {
/* Perform the pending release calls to flush out the queue so that the counts
* in the objects_in_use table are accurate. */
/* Clean up state for objects and memory pages in use. NOTE: We purposefully
* do not finish sending release calls for objects in use, so that we don't
* duplicate plasma_release calls (when handling a SIGTERM, for example). */
pending_release *element, *temp;
DL_FOREACH_SAFE(conn->release_history, element, temp) {
plasma_perform_release(conn, element->object_id);
DL_DELETE(conn->release_history, element);
free(element);
}
/* Loop over the objects in use table and release all remaining objects. */
object_in_use_entry *current_entry, *temp_entry;
HASH_ITER(hh, conn->objects_in_use, current_entry, temp_entry) {
ObjectID object_id_to_release = current_entry->object_id;
int count = current_entry->count;
for (int i = 0; i < count; ++i) {
plasma_perform_release(conn, object_id_to_release);
}
HASH_DELETE(hh, conn->objects_in_use, current_entry);
free(current_entry);
}
client_mmap_table_entry *mmap_entry, *temp_mmap_entry;
HASH_ITER(hh, conn->mmap_table, mmap_entry, temp_mmap_entry) {
HASH_DELETE(hh, conn->mmap_table, mmap_entry);
free(mmap_entry);
}
/* Check that we've successfully released everything. */
CHECKM(conn->in_use_object_bytes == 0, "conn->in_use_object_bytes = %" PRId64,
conn->in_use_object_bytes);
free_protocol_builder(conn->builder);
/* Close the connections to Plasma. The Plasma store will release the objects
* that were in use by us when handling the SIGPIPE. */
close(conn->store_conn);
if (conn->manager_conn >= 0) {
close(conn->manager_conn);
src/plasma/plasma_manager.cc
+22 -1
View File
@@ -531,6 +531,14 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name,
}
void PlasmaManagerState_free(PlasmaManagerState *state) {
/* Reset the SIGTERM handler to default behavior, so we try to clean up the
* plasma manager at most once. */
signal(SIGTERM, SIG_DFL);
if (state->db != NULL) {
db_disconnect(state->db);
state->db = NULL;
}
ClientConnection *manager_conn, *tmp_manager_conn;
HASH_ITER(manager_hh, state->manager_connections, manager_conn,
tmp_manager_conn) {
@@ -550,6 +558,18 @@ void PlasmaManagerState_free(PlasmaManagerState *state) {
free(entry);
}
ObjectWaitRequests *wait_reqs, *tmp_wait_reqs;
HASH_ITER(hh, state->object_wait_requests_local, wait_reqs, tmp_wait_reqs) {
HASH_DELETE(hh, state->object_wait_requests_local, wait_reqs);
utarray_free(wait_reqs->wait_requests);
free(wait_reqs);
}
HASH_ITER(hh, state->object_wait_requests_remote, wait_reqs, tmp_wait_reqs) {
HASH_DELETE(hh, state->object_wait_requests_remote, wait_reqs);
utarray_free(wait_reqs->wait_requests);
free(wait_reqs);
}
plasma_disconnect(state->plasma_conn);
event_loop_destroy(state->loop);
free_protocol_builder(state->builder);
@@ -1605,9 +1625,10 @@ void start_server(const char *store_socket_name,
/* Report "success" to valgrind. */
void signal_handler(int signal) {
LOG_DEBUG("Signal was %d", signal);
if (signal == SIGTERM) {
if (g_manager_state) {
db_disconnect(g_manager_state->db);
PlasmaManagerState_free(g_manager_state);
}
exit(0);
}