Local scheduler sends a null heartbeat to global scheduler (#962)

* Local scheduler sends a null heartbeat to global scheduler to notify death

* Add whitespace.

* Speed up component failures test

* Free local scheduler state upon plasma manager disconnection

src/common/format/common.fbs

@@ -129,6 +129,9 @@ table LocalSchedulerInfoMessage {
   // The resource vector of resources currently available to this local
   // scheduler.
   dynamic_resources: [double];
+  // Whether the local scheduler is dead. If true, then all other fields
+  // besides `db_client_id` will not be set.
+  is_dead: bool;
 }
 
 root_type LocalSchedulerInfoMessage;

src/common/state/local_scheduler_table.cc

@@ -27,3 +27,7 @@ void local_scheduler_table_send_info(DBHandle *db_handle,
   init_table_callback(db_handle, NIL_ID, __func__, data, retry, NULL,
                       redis_local_scheduler_table_send_info, NULL);
 }
+
+void local_scheduler_table_disconnect(DBHandle *db_handle) {
+  redis_local_scheduler_table_disconnect(db_handle);
+}

src/common/state/local_scheduler_table.h

@@ -21,6 +21,9 @@ typedef struct {
   /** The resource vector of resources currently available to this local
    *  scheduler. */
   double dynamic_resources[ResourceIndex_MAX];
+  /** Whether the local scheduler is dead. If true, then all other fields
+   *  should be ignored. */
+  bool is_dead;
 } LocalSchedulerInfo;
 
 /*
@@ -58,13 +61,14 @@ typedef struct {
 } LocalSchedulerTableSubscribeData;
 
 /**
- * Send a heartbeat to all subscriers to the local scheduler table. This
+ * Send a heartbeat to all subscribers to the local scheduler table. This
  * heartbeat contains some information about the load on the local scheduler.
  *
  * @param db_handle Database handle.
  * @param info Information about the local scheduler, including the load on the
  *        local scheduler.
  * @param retry Information about retrying the request to the database.
+ * @return Void.
  */
 void local_scheduler_table_send_info(DBHandle *db_handle,
                                      LocalSchedulerInfo *info,
@@ -77,4 +81,14 @@ typedef struct {
   LocalSchedulerInfo info;
 } LocalSchedulerTableSendInfoData;
 
+/**
+ * Send a null heartbeat to all subscribers to the local scheduler table to
+ * notify them that we are about to exit. This operation is performed
+ * synchronously.
+ *
+ * @param db_handle Database handle.
+ * @return Void.
+ */
+void local_scheduler_table_disconnect(DBHandle *db_handle);
+
 #endif /* LOCAL_SCHEDULER_TABLE_H */

src/common/state/redis.cc

@@ -1289,14 +1289,22 @@ void redis_local_scheduler_table_subscribe_callback(redisAsyncContext *c,
     DBClientID client_id = from_flatbuf(message->db_client_id());
     /* Extract the fields of the local scheduler info struct. */
     LocalSchedulerInfo info;
-    info.total_num_workers = message->total_num_workers();
-    info.task_queue_length = message->task_queue_length();
-    info.available_workers = message->available_workers();
-    for (int i = 0; i < ResourceIndex_MAX; ++i) {
-      info.static_resources[i] = message->static_resources()->Get(i);
-    }
-    for (int i = 0; i < ResourceIndex_MAX; ++i) {
-      info.dynamic_resources[i] = message->dynamic_resources()->Get(i);
+    memset(&info, 0, sizeof(info));
+    if (message->is_dead()) {
+      /* If the local scheduler is dead, then ignore all other fields in the
+       * message. */
+      info.is_dead = true;
+    } else {
+      /* If the local scheduler is alive, collect load information. */
+      info.total_num_workers = message->total_num_workers();
+      info.task_queue_length = message->task_queue_length();
+      info.available_workers = message->available_workers();
+      for (int i = 0; i < ResourceIndex_MAX; ++i) {
+        info.static_resources[i] = message->static_resources()->Get(i);
+      }
+      for (int i = 0; i < ResourceIndex_MAX; ++i) {
+        info.dynamic_resources[i] = message->dynamic_resources()->Get(i);
+      }
     }
 
     /* Call the subscribe callback. */
@@ -1355,7 +1363,7 @@ void redis_local_scheduler_table_send_info(TableCallbackData *callback_data) {
       fbb, to_flatbuf(fbb, db->client), info.total_num_workers,
       info.task_queue_length, info.available_workers,
       fbb.CreateVector(info.static_resources, ResourceIndex_MAX),
-      fbb.CreateVector(info.dynamic_resources, ResourceIndex_MAX));
+      fbb.CreateVector(info.dynamic_resources, ResourceIndex_MAX), false);
   fbb.Finish(message);
 
   int status = redisAsyncCommand(
@@ -1368,6 +1376,22 @@ void redis_local_scheduler_table_send_info(TableCallbackData *callback_data) {
   }
 }
 
+void redis_local_scheduler_table_disconnect(DBHandle *db) {
+  flatbuffers::FlatBufferBuilder fbb;
+  LocalSchedulerInfoMessageBuilder builder(fbb);
+  builder.add_db_client_id(to_flatbuf(fbb, db->client));
+  builder.add_is_dead(true);
+  auto message = builder.Finish();
+  fbb.Finish(message);
+  redisReply *reply = (redisReply *) redisCommand(
+      db->sync_context, "PUBLISH local_schedulers %b", fbb.GetBufferPointer(),
+      fbb.GetSize());
+  CHECK(reply->type != REDIS_REPLY_ERROR);
+  CHECK(reply->type == REDIS_REPLY_INTEGER);
+  LOG_DEBUG("%" PRId64 " subscribers received this publish.\n", reply->integer);
+  freeReplyObject(reply);
+}
+
 void redis_driver_table_subscribe_callback(redisAsyncContext *c,
                                            void *r,
                                            void *privdata) {

src/common/state/redis.h

@@ -291,6 +291,15 @@ void redis_local_scheduler_table_subscribe(TableCallbackData *callback_data);
  */
 void redis_local_scheduler_table_send_info(TableCallbackData *callback_data);
 
+/**
+ * Synchronously publish a null update to the local scheduler table signifying
+ * that we are about to exit.
+ *
+ * @param db The database handle of the dying local scheduler.
+ * @return Void.
+ */
+void redis_local_scheduler_table_disconnect(DBHandle *db);
+
 /**
  * Subscribe to updates from the driver table.
  *