Convert local scheduler messages to flatbuffers (#340)

* use flatbuffer messages for local scheduler

* make sure constructor gets called for C++ object ObjectInfoT

* fix typo

* fix Robert's comments

* Small change to actor test.

* fix valgrind error

* linting

* free notification

* fix

* valgrind

* fix valgrind

* fix other bugs

* valgrind fix

* fixes

* more fixes

* Small changes to comments.

src/common/CMakeLists.txt

@@ -19,7 +19,11 @@ add_custom_target(gen_common_fbs ALL)
 
 add_custom_command(
   TARGET gen_common_fbs
-  COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${COMMON_FBS_SRC}
+  # The --gen-object-api flag generates a C++ class MessageT for each
+  # flatbuffers message Message, which can be used to store deserialized
+  # messages in data structures. This is currently used for ObjectInfo for
+  # example.
+  COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${COMMON_FBS_SRC} --gen-object-api
   DEPENDS ${FBS_DEPENDS}
   COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
   VERBATIM)

src/common/format/common.fbs

@@ -49,6 +49,24 @@ table TaskInfo {
   required_resources: [double];
 }
 
+// Object information data structure.
+table ObjectInfo {
+  // Object ID of this object.
+  object_id: string;
+  // Number of bytes the content of this object occupies in memory.
+  data_size: long;
+  // Number of bytes the metadata of this object occupies in memory.
+  metadata_size: long;
+  // Unix epoch of when this object was created.
+  create_time: long;
+  // How long creation of this object took.
+  construct_duration: long;
+  // Hash of the object content.
+  digest: string;
+  // Specifies if this object was deleted or added.
+  is_deletion: bool;
+}
+
 root_type TaskInfo;
 
 table SubscribeToNotificationsReply {

src/common/io.cc

@@ -14,6 +14,7 @@
 #include <netdb.h>
 
 #include "common.h"
+#include "event_loop.h"
 
 #ifndef _WIN32
 /* This function is actually not declared in standard POSIX, so declare it. */
@@ -318,6 +319,32 @@ disconnected:
   return;
 }
 
+uint8_t *read_message_async(event_loop *loop, int sock) {
+  int64_t size;
+  int error = read_bytes(sock, (uint8_t *) &size, sizeof(int64_t));
+  if (error < 0) {
+    /* The other side has closed the socket. */
+    LOG_DEBUG("Socket has been closed, or some other error has occurred.");
+    if (loop != NULL) {
+      event_loop_remove_file(loop, sock);
+    }
+    close(sock);
+    return NULL;
+  }
+  uint8_t *message = (uint8_t *) malloc(size);
+  error = read_bytes(sock, message, size);
+  if (error < 0) {
+    /* The other side has closed the socket. */
+    LOG_DEBUG("Socket has been closed, or some other error has occurred.");
+    if (loop != NULL) {
+      event_loop_remove_file(loop, sock);
+    }
+    close(sock);
+    return NULL;
+  }
+  return message;
+}
+
 int64_t read_buffer(int fd, int64_t *type, UT_array *buffer) {
   int64_t version;
   int closed = read_bytes(fd, (uint8_t *) &version, sizeof(version));

src/common/io.h

@@ -16,6 +16,9 @@
 #define NUM_CONNECT_ATTEMPTS 50
 #define CONNECT_TIMEOUT_MS 100
 
+struct aeEventLoop;
+typedef aeEventLoop event_loop;
+
 enum common_message_type {
   /** Disconnect a client. */
   DISCONNECT_CLIENT,
@@ -152,6 +155,19 @@ int write_message(int fd, int64_t type, int64_t length, uint8_t *bytes);
  */
 void read_message(int fd, int64_t *type, int64_t *length, uint8_t **bytes);
 
+/**
+ * Read a message from a file descriptor and remove the file descriptor from the
+ * event loop if there is an error. This will actually do two reads. The first
+ * read reads sizeof(int64_t) bytes to determine the number of bytes to read in
+ * the next read.
+ *
+ * @param loop: The event loop.
+ * @param sock: The file descriptor to read from.
+ * @return A byte buffer contining the message or NULL if there was an
+ *         error. The buffer needs to be freed by the user.
+ */
+uint8_t *read_message_async(event_loop *loop, int sock);
+
 /**
  * Read a sequence of bytes written by write_message from a file descriptor.
  * This does not allocate space for the message if the provided buffer is

src/common/object_info.h

@@ -1,21 +0,0 @@
-#ifndef OBJECT_H
-#define OBJECT_H
-
-#include <stdint.h>
-
-#include "common.h"
-
-/**
- * Object information data structure.
- */
-typedef struct {
-  ObjectID obj_id;
-  int64_t data_size;
-  int64_t metadata_size;
-  int64_t create_time;
-  int64_t construct_duration;
-  unsigned char digest[DIGEST_SIZE];
-  bool is_deletion;
-} ObjectInfo;
-
-#endif

src/common/state/object_table.cc

@@ -1,6 +1,5 @@
 #include "object_table.h"
 #include "redis.h"
-#include "object_info.h"
 
 void object_table_lookup(DBHandle *db_handle,
                          ObjectID object_id,

src/common/state/redis.cc

@@ -19,7 +19,6 @@ extern "C" {
 #include "actor_notification_table.h"
 #include "local_scheduler_table.h"
 #include "object_table.h"
-#include "object_info.h"
 #include "task.h"
 #include "task_table.h"
 #include "error_table.h"