Convert task_spec to flatbuffers (#255)

* convert Ray to C++

* convert task_spec to flatbuffers

* fix

* it compiles

* latest

* tests are passing

* task2 -> task

* fix

* fix

* fix

* fix

* fix

* linting

* fix valgrind

* upgrade flatbuffers

* use debug mode for valgrind

* fix naming and comments

* downgrade flatbuffers

* fix linting

* reintroduce TaskSpec_free

* rename TaskSpec -> TaskInfo

* refactoring

* linting

src/plasma/CMakeLists.txt

@@ -30,8 +30,7 @@ add_custom_command(
   VERBATIM
 )
 
-add_custom_target(protocol_fbs)
-add_dependencies(protocol_fbs flatbuffers_ep)
+add_dependencies(gen_plasma_fbs flatbuffers_ep)
 
 if(UNIX AND NOT APPLE)
   link_libraries(rt)
@@ -58,9 +57,9 @@ if(APPLE)
 endif(APPLE)
 
 if(APPLE)
-  target_link_libraries(plasma -Wl,-force_load,${FLATBUFFERS_STATIC_LIB} ${COMMON_LIB} ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB})
+  target_link_libraries(plasma -Wl,-force_load,${FLATBUFFERS_STATIC_LIB} common ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB})
 else(APPLE)
-  target_link_libraries(plasma -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive ${COMMON_LIB} ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB})
+  target_link_libraries(plasma -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive common ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB})
 endif(APPLE)
 
 include_directories("${FLATBUFFERS_INCLUDE_DIR}")

src/plasma/malloc.c

@@ -7,7 +7,6 @@
 #include <unistd.h>
 
 #include "common.h"
-#include "plasma.h"
 #include "uthash.h"
 
 void *fake_mmap(size_t);

src/plasma/plasma_client.cc

@@ -27,10 +27,9 @@
 #include "plasma_client.h"
 #include "uthash.h"
 #include "utlist.h"
-#include "sha256.h"
 
 extern "C" {
-
+#include "sha256.h"
 #include "fling.h"
 
 #define XXH_STATIC_LINKING_ONLY

src/plasma/plasma_protocol.cc

@@ -2,46 +2,9 @@
 #include "format/plasma_generated.h"
 
 #include "plasma_protocol.h"
+#include "common_protocol.h"
 #include "io.h"
 
-/**
- * Convert an object ID to a flatbuffer string.
- *
- * @param fbb Reference to the flatbuffer builder.
- * @param object_id The object ID to be converted.
- * @return The flatbuffer string contining the object ID.
- */
-flatbuffers::Offset<flatbuffers::String> to_flat(
-    flatbuffers::FlatBufferBuilder &fbb,
-    ObjectID object_id) {
-  return fbb.CreateString((char *) &object_id.id[0], sizeof(object_id.id));
-}
-
-/**
- * Convert a flatbuffer string to an object ID.
- *
- * @param string The flatbuffer string.
- * @return The object ID.
- */
-ObjectID from_flat(const flatbuffers::String *string) {
-  ObjectID object_id;
-  CHECK(string->size() == sizeof(object_id.id));
-  memcpy(&object_id.id[0], string->data(), sizeof(object_id.id));
-  return object_id;
-}
-
-flatbuffers::Offset<
-    flatbuffers::Vector<flatbuffers::Offset<flatbuffers::String>>>
-to_flat(flatbuffers::FlatBufferBuilder &fbb,
-        ObjectID object_ids[],
-        int64_t num_objects) {
-  std::vector<flatbuffers::Offset<flatbuffers::String>> results;
-  for (size_t i = 0; i < num_objects; i++) {
-    results.push_back(to_flat(fbb, object_ids[i]));
-  }
-  return fbb.CreateVector(results);
-}
-
 #define FLATBUFFER_BUILDER_DEFAULT_SIZE 1024
 
 protocol_builder *make_protocol_builder(void) {
@@ -68,7 +31,7 @@ int plasma_send_CreateRequest(int sock,
                               int64_t data_size,
                               int64_t metadata_size) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
-  auto message = CreatePlasmaCreateRequest(fbb, to_flat(fbb, object_id),
+  auto message = CreatePlasmaCreateRequest(fbb, to_flatbuf(fbb, object_id),
                                            data_size, metadata_size);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaCreateRequest, fbb.GetSize(),
@@ -83,7 +46,7 @@ void plasma_read_CreateRequest(uint8_t *data,
   auto message = flatbuffers::GetRoot<PlasmaCreateRequest>(data);
   *data_size = message->data_size();
   *metadata_size = message->metadata_size();
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
 }
 
 int plasma_send_CreateReply(int sock,
@@ -95,8 +58,9 @@ int plasma_send_CreateReply(int sock,
   PlasmaObjectSpec plasma_object(
       object->handle.store_fd, object->handle.mmap_size, object->data_offset,
       object->data_size, object->metadata_offset, object->metadata_size);
-  auto message = CreatePlasmaCreateReply(
-      fbb, to_flat(fbb, object_id), &plasma_object, (PlasmaError) error_code);
+  auto message =
+      CreatePlasmaCreateReply(fbb, to_flatbuf(fbb, object_id), &plasma_object,
+                              (PlasmaError) error_code);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaCreateReply, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -108,7 +72,7 @@ void plasma_read_CreateReply(uint8_t *data,
                              int *error_code) {
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaCreateReply>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
   object->handle.store_fd = message->plasma_object()->segment_index();
   object->handle.mmap_size = message->plasma_object()->mmap_size();
   object->data_offset = message->plasma_object()->data_offset();
@@ -127,7 +91,7 @@ int plasma_send_SealRequest(int sock,
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
   auto digest_string = fbb.CreateString((char *) digest, DIGEST_SIZE);
   auto message =
-      CreatePlasmaSealRequest(fbb, to_flat(fbb, object_id), digest_string);
+      CreatePlasmaSealRequest(fbb, to_flatbuf(fbb, object_id), digest_string);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaSealRequest, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -138,7 +102,7 @@ void plasma_read_SealRequest(uint8_t *data,
                              unsigned char *digest) {
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaSealRequest>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
   CHECK(message->digest()->size() == DIGEST_SIZE);
   memcpy(digest, message->digest()->data(), DIGEST_SIZE);
 }
@@ -148,8 +112,8 @@ int plasma_send_SealReply(int sock,
                           ObjectID object_id,
                           int error) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
-  auto message =
-      CreatePlasmaSealReply(fbb, to_flat(fbb, object_id), (PlasmaError) error);
+  auto message = CreatePlasmaSealReply(fbb, to_flatbuf(fbb, object_id),
+                                       (PlasmaError) error);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaSealReply, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -158,7 +122,7 @@ int plasma_send_SealReply(int sock,
 void plasma_read_SealReply(uint8_t *data, ObjectID *object_id, int *error) {
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaSealReply>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
   *error = message->error();
 }
 
@@ -168,7 +132,7 @@ int plasma_send_ReleaseRequest(int sock,
                                protocol_builder *B,
                                ObjectID object_id) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
-  auto message = CreatePlasmaSealRequest(fbb, to_flat(fbb, object_id));
+  auto message = CreatePlasmaSealRequest(fbb, to_flatbuf(fbb, object_id));
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaReleaseRequest, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -177,7 +141,7 @@ int plasma_send_ReleaseRequest(int sock,
 void plasma_read_ReleaseRequest(uint8_t *data, ObjectID *object_id) {
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaReleaseRequest>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
 }
 
 int plasma_send_ReleaseReply(int sock,
@@ -185,7 +149,7 @@ int plasma_send_ReleaseReply(int sock,
                              ObjectID object_id,
                              int error) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
-  auto message = CreatePlasmaReleaseReply(fbb, to_flat(fbb, object_id),
+  auto message = CreatePlasmaReleaseReply(fbb, to_flatbuf(fbb, object_id),
                                           (PlasmaError) error);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaReleaseReply, fbb.GetSize(),
@@ -195,7 +159,7 @@ int plasma_send_ReleaseReply(int sock,
 void plasma_read_ReleaseReply(uint8_t *data, ObjectID *object_id, int *error) {
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
   *error = message->error();
 }
 
@@ -205,7 +169,7 @@ int plasma_send_DeleteRequest(int sock,
                               protocol_builder *B,
                               ObjectID object_id) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
-  auto message = CreatePlasmaDeleteRequest(fbb, to_flat(fbb, object_id));
+  auto message = CreatePlasmaDeleteRequest(fbb, to_flatbuf(fbb, object_id));
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaDeleteRequest, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -214,7 +178,7 @@ int plasma_send_DeleteRequest(int sock,
 void plasma_read_DeleteRequest(uint8_t *data, ObjectID *object_id) {
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaReleaseReply>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
 }
 
 int plasma_send_DeleteReply(int sock,
@@ -222,7 +186,7 @@ int plasma_send_DeleteReply(int sock,
                             ObjectID object_id,
                             int error) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
-  auto message = CreatePlasmaDeleteReply(fbb, to_flat(fbb, object_id),
+  auto message = CreatePlasmaDeleteReply(fbb, to_flatbuf(fbb, object_id),
                                          (PlasmaError) error);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaDeleteReply, fbb.GetSize(),
@@ -232,7 +196,7 @@ int plasma_send_DeleteReply(int sock,
 void plasma_read_DeleteReply(uint8_t *data, ObjectID *object_id, int *error) {
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaDeleteReply>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
   *error = message->error();
 }
 
@@ -244,7 +208,7 @@ int plasma_send_StatusRequest(int sock,
                               int64_t num_objects) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
   auto message =
-      CreatePlasmaStatusRequest(fbb, to_flat(fbb, object_ids, num_objects));
+      CreatePlasmaStatusRequest(fbb, to_flatbuf(fbb, object_ids, num_objects));
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaStatusRequest, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -262,7 +226,7 @@ void plasma_read_StatusRequest(uint8_t *data,
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaStatusRequest>(data);
   for (int64_t i = 0; i < num_objects; ++i) {
-    object_ids[i] = from_flat(message->object_ids()->Get(i));
+    object_ids[i] = from_flatbuf(message->object_ids()->Get(i));
   }
 }
 
@@ -273,7 +237,7 @@ int plasma_send_StatusReply(int sock,
                             int64_t num_objects) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
   auto message =
-      CreatePlasmaStatusReply(fbb, to_flat(fbb, object_ids, num_objects),
+      CreatePlasmaStatusReply(fbb, to_flatbuf(fbb, object_ids, num_objects),
                               fbb.CreateVector(object_status, num_objects));
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaStatusReply, fbb.GetSize(),
@@ -293,7 +257,7 @@ void plasma_read_StatusReply(uint8_t *data,
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaStatusReply>(data);
   for (int64_t i = 0; i < num_objects; ++i) {
-    object_ids[i] = from_flat(message->object_ids()->Get(i));
+    object_ids[i] = from_flatbuf(message->object_ids()->Get(i));
   }
   for (int64_t i = 0; i < num_objects; ++i) {
     object_status[i] = message->status()->data()[i];
@@ -306,7 +270,7 @@ int plasma_send_ContainsRequest(int sock,
                                 protocol_builder *B,
                                 ObjectID object_id) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
-  auto message = CreatePlasmaContainsRequest(fbb, to_flat(fbb, object_id));
+  auto message = CreatePlasmaContainsRequest(fbb, to_flatbuf(fbb, object_id));
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaContainsRequest, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -315,7 +279,7 @@ int plasma_send_ContainsRequest(int sock,
 void plasma_read_ContainsRequest(uint8_t *data, ObjectID *object_id) {
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaContainsRequest>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
 }
 
 int plasma_send_ContainsReply(int sock,
@@ -324,7 +288,7 @@ int plasma_send_ContainsReply(int sock,
                               int has_object) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
   auto message =
-      CreatePlasmaContainsReply(fbb, to_flat(fbb, object_id), has_object);
+      CreatePlasmaContainsReply(fbb, to_flatbuf(fbb, object_id), has_object);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaContainsReply, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -335,7 +299,7 @@ void plasma_read_ContainsReply(uint8_t *data,
                                int *has_object) {
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaContainsReply>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
   *has_object = message->has_object();
 }
 
@@ -406,7 +370,7 @@ int plasma_send_GetRequest(int sock,
                            int64_t timeout_ms) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
   auto message = CreatePlasmaGetRequest(
-      fbb, to_flat(fbb, object_ids, num_objects), timeout_ms);
+      fbb, to_flatbuf(fbb, object_ids, num_objects), timeout_ms);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaGetRequest, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -425,7 +389,7 @@ void plasma_read_GetRequest(uint8_t *data,
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaGetRequest>(data);
   for (int64_t i = 0; i < num_objects; ++i) {
-    object_ids[i] = from_flat(message->object_ids()->Get(i));
+    object_ids[i] = from_flatbuf(message->object_ids()->Get(i));
   }
   *timeout_ms = message->timeout_ms();
 }
@@ -445,7 +409,7 @@ int plasma_send_GetReply(int sock,
         object->data_size, object->metadata_offset, object->metadata_size));
   }
   auto message = CreatePlasmaGetReply(
-      fbb, to_flat(fbb, object_ids, num_objects),
+      fbb, to_flatbuf(fbb, object_ids, num_objects),
       fbb.CreateVectorOfStructs(objects.data(), num_objects));
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaGetReply, fbb.GetSize(),
@@ -459,7 +423,7 @@ void plasma_read_GetReply(uint8_t *data,
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaGetReply>(data);
   for (int64_t i = 0; i < num_objects; ++i) {
-    object_ids[i] = from_flat(message->object_ids()->Get(i));
+    object_ids[i] = from_flatbuf(message->object_ids()->Get(i));
   }
   for (int64_t i = 0; i < num_objects; ++i) {
     const PlasmaObjectSpec *object = message->plasma_objects()->Get(i);
@@ -480,7 +444,7 @@ int plasma_send_FetchRequest(int sock,
                              int64_t num_objects) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
   auto message =
-      CreatePlasmaFetchRequest(fbb, to_flat(fbb, object_ids, num_objects));
+      CreatePlasmaFetchRequest(fbb, to_flatbuf(fbb, object_ids, num_objects));
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaFetchRequest, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -498,7 +462,7 @@ void plasma_read_FetchRequest(uint8_t *data,
   CHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaFetchRequest>(data);
   for (int64_t i = 0; i < num_objects; ++i) {
-    object_ids[i] = from_flat(message->object_ids()->Get(i));
+    object_ids[i] = from_flatbuf(message->object_ids()->Get(i));
   }
 }
 
@@ -514,9 +478,9 @@ int plasma_send_WaitRequest(int sock,
 
   std::vector<flatbuffers::Offset<ObjectRequestSpec>> object_request_specs;
   for (int i = 0; i < num_requests; i++) {
-    object_request_specs.push_back(
-        CreateObjectRequestSpec(fbb, to_flat(fbb, object_requests[i].object_id),
-                                object_requests[i].type));
+    object_request_specs.push_back(CreateObjectRequestSpec(
+        fbb, to_flatbuf(fbb, object_requests[i].object_id),
+        object_requests[i].type));
   }
 
   auto message =
@@ -546,7 +510,7 @@ void plasma_read_WaitRequest(uint8_t *data,
   CHECK(num_object_ids == message->object_requests()->size());
   for (int i = 0; i < num_object_ids; i++) {
     object_requests[i].object_id =
-        from_flat(message->object_requests()->Get(i)->object_id());
+        from_flatbuf(message->object_requests()->Get(i)->object_id());
     object_requests[i].type = message->object_requests()->Get(i)->type();
   }
 }
@@ -560,7 +524,7 @@ int plasma_send_WaitReply(int sock,
   std::vector<flatbuffers::Offset<ObjectReply>> object_replies;
   for (int i = 0; i < num_ready_objects; i++) {
     object_replies.push_back(
-        CreateObjectReply(fbb, to_flat(fbb, object_requests[i].object_id),
+        CreateObjectReply(fbb, to_flatbuf(fbb, object_requests[i].object_id),
                           object_requests[i].status));
   }
 
@@ -581,7 +545,7 @@ void plasma_read_WaitReply(uint8_t *data,
   *num_ready_objects = message->num_ready_objects();
   for (int i = 0; i < *num_ready_objects; i++) {
     object_requests[i].object_id =
-        from_flat(message->object_requests()->Get(i)->object_id());
+        from_flatbuf(message->object_requests()->Get(i)->object_id());
     object_requests[i].status = message->object_requests()->Get(i)->status();
   }
 }
@@ -606,7 +570,7 @@ int plasma_send_DataRequest(int sock,
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
   auto addr = fbb.CreateString((char *) address, strlen(address));
   auto message =
-      CreatePlasmaDataRequest(fbb, to_flat(fbb, object_id), addr, port);
+      CreatePlasmaDataRequest(fbb, to_flatbuf(fbb, object_id), addr, port);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaDataRequest, fbb.GetSize(),
                        fbb.GetBufferPointer());
@@ -619,7 +583,7 @@ void plasma_read_DataRequest(uint8_t *data,
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaDataRequest>(data);
   DCHECK(message->object_id()->size() == sizeof(object_id->id));
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
   *address = strdup(message->address()->c_str());
   *port = message->port();
 }
@@ -630,7 +594,7 @@ int plasma_send_DataReply(int sock,
                           int64_t object_size,
                           int64_t metadata_size) {
   flatbuffers::FlatBufferBuilder fbb(FLATBUFFER_BUILDER_DEFAULT_SIZE);
-  auto message = CreatePlasmaDataReply(fbb, to_flat(fbb, object_id),
+  auto message = CreatePlasmaDataReply(fbb, to_flatbuf(fbb, object_id),
                                        object_size, metadata_size);
   fbb.Finish(message);
   return write_message(sock, MessageType_PlasmaDataReply, fbb.GetSize(),
@@ -643,7 +607,7 @@ void plasma_read_DataReply(uint8_t *data,
                            int64_t *metadata_size) {
   DCHECK(data);
   auto message = flatbuffers::GetRoot<PlasmaDataReply>(data);
-  *object_id = from_flat(message->object_id());
+  *object_id = from_flatbuf(message->object_id());
   *object_size = (int64_t) message->object_size();
   *metadata_size = (int64_t) message->metadata_size();
 }