Pull Plasma from Apache Arrow and remove Plasma store from Ray. (#692)

* Rebase Ray on top of Plasma in Apache Arrow

* add thirdparty building scripts

* use rebased arrow

* fix

* fix build

* fix python visibility

* comment out C tests for now

* fix multithreading

* fix

* reduce logging

* fix plasma manager multithreading

* make sure old and new object IDs can coexist peacefully

* more rebasing

* update

* fixes

* fix

* install pyarrow

* install cython

* fix

* install newer cmake

* fix

* rebase on top of latest arrow

* getting runtest.py run locally (needed to comment out a test for that to work)

* work on plasma tests

* more fixes

* fix local scheduler tests

* fix global scheduler test

* more fixes

* fix python 3 bytes vs string

* fix manager tests valgrind

* fix documentation building

* fix linting

* fix c++ linting

* fix linting

* add tests back in

* Install without sudo.

* Set PKG_CONFIG_PATH in build.sh so that Ray can find plasma.

* Install pkg-config

* Link -lpthread, note that find_package(Threads) doesn't seem to work reliably.

* Comment in testGPUIDs in runtest.py.

* Set PKG_CONFIG_PATH when building pyarrow.

* Pull apache/arrow and not pcmoritz/arrow.

* Fix installation in docker image.

* adapt to changes of the plasma api

* Fix installation of pyarrow module.

* Fix linting.

* Use correct python executable to build pyarrow.

src/common/CMakeLists.txt

@@ -78,7 +78,7 @@ target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhi
 function(define_test test_name library)
   add_executable(${test_name} test/${test_name}.cc ${ARGN})
   add_dependencies(${test_name} hiredis flatbuffers_ep)
-  target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ${library})
+  target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a ${library} -lpthread)
   target_compile_options(${test_name} PUBLIC "-DPLASMA_TEST -DLOCAL_SCHEDULER_TEST -DCOMMON_TEST -DRAY_COMMON_LOG_LEVEL=4")
 endfunction()
 

src/common/cmake/Common.cmake

@@ -5,7 +5,7 @@ include(CMakeParseArguments)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 
-set(FLATBUFFERS_VERSION "1.6.0")
+set(FLATBUFFERS_VERSION "1.7.1")
 
 set(FLATBUFFERS_PREFIX "${CMAKE_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
 if (NOT TARGET flatbuffers_ep)

src/common/common.cc

@@ -15,8 +15,7 @@
 /* This is used to define the array of object IDs. */
 const UT_icd object_id_icd = {sizeof(ObjectID), NULL, NULL, NULL};
 
-const UniqueID NIL_ID = {{255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
-                          255, 255, 255, 255, 255, 255, 255, 255, 255, 255}};
+const UniqueID NIL_ID = UniqueID::nil();
 
 const unsigned char NIL_DIGEST[DIGEST_SIZE] = {0};
 

src/common/common.h

@@ -22,6 +22,9 @@ extern "C" {
 }
 #endif
 
+#include "plasma/common.h"
+#include "arrow/util/macros.h"
+
 /** The duration between heartbeats. These are sent by the plasma manager and
  *  local scheduler. */
 #define HEARTBEAT_TIMEOUT_MILLISECONDS 100
@@ -46,12 +49,6 @@ extern "C" {
 #define RAY_COMMON_LOG_LEVEL RAY_COMMON_INFO
 #endif
 
-/* Arrow defines the same macro, only define it if it has not already been
- * defined. */
-#ifndef UNUSED
-#define UNUSED(x) ((void) (x))
-#endif
-
 /**
  * Macros to enable each level of Ray logging statements depending on the
  * current logging level. */
@@ -113,19 +110,7 @@ extern "C" {
 
 #define CHECK(COND) CHECKM(COND, "")
 
-/* This should be defined if we want to check calls to DCHECK. */
-#define RAY_DCHECK
-
-/* Arrow also defines the DCHECK macro, so undo that definition. */
-#ifdef DCHECK
-#undef DCHECK
-#endif
-
-#ifdef RAY_DCHECK
-#define DCHECK(COND) CHECK(COND)
-#else
-#define DCHECK(COND)
-#endif
+#define RAY_DCHECK(COND) CHECK(COND)
 
 /* These are exit codes for common errors that can occur in Ray components. */
 #define EXIT_COULD_NOT_BIND_PORT -2
@@ -141,7 +126,23 @@ extern "C" {
 
 #define IS_NIL_ID(id) UNIQUE_ID_EQ(id, NIL_ID)
 
-typedef struct { unsigned char id[UNIQUE_ID_SIZE]; } UniqueID;
+struct UniqueID {
+  unsigned char id[UNIQUE_ID_SIZE];
+  UniqueID(const plasma::UniqueID &from) {
+    memcpy(&id[0], from.data(), UNIQUE_ID_SIZE);
+  }
+  UniqueID() {}
+  static const UniqueID nil() {
+    UniqueID result;
+    std::fill_n(result.id, UNIQUE_ID_SIZE, 255);
+    return result;
+  }
+  plasma::UniqueID to_plasma_id() {
+    plasma::UniqueID result;
+    memcpy(result.mutable_data(), &id[0], UNIQUE_ID_SIZE);
+    return result;
+  }
+};
 
 extern const UniqueID NIL_ID;
 

src/common/task.cc

@@ -11,7 +11,7 @@ extern "C" {
 ObjectID task_compute_return_id(TaskID task_id, int64_t return_index) {
   /* Here, return_indices need to be >= 0, so we can use negative
    * indices for put. */
-  DCHECK(return_index >= 0);
+  RAY_DCHECK(return_index >= 0);
   /* TODO(rkn): This line requires object and task IDs to be the same size. */
   ObjectID return_id = task_id;
   int64_t *first_bytes = (int64_t *) &return_id;
@@ -22,7 +22,7 @@ ObjectID task_compute_return_id(TaskID task_id, int64_t return_index) {
 }
 
 ObjectID task_compute_put_id(TaskID task_id, int64_t put_index) {
-  DCHECK(put_index >= 0);
+  RAY_DCHECK(put_index >= 0);
   /* TODO(pcm): This line requires object and task IDs to be the same size. */
   ObjectID put_id = task_id;
   int64_t *first_bytes = (int64_t *) &put_id;