Pull Plasma from Apache Arrow and remove Plasma store from Ray. (#692)

* Rebase Ray on top of Plasma in Apache Arrow

* add thirdparty building scripts

* use rebased arrow

* fix

* fix build

* fix python visibility

* comment out C tests for now

* fix multithreading

* fix

* reduce logging

* fix plasma manager multithreading

* make sure old and new object IDs can coexist peacefully

* more rebasing

* update

* fixes

* fix

* install pyarrow

* install cython

* fix

* install newer cmake

* fix

* rebase on top of latest arrow

* getting runtest.py run locally (needed to comment out a test for that to work)

* work on plasma tests

* more fixes

* fix local scheduler tests

* fix global scheduler test

* more fixes

* fix python 3 bytes vs string

* fix manager tests valgrind

* fix documentation building

* fix linting

* fix c++ linting

* fix linting

* add tests back in

* Install without sudo.

* Set PKG_CONFIG_PATH in build.sh so that Ray can find plasma.

* Install pkg-config

* Link -lpthread, note that find_package(Threads) doesn't seem to work reliably.

* Comment in testGPUIDs in runtest.py.

* Set PKG_CONFIG_PATH when building pyarrow.

* Pull apache/arrow and not pcmoritz/arrow.

* Fix installation in docker image.

* adapt to changes of the plasma api

* Fix installation of pyarrow module.

* Fix linting.

* Use correct python executable to build pyarrow.

src/thirdparty/build_thirdparty.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+if [[ -z  "$1" ]]; then
+  PYTHON_EXECUTABLE=`which python`
+else
+  PYTHON_EXECUTABLE=$1
+fi
+echo "Using Python executable $PYTHON_EXECUTABLE."
+
+# Determine how many parallel jobs to use for make based on the number of cores
+unamestr="$(uname)"
+if [[ "$unamestr" == "Linux" ]]; then
+  PARALLEL=$(nproc)
+elif [[ "$unamestr" == "Darwin" ]]; then
+  PARALLEL=$(sysctl -n hw.ncpu)
+  echo "Platform is macosx."
+else
+  echo "Unrecognized platform."
+  exit 1
+fi
+
+echo "building arrow"
+cd $TP_DIR/arrow/cpp
+mkdir -p $TP_DIR/arrow/cpp/build
+cd $TP_DIR/arrow/cpp/build
+export ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_C_FLAGS="-g -O3" \
+      -DCMAKE_CXX_FLAGS="-g -O3" \
+      -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+      -DARROW_BUILD_TESTS=off \
+      -DARROW_HDFS=on \
+      -DARROW_PYTHON=on \
+      -DARROW_PLASMA=on \
+      -DPLASMA_PYTHON=on \
+      -DARROW_JEMALLOC=off \
+      -DARROW_WITH_BROTLI=off \
+      -DARROW_WITH_LZ4=off \
+      -DARROW_WITH_SNAPPY=off \
+      -DARROW_WITH_ZLIB=off \
+      -DARROW_WITH_ZSTD=off \
+      ..
+make VERBOSE=1 -j$PARALLEL
+make install
+
+echo "installing pyarrow"
+cd $TP_DIR/arrow/python
+# We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
+# find plasma.
+ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
+PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig PYARROW_WITH_PLASMA=1 PYARROW_BUNDLE_ARROW_CPP=1 $PYTHON_EXECUTABLE setup.py install

src/thirdparty/download_thirdparty.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+if [ ! -d $TP_DIR/arrow ]; then
+  git clone https://github.com/apache/arrow/ "$TP_DIR/arrow"
+fi
+cd $TP_DIR/arrow
+git fetch origin master
+
+git checkout dca5d96c7a029c079183e2903db425e486e2deb9