Pull Plasma from Apache Arrow and remove Plasma store from Ray. (#692)

* Rebase Ray on top of Plasma in Apache Arrow

* add thirdparty building scripts

* use rebased arrow

* fix

* fix build

* fix python visibility

* comment out C tests for now

* fix multithreading

* fix

* reduce logging

* fix plasma manager multithreading

* make sure old and new object IDs can coexist peacefully

* more rebasing

* update

* fixes

* fix

* install pyarrow

* install cython

* fix

* install newer cmake

* fix

* rebase on top of latest arrow

* getting runtest.py run locally (needed to comment out a test for that to work)

* work on plasma tests

* more fixes

* fix local scheduler tests

* fix global scheduler test

* more fixes

* fix python 3 bytes vs string

* fix manager tests valgrind

* fix documentation building

* fix linting

* fix c++ linting

* fix linting

* add tests back in

* Install without sudo.

* Set PKG_CONFIG_PATH in build.sh so that Ray can find plasma.

* Install pkg-config

* Link -lpthread, note that find_package(Threads) doesn't seem to work reliably.

* Comment in testGPUIDs in runtest.py.

* Set PKG_CONFIG_PATH when building pyarrow.

* Pull apache/arrow and not pcmoritz/arrow.

* Fix installation in docker image.

* adapt to changes of the plasma api

* Fix installation of pyarrow module.

* Fix linting.

* Use correct python executable to build pyarrow.

python/ray/local_scheduler/test/test.py

@@ -12,6 +12,7 @@ import unittest
 
 import ray.local_scheduler as local_scheduler
 import ray.plasma as plasma
+import pyarrow as pa
 
 USE_VALGRIND = False
 ID_SIZE = 20
@@ -41,8 +42,7 @@ class TestLocalSchedulerClient(unittest.TestCase):
     def setUp(self):
         # Start Plasma store.
         plasma_store_name, self.p1 = plasma.start_plasma_store()
-        self.plasma_client = plasma.PlasmaClient(plasma_store_name,
-                                                 release_delay=0)
+        self.plasma_client = pa.plasma.connect(plasma_store_name, "", 0)
         # Start a local scheduler.
         scheduler_name, self.p2 = local_scheduler.start_local_scheduler(
             plasma_store_name, use_valgrind=USE_VALGRIND)
@@ -72,8 +72,8 @@ class TestLocalSchedulerClient(unittest.TestCase):
         # Create and seal the objects in the object store so that we can
         # schedule all of the subsequent tasks.
         for object_id in object_ids:
-            self.plasma_client.create(object_id.id(), 0)
-            self.plasma_client.seal(object_id.id())
+            self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0)
+            self.plasma_client.seal(pa.plasma.ObjectID(object_id.id()))
         # Define some arguments to use for the tasks.
         args_list = [
             [],
@@ -153,8 +153,8 @@ class TestLocalSchedulerClient(unittest.TestCase):
         time.sleep(0.1)
         # Create and seal the object ID in the object store. This should
         # trigger a scheduling event.
-        self.plasma_client.create(object_id.id(), 0)
-        self.plasma_client.seal(object_id.id())
+        self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id.id()))
         # Wait until the thread finishes so that we know the task was
         # scheduled.
         t.join()
@@ -175,8 +175,8 @@ class TestLocalSchedulerClient(unittest.TestCase):
         t.start()
 
         # Make one of the dependencies available.
-        buf = self.plasma_client.create(object_id1.id(), 1)
-        self.plasma_client.seal(object_id1.id())
+        buf = self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id()))
         # Release the object.
         del buf
         # Check that the thread is still waiting for a task.
@@ -188,23 +188,24 @@ class TestLocalSchedulerClient(unittest.TestCase):
         time.sleep(0.1)
         self.assertTrue(t.is_alive())
         # Check that the first object dependency was evicted.
-        object1 = self.plasma_client.get([object_id1.id()], timeout_ms=0)
+        object1 = self.plasma_client.get([pa.plasma.ObjectID(object_id1.id())],
+                                         timeout_ms=0)
         self.assertEqual(object1, [None])
         # Check that the thread is still waiting for a task.
         time.sleep(0.1)
         self.assertTrue(t.is_alive())
 
         # Create the second dependency.
-        self.plasma_client.create(object_id2.id(), 1)
-        self.plasma_client.seal(object_id2.id())
+        self.plasma_client.create(pa.plasma.ObjectID(object_id2.id()), 1)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id2.id()))
         # Check that the thread is still waiting for a task.
         time.sleep(0.1)
         self.assertTrue(t.is_alive())
 
         # Create the first dependency again. Both dependencies are now
         # available.
-        self.plasma_client.create(object_id1.id(), 1)
-        self.plasma_client.seal(object_id1.id())
+        self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id()))
 
         # Wait until the thread finishes so that we know the task was
         # scheduled.