Pull Plasma from Apache Arrow and remove Plasma store from Ray. (#692)

* Rebase Ray on top of Plasma in Apache Arrow

* add thirdparty building scripts

* use rebased arrow

* fix

* fix build

* fix python visibility

* comment out C tests for now

* fix multithreading

* fix

* reduce logging

* fix plasma manager multithreading

* make sure old and new object IDs can coexist peacefully

* more rebasing

* update

* fixes

* fix

* install pyarrow

* install cython

* fix

* install newer cmake

* fix

* rebase on top of latest arrow

* getting runtest.py run locally (needed to comment out a test for that to work)

* work on plasma tests

* more fixes

* fix local scheduler tests

* fix global scheduler test

* more fixes

* fix python 3 bytes vs string

* fix manager tests valgrind

* fix documentation building

* fix linting

* fix c++ linting

* fix linting

* add tests back in

* Install without sudo.

* Set PKG_CONFIG_PATH in build.sh so that Ray can find plasma.

* Install pkg-config

* Link -lpthread, note that find_package(Threads) doesn't seem to work reliably.

* Comment in testGPUIDs in runtest.py.

* Set PKG_CONFIG_PATH when building pyarrow.

* Pull apache/arrow and not pcmoritz/arrow.

* Fix installation in docker image.

* adapt to changes of the plasma api

* Fix installation of pyarrow module.

* Fix linting.

* Use correct python executable to build pyarrow.

python/ray/global_scheduler/test/test.py

@@ -10,6 +10,7 @@ import sys
 import time
 import unittest
 
+import pyarrow as pa
 import ray.global_scheduler as global_scheduler
 import ray.local_scheduler as local_scheduler
 import ray.plasma as plasma
@@ -87,8 +88,8 @@ class TestGlobalScheduler(unittest.TestCase):
             self.plasma_manager_pids.append(p3)
             plasma_address = "{}:{}".format(self.node_ip_address,
                                             plasma_manager_port)
-            plasma_client = plasma.PlasmaClient(plasma_store_name,
-                                                plasma_manager_name)
+            plasma_client = pa.plasma.connect(plasma_store_name,
+                                              plasma_manager_name, 64)
             self.plasma_clients.append(plasma_client)
             # Start the local scheduler.
             local_scheduler_name, p4 = local_scheduler.start_local_scheduler(
@@ -203,9 +204,10 @@ class TestGlobalScheduler(unittest.TestCase):
         # Sleep before submitting task to local scheduler.
         time.sleep(0.1)
         # Submit a task to Redis.
-        task = local_scheduler.Task(random_driver_id(), random_function_id(),
-                                    [local_scheduler.ObjectID(object_dep)],
-                                    num_return_vals[0], random_task_id(), 0)
+        task = local_scheduler.Task(
+            random_driver_id(), random_function_id(),
+            [local_scheduler.ObjectID(object_dep.binary())],
+            num_return_vals[0], random_task_id(), 0)
         self.local_scheduler_clients[0].submit(task)
         time.sleep(0.1)
         # There should now be a task in Redis, and it should get assigned to
@@ -256,11 +258,11 @@ class TestGlobalScheduler(unittest.TestCase):
                 # Give 10ms for object info handler to fire (long enough to
                 # yield CPU).
                 time.sleep(0.010)
-            task = local_scheduler.Task(random_driver_id(),
-                                        random_function_id(),
-                                        [local_scheduler.ObjectID(object_dep)],
-                                        num_return_vals[0], random_task_id(),
-                                        0)
+            task = local_scheduler.Task(
+                random_driver_id(),
+                random_function_id(),
+                [local_scheduler.ObjectID(object_dep.binary())],
+                num_return_vals[0], random_task_id(), 0)
             self.local_scheduler_clients[0].submit(task)
         # Check that there are the correct number of tasks in Redis and that
         # they all get assigned to the local scheduler.