Pull Plasma from Apache Arrow and remove Plasma store from Ray. (#692)

* Rebase Ray on top of Plasma in Apache Arrow

* add thirdparty building scripts

* use rebased arrow

* fix

* fix build

* fix python visibility

* comment out C tests for now

* fix multithreading

* fix

* reduce logging

* fix plasma manager multithreading

* make sure old and new object IDs can coexist peacefully

* more rebasing

* update

* fixes

* fix

* install pyarrow

* install cython

* fix

* install newer cmake

* fix

* rebase on top of latest arrow

* getting runtest.py run locally (needed to comment out a test for that to work)

* work on plasma tests

* more fixes

* fix local scheduler tests

* fix global scheduler test

* more fixes

* fix python 3 bytes vs string

* fix manager tests valgrind

* fix documentation building

* fix linting

* fix c++ linting

* fix linting

* add tests back in

* Install without sudo.

* Set PKG_CONFIG_PATH in build.sh so that Ray can find plasma.

* Install pkg-config

* Link -lpthread, note that find_package(Threads) doesn't seem to work reliably.

* Comment in testGPUIDs in runtest.py.

* Set PKG_CONFIG_PATH when building pyarrow.

* Pull apache/arrow and not pcmoritz/arrow.

* Fix installation in docker image.

* adapt to changes of the plasma api

* Fix installation of pyarrow module.

* Fix linting.

* Use correct python executable to build pyarrow.

.travis/install-dependencies.sh

@@ -19,20 +19,20 @@ fi
 
 if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
-  sudo apt-get install -y cmake build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip
+  sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool python-dev python-numpy python-pip libboost-all-dev unzip
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh -O miniconda.sh
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers
+  pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
-  sudo apt-get install -y cmake python-dev python-numpy build-essential autoconf curl libtool libboost-all-dev unzip
+  sudo apt-get install -y cmake pkg-config python-dev python-numpy build-essential autoconf curl libtool libboost-all-dev unzip
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -O miniconda.sh
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers
+  pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers
 elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
   which -s brew
@@ -43,12 +43,12 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
     echo "Updating brew."
     brew update
   fi
-  brew install cmake automake autoconf libtool boost
+  brew install cmake pkg-config automake autoconf libtool boost
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh -O miniconda.sh
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers
+  pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
   which -s brew
@@ -59,12 +59,12 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
     echo "Updating brew."
     brew update
   fi
-  brew install cmake automake autoconf libtool boost
+  brew install cmake pkg-config automake autoconf libtool boost
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install numpy cloudpickle funcsigs click colorama psutil redis tensorflow flatbuffers
+  pip install numpy cloudpickle cython cmake funcsigs click colorama psutil redis tensorflow flatbuffers
 elif [[ "$LINT" == "1" ]]; then
   sudo apt-get update
   sudo apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip

CMakeLists.txt

@@ -2,6 +2,11 @@ cmake_minimum_required(VERSION 2.8)
 
 project(ray)
 
+set(ARROW_DIR "${CMAKE_CURRENT_LIST_DIR}/src/thirdparty/arrow/"
+    CACHE STRING "Path of the arrow source directory")
+
+include_directories("${ARROW_DIR}/cpp/src/")
+
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/)
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/)
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/local_scheduler/)

build.sh

@@ -1,10 +1,19 @@
 #!/usr/bin/env bash
 
+set -x
+
 # Cause the script to exit if a single command fails.
 set -e
 
 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
+if [[ -z  "$1" ]]; then
+  PYTHON_EXECUTABLE=`which python`
+else
+  PYTHON_EXECUTABLE=$1
+fi
+echo "Using Python executable $PYTHON_EXECUTABLE."
+
 # Determine how many parallel jobs to use for make based on the number of cores
 unamestr="$(uname)"
 if [[ "$unamestr" == "Linux" ]]; then
@@ -20,17 +29,25 @@ pushd "$ROOT_DIR/src/common/thirdparty/"
   bash build-redis.sh
 popd
 
-bash "$ROOT_DIR/src/numbuf/thirdparty/download_thirdparty.sh"
-bash "$ROOT_DIR/src/numbuf/thirdparty/build_thirdparty.sh"
+bash "$ROOT_DIR/src/thirdparty/download_thirdparty.sh"
+bash "$ROOT_DIR/src/thirdparty/build_thirdparty.sh" $PYTHON_EXECUTABLE
 
 # Now build everything.
 pushd "$ROOT_DIR/python/ray/core"
+  # We use these variables to set PKG_CONFIG_PATH, which is important so that
+  # in cmake, pkg-config can find plasma.
+  TP_DIR=$ROOT_DIR/src/thirdparty
+  ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
   if [ "$VALGRIND" = "1" ]
   then
-    cmake -DCMAKE_BUILD_TYPE=Debug ../../..
+    PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig cmake -DCMAKE_BUILD_TYPE=Debug ../../..
   else
-    cmake -DCMAKE_BUILD_TYPE=Release ../../..
+    PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig cmake -DCMAKE_BUILD_TYPE=Release ../../..
   fi
   make clean
   make -j${PARALLEL}
 popd
+
+# Move stuff from Arrow to Ray.
+
+mv $ROOT_DIR/src/thirdparty/arrow/cpp/build/release/plasma_store $ROOT_DIR/python/ray/core/src/plasma/

doc/source/conf.py

@@ -18,7 +18,9 @@ import shlex
 
 # These lines added to enable Sphinx to work without installing Ray.
 import mock
-MOCK_MODULES = ["ray.numbuf",
+MOCK_MODULES = ["pyarrow",
+                "pyarrow.plasma",
+                "ray.numbuf",
                 "ray.local_scheduler",
                 "ray.plasma",
                 "ray.core.generated.TaskInfo",

doc/source/install-on-macosx.rst

@@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using
 .. code-block:: bash
 
   brew update
-  brew install cmake automake autoconf libtool boost wget
+  brew install cmake pkg-config automake autoconf libtool boost wget
 
   pip install numpy cloudpickle funcsigs click colorama psutil redis flatbuffers --ignore-installed six
 

doc/source/install-on-ubuntu.rst

@@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using
 .. code-block:: bash
 
   sudo apt-get update
-  sudo apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip
+  sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool libboost-all-dev unzip
 
   # If you are not using Anaconda, you need the following.
   sudo apt-get install python-dev  # For Python 2.

docker/base-deps/Dockerfile

@@ -4,7 +4,7 @@
 FROM ubuntu:xenial
 RUN apt-get update \
     && apt-get install -y vim git wget \
-    && apt-get install -y cmake build-essential autoconf curl libtool libboost-all-dev unzip
+    && apt-get install -y cmake pkg-config build-essential autoconf curl libtool libboost-all-dev unzip
 RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \
     && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \
     && /bin/bash /tmp/anaconda.sh -b -p /opt/conda \

python/ray/experimental/state.py

@@ -465,7 +465,7 @@ class GlobalState(object):
                         **params)
 
             for (event, score) in event_list:
-                event_dict = json.loads(event)
+                event_dict = json.loads(event.decode())
                 task_id = ""
                 for event in event_dict:
                     if "task_id" in event[3]:

python/ray/global_scheduler/test/test.py

@@ -10,6 +10,7 @@ import sys
 import time
 import unittest
 
+import pyarrow as pa
 import ray.global_scheduler as global_scheduler
 import ray.local_scheduler as local_scheduler
 import ray.plasma as plasma
@@ -87,8 +88,8 @@ class TestGlobalScheduler(unittest.TestCase):
             self.plasma_manager_pids.append(p3)
             plasma_address = "{}:{}".format(self.node_ip_address,
                                             plasma_manager_port)
-            plasma_client = plasma.PlasmaClient(plasma_store_name,
-                                                plasma_manager_name)
+            plasma_client = pa.plasma.connect(plasma_store_name,
+                                              plasma_manager_name, 64)
             self.plasma_clients.append(plasma_client)
             # Start the local scheduler.
             local_scheduler_name, p4 = local_scheduler.start_local_scheduler(
@@ -203,9 +204,10 @@ class TestGlobalScheduler(unittest.TestCase):
         # Sleep before submitting task to local scheduler.
         time.sleep(0.1)
         # Submit a task to Redis.
-        task = local_scheduler.Task(random_driver_id(), random_function_id(),
-                                    [local_scheduler.ObjectID(object_dep)],
-                                    num_return_vals[0], random_task_id(), 0)
+        task = local_scheduler.Task(
+            random_driver_id(), random_function_id(),
+            [local_scheduler.ObjectID(object_dep.binary())],
+            num_return_vals[0], random_task_id(), 0)
         self.local_scheduler_clients[0].submit(task)
         time.sleep(0.1)
         # There should now be a task in Redis, and it should get assigned to
@@ -256,11 +258,11 @@ class TestGlobalScheduler(unittest.TestCase):
                 # Give 10ms for object info handler to fire (long enough to
                 # yield CPU).
                 time.sleep(0.010)
-            task = local_scheduler.Task(random_driver_id(),
-                                        random_function_id(),
-                                        [local_scheduler.ObjectID(object_dep)],
-                                        num_return_vals[0], random_task_id(),
-                                        0)
+            task = local_scheduler.Task(
+                random_driver_id(),
+                random_function_id(),
+                [local_scheduler.ObjectID(object_dep.binary())],
+                num_return_vals[0], random_task_id(), 0)
             self.local_scheduler_clients[0].submit(task)
         # Check that there are the correct number of tasks in Redis and that
         # they all get assigned to the local scheduler.

python/ray/local_scheduler/test/test.py

@@ -12,6 +12,7 @@ import unittest
 
 import ray.local_scheduler as local_scheduler
 import ray.plasma as plasma
+import pyarrow as pa
 
 USE_VALGRIND = False
 ID_SIZE = 20
@@ -41,8 +42,7 @@ class TestLocalSchedulerClient(unittest.TestCase):
     def setUp(self):
         # Start Plasma store.
         plasma_store_name, self.p1 = plasma.start_plasma_store()
-        self.plasma_client = plasma.PlasmaClient(plasma_store_name,
-                                                 release_delay=0)
+        self.plasma_client = pa.plasma.connect(plasma_store_name, "", 0)
         # Start a local scheduler.
         scheduler_name, self.p2 = local_scheduler.start_local_scheduler(
             plasma_store_name, use_valgrind=USE_VALGRIND)
@@ -72,8 +72,8 @@ class TestLocalSchedulerClient(unittest.TestCase):
         # Create and seal the objects in the object store so that we can
         # schedule all of the subsequent tasks.
         for object_id in object_ids:
-            self.plasma_client.create(object_id.id(), 0)
-            self.plasma_client.seal(object_id.id())
+            self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0)
+            self.plasma_client.seal(pa.plasma.ObjectID(object_id.id()))
         # Define some arguments to use for the tasks.
         args_list = [
             [],
@@ -153,8 +153,8 @@ class TestLocalSchedulerClient(unittest.TestCase):
         time.sleep(0.1)
         # Create and seal the object ID in the object store. This should
         # trigger a scheduling event.
-        self.plasma_client.create(object_id.id(), 0)
-        self.plasma_client.seal(object_id.id())
+        self.plasma_client.create(pa.plasma.ObjectID(object_id.id()), 0)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id.id()))
         # Wait until the thread finishes so that we know the task was
         # scheduled.
         t.join()
@@ -175,8 +175,8 @@ class TestLocalSchedulerClient(unittest.TestCase):
         t.start()
 
         # Make one of the dependencies available.
-        buf = self.plasma_client.create(object_id1.id(), 1)
-        self.plasma_client.seal(object_id1.id())
+        buf = self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id()))
         # Release the object.
         del buf
         # Check that the thread is still waiting for a task.
@@ -188,23 +188,24 @@ class TestLocalSchedulerClient(unittest.TestCase):
         time.sleep(0.1)
         self.assertTrue(t.is_alive())
         # Check that the first object dependency was evicted.
-        object1 = self.plasma_client.get([object_id1.id()], timeout_ms=0)
+        object1 = self.plasma_client.get([pa.plasma.ObjectID(object_id1.id())],
+                                         timeout_ms=0)
         self.assertEqual(object1, [None])
         # Check that the thread is still waiting for a task.
         time.sleep(0.1)
         self.assertTrue(t.is_alive())
 
         # Create the second dependency.
-        self.plasma_client.create(object_id2.id(), 1)
-        self.plasma_client.seal(object_id2.id())
+        self.plasma_client.create(pa.plasma.ObjectID(object_id2.id()), 1)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id2.id()))
         # Check that the thread is still waiting for a task.
         time.sleep(0.1)
         self.assertTrue(t.is_alive())
 
         # Create the first dependency again. Both dependencies are now
         # available.
-        self.plasma_client.create(object_id1.id(), 1)
-        self.plasma_client.seal(object_id1.id())
+        self.plasma_client.create(pa.plasma.ObjectID(object_id1.id()), 1)
+        self.plasma_client.seal(pa.plasma.ObjectID(object_id1.id()))
 
         # Wait until the thread finishes so that we know the task was
         # scheduled.

python/ray/plasma/__init__.py

@@ -2,13 +2,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from ray.plasma.plasma import (PlasmaBuffer, buffers_equal, PlasmaClient,
-                               start_plasma_store, start_plasma_manager,
-                               plasma_object_exists_error,
-                               plasma_out_of_memory_error,
+from ray.plasma.plasma import (start_plasma_store, start_plasma_manager,
                                DEFAULT_PLASMA_STORE_MEMORY)
 
-__all__ = ["PlasmaBuffer", "buffers_equal", "PlasmaClient",
-           "start_plasma_store", "start_plasma_manager",
-           "plasma_object_exists_error", "plasma_out_of_memory_error",
+__all__ = ["start_plasma_store", "start_plasma_manager",
            "DEFAULT_PLASMA_STORE_MEMORY"]

python/ray/plasma/plasma.py

@@ -5,315 +5,14 @@ from __future__ import print_function
 import os
 import random
 import subprocess
-import sys
 import time
 
-import ray.core.src.plasma.libplasma as libplasma
-from ray.core.src.plasma.libplasma import plasma_object_exists_error
-from ray.core.src.plasma.libplasma import plasma_out_of_memory_error
 
-__all__ = ["PlasmaBuffer", "buffers_equal", "PlasmaClient",
-           "start_plasma_store", "start_plasma_manager",
-           "plasma_object_exists_error", "plasma_out_of_memory_error",
+__all__ = ["start_plasma_store", "start_plasma_manager",
            "DEFAULT_PLASMA_STORE_MEMORY"]
 
 PLASMA_WAIT_TIMEOUT = 2 ** 30
 
-
-class PlasmaBuffer(object):
-    """This is the type returned by calls to get with a PlasmaClient.
-
-    We define our own class instead of directly returning a buffer object so
-    that we can add a custom destructor which notifies Plasma that the object
-    is no longer being used, so the memory in the Plasma store backing the
-    object can potentially be freed.
-
-    Attributes:
-      buffer (buffer): A buffer containing an object in the Plasma store.
-      plasma_id (PlasmaID): The ID of the object in the buffer.
-      plasma_client (PlasmaClient): The PlasmaClient that we use to communicate
-        with the store and manager.
-    """
-    def __init__(self, buff, plasma_id, plasma_client):
-        """Initialize a PlasmaBuffer."""
-        self.buffer = buff
-        self.plasma_id = plasma_id
-        self.plasma_client = plasma_client
-
-    def __del__(self):
-        """Notify Plasma that the object is no longer needed.
-
-        If the plasma client has been shut down, then don't do anything.
-        """
-        if self.plasma_client.alive:
-            libplasma.release(self.plasma_client.conn, self.plasma_id)
-
-    def __getitem__(self, index):
-        """Read from the PlasmaBuffer as if it were just a regular buffer."""
-        # We currently don't allow slicing plasma buffers. We should handle
-        # this better, but it requires some care because the slice may be
-        # backed by the same memory in the object store, but the original
-        # plasma buffer may go out of scope causing the memory to no longer be
-        # accessible.
-        assert not isinstance(index, slice)
-        value = self.buffer[index]
-        if sys.version_info >= (3, 0) and not isinstance(index, slice):
-            value = chr(value)
-        return value
-
-    def __setitem__(self, index, value):
-        """Write to the PlasmaBuffer as if it were just a regular buffer.
-
-        This should fail because the buffer should be read only.
-        """
-        # We currently don't allow slicing plasma buffers. We should handle
-        # this better, but it requires some care because the slice may be
-        # backed by the same memory in the object store, but the original
-        # plasma buffer may go out of scope causing the memory to no longer be
-        # accessible.
-        assert not isinstance(index, slice)
-        if sys.version_info >= (3, 0) and not isinstance(index, slice):
-            value = ord(value)
-        self.buffer[index] = value
-
-    def __len__(self):
-        """Return the length of the buffer."""
-        return len(self.buffer)
-
-
-def buffers_equal(buff1, buff2):
-    """Compare two buffers. These buffers may be PlasmaBuffer objects.
-
-    This method should only be used in the tests. We implement a special helper
-    method for doing this because doing comparisons by slicing is much faster,
-    but we don't want to expose slicing of PlasmaBuffer objects because it
-    currently is not safe.
-    """
-    buff1_to_compare = (buff1.buffer if isinstance(buff1, PlasmaBuffer)
-                        else buff1)
-    buff2_to_compare = (buff2.buffer if isinstance(buff2, PlasmaBuffer)
-                        else buff2)
-    return buff1_to_compare[:] == buff2_to_compare[:]
-
-
-class PlasmaClient(object):
-    """The PlasmaClient is used to interface with a plasma store and manager.
-
-    The PlasmaClient can ask the PlasmaStore to allocate a new buffer, seal a
-    buffer, and get a buffer. Buffers are referred to by object IDs, which are
-    strings.
-    """
-
-    def __init__(self, store_socket_name, manager_socket_name=None,
-                 release_delay=64):
-        """Initialize the PlasmaClient.
-
-        Args:
-          store_socket_name (str): Name of the socket the plasma store is
-            listening at.
-          manager_socket_name (str): Name of the socket the plasma manager is
-            listening at.
-          release_delay (int): The maximum number of objects that the client
-            will keep and delay releasing (for caching reasons).
-        """
-        self.store_socket_name = store_socket_name
-        self.manager_socket_name = manager_socket_name
-        self.alive = True
-
-        if manager_socket_name is not None:
-            self.conn = libplasma.connect(store_socket_name,
-                                          manager_socket_name,
-                                          release_delay)
-        else:
-            self.conn = libplasma.connect(store_socket_name, "", release_delay)
-
-    def shutdown(self):
-        """Shutdown the client so that it does not send messages.
-
-        If we kill the Plasma store and Plasma manager that this client is
-        connected to, then we can use this method to prevent the client from
-        trying to send messages to the killed processes.
-        """
-        if self.alive:
-            libplasma.disconnect(self.conn)
-        self.alive = False
-
-    def create(self, object_id, size, metadata=None):
-        """Create a new buffer in the PlasmaStore for a particular object ID.
-
-        The returned buffer is mutable until seal is called.
-
-        Args:
-          object_id (str): A string used to identify an object.
-          size (int): The size in bytes of the created buffer.
-          metadata (buffer): An optional buffer encoding whatever metadata the
-            user wishes to encode.
-
-        Raises:
-          plasma_object_exists_error: This exception is raised if the object
-            could not be created because there already is an object with the
-            same ID in the plasma store.
-          plasma_out_of_memory_error: This exception is raised if the object
-            could not be created because the plasma store is unable to evict
-            enough objects to create room for it.
-        """
-        # Turn the metadata into the right type.
-        metadata = bytearray(b"") if metadata is None else metadata
-        buff = libplasma.create(self.conn, object_id, size, metadata)
-        return PlasmaBuffer(buff, object_id, self)
-
-    def get(self, object_ids, timeout_ms=-1):
-        """Create a buffer from the PlasmaStore based on object ID.
-
-        If the object has not been sealed yet, this call will block. The
-        retrieved buffer is immutable.
-
-        Args:
-          object_ids (List[str]): A list of strings used to identify some
-            objects.
-          timeout_ms (int): The number of milliseconds that the get call should
-            block before timing out and returning. Pass -1 if the call should
-            block and 0 if the call should return immediately.
-        """
-        results = libplasma.get(self.conn, object_ids, timeout_ms)
-        assert len(object_ids) == len(results)
-        returns = []
-        for i in range(len(object_ids)):
-            if results[i] is None:
-                returns.append(None)
-            else:
-                returns.append(PlasmaBuffer(results[i][0], object_ids[i],
-                               self))
-        return returns
-
-    def get_metadata(self, object_ids, timeout_ms=-1):
-        """Create a buffer from the PlasmaStore based on object ID.
-
-        If the object has not been sealed yet, this call will block until the
-        object has been sealed. The retrieved buffer is immutable.
-
-        Args:
-          object_ids (List[str]): A list of strings used to identify some
-            objects.
-          timeout_ms (int): The number of milliseconds that the get call should
-            block before timing out and returning. Pass -1 if the call should
-            block and 0 if the call should return immediately.
-        """
-        results = libplasma.get(self.conn, object_ids, timeout_ms)
-        assert len(object_ids) == len(results)
-        returns = []
-        for i in range(len(object_ids)):
-            if results[i] is None:
-                returns.append(None)
-            else:
-                returns.append(PlasmaBuffer(results[i][1], object_ids[i],
-                                            self))
-        return returns
-
-    def contains(self, object_id):
-        """Check if the object is present and has been sealed.
-
-        Args:
-          object_id (str): A string used to identify an object.
-        """
-        return libplasma.contains(self.conn, object_id)
-
-    def hash(self, object_id):
-        """Compute the hash of an object in the object store.
-
-        Args:
-          object_id (str): A string used to identify an object.
-
-        Returns:
-          A digest string object's SHA256 hash. If the object isn't in the
-            object store, the string will have length zero.
-        """
-        return libplasma.hash(self.conn, object_id)
-
-    def seal(self, object_id):
-        """Seal the buffer in the PlasmaStore for a particular object ID.
-
-        Once a buffer has been sealed, the buffer is immutable and can only be
-        accessed through get.
-
-        Args:
-          object_id (str): A string used to identify an object.
-        """
-        libplasma.seal(self.conn, object_id)
-
-    def delete(self, object_id):
-        """Delete the buffer in the PlasmaStore for a particular object ID.
-
-        Once a buffer has been deleted, the buffer is no longer accessible.
-
-        Args:
-          object_id (str): A string used to identify an object.
-        """
-        libplasma.delete(self.conn, object_id)
-
-    def evict(self, num_bytes):
-        """Evict some objects until to recover some bytes.
-
-        Recover at least num_bytes bytes if possible.
-
-        Args:
-          num_bytes (int): The number of bytes to attempt to recover.
-        """
-        return libplasma.evict(self.conn, num_bytes)
-
-    def transfer(self, addr, port, object_id):
-        """Transfer local object with id object_id to another plasma instance
-
-        Args:
-          addr (str): IPv4 address of the plasma instance the object is sent
-            to.
-          port (int): Port number of the plasma instance the object is sent to.
-          object_id (str): A string used to identify an object.
-        """
-        return libplasma.transfer(self.conn, object_id, addr, port)
-
-    def fetch(self, object_ids):
-        """Fetch the objects with the given IDs from other plasma managers.
-
-        Args:
-          object_ids (List[str]): A list of strings used to identify the
-            objects.
-        """
-        return libplasma.fetch(self.conn, object_ids)
-
-    def wait(self, object_ids, timeout=PLASMA_WAIT_TIMEOUT, num_returns=1):
-        """Wait until num_returns objects in object_ids are ready.
-
-        Currently, the object ID arguments to wait must be unique.
-
-        Args:
-          object_ids (List[str]): List of object IDs to wait for.
-          timeout (int): Return to the caller after timeout milliseconds.
-          num_returns (int): We are waiting for this number of objects to be
-            ready.
-
-        Returns:
-          ready_ids, waiting_ids (List[str], List[str]): List of object IDs
-            that are ready and list of object IDs we might still wait on
-            respectively.
-        """
-        # Check that the object ID arguments are unique. The plasma manager
-        # currently crashes if given duplicate object IDs.
-        if len(object_ids) != len(set(object_ids)):
-            raise Exception("Wait requires a list of unique object IDs.")
-        ready_ids, waiting_ids = libplasma.wait(self.conn, object_ids, timeout,
-                                                num_returns)
-        return ready_ids, list(waiting_ids)
-
-    def subscribe(self):
-        """Subscribe to notifications about sealed objects."""
-        self.notification_fd = libplasma.subscribe(self.conn)
-
-    def get_next_notification(self):
-        """Get the next notification from the notification socket."""
-        return libplasma.receive_notification(self.notification_fd)
-
-
 DEFAULT_PLASMA_STORE_MEMORY = 10 ** 9
 
 

python/ray/plasma/test/test.py

@@ -3,16 +3,20 @@ from __future__ import division
 from __future__ import print_function
 
 import numpy as np
+from numpy.testing import assert_equal
 import os
 import random
 import signal
+import subprocess
 import sys
 import threading
 import time
 import unittest
 
-import ray.plasma as plasma
-from ray.plasma.utils import (random_object_id, generate_metadata,
+import pyarrow as pa
+import pyarrow.plasma as plasma
+import ray
+from ray.plasma.utils import (random_object_id,
                               create_object_with_id, create_object)
 from ray import services
 
@@ -20,6 +24,10 @@ USE_VALGRIND = False
 PLASMA_STORE_MEMORY = 1000000000
 
 
+def random_name():
+    return str(random.randint(0, 99999999))
+
+
 def assert_get_object_equal(unit_test, client1, client2, object_id,
                             memory_buffer=None, metadata=None):
     client1_buff = client1.get([object_id])[0]
@@ -29,473 +37,88 @@ def assert_get_object_equal(unit_test, client1, client2, object_id,
     unit_test.assertEqual(len(client1_buff), len(client2_buff))
     unit_test.assertEqual(len(client1_metadata), len(client2_metadata))
     # Check that the buffers from the two clients are the same.
-    unit_test.assertTrue(plasma.buffers_equal(client1_buff, client2_buff))
+    assert_equal(np.frombuffer(client1_buff, dtype="uint8"),
+                 np.frombuffer(client2_buff, dtype="uint8"))
     # Check that the metadata buffers from the two clients are the same.
-    unit_test.assertTrue(plasma.buffers_equal(client1_metadata,
-                                              client2_metadata))
+    assert_equal(np.frombuffer(client1_metadata, dtype="uint8"),
+                 np.frombuffer(client2_metadata, dtype="uint8"))
     # If a reference buffer was provided, check that it is the same as well.
     if memory_buffer is not None:
-        unit_test.assertTrue(plasma.buffers_equal(memory_buffer, client1_buff))
+        assert_equal(np.frombuffer(memory_buffer, dtype="uint8"),
+                     np.frombuffer(client1_buff, dtype="uint8"))
     # If reference metadata was provided, check that it is the same as well.
     if metadata is not None:
-        unit_test.assertTrue(plasma.buffers_equal(metadata, client1_metadata))
+        assert_equal(np.frombuffer(metadata, dtype="uint8"),
+                     np.frombuffer(client1_metadata, dtype="uint8"))
 
 
-class TestPlasmaClient(unittest.TestCase):
+DEFAULT_PLASMA_STORE_MEMORY = 10 ** 9
 
-    def setUp(self):
-        # Start Plasma store.
-        plasma_store_name, self.p = plasma.start_plasma_store(
-            use_valgrind=USE_VALGRIND)
-        # Connect to Plasma.
-        self.plasma_client = plasma.PlasmaClient(plasma_store_name, None, 64)
-        # For the eviction test
-        self.plasma_client2 = plasma.PlasmaClient(plasma_store_name, None, 0)
 
-    def tearDown(self):
-        # Check that the Plasma store is still alive.
-        self.assertEqual(self.p.poll(), None)
-        # Kill the plasma store process.
-        if USE_VALGRIND:
-            self.p.send_signal(signal.SIGTERM)
-            self.p.wait()
-            if self.p.returncode != 0:
-                os._exit(-1)
-        else:
-            self.p.kill()
+def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
+                       use_valgrind=False, use_profiler=False,
+                       stdout_file=None, stderr_file=None):
+    """Start a plasma store process.
+    Args:
+        use_valgrind (bool): True if the plasma store should be started inside
+            of valgrind. If this is True, use_profiler must be False.
+        use_profiler (bool): True if the plasma store should be started inside
+            a profiler. If this is True, use_valgrind must be False.
+        stdout_file: A file handle opened for writing to redirect stdout to. If
+            no redirection should happen, then this should be None.
+        stderr_file: A file handle opened for writing to redirect stderr to. If
+            no redirection should happen, then this should be None.
+    Return:
+        A tuple of the name of the plasma store socket and the process ID of
+            the plasma store process.
+    """
+    if use_valgrind and use_profiler:
+        raise Exception("Cannot use valgrind and profiler at the same time.")
+    plasma_store_executable = os.path.join(pa.__path__[0], "plasma_store")
+    plasma_store_name = "/tmp/plasma_store{}".format(random_name())
+    command = [plasma_store_executable,
+               "-s", plasma_store_name,
+               "-m", str(plasma_store_memory)]
+    if use_valgrind:
+        pid = subprocess.Popen(["valgrind",
+                                "--track-origins=yes",
+                                "--leak-check=full",
+                                "--show-leak-kinds=all",
+                                "--leak-check-heuristics=stdstring",
+                                "--error-exitcode=1"] + command,
+                               stdout=stdout_file, stderr=stderr_file)
+        time.sleep(1.0)
+    elif use_profiler:
+        pid = subprocess.Popen(["valgrind", "--tool=callgrind"] + command,
+                               stdout=stdout_file, stderr=stderr_file)
+        time.sleep(1.0)
+    else:
+        pid = subprocess.Popen(command, stdout=stdout_file, stderr=stderr_file)
+        time.sleep(0.1)
+    return plasma_store_name, pid
 
-    def test_create(self):
-        # Create an object id string.
-        object_id = random_object_id()
-        # Create a new buffer and write to it.
-        length = 50
-        memory_buffer = self.plasma_client.create(object_id, length)
-        for i in range(length):
-            memory_buffer[i] = chr(i % 256)
-        # Seal the object.
-        self.plasma_client.seal(object_id)
-        # Get the object.
-        memory_buffer = self.plasma_client.get([object_id])[0]
-        for i in range(length):
-            self.assertEqual(memory_buffer[i], chr(i % 256))
 
-    def test_create_with_metadata(self):
-        for length in range(1000):
-            # Create an object id string.
-            object_id = random_object_id()
-            # Create a random metadata string.
-            metadata = generate_metadata(length)
-            # Create a new buffer and write to it.
-            memory_buffer = self.plasma_client.create(object_id, length,
-                                                      metadata)
-            for i in range(length):
-                memory_buffer[i] = chr(i % 256)
-            # Seal the object.
-            self.plasma_client.seal(object_id)
-            # Get the object.
-            memory_buffer = self.plasma_client.get([object_id])[0]
-            for i in range(length):
-                self.assertEqual(memory_buffer[i], chr(i % 256))
-            # Get the metadata.
-            metadata_buffer = self.plasma_client.get_metadata([object_id])[0]
-            self.assertEqual(len(metadata), len(metadata_buffer))
-            for i in range(len(metadata)):
-                self.assertEqual(chr(metadata[i]), metadata_buffer[i])
-
-    def test_create_existing(self):
-        # This test is partially used to test the code path in which we create
-        # an object with an ID that already exists
-        length = 100
-        for _ in range(1000):
-            object_id = random_object_id()
-            self.plasma_client.create(object_id, length,
-                                      generate_metadata(length))
-            try:
-                self.plasma_client.create(object_id, length,
-                                          generate_metadata(length))
-            except plasma.plasma_object_exists_error as e:
-                pass
-            else:
-                self.assertTrue(False)
-
-    def test_get(self):
-        num_object_ids = 100
-        # Test timing out of get with various timeouts.
-        for timeout in [0, 10, 100, 1000]:
-            object_ids = [random_object_id() for _ in range(num_object_ids)]
-            results = self.plasma_client.get(object_ids, timeout_ms=timeout)
-            self.assertEqual(results, num_object_ids * [None])
-
-        data_buffers = []
-        metadata_buffers = []
-        for i in range(num_object_ids):
-            if i % 2 == 0:
-                data_buffer, metadata_buffer = create_object_with_id(
-                    self.plasma_client, object_ids[i], 2000, 2000)
-                data_buffers.append(data_buffer)
-                metadata_buffers.append(metadata_buffer)
-
-        # Test timing out from some but not all get calls with various
-        # timeouts.
-        for timeout in [0, 10, 100, 1000]:
-            data_results = self.plasma_client.get(object_ids,
-                                                  timeout_ms=timeout)
-            for i in range(num_object_ids):
-                if i % 2 == 0:
-                    self.assertTrue(plasma.buffers_equal(data_buffers[i // 2],
-                                                         data_results[i]))
-                else:
-                    self.assertIsNone(results[i])
-
-    def test_store_full(self):
-        # The store is started with 1GB, so make sure that create throws an
-        # exception when it is full.
-        def assert_create_raises_plasma_full(unit_test, size):
-            partial_size = np.random.randint(size)
-            try:
-                _, memory_buffer, _ = create_object(unit_test.plasma_client,
-                                                    partial_size,
-                                                    size - partial_size)
-            except plasma.plasma_out_of_memory_error as e:
-                pass
-            else:
-                # For some reason the above didn't throw an exception, so fail.
-                unit_test.assertTrue(False)
-
-        # Create a list to keep some of the buffers in scope.
-        memory_buffers = []
-        _, memory_buffer, _ = create_object(self.plasma_client, 5 * 10 ** 8, 0)
-        memory_buffers.append(memory_buffer)
-        # Remaining space is 5 * 10 ** 8. Make sure that we can't create an
-        # object of size 5 * 10 ** 8 + 1, but we can create one of size
-        # 2 * 10 ** 8.
-        assert_create_raises_plasma_full(self, 5 * 10 ** 8 + 1)
-        _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0)
-        del memory_buffer
-        _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0)
-        del memory_buffer
-        assert_create_raises_plasma_full(self, 5 * 10 ** 8 + 1)
-
-        _, memory_buffer, _ = create_object(self.plasma_client, 2 * 10 ** 8, 0)
-        memory_buffers.append(memory_buffer)
-        # Remaining space is 3 * 10 ** 8.
-        assert_create_raises_plasma_full(self, 3 * 10 ** 8 + 1)
-
-        _, memory_buffer, _ = create_object(self.plasma_client, 10 ** 8, 0)
-        memory_buffers.append(memory_buffer)
-        # Remaining space is 2 * 10 ** 8.
-        assert_create_raises_plasma_full(self, 2 * 10 ** 8 + 1)
-
-    def test_contains(self):
-        fake_object_ids = [random_object_id() for _ in range(100)]
-        real_object_ids = [random_object_id() for _ in range(100)]
-        for object_id in real_object_ids:
-            self.assertFalse(self.plasma_client.contains(object_id))
-            self.plasma_client.create(object_id, 100)
-            self.plasma_client.seal(object_id)
-            self.assertTrue(self.plasma_client.contains(object_id))
-        for object_id in fake_object_ids:
-            self.assertFalse(self.plasma_client.contains(object_id))
-        for object_id in real_object_ids:
-            self.assertTrue(self.plasma_client.contains(object_id))
-
-    def test_hash(self):
-        # Check the hash of an object that doesn't exist.
-        object_id1 = random_object_id()
-        self.plasma_client.hash(object_id1)
-
-        length = 1000
-        # Create a random object, and check that the hash function always
-        # returns the same value.
-        metadata = generate_metadata(length)
-        memory_buffer = self.plasma_client.create(object_id1, length, metadata)
-        for i in range(length):
-            memory_buffer[i] = chr(i % 256)
-        self.plasma_client.seal(object_id1)
-        self.assertEqual(self.plasma_client.hash(object_id1),
-                         self.plasma_client.hash(object_id1))
-
-        # Create a second object with the same value as the first, and check
-        # that their hashes are equal.
-        object_id2 = random_object_id()
-        memory_buffer = self.plasma_client.create(object_id2, length, metadata)
-        for i in range(length):
-            memory_buffer[i] = chr(i % 256)
-        self.plasma_client.seal(object_id2)
-        self.assertEqual(self.plasma_client.hash(object_id1),
-                         self.plasma_client.hash(object_id2))
-
-        # Create a third object with a different value from the first two, and
-        # check that its hash is different.
-        object_id3 = random_object_id()
-        metadata = generate_metadata(length)
-        memory_buffer = self.plasma_client.create(object_id3, length, metadata)
-        for i in range(length):
-            memory_buffer[i] = chr((i + 1) % 256)
-        self.plasma_client.seal(object_id3)
-        self.assertNotEqual(self.plasma_client.hash(object_id1),
-                            self.plasma_client.hash(object_id3))
-
-        # Create a fourth object with the same value as the third, but
-        # different metadata. Check that its hash is different from any of the
-        # previous three.
-        object_id4 = random_object_id()
-        metadata4 = generate_metadata(length)
-        memory_buffer = self.plasma_client.create(object_id4, length,
-                                                  metadata4)
-        for i in range(length):
-            memory_buffer[i] = chr((i + 1) % 256)
-        self.plasma_client.seal(object_id4)
-        self.assertNotEqual(self.plasma_client.hash(object_id1),
-                            self.plasma_client.hash(object_id4))
-        self.assertNotEqual(self.plasma_client.hash(object_id3),
-                            self.plasma_client.hash(object_id4))
-
-    def test_many_hashes(self):
-        hashes = []
-        length = 2 ** 10
-
-        for i in range(256):
-            object_id = random_object_id()
-            memory_buffer = self.plasma_client.create(object_id, length)
-            for j in range(length):
-                memory_buffer[j] = chr(i)
-            self.plasma_client.seal(object_id)
-            hashes.append(self.plasma_client.hash(object_id))
-
-        # Create objects of varying length. Each pair has two bits different.
-        for i in range(length):
-            object_id = random_object_id()
-            memory_buffer = self.plasma_client.create(object_id, length)
-            for j in range(length):
-                memory_buffer[j] = chr(0)
-            memory_buffer[i] = chr(1)
-            self.plasma_client.seal(object_id)
-            hashes.append(self.plasma_client.hash(object_id))
-
-        # Create objects of varying length, all with value 0.
-        for i in range(length):
-            object_id = random_object_id()
-            memory_buffer = self.plasma_client.create(object_id, i)
-            for j in range(i):
-                memory_buffer[j] = chr(0)
-            self.plasma_client.seal(object_id)
-            hashes.append(self.plasma_client.hash(object_id))
-
-        # Check that all hashes were unique.
-        self.assertEqual(len(set(hashes)), 256 + length + length)
-
-    # def test_individual_delete(self):
-    #   length = 100
-    #   # Create an object id string.
-    #   object_id = random_object_id()
-    #   # Create a random metadata string.
-    #   metadata = generate_metadata(100)
-    #   # Create a new buffer and write to it.
-    #   memory_buffer = self.plasma_client.create(object_id, length, metadata)
-    #   for i in range(length):
-    #     memory_buffer[i] = chr(i % 256)
-    #   # Seal the object.
-    #   self.plasma_client.seal(object_id)
-    #   # Check that the object is present.
-    #   self.assertTrue(self.plasma_client.contains(object_id))
-    #   # Delete the object.
-    #   self.plasma_client.delete(object_id)
-    #   # Make sure the object is no longer present.
-    #   self.assertFalse(self.plasma_client.contains(object_id))
-    #
-    # def test_delete(self):
-    #   # Create some objects.
-    #   object_ids = [random_object_id() for _ in range(100)]
-    #   for object_id in object_ids:
-    #     length = 100
-    #     # Create a random metadata string.
-    #     metadata = generate_metadata(100)
-    #     # Create a new buffer and write to it.
-    #     memory_buffer = self.plasma_client.create(object_id, length,
-    #                                               metadata)
-    #     for i in range(length):
-    #       memory_buffer[i] = chr(i % 256)
-    #     # Seal the object.
-    #     self.plasma_client.seal(object_id)
-    #     # Check that the object is present.
-    #     self.assertTrue(self.plasma_client.contains(object_id))
-    #
-    #   # Delete the objects and make sure they are no longer present.
-    #   for object_id in object_ids:
-    #     # Delete the object.
-    #     self.plasma_client.delete(object_id)
-    #     # Make sure the object is no longer present.
-    #     self.assertFalse(self.plasma_client.contains(object_id))
-
-    def test_illegal_functionality(self):
-        # Create an object id string.
-        object_id = random_object_id()
-        # Create a new buffer and write to it.
-        length = 1000
-        memory_buffer = self.plasma_client.create(object_id, length)
-        # Make sure we cannot access memory out of bounds.
-        self.assertRaises(Exception, lambda: memory_buffer[length])
-        # Seal the object.
-        self.plasma_client.seal(object_id)
-        # This test is commented out because it currently fails.
-        # # Make sure the object is ready only now.
-        # def illegal_assignment():
-        #   memory_buffer[0] = chr(0)
-        # self.assertRaises(Exception, illegal_assignment)
-        # Get the object.
-        memory_buffer = self.plasma_client.get([object_id])[0]
-
-        # Make sure the object is read only.
-        def illegal_assignment():
-            memory_buffer[0] = chr(0)
-        self.assertRaises(Exception, illegal_assignment)
-
-    def test_evict(self):
-        client = self.plasma_client2
-        object_id1 = random_object_id()
-        b1 = client.create(object_id1, 1000)
-        client.seal(object_id1)
-        del b1
-        self.assertEqual(client.evict(1), 1000)
-
-        object_id2 = random_object_id()
-        object_id3 = random_object_id()
-        b2 = client.create(object_id2, 999)
-        b3 = client.create(object_id3, 998)
-        client.seal(object_id3)
-        del b3
-        self.assertEqual(client.evict(1000), 998)
-
-        object_id4 = random_object_id()
-        b4 = client.create(object_id4, 997)
-        client.seal(object_id4)
-        del b4
-        client.seal(object_id2)
-        del b2
-        self.assertEqual(client.evict(1), 997)
-        self.assertEqual(client.evict(1), 999)
-
-        object_id5 = random_object_id()
-        object_id6 = random_object_id()
-        object_id7 = random_object_id()
-        b5 = client.create(object_id5, 996)
-        b6 = client.create(object_id6, 995)
-        b7 = client.create(object_id7, 994)
-        client.seal(object_id5)
-        client.seal(object_id6)
-        client.seal(object_id7)
-        del b5
-        del b6
-        del b7
-        self.assertEqual(client.evict(2000), 996 + 995 + 994)
-
-    def test_subscribe(self):
-        # Subscribe to notifications from the Plasma Store.
-        self.plasma_client.subscribe()
-        for i in [1, 10, 100, 1000, 10000, 100000]:
-            object_ids = [random_object_id() for _ in range(i)]
-            metadata_sizes = [np.random.randint(1000) for _ in range(i)]
-            data_sizes = [np.random.randint(1000) for _ in range(i)]
-            for j in range(i):
-                self.plasma_client.create(
-                    object_ids[j], size=data_sizes[j],
-                    metadata=bytearray(np.random.bytes(metadata_sizes[j])))
-                self.plasma_client.seal(object_ids[j])
-            # Check that we received notifications for all of the objects.
-            for j in range(i):
-                notification_info = self.plasma_client.get_next_notification()
-                recv_objid, recv_dsize, recv_msize = notification_info
-                self.assertEqual(object_ids[j], recv_objid)
-                self.assertEqual(data_sizes[j], recv_dsize)
-                self.assertEqual(metadata_sizes[j], recv_msize)
-
-    def test_subscribe_deletions(self):
-        # Subscribe to notifications from the Plasma Store. We use
-        # plasma_client2 to make sure that all used objects will get evicted
-        # properly.
-        self.plasma_client2.subscribe()
-        for i in [1, 10, 100, 1000, 10000, 100000]:
-            object_ids = [random_object_id() for _ in range(i)]
-            # Add 1 to the sizes to make sure we have nonzero object sizes.
-            metadata_sizes = [np.random.randint(1000) + 1 for _ in range(i)]
-            data_sizes = [np.random.randint(1000) + 1 for _ in range(i)]
-            for j in range(i):
-                x = self.plasma_client2.create(
-                    object_ids[j], size=data_sizes[j],
-                    metadata=bytearray(np.random.bytes(metadata_sizes[j])))
-                self.plasma_client2.seal(object_ids[j])
-            del x
-            # Check that we received notifications for creating all of the
-            # objects.
-            for j in range(i):
-                notification_info = self.plasma_client2.get_next_notification()
-                recv_objid, recv_dsize, recv_msize = notification_info
-                self.assertEqual(object_ids[j], recv_objid)
-                self.assertEqual(data_sizes[j], recv_dsize)
-                self.assertEqual(metadata_sizes[j], recv_msize)
-
-            # Check that we receive notifications for deleting all objects, as
-            # we evict them.
-            for j in range(i):
-                self.assertEqual(self.plasma_client2.evict(1),
-                                 data_sizes[j] + metadata_sizes[j])
-                notification_info = self.plasma_client2.get_next_notification()
-                recv_objid, recv_dsize, recv_msize = notification_info
-                self.assertEqual(object_ids[j], recv_objid)
-                self.assertEqual(-1, recv_dsize)
-                self.assertEqual(-1, recv_msize)
-
-        # Test multiple deletion notifications. The first 9 object IDs have
-        # size 0, and the last has a nonzero size. When Plasma evicts 1 byte,
-        # it will evict all objects, so we should receive deletion
-        # notifications for each.
-        num_object_ids = 10
-        object_ids = [random_object_id() for _ in range(num_object_ids)]
-        metadata_sizes = [0] * (num_object_ids - 1)
-        data_sizes = [0] * (num_object_ids - 1)
-        metadata_sizes.append(np.random.randint(1000))
-        data_sizes.append(np.random.randint(1000))
-        for i in range(num_object_ids):
-            x = self.plasma_client2.create(
-                object_ids[i], size=data_sizes[i],
-                metadata=bytearray(np.random.bytes(metadata_sizes[i])))
-            self.plasma_client2.seal(object_ids[i])
-        del x
-        for i in range(num_object_ids):
-            notification_info = self.plasma_client2.get_next_notification()
-            recv_objid, recv_dsize, recv_msize = notification_info
-            self.assertEqual(object_ids[i], recv_objid)
-            self.assertEqual(data_sizes[i], recv_dsize)
-            self.assertEqual(metadata_sizes[i], recv_msize)
-        self.assertEqual(self.plasma_client2.evict(1),
-                         data_sizes[-1] + metadata_sizes[-1])
-        for i in range(num_object_ids):
-            notification_info = self.plasma_client2.get_next_notification()
-            recv_objid, recv_dsize, recv_msize = notification_info
-            self.assertEqual(object_ids[i], recv_objid)
-            self.assertEqual(-1, recv_dsize)
-            self.assertEqual(-1, recv_msize)
+# Plasma client tests were moved into arrow
 
 
 class TestPlasmaManager(unittest.TestCase):
 
     def setUp(self):
         # Start two PlasmaStores.
-        store_name1, self.p2 = plasma.start_plasma_store(
+        store_name1, self.p2 = start_plasma_store(
             use_valgrind=USE_VALGRIND)
-        store_name2, self.p3 = plasma.start_plasma_store(
+        store_name2, self.p3 = start_plasma_store(
             use_valgrind=USE_VALGRIND)
         # Start a Redis server.
         redis_address, _ = services.start_redis("127.0.0.1")
         # Start two PlasmaManagers.
-        manager_name1, self.p4, self.port1 = plasma.start_plasma_manager(
+        manager_name1, self.p4, self.port1 = ray.plasma.start_plasma_manager(
             store_name1, redis_address, use_valgrind=USE_VALGRIND)
-        manager_name2, self.p5, self.port2 = plasma.start_plasma_manager(
+        manager_name2, self.p5, self.port2 = ray.plasma.start_plasma_manager(
             store_name2, redis_address, use_valgrind=USE_VALGRIND)
         # Connect two PlasmaClients.
-        self.client1 = plasma.PlasmaClient(store_name1, manager_name1)
-        self.client2 = plasma.PlasmaClient(store_name2, manager_name2)
+        self.client1 = plasma.connect(store_name1, manager_name1, 64)
+        self.client2 = plasma.connect(store_name2, manager_name2, 64)
 
         # Store the processes that will be explicitly killed during tearDown so
         # that a test case can remove ones that will be killed during the test.
@@ -719,7 +342,8 @@ class TestPlasmaManager(unittest.TestCase):
 
         # Make sure that wait returns when the requested number of object IDs
         # are available and does not wait for all object IDs to be available.
-        object_ids = [random_object_id() for _ in range(9)] + [20 * b'\x00']
+        object_ids = [random_object_id() for _ in range(9)] + \
+                     [plasma.ObjectID(20 * b'\x00')]
         object_ids_perm = object_ids[:]
         random.shuffle(object_ids_perm)
         for i in range(10):
@@ -812,17 +436,17 @@ class TestPlasmaManagerRecovery(unittest.TestCase):
 
     def setUp(self):
         # Start a Plasma store.
-        self.store_name, self.p2 = plasma.start_plasma_store(
+        self.store_name, self.p2 = start_plasma_store(
             use_valgrind=USE_VALGRIND)
         # Start a Redis server.
         self.redis_address, _ = services.start_redis("127.0.0.1")
         # Start a PlasmaManagers.
-        manager_name, self.p3, self.port1 = plasma.start_plasma_manager(
+        manager_name, self.p3, self.port1 = ray.plasma.start_plasma_manager(
             self.store_name,
             self.redis_address,
             use_valgrind=USE_VALGRIND)
         # Connect a PlasmaClient.
-        self.client = plasma.PlasmaClient(self.store_name, manager_name)
+        self.client = plasma.connect(self.store_name, manager_name, 64)
 
         # Store the processes that will be explicitly killed during tearDown so
         # that a test case can remove ones that will be killed during the test.
@@ -865,12 +489,12 @@ class TestPlasmaManagerRecovery(unittest.TestCase):
         self.assertEqual(waiting, [])
 
         # Start a second plasma manager attached to the same store.
-        manager_name, self.p5, self.port2 = plasma.start_plasma_manager(
+        manager_name, self.p5, self.port2 = ray.plasma.start_plasma_manager(
             self.store_name, self.redis_address, use_valgrind=USE_VALGRIND)
         self.processes_to_kill = [self.p5] + self.processes_to_kill
 
         # Check that the second manager knows about existing objects.
-        client2 = plasma.PlasmaClient(self.store_name, manager_name)
+        client2 = plasma.connect(self.store_name, manager_name, 64)
         ready, waiting = [], object_ids
         while True:
             ready, waiting = client2.wait(object_ids, num_returns=num_objects,

python/ray/plasma/utils.py

@@ -5,9 +5,11 @@ from __future__ import print_function
 import numpy as np
 import random
 
+import pyarrow.plasma as plasma
+
 
 def random_object_id():
-    return np.random.bytes(20)
+    return plasma.ObjectID(np.random.bytes(20))
 
 
 def generate_metadata(length):
@@ -22,11 +24,12 @@ def generate_metadata(length):
 
 
 def write_to_data_buffer(buff, length):
+    array = np.frombuffer(buff, dtype="uint8")
     if length > 0:
-        buff[0] = chr(random.randint(0, 255))
-        buff[-1] = chr(random.randint(0, 255))
+        array[0] = random.randint(0, 255)
+        array[-1] = random.randint(0, 255)
         for _ in range(100):
-            buff[random.randint(0, length - 1)] = chr(random.randint(0, 255))
+            array[random.randint(0, length - 1)] = random.randint(0, 255)
 
 
 def create_object_with_id(client, object_id, data_size, metadata_size,

python/ray/worker.py

@@ -20,6 +20,7 @@ import time
 import traceback
 
 # Ray modules
+import pyarrow.plasma as plasma
 import ray.experimental.state as state
 import ray.serialization as serialization
 import ray.services as services
@@ -300,7 +301,8 @@ class Worker(object):
                                 "type {}.".format(type(value)))
             counter += 1
             try:
-                ray.numbuf.store_list(object_id.id(), self.plasma_client.conn,
+                ray.numbuf.store_list(object_id.id(),
+                                      self.plasma_client.to_capsule(),
                                       [value])
                 break
             except serialization.RaySerializationException as e:
@@ -375,7 +377,7 @@ class Worker(object):
                 for i in range(0, len(object_ids), get_request_size):
                     results += ray.numbuf.retrieve_list(
                         object_ids[i:(i + get_request_size)],
-                        self.plasma_client.conn,
+                        self.plasma_client.to_capsule(),
                         timeout)
                 return results
             except serialization.RayDeserializationException as e:
@@ -420,7 +422,8 @@ class Worker(object):
         # smaller fetches so as to not block the manager for a prolonged period
         # of time in a single call.
         fetch_request_size = 10000
-        plain_object_ids = [object_id.id() for object_id in object_ids]
+        plain_object_ids = [plasma.ObjectID(object_id.id())
+                            for object_id in object_ids]
         for i in range(0, len(object_ids), fetch_request_size):
             self.plasma_client.fetch(
                 plain_object_ids[i:(i + fetch_request_size)])
@@ -443,7 +446,8 @@ class Worker(object):
             # in case they were evicted since the last fetch. We divide the
             # fetch into smaller fetches so as to not block the manager for a
             # prolonged period of time in a single call.
-            object_ids_to_fetch = list(unready_ids.keys())
+            object_ids_to_fetch = list(map(
+                plasma.ObjectID, unready_ids.keys()))
             for i in range(0, len(object_ids_to_fetch), fetch_request_size):
                 self.plasma_client.fetch(
                     object_ids_to_fetch[i:(i + fetch_request_size)])
@@ -1026,7 +1030,7 @@ def cleanup(worker=global_worker):
     if hasattr(worker, "local_scheduler_client"):
         del worker.local_scheduler_client
     if hasattr(worker, "plasma_client"):
-        worker.plasma_client.shutdown()
+        worker.plasma_client.disconnect()
 
     if worker.mode in [SCRIPT_MODE, SILENT_MODE]:
         # If this is a driver, push the finish time to Redis and clean up any
@@ -1371,8 +1375,9 @@ def connect(info, object_id_seed=None, mode=WORKER_MODE, worker=global_worker,
         raise Exception("This code should be unreachable.")
 
     # Create an object store client.
-    worker.plasma_client = ray.plasma.PlasmaClient(info["store_socket_name"],
-                                                   info["manager_socket_name"])
+    worker.plasma_client = plasma.connect(info["store_socket_name"],
+                                          info["manager_socket_name"],
+                                          64)
     # Create the local scheduler client.
     if worker.actor_id != NIL_ACTOR_ID:
         num_gpus = int(worker.redis_client.hget(b"Actor:" + actor_id,
@@ -1713,14 +1718,15 @@ def wait(object_ids, num_returns=1, timeout=None, worker=global_worker):
     check_connected(worker)
     with log_span("ray:wait", worker=worker):
         check_main_thread()
-        object_id_strs = [object_id.id() for object_id in object_ids]
+        object_id_strs = [plasma.ObjectID(object_id.id())
+                          for object_id in object_ids]
         timeout = timeout if timeout is not None else 2 ** 30
         ready_ids, remaining_ids = worker.plasma_client.wait(object_id_strs,
                                                              timeout,
                                                              num_returns)
-        ready_ids = [ray.local_scheduler.ObjectID(object_id)
+        ready_ids = [ray.local_scheduler.ObjectID(object_id.binary())
                      for object_id in ready_ids]
-        remaining_ids = [ray.local_scheduler.ObjectID(object_id)
+        remaining_ids = [ray.local_scheduler.ObjectID(object_id.binary())
                          for object_id in remaining_ids]
         return ready_ids, remaining_ids
 

python/setup.py

@@ -5,14 +5,35 @@ from __future__ import print_function
 import os
 import shutil
 import subprocess
+import sys
 
 from setuptools import setup, find_packages, Distribution
 import setuptools.command.build_ext as _build_ext
 
 
+# This used to be the first line of the run method in the build_ext class.
+# However, we moved it here because the previous approach seemed to fail in
+# Docker. Inside of the build.sh script, we install the pyarrow Python module.
+# Something about calling "python setup.py install" inside of the build_ext
+# run method doesn't work (this is easily reproducible in Docker with just a
+# couple files to simulate two Python modules). The problem is that the pyarrow
+# module doesn't get added to the easy-install.pth file, so it never gets added
+# to the Python path even though the package is built and copied to the right
+# location. An alternative fix would be to manually modify the easy-install.pth
+# file. TODO(rkn): Fix all of this.
+#
+# Note: We are passing in sys.executable so that we use the same version of
+# Python to build pyarrow inside the build.sh script. Note that certain flags
+# will not be passed along such as --user or sudo. TODO(rkn): Fix this.
+subprocess.check_call(["../build.sh", sys.executable])
+
+
 class build_ext(_build_ext.build_ext):
     def run(self):
-        subprocess.check_call(["../build.sh"])
+        # The line below has been moved outside of the build_ext class. See the
+        # explanation there.
+        # subprocess.check_call(["../build.sh"])
+
         # Ideally, we could include these files by putting them in a
         # MANIFEST.in or using the package_data argument to setup, but the
         # MANIFEST.in gets applied at the very beginning when setup.py runs
@@ -46,7 +67,6 @@ files_to_include = [
     "ray/core/src/common/redis_module/libray_redis_module.so",
     "ray/core/src/plasma/plasma_store",
     "ray/core/src/plasma/plasma_manager",
-    "ray/core/src/plasma/libplasma.so",
     "ray/core/src/local_scheduler/local_scheduler",
     "ray/core/src/local_scheduler/liblocal_scheduler_library.so",
     "ray/core/src/numbuf/libnumbuf.so",

src/common/CMakeLists.txt

@@ -78,7 +78,7 @@ target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhi
 function(define_test test_name library)
   add_executable(${test_name} test/${test_name}.cc ${ARGN})
   add_dependencies(${test_name} hiredis flatbuffers_ep)
-  target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ${library})
+  target_link_libraries(${test_name} common ${FLATBUFFERS_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a ${library} -lpthread)
   target_compile_options(${test_name} PUBLIC "-DPLASMA_TEST -DLOCAL_SCHEDULER_TEST -DCOMMON_TEST -DRAY_COMMON_LOG_LEVEL=4")
 endfunction()
 

src/common/cmake/Common.cmake

@@ -5,7 +5,7 @@ include(CMakeParseArguments)
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
 
-set(FLATBUFFERS_VERSION "1.6.0")
+set(FLATBUFFERS_VERSION "1.7.1")
 
 set(FLATBUFFERS_PREFIX "${CMAKE_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
 if (NOT TARGET flatbuffers_ep)

src/common/common.cc

@@ -15,8 +15,7 @@
 /* This is used to define the array of object IDs. */
 const UT_icd object_id_icd = {sizeof(ObjectID), NULL, NULL, NULL};
 
-const UniqueID NIL_ID = {{255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
-                          255, 255, 255, 255, 255, 255, 255, 255, 255, 255}};
+const UniqueID NIL_ID = UniqueID::nil();
 
 const unsigned char NIL_DIGEST[DIGEST_SIZE] = {0};
 

src/common/common.h

@@ -22,6 +22,9 @@ extern "C" {
 }
 #endif
 
+#include "plasma/common.h"
+#include "arrow/util/macros.h"
+
 /** The duration between heartbeats. These are sent by the plasma manager and
  *  local scheduler. */
 #define HEARTBEAT_TIMEOUT_MILLISECONDS 100
@@ -46,12 +49,6 @@ extern "C" {
 #define RAY_COMMON_LOG_LEVEL RAY_COMMON_INFO
 #endif
 
-/* Arrow defines the same macro, only define it if it has not already been
- * defined. */
-#ifndef UNUSED
-#define UNUSED(x) ((void) (x))
-#endif
-
 /**
  * Macros to enable each level of Ray logging statements depending on the
  * current logging level. */
@@ -113,19 +110,7 @@ extern "C" {
 
 #define CHECK(COND) CHECKM(COND, "")
 
-/* This should be defined if we want to check calls to DCHECK. */
-#define RAY_DCHECK
-
-/* Arrow also defines the DCHECK macro, so undo that definition. */
-#ifdef DCHECK
-#undef DCHECK
-#endif
-
-#ifdef RAY_DCHECK
-#define DCHECK(COND) CHECK(COND)
-#else
-#define DCHECK(COND)
-#endif
+#define RAY_DCHECK(COND) CHECK(COND)
 
 /* These are exit codes for common errors that can occur in Ray components. */
 #define EXIT_COULD_NOT_BIND_PORT -2
@@ -141,7 +126,23 @@ extern "C" {
 
 #define IS_NIL_ID(id) UNIQUE_ID_EQ(id, NIL_ID)
 
-typedef struct { unsigned char id[UNIQUE_ID_SIZE]; } UniqueID;
+struct UniqueID {
+  unsigned char id[UNIQUE_ID_SIZE];
+  UniqueID(const plasma::UniqueID &from) {
+    memcpy(&id[0], from.data(), UNIQUE_ID_SIZE);
+  }
+  UniqueID() {}
+  static const UniqueID nil() {
+    UniqueID result;
+    std::fill_n(result.id, UNIQUE_ID_SIZE, 255);
+    return result;
+  }
+  plasma::UniqueID to_plasma_id() {
+    plasma::UniqueID result;
+    memcpy(result.mutable_data(), &id[0], UNIQUE_ID_SIZE);
+    return result;
+  }
+};
 
 extern const UniqueID NIL_ID;
 

src/common/task.cc

@@ -11,7 +11,7 @@ extern "C" {
 ObjectID task_compute_return_id(TaskID task_id, int64_t return_index) {
   /* Here, return_indices need to be >= 0, so we can use negative
    * indices for put. */
-  DCHECK(return_index >= 0);
+  RAY_DCHECK(return_index >= 0);
   /* TODO(rkn): This line requires object and task IDs to be the same size. */
   ObjectID return_id = task_id;
   int64_t *first_bytes = (int64_t *) &return_id;
@@ -22,7 +22,7 @@ ObjectID task_compute_return_id(TaskID task_id, int64_t return_index) {
 }
 
 ObjectID task_compute_put_id(TaskID task_id, int64_t put_index) {
-  DCHECK(put_index >= 0);
+  RAY_DCHECK(put_index >= 0);
   /* TODO(pcm): This line requires object and task IDs to be the same size. */
   ObjectID put_id = task_id;
   int64_t *first_bytes = (int64_t *) &put_id;

src/local_scheduler/CMakeLists.txt

@@ -5,6 +5,12 @@ project(local_scheduler)
 # Recursively include common
 include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
 
+# Include plasma
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../thirdparty/arrow/python/cmake_modules)
+
+find_package(Plasma)
+include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
+
 add_definitions(-fPIC)
 
 if(APPLE)
@@ -21,7 +27,9 @@ endif()
 
 include_directories("${CMAKE_CURRENT_LIST_DIR}/")
 include_directories("${CMAKE_CURRENT_LIST_DIR}/../")
+# TODO(pcm): get rid of this:
 include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/")
+include_directories("${ARROW_DIR}/cpp/src/")
 include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/format/")
 
 # Compile flatbuffers
@@ -60,10 +68,10 @@ endif(APPLE)
 add_dependencies(local_scheduler_library gen_local_scheduler_fbs)
 
 add_executable(local_scheduler local_scheduler.cc local_scheduler_algorithm.cc)
-target_link_libraries(local_scheduler local_scheduler_client common ${HIREDIS_LIB} plasma_lib)
+target_link_libraries(local_scheduler local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a -lpthread)
 
 add_executable(local_scheduler_tests test/local_scheduler_tests.cc local_scheduler.cc local_scheduler_algorithm.cc)
-target_link_libraries(local_scheduler_tests local_scheduler_client common ${HIREDIS_LIB} plasma_lib)
+target_link_libraries(local_scheduler_tests local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a -lpthread)
 target_compile_options(local_scheduler_tests PUBLIC "-DLOCAL_SCHEDULER_TEST")
 
 install(TARGETS local_scheduler_library DESTINATION ${CMAKE_SOURCE_DIR}/local_scheduler)

src/local_scheduler/local_scheduler.cc

@@ -359,7 +359,7 @@ LocalSchedulerState *LocalSchedulerState_init(
     state->db = NULL;
   }
   /* Connect to Plasma. This method will retry if Plasma hasn't started yet. */
-  state->plasma_conn = new PlasmaClient();
+  state->plasma_conn = new plasma::PlasmaClient();
   if (plasma_manager_socket_name != NULL) {
     ARROW_CHECK_OK(state->plasma_conn->Connect(plasma_store_socket_name,
                                                plasma_manager_socket_name,
@@ -370,7 +370,7 @@ LocalSchedulerState *LocalSchedulerState_init(
   }
   /* Subscribe to notifications about sealed objects. */
   int plasma_fd;
-  ARROW_CHECK_OK(state->plasma_conn->Subscribe(plasma_fd));
+  ARROW_CHECK_OK(state->plasma_conn->Subscribe(&plasma_fd));
   /* Add the callback that processes the notification to the event loop. */
   event_loop_add_file(loop, plasma_fd, EVENT_LOOP_READ,
                       process_plasma_notification, state);

src/local_scheduler/local_scheduler_algorithm.cc

@@ -456,11 +456,11 @@ void add_task_to_actor_queue(LocalSchedulerState *state,
 void fetch_missing_dependency(LocalSchedulerState *state,
                               SchedulingAlgorithmState *algorithm_state,
                               std::list<TaskQueueEntry>::iterator task_entry_it,
-                              ObjectID obj_id) {
+                              plasma::ObjectID obj_id) {
   if (algorithm_state->remote_objects.count(obj_id) == 0) {
     /* We weren't actively fetching this object. Try the fetch once
      * immediately. */
-    if (plasma_manager_is_connected(state->plasma_conn)) {
+    if (state->plasma_conn->get_manager_fd() != -1) {
       ARROW_CHECK_OK(state->plasma_conn->Fetch(1, &obj_id));
     }
     /* Create an entry and add it to the list of active fetch requests to
@@ -497,7 +497,8 @@ void fetch_missing_dependencies(
       ObjectID obj_id = TaskSpec_arg_id(task, i);
       if (algorithm_state->local_objects.count(obj_id) == 0) {
         /* If the entry is not yet available locally, record the dependency. */
-        fetch_missing_dependency(state, algorithm_state, task_entry_it, obj_id);
+        fetch_missing_dependency(state, algorithm_state, task_entry_it,
+                                 obj_id.to_plasma_id());
         ++num_missing_dependencies;
       }
     }
@@ -536,7 +537,7 @@ int fetch_object_timeout_handler(event_loop *loop, timer_id id, void *context) {
 
   LocalSchedulerState *state = (LocalSchedulerState *) context;
   /* Only try the fetches if we are connected to the object store manager. */
-  if (!plasma_manager_is_connected(state->plasma_conn)) {
+  if (state->plasma_conn->get_manager_fd() == -1) {
     LOG_INFO("Local scheduler is not connected to a object store manager");
     return kLocalSchedulerFetchTimeoutMilliseconds;
   }
@@ -555,8 +556,9 @@ int fetch_object_timeout_handler(event_loop *loop, timer_id id, void *context) {
   for (int64_t j = 0; j < num_object_ids; j += fetch_request_size) {
     int num_objects_in_request =
         std::min(num_object_ids, j + fetch_request_size) - j;
-    ARROW_CHECK_OK(
-        state->plasma_conn->Fetch(num_objects_in_request, &object_ids[j]));
+    ARROW_CHECK_OK(state->plasma_conn->Fetch(
+        num_objects_in_request,
+        reinterpret_cast<plasma::ObjectID *>(&object_ids[j])));
   }
 
   /* Print a warning if this method took too long. */
@@ -1237,7 +1239,7 @@ void handle_object_removed(LocalSchedulerState *state,
         ObjectID arg_id = TaskSpec_arg_id(it->spec, i);
         if (ObjectID_equal(arg_id, removed_object_id)) {
           fetch_missing_dependency(state, algorithm_state, it,
-                                   removed_object_id);
+                                   removed_object_id.to_plasma_id());
         }
       }
     }

src/local_scheduler/local_scheduler_client.cc

@@ -6,6 +6,8 @@
 #include "common/io.h"
 #include "common/task.h"
 #include <stdlib.h>
+#include <sys/types.h>
+#include <unistd.h>
 
 LocalSchedulerConnection *LocalSchedulerConnection_init(
     const char *local_scheduler_socket,

src/local_scheduler/local_scheduler_shared.h

@@ -4,7 +4,7 @@
 #include "common/task.h"
 #include "common/state/table.h"
 #include "common/state/db.h"
-#include "plasma_client.h"
+#include "plasma/client.h"
 
 #include <list>
 #include <unordered_map>
@@ -57,7 +57,7 @@ struct LocalSchedulerState {
   /** The handle to the database. */
   DBHandle *db;
   /** The Plasma client. */
-  PlasmaClient *plasma_conn;
+  plasma::PlasmaClient *plasma_conn;
   /** State for the scheduling algorithm. */
   SchedulingAlgorithmState *algorithm_state;
   /** Input buffer, used for reading input in process_message to avoid

src/numbuf/CMakeLists.txt

@@ -6,6 +6,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
 
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)
 
+# Include plasma
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../thirdparty/arrow/python/cmake_modules)
+
+find_package(Plasma)
+include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
+
 option(HAS_PLASMA
   "Are we linking with the plasma object store? Recommended if numbuf is used as part of ray."
   ON)
@@ -29,24 +35,15 @@ if(UNIX AND NOT APPLE)
   link_libraries(rt)
 endif()
 
-set(ARROW_DIR "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/"
-    CACHE STRING "Path of the arrow source directory")
-
-set(ARROW_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a"
+set(ARROW_LIB "${ARROW_DIR}/cpp/build/release/libarrow.a"
     CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)")
-set(ARROW_PYTHON_LIB "${CMAKE_CURRENT_LIST_DIR}/thirdparty/arrow/cpp/build/release/libarrow_python.a"
+set(ARROW_PYTHON_LIB "${ARROW_DIR}/cpp/build/release/libarrow_python.a"
     CACHE STRING "Path to libarrow_python.a (needs to be changed if arrow is build in debug mode)")
 
-include_directories("${ARROW_DIR}/cpp/src/")
+# include_directories("${ARROW_DIR}/cpp/src/")
 include_directories("cpp/src/")
 include_directories("python/src/")
 
-if(HAS_PLASMA)
-  include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma")
-  include_directories("${CMAKE_CURRENT_LIST_DIR}/../common")
-  include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty")
-endif()
-
 add_definitions(-fPIC)
 
 add_library(numbuf SHARED
@@ -60,11 +57,11 @@ add_library(numbuf SHARED
 if(APPLE)
   target_link_libraries(numbuf "-undefined dynamic_lookup" ${ARROW_LIB} ${ARROW_PYTHON_LIB} -lpthread)
 else()
-  target_link_libraries(numbuf -Wl,--whole-archive ${ARROW_LIB} -Wl,--no-whole-archive ${ARROW_PYTHON_LIB} -lpthread)
+  target_link_libraries(numbuf -Wl,--whole-archive ${ARROW_LIB} -Wl,--no-whole-archive ${ARROW_PYTHON_LIB} -lpthread -lboost_system -lboost_filesystem)
 endif()
 
 if(HAS_PLASMA)
-  target_link_libraries(numbuf plasma_lib common)
+  target_link_libraries(numbuf ${PLASMA_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a common)
 endif()
 
 install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}/numbuf/)

src/numbuf/cpp/src/numbuf/sequence.cc

@@ -104,17 +104,18 @@ Status SequenceBuilder::AppendDict(int32_t size) {
 #define ADD_SUBSEQUENCE(DATA, OFFSETS, BUILDER, TAG, NAME)                    \
   if (DATA) {                                                                 \
     DCHECK(DATA->length() == OFFSETS.back());                                 \
-    auto list_builder = std::make_shared<ListBuilder>(pool_, DATA);           \
-    auto field = std::make_shared<Field>(NAME, list_builder->type());         \
+    std::shared_ptr<Array> offset_array;                                      \
+    Int32Builder builder(pool_, std::make_shared<Int32Type>());               \
+    RETURN_NOT_OK(builder.Append(OFFSETS.data(), OFFSETS.size()));            \
+    RETURN_NOT_OK(builder.Finish(&offset_array));                             \
+    std::shared_ptr<Array> list_array;                                        \
+    ListArray::FromArrays(*offset_array, *DATA, pool_, &list_array);          \
+    auto field = std::make_shared<Field>(NAME, list_array->type());           \
     auto type = std::make_shared<StructType>(std::vector<FieldPtr>({field})); \
-    auto lists = std::vector<std::shared_ptr<ArrayBuilder>>({list_builder});  \
-    StructBuilder builder(pool_, type, lists);                                \
-    OFFSETS.pop_back();                                                       \
-    RETURN_NOT_OK(list_builder->Append(OFFSETS.data(), OFFSETS.size()));      \
-    for (int i = 0; i < list_builder->length(); ++i) {                        \
-      RETURN_NOT_OK(builder.Append());                                        \
-    }                                                                         \
-    ADD_ELEMENT(builder, TAG);                                                \
+    types[TAG] = std::make_shared<Field>("", type);                           \
+    children[TAG] = std::shared_ptr<StructArray>(                             \
+        new StructArray(type, list_array->length(), {list_array}));           \
+    RETURN_NOT_OK(nones_.AppendToBitmap(true));                               \
     type_ids.push_back(TAG);                                                  \
   } else {                                                                    \
     DCHECK(OFFSETS.size() == 1);                                              \

src/numbuf/python/src/pynumbuf/numbuf.cc

@@ -12,24 +12,23 @@
 // plasma_protocol, because that file is used both with the store and the
 // manager, the store uses it the ObjectID from plasma_common.h and the
 // manager uses it with the ObjectID from common.h.
-#include "plasma_common.h"
+#include "plasma/common.h"
 
-#include "plasma_client.h"
-#include "plasma_protocol.h"
+#include "plasma/client.h"
+#include "plasma/protocol.h"
 
 extern "C" {
 PyObject* NumbufPlasmaOutOfMemoryError;
 PyObject* NumbufPlasmaObjectExistsError;
 }
 
-#include "plasma_extension.h"
+using namespace plasma;
 
 #endif
 
 #include <arrow/api.h>
 #include <arrow/io/memory.h>
 #include <arrow/ipc/api.h>
-#include <arrow/ipc/util.h>
 #include <arrow/ipc/writer.h>
 #include <arrow/python/numpy_convert.h>
 
@@ -85,7 +84,7 @@ Status read_batch_and_tensors(uint8_t* data, int64_t size,
   auto source = std::make_shared<arrow::io::BufferReader>(
       LENGTH_PREFIX_SIZE + data, size - LENGTH_PREFIX_SIZE);
   RETURN_NOT_OK(arrow::ipc::FileReader::Open(source, batch_size, &reader));
-  RETURN_NOT_OK(reader->GetRecordBatch(0, batch_out));
+  RETURN_NOT_OK(reader->ReadRecordBatch(0, batch_out));
   int64_t offset = batch_size;
   while (true) {
     std::shared_ptr<Tensor> tensor;
@@ -129,6 +128,26 @@ static void ArrowCapsule_Destructor(PyObject* capsule) {
   delete reinterpret_cast<RayObject*>(PyCapsule_GetPointer(capsule, "arrow"));
 }
 
+static int PyObjectToPlasmaClient(PyObject* object, PlasmaClient** client) {
+  if (PyCapsule_IsValid(object, "plasma")) {
+    *client = reinterpret_cast<PlasmaClient*>(PyCapsule_GetPointer(object, "plasma"));
+    return 1;
+  } else {
+    PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule");
+    return 0;
+  }
+}
+
+int PyStringToUniqueID(PyObject* object, ObjectID* object_id) {
+  if (PyBytes_Check(object)) {
+    memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID));
+    return 1;
+  } else {
+    PyErr_SetString(PyExc_TypeError, "must be a 20 character string");
+    return 0;
+  }
+}
+
 /* Documented in doc/numbuf.rst in ray-core */
 static PyObject* serialize_list(PyObject* self, PyObject* args) {
   PyObject* value;
@@ -152,7 +171,7 @@ static PyObject* serialize_list(PyObject* self, PyObject* args) {
     object->batch = make_batch(array);
 
     int64_t data_size, total_size;
-    auto mock = std::make_shared<arrow::ipc::MockOutputStream>();
+    auto mock = std::make_shared<arrow::io::MockOutputStream>();
     write_batch_and_tensors(
         mock.get(), object->batch, object->arrays, &data_size, &total_size);
 
@@ -253,14 +272,15 @@ static PyObject* register_callbacks(PyObject* self, PyObject* args) {
  * @return Void.
  */
 static void BufferCapsule_Destructor(PyObject* capsule) {
-  ObjectID* id = reinterpret_cast<ObjectID*>(PyCapsule_GetPointer(capsule, "buffer"));
+  plasma::ObjectID* id =
+      reinterpret_cast<plasma::ObjectID*>(PyCapsule_GetPointer(capsule, "buffer"));
   auto context = reinterpret_cast<PyObject*>(PyCapsule_GetContext(capsule));
   /* We use the context of the connection capsule to indicate if the connection
    * is still active (if the context is NULL) or if it is closed (if the context
    * is (void*) 0x1). This is neccessary because the primary pointer of the
    * capsule cannot be NULL. */
   if (PyCapsule_GetContext(context) == NULL) {
-    PlasmaClient* client;
+    plasma::PlasmaClient* client;
     ARROW_CHECK(PyObjectToPlasmaClient(context, &client));
     ARROW_CHECK_OK(client->Release(*id));
   }
@@ -282,7 +302,7 @@ static void BufferCapsule_Destructor(PyObject* capsule) {
  */
 static PyObject* store_list(PyObject* self, PyObject* args) {
   ObjectID obj_id;
-  PlasmaClient* client;
+  plasma::PlasmaClient* client;
   PyObject* value;
   if (!PyArg_ParseTuple(args, "O&O&O", PyStringToUniqueID, &obj_id,
           PyObjectToPlasmaClient, &client, &value)) {
@@ -300,7 +320,7 @@ static PyObject* store_list(PyObject* self, PyObject* args) {
   std::shared_ptr<RecordBatch> batch = make_batch(array);
 
   int64_t data_size, total_size;
-  auto mock = std::make_shared<arrow::ipc::MockOutputStream>();
+  auto mock = std::make_shared<arrow::io::MockOutputStream>();
   write_batch_and_tensors(mock.get(), batch, tensors, &data_size, &total_size);
 
   uint8_t* data;
@@ -363,7 +383,7 @@ static PyObject* retrieve_list(PyObject* self, PyObject* args) {
   if (!PyArg_ParseTuple(args, "OOL", &object_id_list, &plasma_client, &timeout_ms)) {
     return NULL;
   }
-  PlasmaClient* client;
+  plasma::PlasmaClient* client;
   if (!PyObjectToPlasmaClient(plasma_client, &client)) { return NULL; }
 
   Py_ssize_t num_object_ids = PyList_Size(object_id_list);

src/numbuf/thirdparty/build_thirdparty.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-set -x
-
-# Cause the script to exit if a single command fails.
-set -e
-
-TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-PREFIX=$TP_DIR/installed
-
-# Determine how many parallel jobs to use for make based on the number of cores
-unamestr="$(uname)"
-if [[ "$unamestr" == "Linux" ]]; then
-  PARALLEL=$(nproc)
-elif [[ "$unamestr" == "Darwin" ]]; then
-  PARALLEL=$(sysctl -n hw.ncpu)
-  echo "Platform is macosx."
-else
-  echo "Unrecognized platform."
-  exit 1
-fi
-
-echo "building arrow"
-cd $TP_DIR/arrow/cpp
-mkdir -p $TP_DIR/arrow/cpp/build
-cd $TP_DIR/arrow/cpp/build
-cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_FLAGS="-g -O3" -DCMAKE_CXX_FLAGS="-g -O3" -DARROW_BUILD_TESTS=OFF -DARROW_HDFS=OFF -DARROW_PYTHON=on ..
-make VERBOSE=1 -j$PARALLEL

src/plasma/CMakeLists.txt

@@ -5,102 +5,30 @@ project(plasma)
 # Recursively include common
 include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
 
-if(APPLE)
-  SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
-endif(APPLE)
+# Include plasma
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../thirdparty/arrow/python/cmake_modules)
 
-include_directories("${PYTHON_INCLUDE_DIRS}" thirdparty)
+find_package(Plasma)
+include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --std=c99 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -O3")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++11 -D_XOPEN_SOURCE=500 -D_POSIX_C_SOURCE=200809L -O3")
 
-# Compile flatbuffers
-
-set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs")
-set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
-
-set(PLASMA_FBS_OUTPUT_FILES
-  "${OUTPUT_DIR}/plasma_generated.h")
-
-add_custom_command(
-  OUTPUT ${PLASMA_FBS_OUTPUT_FILES}
-  COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${PLASMA_FBS_SRC}
-  DEPENDS ${PLASMA_FBS_SRC}
-  COMMENT "Running flatc compiler on ${PLASMA_FBS_SRC}"
-  VERBATIM)
-
-add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
-
-add_dependencies(gen_plasma_fbs flatbuffers_ep)
-
 if(UNIX AND NOT APPLE)
   link_libraries(rt)
 endif()
 
-include_directories("${CMAKE_CURRENT_LIST_DIR}/")
-include_directories("${CMAKE_CURRENT_LIST_DIR}/../")
-
-add_library(plasma SHARED
-  plasma.cc
-  plasma_extension.cc
-  plasma_protocol.cc
-  plasma_client.cc
-  thirdparty/xxhash.c
-  fling.c)
-
-add_dependencies(plasma gen_plasma_fbs)
-
-if(APPLE)
-  target_link_libraries(plasma plasma_lib "-undefined dynamic_lookup" -Wl,-force_load,${FLATBUFFERS_STATIC_LIB} ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB} -lpthread)
-else(APPLE)
-  target_link_libraries(plasma plasma_lib -Wl,--whole-archive ${FLATBUFFERS_STATIC_LIB} -Wl,--no-whole-archive ${PYTHON_LIBRARIES} ${FLATBUFFERS_STATIC_LIB} -lpthread)
-endif(APPLE)
+include_directories("${ARROW_DIR}/cpp/src/")
+# include_directories("${CMAKE_CURRENT_LIST_DIR}/../")
 
 include_directories("${FLATBUFFERS_INCLUDE_DIR}")
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
 
-set_source_files_properties(thirdparty/dlmalloc.c PROPERTIES COMPILE_FLAGS -Wno-all)
-
-add_library(plasma_lib STATIC
-  plasma_client.cc
-  plasma.cc
-  plasma_common.cc
-  plasma_io.cc
-  plasma_protocol.cc
-  status.cc
-  fling.c
-  thirdparty/xxhash.c)
-
-target_link_libraries(plasma_lib ${FLATBUFFERS_STATIC_LIB} -lpthread)
-add_dependencies(plasma_lib gen_plasma_fbs)
-
-add_executable(plasma_store
-  plasma_store.cc
-  thirdparty/ae/ae.c
-  plasma.cc
-  plasma_events.cc
-  plasma_protocol.cc
-  eviction_policy.cc
-  fling.c
-  malloc.cc)
-
-add_dependencies(plasma_store hiredis gen_plasma_fbs)
-
-target_link_libraries(plasma_store plasma_lib ${FLATBUFFERS_STATIC_LIB})
-
-add_dependencies(plasma protocol_fbs)
-
 add_executable(plasma_manager
   plasma_manager.cc)
 
-target_link_libraries(plasma_manager common plasma_lib ${FLATBUFFERS_STATIC_LIB})
+target_link_libraries(plasma_manager common ${PLASMA_STATIC_LIB} ${ARROW_DIR}/cpp/build/release/libarrow.a -lpthread)
 
-add_library(plasma_client SHARED plasma_client.cc)
-target_link_libraries(plasma_client ${FLATBUFFERS_STATIC_LIB})
-
-target_link_libraries(plasma_client plasma_lib ${FLATBUFFERS_STATIC_LIB})
-
-define_test(client_tests plasma_lib)
-define_test(manager_tests plasma_lib plasma_manager.cc)
-define_test(serialization_tests plasma_lib)
+define_test(client_tests ${PLASMA_STATIC_LIB})
+define_test(manager_tests ${PLASMA_STATIC_LIB} plasma_manager.cc)

src/plasma/eviction_policy.cc

@@ -1,95 +0,0 @@
-#include "eviction_policy.h"
-
-void LRUCache::add(const ObjectID &key, int64_t size) {
-  auto it = item_map_.find(key);
-  ARROW_CHECK(it == item_map_.end());
-  /* Note that it is important to use a list so the iterators stay valid. */
-  item_list_.emplace_front(key, size);
-  item_map_.emplace(key, item_list_.begin());
-}
-
-void LRUCache::remove(const ObjectID &key) {
-  auto it = item_map_.find(key);
-  ARROW_CHECK(it != item_map_.end());
-  item_list_.erase(it->second);
-  item_map_.erase(it);
-}
-
-int64_t LRUCache::choose_objects_to_evict(
-    int64_t num_bytes_required,
-    std::vector<ObjectID> &objects_to_evict) {
-  int64_t bytes_evicted = 0;
-  auto it = item_list_.end();
-  while (bytes_evicted < num_bytes_required && it != item_list_.begin()) {
-    it--;
-    objects_to_evict.push_back(it->first);
-    bytes_evicted += it->second;
-  }
-  return bytes_evicted;
-}
-
-EvictionPolicy::EvictionPolicy(PlasmaStoreInfo *store_info)
-    : memory_used_(0), store_info_(store_info) {}
-
-int64_t EvictionPolicy::choose_objects_to_evict(
-    int64_t num_bytes_required,
-    std::vector<ObjectID> &objects_to_evict) {
-  int64_t bytes_evicted =
-      cache_.choose_objects_to_evict(num_bytes_required, objects_to_evict);
-  /* Update the LRU cache. */
-  for (auto &object_id : objects_to_evict) {
-    cache_.remove(object_id);
-  }
-  /* Update the number of bytes used. */
-  memory_used_ -= bytes_evicted;
-  return bytes_evicted;
-}
-
-void EvictionPolicy::object_created(ObjectID object_id) {
-  auto entry = store_info_->objects[object_id].get();
-  cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
-}
-
-bool EvictionPolicy::require_space(int64_t size,
-                                   std::vector<ObjectID> &objects_to_evict) {
-  /* Check if there is enough space to create the object. */
-  int64_t required_space = memory_used_ + size - store_info_->memory_capacity;
-  int64_t num_bytes_evicted;
-  if (required_space > 0) {
-    /* Try to free up at least as much space as we need right now but ideally
-     * up to 20% of the total capacity. */
-    int64_t space_to_free = std::max(size, store_info_->memory_capacity / 5);
-    ARROW_LOG(DEBUG)
-        << "not enough space to create this object, so evicting objects";
-    /* Choose some objects to evict, and update the return pointers. */
-    num_bytes_evicted =
-        choose_objects_to_evict(space_to_free, objects_to_evict);
-    ARROW_LOG(INFO)
-        << "There is not enough space to create this object, so evicting "
-        << objects_to_evict.size() << " objects to free up "
-        << num_bytes_evicted << " bytes.";
-  } else {
-    num_bytes_evicted = 0;
-  }
-  if (num_bytes_evicted >= required_space) {
-    /* We only increment the space used if there is enough space to create the
-     * object. */
-    memory_used_ += size;
-  }
-  return num_bytes_evicted >= required_space;
-}
-
-void EvictionPolicy::begin_object_access(
-    ObjectID object_id,
-    std::vector<ObjectID> &objects_to_evict) {
-  /* If the object is in the LRU cache, remove it. */
-  cache_.remove(object_id);
-}
-
-void EvictionPolicy::end_object_access(
-    ObjectID object_id,
-    std::vector<ObjectID> &objects_to_evict) {
-  auto entry = store_info_->objects[object_id].get();
-  /* Add the object to the LRU cache.*/
-  cache_.add(object_id, entry->info.data_size + entry->info.metadata_size);
-}

src/plasma/eviction_policy.h

@@ -1,128 +0,0 @@
-#ifndef EVICTION_POLICY_H
-#define EVICTION_POLICY_H
-
-#include <list>
-#include <unordered_map>
-
-#include "plasma_common.h"
-#include "plasma.h"
-
-/* ==== The eviction policy ====
- *
- * This file contains declaration for all functions and data structures that
- * need to be provided if you want to implement a new eviction algorithm for the
- * Plasma store.
- */
-
-class LRUCache {
- private:
-  /** A doubly-linked list containing the items in the cache and
-   *  their sizes in LRU order. */
-  typedef std::list<std::pair<ObjectID, int64_t>> ItemList;
-  ItemList item_list_;
-  /** A hash table mapping the object ID of an object in the cache to its
-   *  location in the doubly linked list item_list_. */
-  std::unordered_map<ObjectID, ItemList::iterator, UniqueIDHasher> item_map_;
-
- public:
-  LRUCache(){};
-
-  void add(const ObjectID &key, int64_t size);
-
-  void remove(const ObjectID &key);
-
-  int64_t choose_objects_to_evict(int64_t num_bytes_required,
-                                  std::vector<ObjectID> &objects_to_evict);
-};
-
-/** The eviction policy. */
-class EvictionPolicy {
- public:
-  /**
-   * Construct an eviction policy.
-   *
-   * @param store_info Information about the Plasma store that is exposed
-   *        to the eviction policy.
-   */
-  EvictionPolicy(PlasmaStoreInfo *store_info);
-
-  /**
-   * This method will be called whenever an object is first created in order to
-   * add it to the LRU cache. This is done so that the first time, the Plasma
-   * store calls begin_object_access, we can remove the object from the LRU
-   * cache.
-   *
-   * @param object_id The object ID of the object that was created.
-   * @return Void.
-   */
-  void object_created(ObjectID object_id);
-
-  /**
-   * This method will be called when the Plasma store needs more space, perhaps
-   * to create a new object. If the required amount of space cannot be freed up,
-   * then a fatal error will be thrown. When this method is called, the eviction
-   * policy will assume that the objects chosen to be evicted will in fact be
-   * evicted from the Plasma store by the caller.
-   *
-   * @param size The size in bytes of the new object, including both data and
-   *        metadata.
-   * @param objects_to_evict The object IDs that were chosen for eviction will
-   *        be stored into this vector.
-   * @return True if enough space can be freed and false otherwise.
-   */
-  bool require_space(int64_t size, std::vector<ObjectID> &objects_to_evict);
-
-  /**
-   * This method will be called whenever an unused object in the Plasma store
-   * starts to be used. When this method is called, the eviction policy will
-   * assume that the objects chosen to be evicted will in fact be evicted from
-   * the Plasma store by the caller.
-   *
-   * @param object_id The ID of the object that is now being used.
-   * @param objects_to_evict The object IDs that were chosen for eviction will
-   *        be stored into this vector.
-   * @return Void.
-   */
-  void begin_object_access(ObjectID object_id,
-                           std::vector<ObjectID> &objects_to_evict);
-
-  /**
-   * This method will be called whenever an object in the Plasma store that was
-   * being used is no longer being used. When this method is called, the
-   * eviction policy will assume that the objects chosen to be evicted will in
-   * fact be evicted from the Plasma store by the caller.
-   *
-   * @param object_id The ID of the object that is no longer being used.
-   * @param objects_to_evict The object IDs that were chosen for eviction will
-   *        be stored into this vector.
-   * @return Void.
-   */
-  void end_object_access(ObjectID object_id,
-                         std::vector<ObjectID> &objects_to_evict);
-
-  /**
-  * Choose some objects to evict from the Plasma store. When this method is
-  * called, the eviction policy will assume that the objects chosen to be
-  * evicted will in fact be evicted from the Plasma store by the caller.
-  *
-  * @note This method is not part of the API. It is exposed in the header file
-  * only for testing.
-  *
-  * @param num_bytes_required The number of bytes of space to try to free up.
-  * @param objects_to_evict The object IDs that were chosen for eviction will
-  *        be stored into this vector.
-  * @return The total number of bytes of space chosen to be evicted.
-  */
-  int64_t choose_objects_to_evict(int64_t num_bytes_required,
-                                  std::vector<ObjectID> &objects_to_evict);
-
- private:
-  /** Pointer to the plasma store info. */
-  PlasmaStoreInfo *store_info_;
-  /** The amount of memory (in bytes) currently being used. */
-  int64_t memory_used_;
-  /** Datastructure for the LRU cache. */
-  LRUCache cache_;
-};
-
-#endif /* EVICTION_POLICY_H */

src/plasma/fling.c

@@ -1,76 +0,0 @@
-#include "fling.h"
-
-#include <string.h>
-
-void init_msg(struct msghdr *msg,
-              struct iovec *iov,
-              char *buf,
-              size_t buf_len) {
-  iov->iov_base = buf;
-  iov->iov_len = 1;
-
-  msg->msg_iov = iov;
-  msg->msg_iovlen = 1;
-  msg->msg_control = buf;
-  msg->msg_controllen = buf_len;
-  msg->msg_name = NULL;
-  msg->msg_namelen = 0;
-}
-
-int send_fd(int conn, int fd) {
-  struct msghdr msg;
-  struct iovec iov;
-  char buf[CMSG_SPACE(sizeof(int))];
-  memset(&buf, 0, CMSG_SPACE(sizeof(int)));
-
-  init_msg(&msg, &iov, buf, sizeof(buf));
-
-  struct cmsghdr *header = CMSG_FIRSTHDR(&msg);
-  header->cmsg_level = SOL_SOCKET;
-  header->cmsg_type = SCM_RIGHTS;
-  header->cmsg_len = CMSG_LEN(sizeof(int));
-  *(int *) CMSG_DATA(header) = fd;
-
-  /* Send file descriptor. */
-  return sendmsg(conn, &msg, 0);
-}
-
-int recv_fd(int conn) {
-  struct msghdr msg;
-  struct iovec iov;
-  char buf[CMSG_SPACE(sizeof(int))];
-  init_msg(&msg, &iov, buf, sizeof(buf));
-
-  if (recvmsg(conn, &msg, 0) == -1)
-    return -1;
-
-  int found_fd = -1;
-  int oh_noes = 0;
-  for (struct cmsghdr *header = CMSG_FIRSTHDR(&msg); header != NULL;
-       header = CMSG_NXTHDR(&msg, header))
-    if (header->cmsg_level == SOL_SOCKET && header->cmsg_type == SCM_RIGHTS) {
-      int count =
-          (header->cmsg_len - (CMSG_DATA(header) - (unsigned char *) header)) /
-          sizeof(int);
-      for (int i = 0; i < count; ++i) {
-        int fd = ((int *) CMSG_DATA(header))[i];
-        if (found_fd == -1) {
-          found_fd = fd;
-        } else {
-          close(fd);
-          oh_noes = 1;
-        }
-      }
-    }
-
-  /* The sender sent us more than one file descriptor. We've closed
-   * them all to prevent fd leaks but notify the caller that we got
-   * a bad message. */
-  if (oh_noes) {
-    close(found_fd);
-    errno = EBADMSG;
-    return -1;
-  }
-
-  return found_fd;
-}

src/plasma/fling.h

@@ -1,43 +0,0 @@
-/* FLING: Exchanging file descriptors over sockets
- *
- * This is a little library for sending file descriptors over a socket
- * between processes. The reason for doing that (as opposed to using
- * filenames to share the files) is so (a) no files remain in the
- * filesystem after all the processes terminate, (b) to make sure that
- * there are no name collisions and (c) to be able to control who has
- * access to the data.
- *
- * Most of the code is from https://github.com/sharvil/flingfd */
-
-#include <unistd.h>
-#include <errno.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-
-/* This is neccessary for Mac OS X, see http://www.apuebook.com/faqs2e.html
- * (10). */
-#if !defined(CMSG_SPACE) && !defined(CMSG_LEN)
-#define CMSG_SPACE(len) \
-  (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + __DARWIN_ALIGN32(len))
-#define CMSG_LEN(len) (__DARWIN_ALIGN32(sizeof(struct cmsghdr)) + (len))
-#endif
-
-void init_msg(struct msghdr *msg, struct iovec *iov, char *buf, size_t buf_len);
-
-/**
- * Send a file descriptor over a unix domain socket.
- *
- * @param conn Unix domain socket to send the file descriptor over.
- * @param fd File descriptor to send over.
- * @return Status code which is < 0 on failure.
- */
-int send_fd(int conn, int fd);
-
-/**
- * Receive a file descriptor over a unix domain socket.
- *
- * @param conn Unix domain socket to receive the file descriptor from.
- * @return File descriptor or a value < 0 on failure.
- */
-int recv_fd(int conn);

src/plasma/logging.h

@@ -1,147 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#ifndef ARROW_UTIL_LOGGING_H
-#define ARROW_UTIL_LOGGING_H
-
-#include <cstdlib>
-#include <iostream>
-
-namespace arrow {
-
-// Stubbed versions of macros defined in glog/logging.h, intended for
-// environments where glog headers aren't available.
-//
-// Add more as needed.
-
-// Log levels. LOG ignores them, so their values are abitrary.
-
-#define ARROW_DEBUG (-1)
-#define ARROW_INFO 0
-#define ARROW_WARNING 1
-#define ARROW_ERROR 2
-#define ARROW_FATAL 3
-
-#define ARROW_LOG_INTERNAL(level) ::arrow::internal::CerrLog(level)
-#define ARROW_LOG(level) ARROW_LOG_INTERNAL(ARROW_##level)
-
-#define ARROW_CHECK(condition)                               \
-  (condition) ? 0 : ::arrow::internal::FatalLog(ARROW_FATAL) \
-                        << __FILE__ << __LINE__              \
-                        << " Check failed: " #condition " "
-
-#ifdef NDEBUG
-#define ARROW_DFATAL ARROW_WARNING
-
-#define DCHECK(condition) \
-  while (false)           \
-  ::arrow::internal::NullLog()
-#define DCHECK_EQ(val1, val2) \
-  while (false)               \
-  ::arrow::internal::NullLog()
-#define DCHECK_NE(val1, val2) \
-  while (false)               \
-  ::arrow::internal::NullLog()
-#define DCHECK_LE(val1, val2) \
-  while (false)               \
-  ::arrow::internal::NullLog()
-#define DCHECK_LT(val1, val2) \
-  while (false)               \
-  ::arrow::internal::NullLog()
-#define DCHECK_GE(val1, val2) \
-  while (false)               \
-  ::arrow::internal::NullLog()
-#define DCHECK_GT(val1, val2) \
-  while (false)               \
-  ::arrow::internal::NullLog()
-
-#else
-#define ARROW_DFATAL ARROW_FATAL
-
-#define DCHECK(condition) ARROW_CHECK(condition)
-#define DCHECK_EQ(val1, val2) ARROW_CHECK((val1) == (val2))
-#define DCHECK_NE(val1, val2) ARROW_CHECK((val1) != (val2))
-#define DCHECK_LE(val1, val2) ARROW_CHECK((val1) <= (val2))
-#define DCHECK_LT(val1, val2) ARROW_CHECK((val1) < (val2))
-#define DCHECK_GE(val1, val2) ARROW_CHECK((val1) >= (val2))
-#define DCHECK_GT(val1, val2) ARROW_CHECK((val1) > (val2))
-
-#endif  // NDEBUG
-
-namespace internal {
-
-class NullLog {
- public:
-  template <class T>
-  NullLog &operator<<(const T &t) {
-    return *this;
-  }
-};
-
-class CerrLog {
- public:
-  CerrLog(int severity)  // NOLINT(runtime/explicit)
-      : severity_(severity),
-        has_logged_(false) {}
-
-  virtual ~CerrLog() {
-    if (has_logged_) {
-      std::cerr << std::endl;
-    }
-    if (severity_ == ARROW_FATAL) {
-      std::exit(1);
-    }
-  }
-
-  template <class T>
-  CerrLog &operator<<(const T &t) {
-    // TODO(pcm): Print this if in debug mode, but not if in valgrind
-    // mode
-    if (severity_ == ARROW_DEBUG) {
-      return *this;
-    }
-
-    has_logged_ = true;
-    std::cerr << t;
-    return *this;
-  }
-
- protected:
-  const int severity_;
-  bool has_logged_;
-};
-
-// Clang-tidy isn't smart enough to determine that DCHECK using CerrLog doesn't
-// return so we create a new class to give it a hint.
-class FatalLog : public CerrLog {
- public:
-  explicit FatalLog(int /* severity */)  // NOLINT
-      : CerrLog(ARROW_FATAL){}           // NOLINT
-
-            [[noreturn]] ~FatalLog() {
-    if (has_logged_) {
-      std::cerr << std::endl;
-    }
-    std::exit(1);
-  }
-};
-
-}  // namespace internal
-
-}  // namespace arrow
-
-#endif  // ARROW_UTIL_LOGGING_H

src/plasma/malloc.cc

@@ -1,168 +0,0 @@
-#include <assert.h>
-#include <stdlib.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <unistd.h>
-
-#include <unordered_map>
-
-#include "common.h"
-
-extern "C" {
-void *fake_mmap(size_t);
-int fake_munmap(void *, size_t);
-
-#define MMAP(s) fake_mmap(s)
-#define MUNMAP(a, s) fake_munmap(a, s)
-#define DIRECT_MMAP(s) fake_mmap(s)
-#define DIRECT_MUNMAP(a, s) fake_munmap(a, s)
-#define USE_DL_PREFIX
-#define HAVE_MORECORE 0
-#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
-#define DEFAULT_GRANULARITY ((size_t) 128U * 1024U)
-
-#include "thirdparty/dlmalloc.c"
-
-#undef MMAP
-#undef MUNMAP
-#undef DIRECT_MMAP
-#undef DIRECT_MUNMAP
-#undef USE_DL_PREFIX
-#undef HAVE_MORECORE
-#undef DEFAULT_GRANULARITY
-}
-
-struct mmap_record {
-  int fd;
-  int64_t size;
-};
-
-namespace {
-
-/** Hashtable that contains one entry per segment that we got from the OS
- *  via mmap. Associates the address of that segment with its file descriptor
- *  and size. */
-std::unordered_map<void *, mmap_record> mmap_records;
-
-} /* namespace */
-
-constexpr int GRANULARITY_MULTIPLIER = 2;
-
-static void *pointer_advance(void *p, ptrdiff_t n) {
-  return (unsigned char *) p + n;
-}
-
-static void *pointer_retreat(void *p, ptrdiff_t n) {
-  return (unsigned char *) p - n;
-}
-
-static ptrdiff_t pointer_distance(void const *pfrom, void const *pto) {
-  return (unsigned char const *) pto - (unsigned char const *) pfrom;
-}
-
-/* Create a buffer. This is creating a temporary file and then
- * immediately unlinking it so we do not leave traces in the system. */
-int create_buffer(int64_t size) {
-  int fd;
-#ifdef _WIN32
-  if (!CreateFileMapping(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE,
-                         (DWORD)((uint64_t) size >> (CHAR_BIT * sizeof(DWORD))),
-                         (DWORD)(uint64_t) size, NULL)) {
-    fd = -1;
-  }
-#else
-#ifdef __linux__
-  constexpr char file_template[] = "/dev/shm/plasmaXXXXXX";
-#else
-  constexpr char file_template[] = "/tmp/plasmaXXXXXX";
-#endif
-  char file_name[32];
-  strncpy(file_name, file_template, 32);
-  fd = mkstemp(file_name);
-  if (fd < 0)
-    return -1;
-  FILE *file = fdopen(fd, "a+");
-  if (!file) {
-    close(fd);
-    return -1;
-  }
-  if (unlink(file_name) != 0) {
-    LOG_ERROR("unlink error");
-    return -1;
-  }
-  if (ftruncate(fd, (off_t) size) != 0) {
-    LOG_ERROR("ftruncate error");
-    return -1;
-  }
-#endif
-  return fd;
-}
-
-void *fake_mmap(size_t size) {
-  /* Add sizeof(size_t) so that the returned pointer is deliberately not
-   * page-aligned. This ensures that the segments of memory returned by
-   * fake_mmap are never contiguous. */
-  size += sizeof(size_t);
-
-  int fd = create_buffer(size);
-  CHECKM(fd >= 0, "Failed to create buffer during mmap");
-  void *pointer = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
-  if (pointer == MAP_FAILED) {
-    return pointer;
-  }
-
-  /* Increase dlmalloc's allocation granularity directly. */
-  mparams.granularity *= GRANULARITY_MULTIPLIER;
-
-  mmap_record &record = mmap_records[pointer];
-  record.fd = fd;
-  record.size = size;
-
-  /* We lie to dlmalloc about where mapped memory actually lives. */
-  pointer = pointer_advance(pointer, sizeof(size_t));
-  LOG_DEBUG("%p = fake_mmap(%lu)", pointer, size);
-  return pointer;
-}
-
-int fake_munmap(void *addr, size_t size) {
-  LOG_DEBUG("fake_munmap(%p, %lu)", addr, size);
-  addr = pointer_retreat(addr, sizeof(size_t));
-  size += sizeof(size_t);
-
-  auto entry = mmap_records.find(addr);
-
-  if (entry == mmap_records.end() || entry->second.size != size) {
-    /* Reject requests to munmap that don't directly match previous
-     * calls to mmap, to prevent dlmalloc from trimming. */
-    return -1;
-  }
-
-  int r = munmap(addr, size);
-  if (r == 0) {
-    close(entry->second.fd);
-  }
-
-  mmap_records.erase(entry);
-  return r;
-}
-
-void get_malloc_mapinfo(void *addr,
-                        int *fd,
-                        int64_t *map_size,
-                        ptrdiff_t *offset) {
-  /* TODO(rshin): Implement a more efficient search through mmap_records. */
-  for (const auto &entry : mmap_records) {
-    if (addr >= entry.first &&
-        addr < pointer_advance(entry.first, entry.second.size)) {
-      *fd = entry.second.fd;
-      *map_size = entry.second.size;
-      *offset = pointer_distance(entry.first, addr);
-      return;
-    }
-  }
-  *fd = -1;
-  *map_size = 0;
-  *offset = 0;
-}

src/plasma/malloc.h

@@ -1,9 +0,0 @@
-#ifndef MALLOC_H
-#define MALLOC_H
-
-void get_malloc_mapinfo(void *addr,
-                        int *fd,
-                        int64_t *map_length,
-                        ptrdiff_t *offset);
-
-#endif /* MALLOC_H */

src/plasma/plasma.cc

@@ -1,53 +0,0 @@
-#include "plasma_common.h"
-#include "plasma.h"
-
-#include "io.h"
-#include <sys/types.h>
-#include <sys/socket.h>
-#include <unistd.h>
-
-#include "plasma_protocol.h"
-
-int warn_if_sigpipe(int status, int client_sock) {
-  if (status >= 0) {
-    return 0;
-  }
-  if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {
-    ARROW_LOG(WARNING)
-        << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when "
-           "sending a message to client on fd "
-        << client_sock << ". The client on the other end may "
-                          "have hung up.";
-    return errno;
-  }
-  ARROW_LOG(FATAL) << "Failed to write message to client on fd " << client_sock
-                   << ".";
-}
-
-/**
- * This will create a new ObjectInfo buffer. The first sizeof(int64_t) bytes
- * of this buffer are the length of the remaining message and the
- * remaining message is a serialized version of the object info.
- *
- * @param object_info The object info to be serialized
- * @return The object info buffer. It is the caller's responsibility to free
- *         this buffer with "free" after it has been used.
- */
-uint8_t *create_object_info_buffer(ObjectInfoT *object_info) {
-  flatbuffers::FlatBufferBuilder fbb;
-  auto message = CreateObjectInfo(fbb, object_info);
-  fbb.Finish(message);
-  uint8_t *notification = (uint8_t *) malloc(sizeof(int64_t) + fbb.GetSize());
-  *((int64_t *) notification) = fbb.GetSize();
-  memcpy(notification + sizeof(int64_t), fbb.GetBufferPointer(), fbb.GetSize());
-  return notification;
-}
-
-ObjectTableEntry *get_object_table_entry(PlasmaStoreInfo *store_info,
-                                         ObjectID object_id) {
-  auto it = store_info->objects.find(object_id);
-  if (it == store_info->objects.end()) {
-    return NULL;
-  }
-  return it->second.get();
-}

src/plasma/plasma.h

@@ -1,186 +0,0 @@
-#ifndef PLASMA_H
-#define PLASMA_H
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-#include <stddef.h>
-#include <string.h>
-#include <unistd.h> /* pid_t */
-
-extern "C" {
-#include "sha256.h"
-}
-
-#include <unordered_map>
-#include <unordered_set>
-
-#include "format/common_generated.h"
-#include "logging.h"
-#include "status.h"
-
-#include <inttypes.h>
-
-#define HANDLE_SIGPIPE(s, fd_)                                              \
-  do {                                                                      \
-    Status _s = (s);                                                        \
-    if (!_s.ok()) {                                                         \
-      if (errno == EPIPE || errno == EBADF || errno == ECONNRESET) {        \
-        ARROW_LOG(WARNING)                                                  \
-            << "Received SIGPIPE, BAD FILE DESCRIPTOR, or ECONNRESET when " \
-               "sending a message to client on fd "                         \
-            << fd_ << ". "                                                  \
-                      "The client on the other end may have hung up.";      \
-      } else {                                                              \
-        return _s;                                                          \
-      }                                                                     \
-    }                                                                       \
-  } while (0);
-
-/** Allocation granularity used in plasma for object allocation. */
-#define BLOCK_SIZE 64
-
-// Size of object hash digests.
-constexpr int64_t kDigestSize = SHA256_BLOCK_SIZE;
-
-struct Client;
-
-/**
- * Object request data structure. Used in the plasma_wait_for_objects()
- * argument.
- */
-typedef struct {
-  /** The ID of the requested object. If ID_NIL request any object. */
-  ObjectID object_id;
-  /** Request associated to the object. It can take one of the following values:
-   *  - PLASMA_QUERY_LOCAL: return if or when the object is available in the
-   *    local Plasma Store.
-   *  - PLASMA_QUERY_ANYWHERE: return if or when the object is available in
-   *    the system (i.e., either in the local or a remote Plasma Store). */
-  int type;
-  /** Object status. Same as the status returned by plasma_status() function
-   *  call. This is filled in by plasma_wait_for_objects1():
-   *  - ObjectStatus_Local: object is ready at the local Plasma Store.
-   *  - ObjectStatus_Remote: object is ready at a remote Plasma Store.
-   *  - ObjectStatus_Nonexistent: object does not exist in the system.
-   *  - PLASMA_CLIENT_IN_TRANSFER, if the object is currently being scheduled
-   *    for being transferred or it is transferring. */
-  int status;
-} ObjectRequest;
-
-/** Mapping from object IDs to type and status of the request. */
-typedef std::unordered_map<ObjectID, ObjectRequest, UniqueIDHasher>
-    ObjectRequestMap;
-
-/* Handle to access memory mapped file and map it into client address space. */
-typedef struct {
-  /** The file descriptor of the memory mapped file in the store. It is used as
-   *  a unique identifier of the file in the client to look up the corresponding
-   *  file descriptor on the client's side. */
-  int store_fd;
-  /** The size in bytes of the memory mapped file. */
-  int64_t mmap_size;
-} object_handle;
-
-typedef struct {
-  /** Handle for memory mapped file the object is stored in. */
-  object_handle handle;
-  /** The offset in bytes in the memory mapped file of the data. */
-  ptrdiff_t data_offset;
-  /** The offset in bytes in the memory mapped file of the metadata. */
-  ptrdiff_t metadata_offset;
-  /** The size in bytes of the data. */
-  int64_t data_size;
-  /** The size in bytes of the metadata. */
-  int64_t metadata_size;
-} PlasmaObject;
-
-typedef enum {
-  /** Object was created but not sealed in the local Plasma Store. */
-  PLASMA_CREATED = 1,
-  /** Object is sealed and stored in the local Plasma Store. */
-  PLASMA_SEALED
-} object_state;
-
-typedef enum {
-  /** The object was not found. */
-  OBJECT_NOT_FOUND = 0,
-  /** The object was found. */
-  OBJECT_FOUND = 1
-} object_status;
-
-typedef enum {
-  /** Query for object in the local plasma store. */
-  PLASMA_QUERY_LOCAL = 1,
-  /** Query for object in the local plasma store or in a remote plasma store. */
-  PLASMA_QUERY_ANYWHERE
-} object_request_type;
-
-/** This type is used by the Plasma store. It is here because it is exposed to
- *  the eviction policy. */
-struct ObjectTableEntry {
-  /** Object id of this object. */
-  ObjectID object_id;
-  /** Object info like size, creation time and owner. */
-  ObjectInfoT info;
-  /** Memory mapped file containing the object. */
-  int fd;
-  /** Size of the underlying map. */
-  int64_t map_size;
-  /** Offset from the base of the mmap. */
-  ptrdiff_t offset;
-  /** Pointer to the object data. Needed to free the object. */
-  uint8_t *pointer;
-  /** Set of clients currently using this object. */
-  std::unordered_set<Client *> clients;
-  /** The state of the object, e.g., whether it is open or sealed. */
-  object_state state;
-  /** The digest of the object. Used to see if two objects are the same. */
-  unsigned char digest[kDigestSize];
-};
-
-/** The plasma store information that is exposed to the eviction policy. */
-struct PlasmaStoreInfo {
-  /** Objects that are in the Plasma store. */
-  std::unordered_map<ObjectID,
-                     std::unique_ptr<ObjectTableEntry>,
-                     UniqueIDHasher>
-      objects;
-  /** The amount of memory (in bytes) that we allow to be allocated in the
-   *  store. */
-  int64_t memory_capacity;
-};
-
-/**
- * Get an entry from the object table and return NULL if the object_id
- * is not present.
- *
- * @param store_info The PlasmaStoreInfo that contains the object table.
- * @param object_id The object_id of the entry we are looking for.
- * @return The entry associated with the object_id or NULL if the object_id
- *         is not present.
- */
-ObjectTableEntry *get_object_table_entry(PlasmaStoreInfo *store_info,
-                                         ObjectID object_id);
-
-/**
- * Print a warning if the status is less than zero. This should be used to check
- * the success of messages sent to plasma clients. We print a warning instead of
- * failing because the plasma clients are allowed to die. This is used to handle
- * situations where the store writes to a client file descriptor, and the client
- * may already have disconnected. If we have processed the disconnection and
- * closed the file descriptor, we should get a BAD FILE DESCRIPTOR error. If we
- * have not, then we should get a SIGPIPE. If we write to a TCP socket that
- * isn't connected yet, then we should get an ECONNRESET.
- *
- * @param status The status to check. If it is less less than zero, we will
- *        print a warning.
- * @param client_sock The client socket. This is just used to print some extra
- *        information.
- * @return The errno set.
- */
-int warn_if_sigpipe(int status, int client_sock);
-
-uint8_t *create_object_info_buffer(ObjectInfoT *object_info);
-
-#endif /* PLASMA_H */

src/plasma/plasma_client.cc

@@ -1,624 +0,0 @@
-// PLASMA CLIENT: Client library for using the plasma store and manager
-
-#ifdef _WIN32
-#include <Win32_Interop/win32_types.h>
-#endif
-
-#include <assert.h>
-#include <fcntl.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <strings.h>
-#include <netinet/in.h>
-
-#include "plasma_common.h"
-#include "plasma.h"
-#include "plasma_io.h"
-#include "plasma_protocol.h"
-#include "plasma_client.h"
-
-#include <vector>
-#include <thread>
-
-extern "C" {
-#include "sha256.h"
-#include "fling.h"
-
-#define XXH_STATIC_LINKING_ONLY
-#include "xxhash.h"
-
-#define XXH64_DEFAULT_SEED 0
-}
-
-// Number of threads used for memcopy and hash computations.
-constexpr int64_t kThreadPoolSize = 8;
-constexpr int64_t kBytesInMB = 1 << 20;
-static std::vector<std::thread> threadpool_(kThreadPoolSize);
-
-struct ClientMmapTableEntry {
-  /// The result of mmap for this file descriptor.
-  uint8_t *pointer;
-  /// The length of the memory-mapped file.
-  size_t length;
-  /// The number of objects in this memory-mapped file that are currently being
-  /// used by the client. When this count reaches zeros, we unmap the file.
-  int count;
-};
-
-struct ObjectInUseEntry {
-  /// A count of the number of times this client has called PlasmaClient::Create
-  /// or
-  /// PlasmaClient::Get on this object ID minus the number of calls to
-  /// PlasmaClient::Release.
-  /// When this count reaches zero, we remove the entry from the ObjectsInUse
-  /// and decrement a count in the relevant ClientMmapTableEntry.
-  int count;
-  /// Cached information to read the object.
-  PlasmaObject object;
-  /// A flag representing whether the object has been sealed.
-  bool is_sealed;
-};
-
-// If the file descriptor fd has been mmapped in this client process before,
-// return the pointer that was returned by mmap, otherwise mmap it and store the
-// pointer in a hash table.
-uint8_t *lookup_or_mmap(PlasmaClient *conn,
-                        int fd,
-                        int store_fd_val,
-                        int64_t map_size) {
-  auto entry = conn->mmap_table.find(store_fd_val);
-  if (entry != conn->mmap_table.end()) {
-    close(fd);
-    return entry->second->pointer;
-  } else {
-    uint8_t *result = (uint8_t *) mmap(NULL, map_size, PROT_READ | PROT_WRITE,
-                                       MAP_SHARED, fd, 0);
-    if (result == MAP_FAILED) {
-      ARROW_LOG(FATAL) << "mmap failed";
-    }
-    close(fd);
-    ClientMmapTableEntry *entry = new ClientMmapTableEntry();
-    entry->pointer = result;
-    entry->length = map_size;
-    entry->count = 0;
-    conn->mmap_table[store_fd_val] = entry;
-    return result;
-  }
-}
-
-// Get a pointer to a file that we know has been memory mapped in this client
-// process before.
-uint8_t *lookup_mmapped_file(PlasmaClient *conn, int store_fd_val) {
-  auto entry = conn->mmap_table.find(store_fd_val);
-  ARROW_CHECK(entry != conn->mmap_table.end());
-  return entry->second->pointer;
-}
-
-void increment_object_count(PlasmaClient *conn,
-                            ObjectID object_id,
-                            PlasmaObject *object,
-                            bool is_sealed) {
-  // Increment the count of the object to track the fact that it is being used.
-  // The corresponding decrement should happen in PlasmaClient::Release.
-  auto elem = conn->objects_in_use.find(object_id);
-  ObjectInUseEntry *object_entry;
-  if (elem == conn->objects_in_use.end()) {
-    // Add this object ID to the hash table of object IDs in use. The
-    // corresponding call to free happens in PlasmaClient::Release.
-    object_entry = new ObjectInUseEntry();
-    object_entry->object = *object;
-    object_entry->count = 0;
-    object_entry->is_sealed = is_sealed;
-    conn->objects_in_use[object_id] = object_entry;
-    // Increment the count of the number of objects in the memory-mapped file
-    // that are being used. The corresponding decrement should happen in
-    // PlasmaClient::Release.
-    auto entry = conn->mmap_table.find(object->handle.store_fd);
-    ARROW_CHECK(entry != conn->mmap_table.end());
-    ARROW_CHECK(entry->second->count >= 0);
-    // Update the in_use_object_bytes.
-    conn->in_use_object_bytes +=
-        (object_entry->object.data_size + object_entry->object.metadata_size);
-    entry->second->count += 1;
-  } else {
-    object_entry = elem->second;
-    ARROW_CHECK(object_entry->count > 0);
-  }
-  // Increment the count of the number of instances of this object that are
-  // being used by this client. The corresponding decrement should happen in
-  // PlasmaClient::Release.
-  object_entry->count += 1;
-}
-
-Status PlasmaClient::Create(ObjectID object_id,
-                            int64_t data_size,
-                            uint8_t *metadata,
-                            int64_t metadata_size,
-                            uint8_t **data) {
-  ARROW_LOG(DEBUG) << "called plasma_create on conn " << store_conn
-                   << " with size " << data_size << " and metadata size "
-                   << metadata_size;
-  RETURN_NOT_OK(
-      SendCreateRequest(store_conn, object_id, data_size, metadata_size));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(
-      PlasmaReceive(store_conn, MessageType_PlasmaCreateReply, buffer));
-  ObjectID id;
-  PlasmaObject object;
-  RETURN_NOT_OK(ReadCreateReply(buffer.data(), &id, &object));
-  // If the CreateReply included an error, then the store will not send a file
-  // descriptor.
-  int fd = recv_fd(store_conn);
-  ARROW_CHECK(fd >= 0) << "recv not successful";
-  ARROW_CHECK(object.data_size == data_size);
-  ARROW_CHECK(object.metadata_size == metadata_size);
-  // The metadata should come right after the data.
-  ARROW_CHECK(object.metadata_offset == object.data_offset + data_size);
-  *data = lookup_or_mmap(this, fd, object.handle.store_fd,
-                         object.handle.mmap_size) +
-          object.data_offset;
-  // If plasma_create is being called from a transfer, then we will not copy the
-  // metadata here. The metadata will be written along with the data streamed
-  // from the transfer.
-  if (metadata != NULL) {
-    // Copy the metadata to the buffer.
-    memcpy(*data + object.data_size, metadata, metadata_size);
-  }
-  // Increment the count of the number of instances of this object that this
-  // client is using. A call to PlasmaClient::Release is required to decrement
-  // this
-  // count. Cache the reference to the object.
-  increment_object_count(this, object_id, &object, false);
-  // We increment the count a second time (and the corresponding decrement will
-  // happen in a PlasmaClient::Release call in plasma_seal) so even if the
-  // buffer
-  // returned by PlasmaClient::Dreate goes out of scope, the object does not get
-  // released before the call to PlasmaClient::Seal happens.
-  increment_object_count(this, object_id, &object, false);
-  return Status::OK();
-}
-
-Status PlasmaClient::Get(ObjectID object_ids[],
-                         int64_t num_objects,
-                         int64_t timeout_ms,
-                         ObjectBuffer object_buffers[]) {
-  // Fill out the info for the objects that are already in use locally.
-  bool all_present = true;
-  for (int i = 0; i < num_objects; ++i) {
-    auto object_entry = objects_in_use.find(object_ids[i]);
-    if (object_entry == objects_in_use.end()) {
-      // This object is not currently in use by this client, so we need to send
-      // a request to the store.
-      all_present = false;
-      // Make a note to ourselves that the object is not present.
-      object_buffers[i].data_size = -1;
-    } else {
-      // NOTE: If the object is still unsealed, we will deadlock, since we must
-      // have been the one who created it.
-      ARROW_CHECK(object_entry->second->is_sealed)
-          << "Plasma client called get on an unsealed object that it created";
-      PlasmaObject *object = &object_entry->second->object;
-      object_buffers[i].data =
-          lookup_mmapped_file(this, object->handle.store_fd);
-      object_buffers[i].data = object_buffers[i].data + object->data_offset;
-      object_buffers[i].data_size = object->data_size;
-      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
-      object_buffers[i].metadata_size = object->metadata_size;
-      // Increment the count of the number of instances of this object that this
-      // client is using. A call to PlasmaClient::Release is required to
-      // decrement this
-      // count. Cache the reference to the object.
-      increment_object_count(this, object_ids[i], object, true);
-    }
-  }
-
-  if (all_present) {
-    return Status::OK();
-  }
-
-  // If we get here, then the objects aren't all currently in use by this
-  // client, so we need to send a request to the plasma store.
-  RETURN_NOT_OK(
-      SendGetRequest(store_conn, object_ids, num_objects, timeout_ms));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(PlasmaReceive(store_conn, MessageType_PlasmaGetReply, buffer));
-  std::vector<ObjectID> received_object_ids(num_objects);
-  std::vector<PlasmaObject> object_data(num_objects);
-  PlasmaObject *object;
-  RETURN_NOT_OK(ReadGetReply(buffer.data(), received_object_ids.data(),
-                             object_data.data(), num_objects));
-
-  for (int i = 0; i < num_objects; ++i) {
-    DCHECK(received_object_ids[i] == object_ids[i]);
-    object = &object_data[i];
-    if (object_buffers[i].data_size != -1) {
-      // If the object was already in use by the client, then the store should
-      // have returned it.
-      DCHECK(object->data_size != -1);
-      // We won't use this file descriptor, but the store sent us one, so we
-      // need to receive it and then close it right away so we don't leak file
-      // descriptors.
-      int fd = recv_fd(store_conn);
-      close(fd);
-      ARROW_CHECK(fd >= 0);
-      // We've already filled out the information for this object, so we can
-      // just continue.
-      continue;
-    }
-    // If we are here, the object was not currently in use, so we need to
-    // process the reply from the object store.
-    if (object->data_size != -1) {
-      // The object was retrieved. The user will be responsible for releasing
-      // this object.
-      int fd = recv_fd(store_conn);
-      ARROW_CHECK(fd >= 0);
-      object_buffers[i].data = lookup_or_mmap(this, fd, object->handle.store_fd,
-                                              object->handle.mmap_size);
-      // Finish filling out the return values.
-      object_buffers[i].data = object_buffers[i].data + object->data_offset;
-      object_buffers[i].data_size = object->data_size;
-      object_buffers[i].metadata = object_buffers[i].data + object->data_size;
-      object_buffers[i].metadata_size = object->metadata_size;
-      // Increment the count of the number of instances of this object that this
-      // client is using. A call to PlasmaClient::Release is required to
-      // decrement this
-      // count. Cache the reference to the object.
-      increment_object_count(this, received_object_ids[i], object, true);
-    } else {
-      // The object was not retrieved. Make sure we already put a -1 here to
-      // indicate that the object was not retrieved. The caller is not
-      // responsible for releasing this object.
-      DCHECK(object_buffers[i].data_size == -1);
-      object_buffers[i].data_size = -1;
-    }
-  }
-  return Status::OK();
-}
-
-/// This is a helper method for implementing plasma_release. We maintain a
-/// buffer
-/// of release calls and only perform them once the buffer becomes full (as
-/// judged by the aggregate sizes of the objects). There may be multiple release
-/// calls for the same object ID in the buffer. In this case, the first release
-/// calls will not do anything. The client will only send a message to the store
-/// releasing the object when the client is truly done with the object.
-///
-/// @param conn The plasma connection.
-/// @param object_id The object ID to attempt to release.
-Status PlasmaClient::PerformRelease(ObjectID object_id) {
-  // Decrement the count of the number of instances of this object that are
-  // being used by this client. The corresponding increment should have happened
-  // in PlasmaClient::Get.
-  auto object_entry = objects_in_use.find(object_id);
-  ARROW_CHECK(object_entry != objects_in_use.end());
-  object_entry->second->count -= 1;
-  ARROW_CHECK(object_entry->second->count >= 0);
-  // Check if the client is no longer using this object.
-  if (object_entry->second->count == 0) {
-    // Decrement the count of the number of objects in this memory-mapped file
-    // that the client is using. The corresponding increment should have
-    // happened in plasma_get.
-    int fd = object_entry->second->object.handle.store_fd;
-    auto entry = mmap_table.find(fd);
-    ARROW_CHECK(entry != mmap_table.end());
-    entry->second->count -= 1;
-    ARROW_CHECK(entry->second->count >= 0);
-    // If none are being used then unmap the file.
-    if (entry->second->count == 0) {
-      munmap(entry->second->pointer, entry->second->length);
-      // Remove the corresponding entry from the hash table.
-      delete entry->second;
-      mmap_table.erase(fd);
-    }
-    // Tell the store that the client no longer needs the object.
-    RETURN_NOT_OK(SendReleaseRequest(store_conn, object_id));
-    // Update the in_use_object_bytes.
-    in_use_object_bytes -= (object_entry->second->object.data_size +
-                            object_entry->second->object.metadata_size);
-    DCHECK(in_use_object_bytes >= 0);
-    // Remove the entry from the hash table of objects currently in use.
-    delete object_entry->second;
-    objects_in_use.erase(object_id);
-  }
-  return Status::OK();
-}
-
-Status PlasmaClient::Release(ObjectID object_id) {
-  // Add the new object to the release history.
-  release_history.push_front(object_id);
-  // If there are too many bytes in use by the client or if there are too many
-  // pending release calls, and there are at least some pending release calls in
-  // the release_history list, then release some objects.
-  while ((in_use_object_bytes >
-              std::min(kL3CacheSizeBytes, store_capacity / 100) ||
-          release_history.size() > config.release_delay) &&
-         release_history.size() > 0) {
-    // Perform a release for the object ID for the first pending release.
-    RETURN_NOT_OK(PerformRelease(release_history.back()));
-    // Remove the last entry from the release history.
-    release_history.pop_back();
-  }
-  return Status::OK();
-}
-
-// This method is used to query whether the plasma store contains an object.
-Status PlasmaClient::Contains(ObjectID object_id, int *has_object) {
-  // Check if we already have a reference to the object.
-  if (objects_in_use.count(object_id) > 0) {
-    *has_object = 1;
-  } else {
-    // If we don't already have a reference to the object, check with the store
-    // to see if we have the object.
-    RETURN_NOT_OK(SendContainsRequest(store_conn, object_id));
-    std::vector<uint8_t> buffer;
-    RETURN_NOT_OK(
-        PlasmaReceive(store_conn, MessageType_PlasmaContainsReply, buffer));
-    ObjectID object_id2;
-    RETURN_NOT_OK(ReadContainsReply(buffer.data(), &object_id2, has_object));
-  }
-  return Status::OK();
-}
-
-static void compute_block_hash(const unsigned char *data,
-                               int64_t nbytes,
-                               uint64_t *hash) {
-  XXH64_state_t hash_state;
-  XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
-  XXH64_update(&hash_state, data, nbytes);
-  *hash = XXH64_digest(&hash_state);
-}
-
-static inline bool compute_object_hash_parallel(XXH64_state_t *hash_state,
-                                                const unsigned char *data,
-                                                int64_t nbytes) {
-  // Note that this function will likely be faster if the address of data is
-  // aligned on a 64-byte boundary.
-  const uint64_t num_threads = kThreadPoolSize;
-  uint64_t threadhash[num_threads + 1];
-  const uint64_t data_address = reinterpret_cast<uint64_t>(data);
-  const uint64_t num_blocks = nbytes / BLOCK_SIZE;
-  const uint64_t chunk_size = (num_blocks / num_threads) * BLOCK_SIZE;
-  const uint64_t right_address = data_address + chunk_size * num_threads;
-  const uint64_t suffix = (data_address + nbytes) - right_address;
-  // Now the data layout is | k * num_threads * block_size | suffix | ==
-  // | num_threads * chunk_size | suffix |, where chunk_size = k * block_size.
-  // Each thread gets a "chunk" of k blocks, except the suffix thread.
-
-  for (int i = 0; i < num_threads; i++) {
-    threadpool_[i] =
-        std::thread(compute_block_hash,
-                    reinterpret_cast<uint8_t *>(data_address) + i * chunk_size,
-                    chunk_size, &threadhash[i]);
-  }
-  compute_block_hash(reinterpret_cast<uint8_t *>(right_address), suffix,
-                     &threadhash[num_threads]);
-
-  // Join the threads.
-  for (auto &t : threadpool_) {
-    if (t.joinable()) {
-      t.join();
-    }
-  }
-
-  XXH64_update(hash_state, (unsigned char *) threadhash, sizeof(threadhash));
-  return true;
-}
-
-static uint64_t compute_object_hash(const ObjectBuffer &obj_buffer) {
-  XXH64_state_t hash_state;
-  XXH64_reset(&hash_state, XXH64_DEFAULT_SEED);
-  if (obj_buffer.data_size >= kBytesInMB) {
-    compute_object_hash_parallel(&hash_state, (unsigned char *) obj_buffer.data,
-                                 obj_buffer.data_size);
-  } else {
-    XXH64_update(&hash_state, (unsigned char *) obj_buffer.data,
-                 obj_buffer.data_size);
-  }
-  XXH64_update(&hash_state, (unsigned char *) obj_buffer.metadata,
-               obj_buffer.metadata_size);
-  return XXH64_digest(&hash_state);
-}
-
-bool plasma_compute_object_hash(PlasmaClient *conn,
-                                ObjectID obj_id,
-                                unsigned char *digest) {
-  // Get the plasma object data. We pass in a timeout of 0 to indicate that
-  // the operation should timeout immediately.
-  ObjectBuffer obj_buffer;
-  ObjectID obj_id_array[1] = {obj_id};
-  uint64_t hash;
-
-  ARROW_CHECK_OK(conn->Get(obj_id_array, 1, 0, &obj_buffer));
-  // If the object was not retrieved, return false.
-  if (obj_buffer.data_size == -1) {
-    return false;
-  }
-  // Compute the hash.
-  hash = compute_object_hash(obj_buffer);
-  memcpy(digest, &hash, sizeof(hash));
-  // Release the plasma object.
-  ARROW_CHECK_OK(conn->Release(obj_id));
-  return true;
-}
-
-Status PlasmaClient::Seal(ObjectID object_id) {
-  // Make sure this client has a reference to the object before sending the
-  // request to Plasma.
-  auto object_entry = objects_in_use.find(object_id);
-  ARROW_CHECK(object_entry != objects_in_use.end())
-      << "Plasma client called seal an object without a reference to it";
-  ARROW_CHECK(!object_entry->second->is_sealed)
-      << "Plasma client called seal an already sealed object";
-  object_entry->second->is_sealed = true;
-  /// Send the seal request to Plasma.
-  static unsigned char digest[kDigestSize];
-  ARROW_CHECK(plasma_compute_object_hash(this, object_id, &digest[0]));
-  RETURN_NOT_OK(SendSealRequest(store_conn, object_id, &digest[0]));
-  // We call PlasmaClient::Release to decrement the number of instances of this
-  // object
-  // that are currently being used by this client. The corresponding increment
-  // happened in plasma_create and was used to ensure that the object was not
-  // released before the call to PlasmaClient::Seal.
-  return Release(object_id);
-}
-
-Status PlasmaClient::Delete(ObjectID object_id) {
-  // TODO(rkn): In the future, we can use this method to give hints to the
-  // eviction policy about when an object will no longer be needed.
-  return Status::NotImplemented("PlasmaClient::Delete is not implemented.");
-}
-
-Status PlasmaClient::Evict(int64_t num_bytes, int64_t &num_bytes_evicted) {
-  // Send a request to the store to evict objects.
-  RETURN_NOT_OK(SendEvictRequest(store_conn, num_bytes));
-  // Wait for a response with the number of bytes actually evicted.
-  std::vector<uint8_t> buffer;
-  int64_t type;
-  RETURN_NOT_OK(ReadMessage(store_conn, &type, buffer));
-  return ReadEvictReply(buffer.data(), num_bytes_evicted);
-}
-
-Status PlasmaClient::Subscribe(int &fd) {
-  int sock[2];
-  // Create a non-blocking socket pair. This will only be used to send
-  // notifications from the Plasma store to the client.
-  socketpair(AF_UNIX, SOCK_STREAM, 0, sock);
-  // Make the socket non-blocking.
-  int flags = fcntl(sock[1], F_GETFL, 0);
-  ARROW_CHECK(fcntl(sock[1], F_SETFL, flags | O_NONBLOCK) == 0);
-  // Tell the Plasma store about the subscription.
-  RETURN_NOT_OK(SendSubscribeRequest(store_conn));
-  // Send the file descriptor that the Plasma store should use to push
-  // notifications about sealed objects to this client.
-  ARROW_CHECK(send_fd(store_conn, sock[1]) >= 0);
-  close(sock[1]);
-  // Return the file descriptor that the client should use to read notifications
-  // about sealed objects.
-  fd = sock[0];
-  return Status::OK();
-}
-
-Status PlasmaClient::Connect(const std::string &store_socket_name,
-                             const std::string &manager_socket_name,
-                             int release_delay) {
-  store_conn = connect_ipc_sock_retry(store_socket_name, -1, -1);
-  if (manager_socket_name != "") {
-    manager_conn = connect_ipc_sock_retry(manager_socket_name, -1, -1);
-  } else {
-    manager_conn = -1;
-  }
-  config.release_delay = release_delay;
-  in_use_object_bytes = 0;
-  // Send a ConnectRequest to the store to get its memory capacity.
-  RETURN_NOT_OK(SendConnectRequest(store_conn));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(
-      PlasmaReceive(store_conn, MessageType_PlasmaConnectReply, buffer));
-  RETURN_NOT_OK(ReadConnectReply(buffer.data(), &store_capacity));
-  return Status::OK();
-}
-
-Status PlasmaClient::Disconnect() {
-  // NOTE: We purposefully do not finish sending release calls for objects in
-  // use, so that we don't duplicate PlasmaClient::Release calls (when handling
-  // a
-  // SIGTERM, for example).
-  for (auto &entry : objects_in_use) {
-    delete entry.second;
-  }
-  for (auto &entry : mmap_table) {
-    delete entry.second;
-  }
-  // Close the connections to Plasma. The Plasma store will release the objects
-  // that were in use by us when handling the SIGPIPE.
-  close(store_conn);
-  if (manager_conn >= 0) {
-    close(manager_conn);
-  }
-  return Status::OK();
-}
-
-bool plasma_manager_is_connected(PlasmaClient *conn) {
-  return conn->manager_conn >= 0;
-}
-
-#define h_addr h_addr_list[0]
-
-Status PlasmaClient::Transfer(const char *address,
-                              int port,
-                              ObjectID object_id) {
-  return SendDataRequest(manager_conn, object_id, address, port);
-}
-
-Status PlasmaClient::Fetch(int num_object_ids, ObjectID object_ids[]) {
-  ARROW_CHECK(manager_conn >= 0);
-  return SendFetchRequest(manager_conn, object_ids, num_object_ids);
-}
-
-int get_manager_fd(PlasmaClient *conn) {
-  return conn->manager_conn;
-}
-
-Status PlasmaClient::Info(ObjectID object_id, int *object_status) {
-  ARROW_CHECK(manager_conn >= 0);
-
-  RETURN_NOT_OK(SendStatusRequest(manager_conn, &object_id, 1));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(
-      PlasmaReceive(manager_conn, MessageType_PlasmaStatusReply, buffer));
-  return ReadStatusReply(buffer.data(), &object_id, object_status, 1);
-}
-
-Status PlasmaClient::Wait(int num_object_requests,
-                          ObjectRequest object_requests[],
-                          int num_ready_objects,
-                          uint64_t timeout_ms,
-                          int &num_objects_ready) {
-  ARROW_CHECK(manager_conn >= 0);
-  ARROW_CHECK(num_object_requests > 0);
-  ARROW_CHECK(num_ready_objects > 0);
-  ARROW_CHECK(num_ready_objects <= num_object_requests);
-
-  for (int i = 0; i < num_object_requests; ++i) {
-    ARROW_CHECK(object_requests[i].type == PLASMA_QUERY_LOCAL ||
-                object_requests[i].type == PLASMA_QUERY_ANYWHERE);
-  }
-
-  RETURN_NOT_OK(SendWaitRequest(manager_conn, object_requests,
-                                num_object_requests, num_ready_objects,
-                                timeout_ms));
-  std::vector<uint8_t> buffer;
-  RETURN_NOT_OK(
-      PlasmaReceive(manager_conn, MessageType_PlasmaWaitReply, buffer));
-  RETURN_NOT_OK(
-      ReadWaitReply(buffer.data(), object_requests, &num_ready_objects));
-
-  num_objects_ready = 0;
-  for (int i = 0; i < num_object_requests; ++i) {
-    int type = object_requests[i].type;
-    int status = object_requests[i].status;
-    switch (type) {
-    case PLASMA_QUERY_LOCAL:
-      if (status == ObjectStatus_Local) {
-        num_objects_ready += 1;
-      }
-      break;
-    case PLASMA_QUERY_ANYWHERE:
-      if (status == ObjectStatus_Local || status == ObjectStatus_Remote) {
-        num_objects_ready += 1;
-      } else {
-        ARROW_CHECK(status == ObjectStatus_Nonexistent);
-      }
-      break;
-    default:
-      ARROW_LOG(FATAL) << "This code should be unreachable.";
-    }
-  }
-  return Status::OK();
-}

src/plasma/plasma_client.h

@@ -1,334 +0,0 @@
-#ifndef PLASMA_CLIENT_H
-#define PLASMA_CLIENT_H
-
-#include <time.h>
-
-#include <deque>
-
-#include "plasma.h"
-
-using arrow::Status;
-
-#define PLASMA_DEFAULT_RELEASE_DELAY 64
-
-// Use 100MB as an overestimate of the L3 cache size.
-constexpr int64_t kL3CacheSizeBytes = 100000000;
-
-/// Object buffer data structure.
-struct ObjectBuffer {
-  /// The size in bytes of the data object.
-  int64_t data_size;
-  /// The address of the data object.
-  uint8_t *data;
-  /// The metadata size in bytes.
-  int64_t metadata_size;
-  /// The address of the metadata.
-  uint8_t *metadata;
-};
-
-/// Configuration options for the plasma client.
-struct PlasmaClientConfig {
-  /// Number of release calls we wait until the object is actually released.
-  /// This allows us to avoid invalidating the cpu cache on workers if objects
-  /// are reused accross tasks.
-  int release_delay;
-};
-
-struct ClientMmapTableEntry;
-struct ObjectInUseEntry;
-
-class PlasmaClient {
- public:
-  /// Connect to the local plasma store and plasma manager. Return
-  /// the resulting connection.
-  ///
-  /// @param store_socket_name The name of the UNIX domain socket to use to
-  ///        connect to the Plasma store.
-  /// @param manager_socket_name The name of the UNIX domain socket to use to
-  ///        connect to the local Plasma manager. If this is NULL, then this
-  ///        function will not connect to a manager.
-  /// @param release_delay Number of released objects that are kept around
-  ///        and not evicted to avoid too many munmaps.
-  /// @return The return status.
-  Status Connect(const std::string &store_socket_name,
-                 const std::string &manager_socket_name,
-                 int release_delay);
-
-  /// Create an object in the Plasma Store. Any metadata for this object must be
-  /// be passed in when the object is created.
-  ///
-  /// @param object_id The ID to use for the newly created object.
-  /// @param data_size The size in bytes of the space to be allocated for this
-  /// object's
-  ///        data (this does not include space used for metadata).
-  /// @param metadata The object's metadata. If there is no metadata, this
-  /// pointer
-  ///        should be NULL.
-  /// @param metadata_size The size in bytes of the metadata. If there is no
-  ///        metadata, this should be 0.
-  /// @param data The address of the newly created object will be written here.
-  /// @return The return status.
-  Status Create(ObjectID object_id,
-                int64_t data_size,
-                uint8_t *metadata,
-                int64_t metadata_size,
-                uint8_t **data);
-
-  /// Get some objects from the Plasma Store. This function will block until the
-  /// objects have all been created and sealed in the Plasma Store or the
-  /// timeout
-  /// expires. The caller is responsible for releasing any retrieved objects,
-  /// but
-  /// the caller should not release objects that were not retrieved.
-  ///
-  /// @param object_ids The IDs of the objects to get.
-  /// @param num_object_ids The number of object IDs to get.
-  /// @param timeout_ms The amount of time in milliseconds to wait before this
-  ///        request times out. If this value is -1, then no timeout is set.
-  /// @param object_buffers An array where the results will be stored. If the
-  /// data
-  ///        size field is -1, then the object was not retrieved.
-  /// @return The return status.
-  Status Get(ObjectID object_ids[],
-             int64_t num_objects,
-             int64_t timeout_ms,
-             ObjectBuffer object_buffers[]);
-
-  /// Tell Plasma that the client no longer needs the object. This should be
-  /// called
-  /// after Get when the client is done with the object. After this call,
-  /// the address returned by Get is no longer valid. This should be called
-  /// once for each call to Get (with the same object ID).
-  ///
-  /// @param object_id The ID of the object that is no longer needed.
-  /// @return The return status.
-  Status Release(ObjectID object_id);
-
-  /// Check if the object store contains a particular object and the object has
-  /// been sealed. The result will be stored in has_object.
-  ///
-  /// @todo: We may want to indicate if the object has been created but not
-  /// sealed.
-  ///
-  /// @param object_id The ID of the object whose presence we are checking.
-  /// @param has_object The function will write 1 at this address if the object
-  /// is
-  ///        present and 0 if it is not present.
-  /// @return The return status.
-  Status Contains(ObjectID object_id, int *has_object);
-
-  /// Seal an object in the object store. The object will be immutable after
-  /// this
-  /// call.
-  ///
-  /// @param object_id The ID of the object to seal.
-  /// @return The return status.
-  Status Seal(ObjectID object_id);
-
-  /// Delete an object from the object store. This currently assumes that the
-  /// object is present and has been sealed.
-  ///
-  /// @todo We may want to allow the deletion of objects that are not present or
-  ///       haven't been sealed.
-  ///
-  /// @param object_id The ID of the object to delete.
-  /// @return The return status.
-  Status Delete(ObjectID object_id);
-
-  /// Delete objects until we have freed up num_bytes bytes or there are no more
-  /// released objects that can be deleted.
-  ///
-  /// @param num_bytes The number of bytes to try to free up.
-  /// @param num_bytes_evicted Out parameter for total number of bytes of space
-  /// retrieved.
-  /// @return The return status.
-  Status Evict(int64_t num_bytes, int64_t &num_bytes_evicted);
-
-  /// Subscribe to notifications when objects are sealed in the object store.
-  /// Whenever an object is sealed, a message will be written to the client
-  /// socket
-  /// that is returned by this method.
-  ///
-  /// @param fd Out parameter for the file descriptor the client should use to
-  /// read notifications
-  ///         from the object store about sealed objects.
-  /// @return The return status.
-  Status Subscribe(int &fd);
-
-  /// Disconnect from the local plasma instance, including the local store and
-  /// manager.
-  ///
-  /// @return The return status.
-  Status Disconnect();
-
-  /// Attempt to initiate the transfer of some objects from remote Plasma
-  /// Stores.
-  /// This method does not guarantee that the fetched objects will arrive
-  /// locally.
-  ///
-  /// For an object that is available in the local Plasma Store, this method
-  /// will
-  /// not do anything. For an object that is not available locally, it will
-  /// check
-  /// if the object are already being fetched. If so, it will not do anything.
-  /// If
-  /// not, it will query the object table for a list of Plasma Managers that
-  /// have
-  /// the object. The object table will return a non-empty list, and this Plasma
-  /// Manager will attempt to initiate transfers from one of those Plasma
-  /// Managers.
-  ///
-  /// This function is non-blocking.
-  ///
-  /// This method is idempotent in the sense that it is ok to call it multiple
-  /// times.
-  ///
-  /// @param num_object_ids The number of object IDs fetch is being called on.
-  /// @param object_ids The IDs of the objects that fetch is being called on.
-  /// @return The return status.
-  Status Fetch(int num_object_ids, ObjectID object_ids[]);
-
-  /// Wait for (1) a specified number of objects to be available (sealed) in the
-  /// local Plasma Store or in a remote Plasma Store, or (2) for a timeout to
-  /// expire. This is a blocking call.
-  ///
-  /// @param num_object_requests Size of the object_requests array.
-  /// @param object_requests Object event array. Each element contains a request
-  ///        for a particular object_id. The type of request is specified in the
-  ///        "type" field.
-  ///        - A PLASMA_QUERY_LOCAL request is satisfied when object_id becomes
-  ///          available in the local Plasma Store. In this case, this function
-  ///          sets the "status" field to ObjectStatus_Local. Note, if the
-  ///          status
-  ///          is not ObjectStatus_Local, it will be ObjectStatus_Nonexistent,
-  ///          but it may exist elsewhere in the system.
-  ///        - A PLASMA_QUERY_ANYWHERE request is satisfied when object_id
-  ///        becomes
-  ///          available either at the local Plasma Store or on a remote Plasma
-  ///          Store. In this case, the functions sets the "status" field to
-  ///          ObjectStatus_Local or ObjectStatus_Remote.
-  /// @param num_ready_objects The number of requests in object_requests array
-  /// that
-  ///        must be satisfied before the function returns, unless it timeouts.
-  ///        The num_ready_objects should be no larger than num_object_requests.
-  /// @param timeout_ms Timeout value in milliseconds. If this timeout expires
-  ///        before min_num_ready_objects of requests are satisfied, the
-  ///        function
-  ///        returns.
-  /// @param num_objects_ready Out parameter for number of satisfied requests in
-  ///        the object_requests list. If the returned number is less than
-  ///        min_num_ready_objects this means that timeout expired.
-  /// @return The return status.
-  Status Wait(int num_object_requests,
-              ObjectRequest object_requests[],
-              int num_ready_objects,
-              uint64_t timeout_ms,
-              int &num_objects_ready);
-
-  /// Transfer local object to a different plasma manager.
-  ///
-  /// @param conn The object containing the connection state.
-  /// @param addr IP address of the plasma manager we are transfering to.
-  /// @param port Port of the plasma manager we are transfering to.
-  /// @object_id ObjectID of the object we are transfering.
-  /// @return The return status.
-  Status Transfer(const char *addr, int port, ObjectID object_id);
-
-  /// Return the status of a given object. This method may query the object
-  /// table.
-  ///
-  /// @param conn The object containing the connection state.
-  /// @param object_id The ID of the object whose status we query.
-  /// @param object_status Out parameter for object status. Can take the
-  ///         following values.
-  ///         - PLASMA_CLIENT_LOCAL, if object is stored in the local Plasma
-  ///         Store.
-  ///           has been already scheduled by the Plasma Manager.
-  ///         - PLASMA_CLIENT_TRANSFER, if the object is either currently being
-  ///           transferred or just scheduled.
-  ///         - PLASMA_CLIENT_REMOTE, if the object is stored at a remote
-  ///           Plasma Store.
-  ///         - PLASMA_CLIENT_DOES_NOT_EXIST, if the object doesn’t exist in the
-  ///           system.
-  /// @return The return status.
-  Status Info(ObjectID object_id, int *object_status);
-
-  //  private:
-
-  Status PerformRelease(ObjectID object_id);
-
-  /// File descriptor of the Unix domain socket that connects to the store.
-  int store_conn;
-  /// File descriptor of the Unix domain socket that connects to the manager.
-  int manager_conn;
-  /// File descriptor of the Unix domain socket on which client receives event
-  /// notifications for the objects it subscribes for when these objects are
-  /// sealed either locally or remotely.
-  int manager_conn_subscribe;
-  /// Table of dlmalloc buffer files that have been memory mapped so far. This
-  /// is a hash table mapping a file descriptor to a struct containing the
-  /// address of the corresponding memory-mapped file.
-  std::unordered_map<int, ClientMmapTableEntry *> mmap_table;
-  /// A hash table of the object IDs that are currently being used by this
-  /// client.
-  std::unordered_map<ObjectID, ObjectInUseEntry *, UniqueIDHasher>
-      objects_in_use;
-  /// Object IDs of the last few release calls. This is a deque and
-  /// is used to delay releasing objects to see if they can be reused by
-  /// subsequent tasks so we do not unneccessarily invalidate cpu caches.
-  /// TODO(pcm): replace this with a proper lru cache using the size of the L3
-  /// cache.
-  std::deque<ObjectID> release_history;
-  /// The number of bytes in the combined objects that are held in the release
-  /// history doubly-linked list. If this is too large then the client starts
-  /// releasing objects.
-  int64_t in_use_object_bytes;
-  /// Configuration options for the plasma client.
-  PlasmaClientConfig config;
-  /// The amount of memory available to the Plasma store. The client needs this
-  /// information to make sure that it does not delay in releasing so much
-  /// memory that the store is unable to evict enough objects to free up space.
-  int64_t store_capacity;
-};
-
-/// Return true if the plasma manager is connected.
-///
-/// @param conn The connection to the local plasma store and plasma manager.
-/// @return True if the plasma manager is connected and false otherwise.
-bool plasma_manager_is_connected(PlasmaClient *conn);
-
-/// Compute the hash of an object in the object store.
-///
-/// @param conn The object containing the connection state.
-/// @param object_id The ID of the object we want to hash.
-/// @param digest A pointer at which to return the hash digest of the object.
-///        The pointer must have at least DIGEST_SIZE bytes allocated.
-/// @return A boolean representing whether the hash operation succeeded.
-bool plasma_compute_object_hash(PlasmaClient *conn,
-                                ObjectID object_id,
-                                unsigned char *digest);
-
-/**
- * Get the file descriptor for the socket connection to the plasma manager.
- *
- * @param conn The plasma connection.
- * @return The file descriptor for the manager connection. If there is no
- *         connection to the manager, this is -1.
- */
-int get_manager_fd(PlasmaClient *conn);
-
-/**
- * Return the information associated to a given object.
- *
- * @param conn The object containing the connection state.
- * @param object_id The ID of the object whose info the client queries.
- * @param object_info The object's infirmation.
- * @return PLASMA_CLIENT_LOCAL, if the object is in the local Plasma Store.
- *         PLASMA_CLIENT_NOT_LOCAL, if not. In this case, the caller needs to
- *         ignore data, metadata_size, and metadata fields.
- */
-// int plasma_info(PlasmaConnection *conn,
-//                ObjectID object_id,
-//                ObjectInfo *object_info);
-
-#endif /* PLASMA_CLIENT_H */

src/plasma/plasma_common.cc

@@ -1,67 +0,0 @@
-#include "plasma_common.h"
-
-#include <random>
-
-#include "format/plasma_generated.h"
-
-using arrow::Status;
-
-UniqueID UniqueID::from_random() {
-  UniqueID id;
-  uint8_t *data = id.mutable_data();
-  std::random_device engine;
-  for (int i = 0; i < kUniqueIDSize; i++) {
-    data[i] = engine();
-  }
-  return id;
-}
-
-UniqueID UniqueID::from_binary(const std::string &binary) {
-  UniqueID id;
-  std::memcpy(&id, binary.data(), sizeof(id));
-  return id;
-}
-
-const uint8_t *UniqueID::data() const {
-  return id_;
-}
-
-uint8_t *UniqueID::mutable_data() {
-  return id_;
-}
-
-std::string UniqueID::binary() const {
-  return std::string(reinterpret_cast<const char *>(id_), kUniqueIDSize);
-}
-
-std::string UniqueID::hex() const {
-  constexpr char hex[] = "0123456789abcdef";
-  std::string result;
-  for (int i = 0; i < sizeof(UniqueID); i++) {
-    unsigned int val = id_[i];
-    result.push_back(hex[val >> 4]);
-    result.push_back(hex[val & 0xf]);
-  }
-  return result;
-}
-
-bool UniqueID::operator==(const UniqueID &rhs) const {
-  return std::memcmp(data(), rhs.data(), kUniqueIDSize) == 0;
-}
-
-Status plasma_error_status(int plasma_error) {
-  switch (plasma_error) {
-  case PlasmaError_OK:
-    return Status::OK();
-  case PlasmaError_ObjectExists:
-    return Status::PlasmaObjectExists(
-        "object already exists in the plasma store");
-  case PlasmaError_ObjectNonexistent:
-    return Status::PlasmaObjectNonexistent(
-        "object does not exist in the plasma store");
-  case PlasmaError_OutOfMemory:
-    return Status::PlasmaStoreFull("object does not fit in the plasma store");
-  default:
-    ARROW_LOG(FATAL) << "unknown plasma error code " << plasma_error;
-  }
-}

src/plasma/plasma_common.h

@@ -1,46 +0,0 @@
-#ifndef PLASMA_COMMON_H
-#define PLASMA_COMMON_H
-
-#include <cstring>
-#include <string>
-// TODO(pcm): Convert getopt and sscanf in the store to use more idiomatic C++
-// and get rid of the next three lines:
-#ifndef __STDC_FORMAT_MACROS
-#define __STDC_FORMAT_MACROS
-#endif
-
-#include "logging.h"
-#include "status.h"
-
-constexpr int64_t kUniqueIDSize = 20;
-
-class UniqueID {
- public:
-  static UniqueID from_random();
-  static UniqueID from_binary(const std::string &binary);
-  bool operator==(const UniqueID &rhs) const;
-  const uint8_t *data() const;
-  uint8_t *mutable_data();
-  std::string binary() const;
-  std::string hex() const;
-
- private:
-  uint8_t id_[kUniqueIDSize];
-};
-
-static_assert(std::is_pod<UniqueID>::value, "UniqueID must be plain old data");
-
-struct UniqueIDHasher {
-  /* ObjectID hashing function. */
-  size_t operator()(const UniqueID &id) const {
-    size_t result;
-    std::memcpy(&result, id.data(), sizeof(size_t));
-    return result;
-  }
-};
-
-typedef UniqueID ObjectID;
-
-arrow::Status plasma_error_status(int plasma_error);
-
-#endif  // PLASMA_COMMON_H

src/plasma/plasma_events.cc

@@ -1,74 +0,0 @@
-#include "plasma_events.h"
-
-#include <errno.h>
-
-void EventLoop::file_event_callback(aeEventLoop *loop,
-                                    int fd,
-                                    void *context,
-                                    int events) {
-  FileCallback *callback = reinterpret_cast<FileCallback *>(context);
-  (*callback)(events);
-}
-
-int EventLoop::timer_event_callback(aeEventLoop *loop,
-                                    long long timer_id,
-                                    void *context) {
-  TimerCallback *callback = reinterpret_cast<TimerCallback *>(context);
-  return (*callback)(timer_id);
-}
-
-constexpr int kInitialEventLoopSize = 1024;
-
-EventLoop::EventLoop() {
-  loop_ = aeCreateEventLoop(kInitialEventLoopSize);
-}
-
-bool EventLoop::add_file_event(int fd, int events, FileCallback callback) {
-  if (file_callbacks_.find(fd) != file_callbacks_.end()) {
-    return false;
-  }
-  auto data = std::unique_ptr<FileCallback>(new FileCallback(callback));
-  void *context = reinterpret_cast<void *>(data.get());
-  // Try to add the file descriptor.
-  int err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback,
-                              context);
-  // If it cannot be added, increase the size of the event loop.
-  if (err == AE_ERR && errno == ERANGE) {
-    err = aeResizeSetSize(loop_, 3 * aeGetSetSize(loop_) / 2);
-    if (err != AE_OK) {
-      return false;
-    }
-    err = aeCreateFileEvent(loop_, fd, events, EventLoop::file_event_callback,
-                            context);
-  }
-  // In any case, test if there were errors.
-  if (err == AE_OK) {
-    file_callbacks_.emplace(fd, std::move(data));
-    return true;
-  }
-  return false;
-}
-
-void EventLoop::remove_file_event(int fd) {
-  aeDeleteFileEvent(loop_, fd, AE_READABLE | AE_WRITABLE);
-  file_callbacks_.erase(fd);
-}
-
-void EventLoop::run() {
-  aeMain(loop_);
-}
-
-int64_t EventLoop::add_timer(int64_t timeout, TimerCallback callback) {
-  auto data = std::unique_ptr<TimerCallback>(new TimerCallback(callback));
-  void *context = reinterpret_cast<void *>(data.get());
-  int64_t timer_id = aeCreateTimeEvent(
-      loop_, timeout, EventLoop::timer_event_callback, context, NULL);
-  timer_callbacks_.emplace(timer_id, std::move(data));
-  return timer_id;
-}
-
-int EventLoop::remove_timer(int64_t timer_id) {
-  int err = aeDeleteTimeEvent(loop_, timer_id);
-  timer_callbacks_.erase(timer_id);
-  return err;
-}

src/plasma/plasma_events.h

@@ -1,85 +0,0 @@
-#ifndef PLASMA_EVENTS
-#define PLASMA_EVENTS
-
-#include <functional>
-#include <memory>
-#include <unordered_map>
-
-extern "C" {
-#include "ae/ae.h"
-}
-
-/// Constant specifying that the timer is done and it will be removed.
-constexpr int kEventLoopTimerDone = AE_NOMORE;
-
-/// Read event on the file descriptor.
-constexpr int kEventLoopRead = AE_READABLE;
-
-/// Write event on the file descriptor.
-constexpr int kEventLoopWrite = AE_WRITABLE;
-
-class EventLoop {
- public:
-  // Signature of the handler that will be called when there is a new event
-  // on the file descriptor that this handler has been registered for.
-  //
-  // The arguments are the event flags (read or write).
-  typedef std::function<void(int)> FileCallback;
-
-  // This handler will be called when a timer times out. The timer id is
-  // passed as an argument. The return is the number of milliseconds the timer
-  // shall be reset to or kEventLoopTimerDone if the timer shall not be
-  // triggered again.
-  typedef std::function<int(int64_t)> TimerCallback;
-
-  EventLoop();
-
-  /// Add a new file event handler to the event loop.
-  ///
-  /// @param fd The file descriptor we are listening to.
-  /// @param events The flags for events we are listening to (read or write).
-  /// @param callback The callback that will be called when the event happens.
-  /// @return Returns true if the event handler was added successfully.
-  bool add_file_event(int fd, int events, FileCallback callback);
-
-  /// Remove a file event handler from the event loop.
-  ///
-  /// @param fd The file descriptor of the event handler.
-  /// @return Void.
-  void remove_file_event(int fd);
-
-  /// Register a handler that will be called after a time slice of
-  ///  "timeout" milliseconds.
-  ///
-  ///  @param timeout The timeout in milliseconds.
-  ///  @param callback The callback for the timeout.
-  ///  @return The ID of the newly created timer.
-  int64_t add_timer(int64_t timeout, TimerCallback callback);
-
-  /// Remove a timer handler from the event loop.
-  ///
-  /// @param timer_id The ID of the timer that is to be removed.
-  /// @return The ae.c error code. TODO(pcm): needs to be standardized
-  int remove_timer(int64_t timer_id);
-
-  /// Run the event loop.
-  ///
-  /// @return Void.
-  void run();
-
- private:
-  static void file_event_callback(aeEventLoop *loop,
-                                  int fd,
-                                  void *context,
-                                  int events);
-
-  static int timer_event_callback(aeEventLoop *loop,
-                                  long long timer_id,
-                                  void *context);
-
-  aeEventLoop *loop_;
-  std::unordered_map<int, std::unique_ptr<FileCallback>> file_callbacks_;
-  std::unordered_map<int64_t, std::unique_ptr<TimerCallback>> timer_callbacks_;
-};
-
-#endif  // PLASMA_EVENTS

src/plasma/plasma_extension.cc

@@ -1,463 +0,0 @@
-#include <Python.h>
-#include "bytesobject.h"
-
-#include "plasma_io.h"
-#include "plasma_common.h"
-#include "plasma_protocol.h"
-#include "plasma_client.h"
-
-PyObject *PlasmaOutOfMemoryError;
-PyObject *PlasmaObjectExistsError;
-
-#include "plasma_extension.h"
-
-PyObject *PyPlasma_connect(PyObject *self, PyObject *args) {
-  const char *store_socket_name;
-  const char *manager_socket_name;
-  int release_delay;
-  if (!PyArg_ParseTuple(args, "ssi", &store_socket_name, &manager_socket_name,
-                        &release_delay)) {
-    return NULL;
-  }
-  PlasmaClient *client = new PlasmaClient();
-  ARROW_CHECK_OK(
-      client->Connect(store_socket_name, manager_socket_name, release_delay));
-
-  return PyCapsule_New(client, "plasma", NULL);
-}
-
-PyObject *PyPlasma_disconnect(PyObject *self, PyObject *args) {
-  PyObject *client_capsule;
-  if (!PyArg_ParseTuple(args, "O", &client_capsule)) {
-    return NULL;
-  }
-  PlasmaClient *client;
-  ARROW_CHECK(PyObjectToPlasmaClient(client_capsule, &client));
-  ARROW_CHECK_OK(client->Disconnect());
-  /* We use the context of the connection capsule to indicate if the connection
-   * is still active (if the context is NULL) or if it is closed (if the context
-   * is (void*) 0x1). This is neccessary because the primary pointer of the
-   * capsule cannot be NULL. */
-  PyCapsule_SetContext(client_capsule, (void *) 0x1);
-  Py_RETURN_NONE;
-}
-
-PyObject *PyPlasma_create(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  ObjectID object_id;
-  long long size;
-  PyObject *metadata;
-  if (!PyArg_ParseTuple(args, "O&O&LO", PyObjectToPlasmaClient, &client,
-                        PyStringToUniqueID, &object_id, &size, &metadata)) {
-    return NULL;
-  }
-  if (!PyByteArray_Check(metadata)) {
-    PyErr_SetString(PyExc_TypeError, "metadata must be a bytearray");
-    return NULL;
-  }
-  uint8_t *data;
-  Status s = client->Create(object_id, size,
-                            (uint8_t *) PyByteArray_AsString(metadata),
-                            PyByteArray_Size(metadata), &data);
-  if (s.IsPlasmaObjectExists()) {
-    PyErr_SetString(PlasmaObjectExistsError,
-                    "An object with this ID already exists in the plasma "
-                    "store.");
-    return NULL;
-  }
-  if (s.IsPlasmaStoreFull()) {
-    PyErr_SetString(PlasmaOutOfMemoryError,
-                    "The plasma store ran out of memory and could not create "
-                    "this object.");
-    return NULL;
-  }
-  ARROW_CHECK(s.ok());
-
-#if PY_MAJOR_VERSION >= 3
-  return PyMemoryView_FromMemory((char *) data, (Py_ssize_t) size, PyBUF_WRITE);
-#else
-  return PyBuffer_FromReadWriteMemory((void *) data, (Py_ssize_t) size);
-#endif
-}
-
-PyObject *PyPlasma_hash(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client,
-                        PyStringToUniqueID, &object_id)) {
-    return NULL;
-  }
-  unsigned char digest[kDigestSize];
-  bool success = plasma_compute_object_hash(client, object_id, digest);
-  if (success) {
-    PyObject *digest_string =
-        PyBytes_FromStringAndSize((char *) digest, kDigestSize);
-    return digest_string;
-  } else {
-    Py_RETURN_NONE;
-  }
-}
-
-PyObject *PyPlasma_seal(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client,
-                        PyStringToUniqueID, &object_id)) {
-    return NULL;
-  }
-  ARROW_CHECK_OK(client->Seal(object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject *PyPlasma_release(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client,
-                        PyStringToUniqueID, &object_id)) {
-    return NULL;
-  }
-  ARROW_CHECK_OK(client->Release(object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject *PyPlasma_get(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  PyObject *object_id_list;
-  long long timeout_ms;
-  if (!PyArg_ParseTuple(args, "O&OL", PyObjectToPlasmaClient, &client,
-                        &object_id_list, &timeout_ms)) {
-    return NULL;
-  }
-
-  Py_ssize_t num_object_ids = PyList_Size(object_id_list);
-  ObjectID *object_ids = (ObjectID *) malloc(sizeof(ObjectID) * num_object_ids);
-  ObjectBuffer *object_buffers =
-      (ObjectBuffer *) malloc(sizeof(ObjectBuffer) * num_object_ids);
-
-  for (int i = 0; i < num_object_ids; ++i) {
-    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
-  }
-
-  Py_BEGIN_ALLOW_THREADS;
-  ARROW_CHECK_OK(
-      client->Get(object_ids, num_object_ids, timeout_ms, object_buffers));
-  Py_END_ALLOW_THREADS;
-  free(object_ids);
-
-  PyObject *returns = PyList_New(num_object_ids);
-  for (int i = 0; i < num_object_ids; ++i) {
-    if (object_buffers[i].data_size != -1) {
-      /* The object was retrieved, so return the object. */
-      PyObject *t = PyTuple_New(2);
-#if PY_MAJOR_VERSION >= 3
-      PyTuple_SetItem(
-          t, 0, PyMemoryView_FromMemory(
-                    (char *) object_buffers[i].data,
-                    (Py_ssize_t) object_buffers[i].data_size, PyBUF_READ));
-      PyTuple_SetItem(
-          t, 1, PyMemoryView_FromMemory(
-                    (char *) object_buffers[i].metadata,
-                    (Py_ssize_t) object_buffers[i].metadata_size, PyBUF_READ));
-#else
-      PyTuple_SetItem(
-          t, 0, PyBuffer_FromMemory((void *) object_buffers[i].data,
-                                    (Py_ssize_t) object_buffers[i].data_size));
-      PyTuple_SetItem(t, 1, PyBuffer_FromMemory(
-                                (void *) object_buffers[i].metadata,
-                                (Py_ssize_t) object_buffers[i].metadata_size));
-#endif
-      PyList_SetItem(returns, i, t);
-    } else {
-      /* The object was not retrieved, so just add None to the list of return
-       * values. */
-      Py_XINCREF(Py_None);
-      PyList_SetItem(returns, i, Py_None);
-    }
-  }
-  free(object_buffers);
-  return returns;
-}
-
-PyObject *PyPlasma_contains(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client,
-                        PyStringToUniqueID, &object_id)) {
-    return NULL;
-  }
-  int has_object;
-  ARROW_CHECK_OK(client->Contains(object_id, &has_object));
-
-  if (has_object)
-    Py_RETURN_TRUE;
-  else
-    Py_RETURN_FALSE;
-}
-
-PyObject *PyPlasma_fetch(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  PyObject *object_id_list;
-  if (!PyArg_ParseTuple(args, "O&O", PyObjectToPlasmaClient, &client,
-                        &object_id_list)) {
-    return NULL;
-  }
-  if (!plasma_manager_is_connected(client)) {
-    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
-    return NULL;
-  }
-  Py_ssize_t n = PyList_Size(object_id_list);
-  ObjectID *object_ids = (ObjectID *) malloc(sizeof(ObjectID) * n);
-  for (int i = 0; i < n; ++i) {
-    PyStringToUniqueID(PyList_GetItem(object_id_list, i), &object_ids[i]);
-  }
-  ARROW_CHECK_OK(client->Fetch((int) n, object_ids));
-  free(object_ids);
-  Py_RETURN_NONE;
-}
-
-PyObject *PyPlasma_wait(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  PyObject *object_id_list;
-  long long timeout;
-  int num_returns;
-  if (!PyArg_ParseTuple(args, "O&OLi", PyObjectToPlasmaClient, &client,
-                        &object_id_list, &timeout, &num_returns)) {
-    return NULL;
-  }
-  Py_ssize_t n = PyList_Size(object_id_list);
-
-  if (!plasma_manager_is_connected(client)) {
-    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
-    return NULL;
-  }
-  if (num_returns < 0) {
-    PyErr_SetString(PyExc_RuntimeError,
-                    "The argument num_returns cannot be less than zero.");
-    return NULL;
-  }
-  if (num_returns > n) {
-    PyErr_SetString(
-        PyExc_RuntimeError,
-        "The argument num_returns cannot be greater than len(object_ids)");
-    return NULL;
-  }
-  int64_t threshold = 1 << 30;
-  if (timeout > threshold) {
-    PyErr_SetString(PyExc_RuntimeError,
-                    "The argument timeout cannot be greater than 2 ** 30.");
-    return NULL;
-  }
-
-  ObjectRequest *object_requests =
-      (ObjectRequest *) malloc(sizeof(ObjectRequest) * n);
-  for (int i = 0; i < n; ++i) {
-    ARROW_CHECK(PyStringToUniqueID(PyList_GetItem(object_id_list, i),
-                                   &object_requests[i].object_id) == 1);
-    object_requests[i].type = PLASMA_QUERY_ANYWHERE;
-  }
-  /* Drop the global interpreter lock while we are waiting, so other threads can
-   * run. */
-  int num_return_objects;
-  Py_BEGIN_ALLOW_THREADS;
-  ARROW_CHECK_OK(client->Wait((int) n, object_requests, num_returns,
-                              (uint64_t) timeout, num_return_objects));
-  Py_END_ALLOW_THREADS;
-
-  int num_to_return = std::min(num_return_objects, num_returns);
-  PyObject *ready_ids = PyList_New(num_to_return);
-  PyObject *waiting_ids = PySet_New(object_id_list);
-  int num_returned = 0;
-  for (int i = 0; i < n; ++i) {
-    if (num_returned == num_to_return) {
-      break;
-    }
-    if (object_requests[i].status == ObjectStatus_Local ||
-        object_requests[i].status == ObjectStatus_Remote) {
-      PyObject *ready =
-          PyBytes_FromStringAndSize((char *) &object_requests[i].object_id,
-                                    sizeof(object_requests[i].object_id));
-      PyList_SetItem(ready_ids, num_returned, ready);
-      PySet_Discard(waiting_ids, ready);
-      num_returned += 1;
-    } else {
-      ARROW_CHECK(object_requests[i].status == ObjectStatus_Nonexistent);
-    }
-  }
-  ARROW_CHECK(num_returned == num_to_return);
-  /* Return both the ready IDs and the remaining IDs. */
-  PyObject *t = PyTuple_New(2);
-  PyTuple_SetItem(t, 0, ready_ids);
-  PyTuple_SetItem(t, 1, waiting_ids);
-  return t;
-}
-
-PyObject *PyPlasma_evict(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  long long num_bytes;
-  if (!PyArg_ParseTuple(args, "O&L", PyObjectToPlasmaClient, &client,
-                        &num_bytes)) {
-    return NULL;
-  }
-  int64_t evicted_bytes;
-  ARROW_CHECK_OK(client->Evict((int64_t) num_bytes, evicted_bytes));
-  return PyLong_FromLong((long) evicted_bytes);
-}
-
-PyObject *PyPlasma_delete(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  ObjectID object_id;
-  if (!PyArg_ParseTuple(args, "O&O&", PyObjectToPlasmaClient, &client,
-                        PyStringToUniqueID, &object_id)) {
-    return NULL;
-  }
-  ARROW_CHECK_OK(client->Delete(object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject *PyPlasma_transfer(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  ObjectID object_id;
-  const char *addr;
-  int port;
-  if (!PyArg_ParseTuple(args, "O&O&si", PyObjectToPlasmaClient, &client,
-                        PyStringToUniqueID, &object_id, &addr, &port)) {
-    return NULL;
-  }
-
-  if (!plasma_manager_is_connected(client)) {
-    PyErr_SetString(PyExc_RuntimeError, "Not connected to the plasma manager");
-    return NULL;
-  }
-
-  ARROW_CHECK_OK(client->Transfer(addr, port, object_id));
-  Py_RETURN_NONE;
-}
-
-PyObject *PyPlasma_subscribe(PyObject *self, PyObject *args) {
-  PlasmaClient *client;
-  if (!PyArg_ParseTuple(args, "O&", PyObjectToPlasmaClient, &client)) {
-    return NULL;
-  }
-
-  int sock;
-  ARROW_CHECK_OK(client->Subscribe(sock));
-  return PyLong_FromLong(sock);
-}
-
-PyObject *PyPlasma_receive_notification(PyObject *self, PyObject *args) {
-  int plasma_sock;
-
-  if (!PyArg_ParseTuple(args, "i", &plasma_sock)) {
-    return NULL;
-  }
-  /* Receive object notification from the plasma connection socket. If the
-   * object was added, return a tuple of its fields: ObjectID, data_size,
-   * metadata_size. If the object was deleted, data_size and metadata_size will
-   * be set to -1. */
-  uint8_t *notification = read_message_async(plasma_sock);
-  if (notification == NULL) {
-    PyErr_SetString(PyExc_RuntimeError,
-                    "Failed to read object notification from Plasma socket");
-    return NULL;
-  }
-  auto object_info = flatbuffers::GetRoot<ObjectInfo>(notification);
-  /* Construct a tuple from object_info and return. */
-  PyObject *t = PyTuple_New(3);
-  PyTuple_SetItem(t, 0,
-                  PyBytes_FromStringAndSize(object_info->object_id()->data(),
-                                            object_info->object_id()->size()));
-  if (object_info->is_deletion()) {
-    PyTuple_SetItem(t, 1, PyLong_FromLong(-1));
-    PyTuple_SetItem(t, 2, PyLong_FromLong(-1));
-  } else {
-    PyTuple_SetItem(t, 1, PyLong_FromLong(object_info->data_size()));
-    PyTuple_SetItem(t, 2, PyLong_FromLong(object_info->metadata_size()));
-  }
-
-  free(notification);
-  return t;
-}
-
-static PyMethodDef plasma_methods[] = {
-    {"connect", PyPlasma_connect, METH_VARARGS, "Connect to plasma."},
-    {"disconnect", PyPlasma_disconnect, METH_VARARGS,
-     "Disconnect from plasma."},
-    {"create", PyPlasma_create, METH_VARARGS, "Create a new plasma object."},
-    {"hash", PyPlasma_hash, METH_VARARGS,
-     "Compute the hash of a plasma object."},
-    {"seal", PyPlasma_seal, METH_VARARGS, "Seal a plasma object."},
-    {"get", PyPlasma_get, METH_VARARGS, "Get a plasma object."},
-    {"contains", PyPlasma_contains, METH_VARARGS,
-     "Does the plasma store contain this plasma object?"},
-    {"fetch", PyPlasma_fetch, METH_VARARGS,
-     "Fetch the object from another plasma manager instance."},
-    {"wait", PyPlasma_wait, METH_VARARGS,
-     "Wait until num_returns objects in object_ids are ready."},
-    {"evict", PyPlasma_evict, METH_VARARGS,
-     "Evict some objects until we recover some number of bytes."},
-    {"release", PyPlasma_release, METH_VARARGS, "Release the plasma object."},
-    {"delete", PyPlasma_delete, METH_VARARGS, "Delete a plasma object."},
-    {"transfer", PyPlasma_transfer, METH_VARARGS,
-     "Transfer object to another plasma manager."},
-    {"subscribe", PyPlasma_subscribe, METH_VARARGS,
-     "Subscribe to the plasma notification socket."},
-    {"receive_notification", PyPlasma_receive_notification, METH_VARARGS,
-     "Receive next notification from plasma notification socket."},
-    {NULL} /* Sentinel */
-};
-
-#if PY_MAJOR_VERSION >= 3
-static struct PyModuleDef moduledef = {
-    PyModuleDef_HEAD_INIT,
-    "libplasma",                           /* m_name */
-    "A Python client library for plasma.", /* m_doc */
-    0,                                     /* m_size */
-    plasma_methods,                        /* m_methods */
-    NULL,                                  /* m_reload */
-    NULL,                                  /* m_traverse */
-    NULL,                                  /* m_clear */
-    NULL,                                  /* m_free */
-};
-#endif
-
-#if PY_MAJOR_VERSION >= 3
-#define INITERROR return NULL
-#else
-#define INITERROR return
-#endif
-
-#ifndef PyMODINIT_FUNC /* declarations for DLL import/export */
-#define PyMODINIT_FUNC void
-#endif
-
-#if PY_MAJOR_VERSION >= 3
-#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
-#else
-#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
-#endif
-
-MOD_INIT(libplasma) {
-#if PY_MAJOR_VERSION >= 3
-  PyObject *m = PyModule_Create(&moduledef);
-#else
-  PyObject *m = Py_InitModule3("libplasma", plasma_methods,
-                               "A Python client library for plasma.");
-#endif
-
-  /* Create a custom exception for when an object ID is reused. */
-  char plasma_object_exists_error[] = "plasma_object_exists.error";
-  PlasmaObjectExistsError =
-      PyErr_NewException(plasma_object_exists_error, NULL, NULL);
-  Py_INCREF(PlasmaObjectExistsError);
-  PyModule_AddObject(m, "plasma_object_exists_error", PlasmaObjectExistsError);
-  /* Create a custom exception for when the plasma store is out of memory. */
-  char plasma_out_of_memory_error[] = "plasma_out_of_memory.error";
-  PlasmaOutOfMemoryError =
-      PyErr_NewException(plasma_out_of_memory_error, NULL, NULL);
-  Py_INCREF(PlasmaOutOfMemoryError);
-  PyModule_AddObject(m, "plasma_out_of_memory_error", PlasmaOutOfMemoryError);
-
-#if PY_MAJOR_VERSION >= 3
-  return m;
-#endif
-}

src/plasma/plasma_extension.h

@@ -1,24 +0,0 @@
-#ifndef PLASMA_EXTENSION_H
-#define PLASMA_EXTENSION_H
-
-static int PyObjectToPlasmaClient(PyObject *object, PlasmaClient **client) {
-  if (PyCapsule_IsValid(object, "plasma")) {
-    *client = (PlasmaClient *) PyCapsule_GetPointer(object, "plasma");
-    return 1;
-  } else {
-    PyErr_SetString(PyExc_TypeError, "must be a 'plasma' capsule");
-    return 0;
-  }
-}
-
-int PyStringToUniqueID(PyObject *object, ObjectID *object_id) {
-  if (PyBytes_Check(object)) {
-    memcpy(object_id, PyBytes_AsString(object), sizeof(ObjectID));
-    return 1;
-  } else {
-    PyErr_SetString(PyExc_TypeError, "must be a 20 character string");
-    return 0;
-  }
-}
-
-#endif /* PLASMA_EXTENSION_H */

src/plasma/plasma_io.cc

@@ -1,220 +0,0 @@
-#include "plasma_io.h"
-#include "plasma_common.h"
-
-using arrow::Status;
-
-/* Number of times we try binding to a socket. */
-#define NUM_BIND_ATTEMPTS 5
-#define BIND_TIMEOUT_MS 100
-
-/* Number of times we try connecting to a socket. */
-#define NUM_CONNECT_ATTEMPTS 50
-#define CONNECT_TIMEOUT_MS 100
-
-Status WriteBytes(int fd, uint8_t *cursor, size_t length) {
-  ssize_t nbytes = 0;
-  size_t bytesleft = length;
-  size_t offset = 0;
-  while (bytesleft > 0) {
-    /* While we haven't written the whole message, write to the file descriptor,
-     * advance the cursor, and decrease the amount left to write. */
-    nbytes = write(fd, cursor + offset, bytesleft);
-    if (nbytes < 0) {
-      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
-        continue;
-      }
-      return Status::IOError(std::string(strerror(errno)));
-    } else if (nbytes == 0) {
-      return Status::IOError("Encountered unexpected EOF");
-    }
-    ARROW_CHECK(nbytes > 0);
-    bytesleft -= nbytes;
-    offset += nbytes;
-  }
-
-  return Status::OK();
-}
-
-Status WriteMessage(int fd, int64_t type, int64_t length, uint8_t *bytes) {
-  int64_t version = PLASMA_PROTOCOL_VERSION;
-  RETURN_NOT_OK(
-      WriteBytes(fd, reinterpret_cast<uint8_t *>(&version), sizeof(version)));
-  RETURN_NOT_OK(
-      WriteBytes(fd, reinterpret_cast<uint8_t *>(&type), sizeof(type)));
-  RETURN_NOT_OK(
-      WriteBytes(fd, reinterpret_cast<uint8_t *>(&length), sizeof(length)));
-  return WriteBytes(fd, bytes, length * sizeof(char));
-}
-
-Status ReadBytes(int fd, uint8_t *cursor, size_t length) {
-  ssize_t nbytes = 0;
-  /* Termination condition: EOF or read 'length' bytes total. */
-  size_t bytesleft = length;
-  size_t offset = 0;
-  while (bytesleft > 0) {
-    nbytes = read(fd, cursor + offset, bytesleft);
-    if (nbytes < 0) {
-      if (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR) {
-        continue;
-      }
-      return Status::IOError(std::string(strerror(errno)));
-    } else if (0 == nbytes) {
-      return Status::IOError("Encountered unexpected EOF");
-    }
-    ARROW_CHECK(nbytes > 0);
-    bytesleft -= nbytes;
-    offset += nbytes;
-  }
-
-  return Status::OK();
-}
-
-Status ReadMessage(int fd, int64_t *type, std::vector<uint8_t> &buffer) {
-  int64_t version;
-  RETURN_NOT_OK_ELSE(
-      ReadBytes(fd, reinterpret_cast<uint8_t *>(&version), sizeof(version)),
-      *type = DISCONNECT_CLIENT);
-  ARROW_CHECK(version == PLASMA_PROTOCOL_VERSION) << "version = " << version;
-  int64_t length;
-  RETURN_NOT_OK_ELSE(
-      ReadBytes(fd, reinterpret_cast<uint8_t *>(type), sizeof(*type)),
-      *type = DISCONNECT_CLIENT);
-  RETURN_NOT_OK_ELSE(
-      ReadBytes(fd, reinterpret_cast<uint8_t *>(&length), sizeof(length)),
-      *type = DISCONNECT_CLIENT);
-  if (length > buffer.size()) {
-    buffer.resize(length);
-  }
-  RETURN_NOT_OK_ELSE(ReadBytes(fd, buffer.data(), length),
-                     *type = DISCONNECT_CLIENT);
-  return Status::OK();
-}
-
-int bind_ipc_sock(const std::string &pathname, bool shall_listen) {
-  struct sockaddr_un socket_address;
-  int socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
-  if (socket_fd < 0) {
-    ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
-    return -1;
-  }
-  /* Tell the system to allow the port to be reused. */
-  int on = 1;
-  if (setsockopt(socket_fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on,
-                 sizeof(on)) < 0) {
-    ARROW_LOG(ERROR) << "setsockopt failed for pathname " << pathname;
-    close(socket_fd);
-    return -1;
-  }
-
-  unlink(pathname.c_str());
-  memset(&socket_address, 0, sizeof(socket_address));
-  socket_address.sun_family = AF_UNIX;
-  if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
-    ARROW_LOG(ERROR) << "Socket pathname is too long.";
-    close(socket_fd);
-    return -1;
-  }
-  strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
-
-  if (bind(socket_fd, (struct sockaddr *) &socket_address,
-           sizeof(socket_address)) != 0) {
-    ARROW_LOG(ERROR) << "Bind failed for pathname " << pathname;
-    close(socket_fd);
-    return -1;
-  }
-  if (shall_listen && listen(socket_fd, 128) == -1) {
-    ARROW_LOG(ERROR) << "Could not listen to socket " << pathname;
-    close(socket_fd);
-    return -1;
-  }
-  return socket_fd;
-}
-
-int connect_ipc_sock_retry(const std::string &pathname,
-                           int num_retries,
-                           int64_t timeout) {
-  /* Pick the default values if the user did not specify. */
-  if (num_retries < 0) {
-    num_retries = NUM_CONNECT_ATTEMPTS;
-  }
-  if (timeout < 0) {
-    timeout = CONNECT_TIMEOUT_MS;
-  }
-
-  int fd = -1;
-  for (int num_attempts = 0; num_attempts < num_retries; ++num_attempts) {
-    fd = connect_ipc_sock(pathname);
-    if (fd >= 0) {
-      break;
-    }
-    if (num_attempts == 0) {
-      ARROW_LOG(ERROR) << "Connection to socket failed for pathname "
-                       << pathname;
-    }
-    /* Sleep for timeout milliseconds. */
-    usleep(timeout * 1000);
-  }
-  /* If we could not connect to the socket, exit. */
-  if (fd == -1) {
-    ARROW_LOG(FATAL) << "Could not connect to socket " << pathname;
-  }
-  return fd;
-}
-
-int connect_ipc_sock(const std::string &pathname) {
-  struct sockaddr_un socket_address;
-  int socket_fd;
-
-  socket_fd = socket(AF_UNIX, SOCK_STREAM, 0);
-  if (socket_fd < 0) {
-    ARROW_LOG(ERROR) << "socket() failed for pathname " << pathname;
-    return -1;
-  }
-
-  memset(&socket_address, 0, sizeof(socket_address));
-  socket_address.sun_family = AF_UNIX;
-  if (pathname.size() + 1 > sizeof(socket_address.sun_path)) {
-    ARROW_LOG(ERROR) << "Socket pathname is too long.";
-    return -1;
-  }
-  strncpy(socket_address.sun_path, pathname.c_str(), pathname.size() + 1);
-
-  if (connect(socket_fd, (struct sockaddr *) &socket_address,
-              sizeof(socket_address)) != 0) {
-    close(socket_fd);
-    return -1;
-  }
-
-  return socket_fd;
-}
-
-int AcceptClient(int socket_fd) {
-  int client_fd = accept(socket_fd, NULL, NULL);
-  if (client_fd < 0) {
-    ARROW_LOG(ERROR) << "Error reading from socket.";
-    return -1;
-  }
-  return client_fd;
-}
-
-uint8_t *read_message_async(int sock) {
-  int64_t size;
-  Status s = ReadBytes(sock, (uint8_t *) &size, sizeof(int64_t));
-  if (!s.ok()) {
-    /* The other side has closed the socket. */
-    ARROW_LOG(DEBUG)
-        << "Socket has been closed, or some other error has occurred.";
-    close(sock);
-    return NULL;
-  }
-  uint8_t *message = (uint8_t *) malloc(size);
-  s = ReadBytes(sock, message, size);
-  if (!s.ok()) {
-    /* The other side has closed the socket. */
-    ARROW_LOG(DEBUG)
-        << "Socket has been closed, or some other error has occurred.";
-    close(sock);
-    return NULL;
-  }
-  return message;
-}

src/plasma/plasma_io.h

@@ -1,38 +0,0 @@
-#include <inttypes.h>
-#include <unistd.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-
-#include <vector>
-#include <string>
-
-#include "status.h"
-
-// TODO(pcm): Replace our own custom message header (message type,
-// message length, plasma protocol verion) with one that is serialized
-// using flatbuffers.
-#define PLASMA_PROTOCOL_VERSION 0x0000000000000000
-#define DISCONNECT_CLIENT 0
-
-arrow::Status WriteBytes(int fd, uint8_t *cursor, size_t length);
-
-arrow::Status WriteMessage(int fd,
-                           int64_t type,
-                           int64_t length,
-                           uint8_t *bytes);
-
-arrow::Status ReadBytes(int fd, uint8_t *cursor, size_t length);
-
-arrow::Status ReadMessage(int fd, int64_t *type, std::vector<uint8_t> &buffer);
-
-int bind_ipc_sock(const std::string &pathname, bool shall_listen);
-
-int connect_ipc_sock(const std::string &pathname);
-
-int connect_ipc_sock_retry(const std::string &pathname,
-                           int num_retries,
-                           int64_t timeout);
-
-int AcceptClient(int socket_fd);
-
-uint8_t *read_message_async(int sock);

src/plasma/plasma_manager.cc

@@ -28,13 +28,14 @@
 #include "utarray.h"
 #include "utstring.h"
 #include "common_protocol.h"
-#include "common.h"
 #include "io.h"
 #include "net.h"
 #include "event_loop.h"
-#include "plasma.h"
-#include "plasma_protocol.h"
-#include "plasma_client.h"
+#include "common.h"
+#include "plasma/plasma.h"
+#include "plasma/events.h"
+#include "plasma/protocol.h"
+#include "plasma/client.h"
 #include "plasma_manager.h"
 #include "state/db.h"
 #include "state/object_table.h"
@@ -168,7 +169,7 @@ struct WaitRequest {
   WaitRequest(ClientConnection *client_conn,
               int64_t timer,
               int64_t num_object_requests,
-              ObjectRequestMap &&object_requests,
+              plasma::ObjectRequestMap &&object_requests,
               int64_t num_objects_to_wait_for,
               int64_t num_satisfied)
       : client_conn(client_conn),
@@ -188,7 +189,7 @@ struct WaitRequest {
   /** The object requests for this wait request. Each object request has a
    *  status field which is either PLASMA_QUERY_LOCAL or PLASMA_QUERY_ANYWHERE.
    */
-  ObjectRequestMap object_requests;
+  plasma::ObjectRequestMap object_requests;
   /** The minimum number of objects to wait for in this request. */
   int64_t num_objects_to_wait_for;
   /** The number of object requests in this wait request that are already
@@ -214,7 +215,7 @@ struct PlasmaManagerState {
   /** Event loop. */
   event_loop *loop;
   /** Connection to the local plasma store for reading or writing data. */
-  PlasmaClient *plasma_conn;
+  plasma::PlasmaClient *plasma_conn;
   /** Hash table of all contexts for active connections to
    *  other plasma managers. These are used for writing data to
    *  other plasma stores. */
@@ -341,9 +342,9 @@ ObjectWaitRequests **object_wait_requests_table_ptr_from_type(
     PlasmaManagerState *manager_state,
     int type) {
   /* We use different types of hash tables for different requests. */
-  if (type == PLASMA_QUERY_LOCAL) {
+  if (type == plasma::PLASMA_QUERY_LOCAL) {
     return &manager_state->object_wait_requests_local;
-  } else if (type == PLASMA_QUERY_ANYWHERE) {
+  } else if (type == plasma::PLASMA_QUERY_ANYWHERE) {
     return &manager_state->object_wait_requests_remote;
   } else {
     LOG_FATAL("This code should be unreachable.");
@@ -415,10 +416,10 @@ void remove_wait_request(PlasmaManagerState *manager_state,
 void return_from_wait(PlasmaManagerState *manager_state,
                       WaitRequest *wait_req) {
   /* Send the reply to the client. */
-  handle_sigpipe(
-      SendWaitReply(wait_req->client_conn->fd, wait_req->object_requests,
-                    wait_req->num_object_requests),
-      wait_req->client_conn->fd);
+  handle_sigpipe(plasma::SendWaitReply(wait_req->client_conn->fd,
+                                       wait_req->object_requests,
+                                       wait_req->num_object_requests),
+                 wait_req->client_conn->fd);
   /* Iterate over all object IDs requested as part of this wait request.
    * Remove the wait request from each of the relevant object_wait_requests hash
    * tables if it is present there. */
@@ -455,7 +456,8 @@ void update_object_wait_requests(PlasmaManagerState *manager_state,
       WaitRequest *wait_req = *wait_req_ptr;
       wait_req->num_satisfied += 1;
       /* Mark the object as present in the wait request. */
-      auto object_request = wait_req->object_requests.find(obj_id);
+      auto object_request =
+          wait_req->object_requests.find(obj_id.to_plasma_id());
       /* Check that we found the object. */
       CHECK(object_request != wait_req->object_requests.end());
       /* Check that the object found was not previously known to us. */
@@ -513,7 +515,7 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name,
   PlasmaManagerState *state =
       (PlasmaManagerState *) malloc(sizeof(PlasmaManagerState));
   state->loop = event_loop_create();
-  state->plasma_conn = new PlasmaClient();
+  state->plasma_conn = new plasma::PlasmaClient();
   ARROW_CHECK_OK(state->plasma_conn->Connect(store_socket_name, "",
                                              PLASMA_DEFAULT_RELEASE_DELAY));
   state->manager_connections = NULL;
@@ -551,7 +553,7 @@ PlasmaManagerState *PlasmaManagerState_init(const char *store_socket_name,
   state->local_available_objects = NULL;
   /* Subscribe to notifications about sealed objects. */
   int plasma_fd;
-  ARROW_CHECK_OK(state->plasma_conn->Subscribe(plasma_fd));
+  ARROW_CHECK_OK(state->plasma_conn->Subscribe(&plasma_fd));
   /* Add the callback that processes the notification to the event loop. */
   event_loop_add_file(state->loop, plasma_fd, EVENT_LOOP_READ,
                       process_object_notification, state);
@@ -642,7 +644,8 @@ int write_object_chunk(ClientConnection *conn, PlasmaRequestBuffer *buf) {
     conn->cursor = 0;
     /* We are done sending the object, so release it. The corresponding call to
      * plasma_get occurred in process_transfer_request. */
-    ARROW_CHECK_OK(conn->manager_state->plasma_conn->Release(buf->object_id));
+    ARROW_CHECK_OK(conn->manager_state->plasma_conn->Release(
+        buf->object_id.to_plasma_id()));
   }
 
   return 0;
@@ -668,7 +671,8 @@ void send_queued_request(event_loop *loop,
   switch (buf->type) {
   case MessageType_PlasmaDataRequest:
     err = handle_sigpipe(
-        SendDataRequest(conn->fd, buf->object_id, state->addr, state->port),
+        plasma::SendDataRequest(conn->fd, buf->object_id.to_plasma_id(),
+                                state->addr, state->port),
         conn->fd);
     break;
   case MessageType_PlasmaDataReply:
@@ -676,9 +680,10 @@ void send_queued_request(event_loop *loop,
     if (conn->cursor == 0) {
       /* If the cursor is zero, we haven't sent any requests for this object
        * yet, so send the initial data request. */
-      err = handle_sigpipe(SendDataReply(conn->fd, buf->object_id,
-                                         buf->data_size, buf->metadata_size),
-                           conn->fd);
+      err = handle_sigpipe(
+          plasma::SendDataReply(conn->fd, buf->object_id.to_plasma_id(),
+                                buf->data_size, buf->metadata_size),
+          conn->fd);
     }
     if (err == 0) {
       err = write_object_chunk(conn, buf);
@@ -759,8 +764,10 @@ void process_data_chunk(event_loop *loop,
   LOG_DEBUG("reading on channel %d finished", data_sock);
   /* The following seal also triggers notification of clients for fetch or
    * wait requests, see process_object_notification. */
-  ARROW_CHECK_OK(conn->manager_state->plasma_conn->Seal(buf->object_id));
-  ARROW_CHECK_OK(conn->manager_state->plasma_conn->Release(buf->object_id));
+  ARROW_CHECK_OK(
+      conn->manager_state->plasma_conn->Seal(buf->object_id.to_plasma_id()));
+  ARROW_CHECK_OK(
+      conn->manager_state->plasma_conn->Release(buf->object_id.to_plasma_id()));
   /* Remove the request buffer used for reading this object's data. */
   DL_DELETE(conn->transfer_queue, buf);
   free(buf);
@@ -838,10 +845,11 @@ void process_transfer_request(event_loop *loop,
   }
 
   /* Allocate and append the request to the transfer queue. */
-  ObjectBuffer object_buffer;
+  plasma::ObjectBuffer object_buffer;
+  plasma::ObjectID object_id = obj_id.to_plasma_id();
   /* We pass in 0 to indicate that the command should return immediately. */
   ARROW_CHECK_OK(
-      conn->manager_state->plasma_conn->Get(&obj_id, 1, 0, &object_buffer));
+      conn->manager_state->plasma_conn->Get(&object_id, 1, 0, &object_buffer));
   if (object_buffer.data_size == -1) {
     /* If the object wasn't locally available, exit immediately. If the object
      * later appears locally, the requesting plasma manager should request the
@@ -908,7 +916,7 @@ void process_data_request(event_loop *loop,
   /* The corresponding call to plasma_release should happen in
    * process_data_chunk. */
   Status s = conn->manager_state->plasma_conn->Create(
-      object_id, data_size, NULL, metadata_size, &(buf->data));
+      object_id.to_plasma_id(), data_size, NULL, metadata_size, &(buf->data));
   /* If success_create == true, a new object has been created.
    * If success_create == false the object creation has failed, possibly
    * due to an object with the same ID already existing in the Plasma Store. */
@@ -1118,7 +1126,8 @@ void object_present_callback(ObjectID object_id,
   CHECK(manager_count >= 1);
 
   /* Update the in-progress remote wait requests. */
-  update_object_wait_requests(manager_state, object_id, PLASMA_QUERY_ANYWHERE,
+  update_object_wait_requests(manager_state, object_id,
+                              plasma::PLASMA_QUERY_ANYWHERE,
                               ObjectStatus_Remote);
 }
 
@@ -1143,7 +1152,7 @@ void object_table_subscribe_callback(ObjectID object_id,
 
 void process_fetch_requests(ClientConnection *client_conn,
                             int num_object_ids,
-                            ObjectID object_ids[]) {
+                            plasma::ObjectID object_ids[]) {
   PlasmaManagerState *manager_state = client_conn->manager_state;
 
   int num_object_ids_to_request = 0;
@@ -1197,7 +1206,7 @@ int wait_timeout_handler(event_loop *loop, timer_id id, void *context) {
 }
 
 void process_wait_request(ClientConnection *client_conn,
-                          ObjectRequestMap &&object_requests,
+                          plasma::ObjectRequestMap &&object_requests,
                           uint64_t timeout_ms,
                           int num_ready_objects) {
   CHECK(client_conn != NULL);
@@ -1231,10 +1240,10 @@ void process_wait_request(ClientConnection *client_conn,
     add_wait_request_for_object(manager_state, obj_id, object_request.type,
                                 wait_req);
 
-    if (object_request.type == PLASMA_QUERY_LOCAL) {
+    if (object_request.type == plasma::PLASMA_QUERY_LOCAL) {
       /* TODO(rkn): If desired, we could issue a fetch command here to retrieve
        * the object. */
-    } else if (object_request.type == PLASMA_QUERY_ANYWHERE) {
+    } else if (object_request.type == plasma::PLASMA_QUERY_ANYWHERE) {
       /* Add this object ID to the list of object IDs to request notifications
        * for from the object table. */
       object_ids_to_request[num_object_ids_to_request] = obj_id;
@@ -1289,8 +1298,10 @@ void request_status_done(ObjectID object_id,
   ClientConnection *client_conn = (ClientConnection *) context;
   int status =
       request_status(object_id, manager_count, manager_vector, context);
-  handle_sigpipe(SendStatusReply(client_conn->fd, &object_id, &status, 1),
-                 client_conn->fd);
+  plasma::ObjectID object_id_copy = object_id.to_plasma_id();
+  handle_sigpipe(
+      plasma::SendStatusReply(client_conn->fd, &object_id_copy, &status, 1),
+      client_conn->fd);
 }
 
 int request_status(ObjectID object_id,
@@ -1318,19 +1329,22 @@ void object_table_lookup_fail_callback(ObjectID object_id,
   CHECK(0);
 }
 
-void process_status_request(ClientConnection *client_conn, ObjectID object_id) {
+void process_status_request(ClientConnection *client_conn,
+                            plasma::ObjectID object_id) {
   /* Return success immediately if we already have this object. */
   if (is_object_local(client_conn->manager_state, object_id)) {
     int status = ObjectStatus_Local;
-    handle_sigpipe(SendStatusReply(client_conn->fd, &object_id, &status, 1),
-                   client_conn->fd);
+    handle_sigpipe(
+        plasma::SendStatusReply(client_conn->fd, &object_id, &status, 1),
+        client_conn->fd);
     return;
   }
 
   if (client_conn->manager_state->db == NULL) {
     int status = ObjectStatus_Nonexistent;
-    handle_sigpipe(SendStatusReply(client_conn->fd, &object_id, &status, 1),
-                   client_conn->fd);
+    handle_sigpipe(
+        plasma::SendStatusReply(client_conn->fd, &object_id, &status, 1),
+        client_conn->fd);
     return;
   }
 
@@ -1426,9 +1440,9 @@ void process_add_object_notification(PlasmaManagerState *state,
   }
 
   /* Update the in-progress local and remote wait requests. */
-  update_object_wait_requests(state, object_id, PLASMA_QUERY_LOCAL,
+  update_object_wait_requests(state, object_id, plasma::PLASMA_QUERY_LOCAL,
                               ObjectStatus_Local);
-  update_object_wait_requests(state, object_id, PLASMA_QUERY_ANYWHERE,
+  update_object_wait_requests(state, object_id, plasma::PLASMA_QUERY_ANYWHERE,
                               ObjectStatus_Local);
 }
 
@@ -1546,46 +1560,47 @@ void process_message(event_loop *loop,
   switch (type) {
   case MessageType_PlasmaDataRequest: {
     LOG_DEBUG("Processing data request");
-    ObjectID object_id;
+    plasma::ObjectID object_id;
     char *address;
     int port;
-    ARROW_CHECK_OK(ReadDataRequest(data, &object_id, &address, &port));
+    ARROW_CHECK_OK(
+        plasma::ReadDataRequest(data, length, &object_id, &address, &port));
     process_transfer_request(loop, object_id, address, port, conn);
     free(address);
   } break;
   case MessageType_PlasmaDataReply: {
     LOG_DEBUG("Processing data reply");
-    ObjectID object_id;
+    plasma::ObjectID object_id;
     int64_t object_size;
     int64_t metadata_size;
-    ARROW_CHECK_OK(
-        ReadDataReply(data, &object_id, &object_size, &metadata_size));
+    ARROW_CHECK_OK(plasma::ReadDataReply(data, length, &object_id, &object_size,
+                                         &metadata_size));
     process_data_request(loop, client_sock, object_id, object_size,
                          metadata_size, conn);
   } break;
   case MessageType_PlasmaFetchRequest: {
     LOG_DEBUG("Processing fetch remote");
-    std::vector<ObjectID> object_ids_to_fetch;
+    std::vector<plasma::ObjectID> object_ids_to_fetch;
     /* TODO(pcm): process_fetch_requests allocates an array of num_objects
      * object_ids too so these should be shared in the future. */
-    ARROW_CHECK_OK(ReadFetchRequest(data, object_ids_to_fetch));
+    ARROW_CHECK_OK(plasma::ReadFetchRequest(data, length, object_ids_to_fetch));
     process_fetch_requests(conn, object_ids_to_fetch.size(),
                            object_ids_to_fetch.data());
   } break;
   case MessageType_PlasmaWaitRequest: {
     LOG_DEBUG("Processing wait");
-    ObjectRequestMap object_requests;
+    plasma::ObjectRequestMap object_requests;
     int64_t timeout_ms;
     int num_ready_objects;
-    ARROW_CHECK_OK(ReadWaitRequest(data, object_requests, &timeout_ms,
-                                   &num_ready_objects));
+    ARROW_CHECK_OK(plasma::ReadWaitRequest(data, length, object_requests,
+                                           &timeout_ms, &num_ready_objects));
     process_wait_request(conn, std::move(object_requests), timeout_ms,
                          num_ready_objects);
   } break;
   case MessageType_PlasmaStatusRequest: {
     LOG_DEBUG("Processing status");
-    ObjectID object_id;
-    ARROW_CHECK_OK(ReadStatusRequest(data, &object_id, 1));
+    plasma::ObjectID object_id;
+    ARROW_CHECK_OK(plasma::ReadStatusRequest(data, length, &object_id, 1));
     process_status_request(conn, object_id);
   } break;
   case DISCONNECT_CLIENT: {

src/plasma/plasma_protocol.h

@@ -1,194 +0,0 @@
-#ifndef PLASMA_PROTOCOL_H
-#define PLASMA_PROTOCOL_H
-
-#include "status.h"
-#include "format/plasma_generated.h"
-#include "plasma.h"
-
-using arrow::Status;
-
-/* Plasma receive message. */
-
-Status PlasmaReceive(int sock,
-                     int64_t message_type,
-                     std::vector<uint8_t> &buffer);
-
-/* Plasma Create message functions. */
-
-Status SendCreateRequest(int sock,
-                         ObjectID object_id,
-                         int64_t data_size,
-                         int64_t metadata_size);
-
-Status ReadCreateRequest(uint8_t *data,
-                         ObjectID *object_id,
-                         int64_t *data_size,
-                         int64_t *metadata_size);
-
-Status SendCreateReply(int sock,
-                       ObjectID object_id,
-                       PlasmaObject *object,
-                       int error);
-
-Status ReadCreateReply(uint8_t *data,
-                       ObjectID *object_id,
-                       PlasmaObject *object);
-
-/* Plasma Seal message functions. */
-
-Status SendSealRequest(int sock, ObjectID object_id, unsigned char *digest);
-
-Status ReadSealRequest(uint8_t *data,
-                       ObjectID *object_id,
-                       unsigned char *digest);
-
-Status SendSealReply(int sock, ObjectID object_id, int error);
-
-Status ReadSealReply(uint8_t *data, ObjectID *object_id);
-
-/* Plasma Get message functions. */
-
-Status SendGetRequest(int sock,
-                      ObjectID object_ids[],
-                      int64_t num_objects,
-                      int64_t timeout_ms);
-
-Status ReadGetRequest(uint8_t *data,
-                      std::vector<ObjectID> &object_ids,
-                      int64_t *timeout_ms);
-
-Status SendGetReply(
-    int sock,
-    ObjectID object_ids[],
-    std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> &plasma_objects,
-    int64_t num_objects);
-
-Status ReadGetReply(uint8_t *data,
-                    ObjectID object_ids[],
-                    PlasmaObject plasma_objects[],
-                    int64_t num_objects);
-
-/* Plasma Release message functions. */
-
-Status SendReleaseRequest(int sock, ObjectID object_id);
-
-Status ReadReleaseRequest(uint8_t *data, ObjectID *object_id);
-
-Status SendReleaseReply(int sock, ObjectID object_id, int error);
-
-Status ReadReleaseReply(uint8_t *data, ObjectID *object_id);
-
-/* Plasma Delete message functions. */
-
-Status SendDeleteRequest(int sock, ObjectID object_id);
-
-Status ReadDeleteRequest(uint8_t *data, ObjectID *object_id);
-
-Status SendDeleteReply(int sock, ObjectID object_id, int error);
-
-Status ReadDeleteReply(uint8_t *data, ObjectID *object_id);
-
-/* Satus messages. */
-
-Status SendStatusRequest(int sock, ObjectID object_ids[], int64_t num_objects);
-
-Status ReadStatusRequest(uint8_t *data,
-                         ObjectID object_ids[],
-                         int64_t num_objects);
-
-Status SendStatusReply(int sock,
-                       ObjectID object_ids[],
-                       int object_status[],
-                       int64_t num_objects);
-
-int64_t ReadStatusReply_num_objects(uint8_t *data);
-
-Status ReadStatusReply(uint8_t *data,
-                       ObjectID object_ids[],
-                       int object_status[],
-                       int64_t num_objects);
-
-/* Plasma Constains message functions. */
-
-Status SendContainsRequest(int sock, ObjectID object_id);
-
-Status ReadContainsRequest(uint8_t *data, ObjectID *object_id);
-
-Status SendContainsReply(int sock, ObjectID object_id, int has_object);
-
-Status ReadContainsReply(uint8_t *data, ObjectID *object_id, int *has_object);
-
-/* Plasma Connect message functions. */
-
-Status SendConnectRequest(int sock);
-
-Status ReadConnectRequest(uint8_t *data);
-
-Status SendConnectReply(int sock, int64_t memory_capacity);
-
-Status ReadConnectReply(uint8_t *data, int64_t *memory_capacity);
-
-/* Plasma Evict message functions (no reply so far). */
-
-Status SendEvictRequest(int sock, int64_t num_bytes);
-
-Status ReadEvictRequest(uint8_t *data, int64_t *num_bytes);
-
-Status SendEvictReply(int sock, int64_t num_bytes);
-
-Status ReadEvictReply(uint8_t *data, int64_t &num_bytes);
-
-/* Plasma Fetch Remote message functions. */
-
-Status SendFetchRequest(int sock, ObjectID object_ids[], int64_t num_objects);
-
-Status ReadFetchRequest(uint8_t *data, std::vector<ObjectID> &object_ids);
-
-/* Plasma Wait message functions. */
-
-Status SendWaitRequest(int sock,
-                       ObjectRequest object_requests[],
-                       int num_requests,
-                       int num_ready_objects,
-                       int64_t timeout_ms);
-
-Status ReadWaitRequest(uint8_t *data,
-                       ObjectRequestMap &object_requests,
-                       int64_t *timeout_ms,
-                       int *num_ready_objects);
-
-Status SendWaitReply(int sock,
-                     const ObjectRequestMap &object_requests,
-                     int num_ready_objects);
-
-Status ReadWaitReply(uint8_t *data,
-                     ObjectRequest object_requests[],
-                     int *num_ready_objects);
-
-/* Plasma Subscribe message functions. */
-
-Status SendSubscribeRequest(int sock);
-
-/* Data messages. */
-
-Status SendDataRequest(int sock,
-                       ObjectID object_id,
-                       const char *address,
-                       int port);
-
-Status ReadDataRequest(uint8_t *data,
-                       ObjectID *object_id,
-                       char **address,
-                       int *port);
-
-Status SendDataReply(int sock,
-                     ObjectID object_id,
-                     int64_t object_size,
-                     int64_t metadata_size);
-
-Status ReadDataReply(uint8_t *data,
-                     ObjectID *object_id,
-                     int64_t *object_size,
-                     int64_t *metadata_size);
-
-#endif /* PLASMA_PROTOCOL */

src/plasma/plasma_store.cc

@@ -1,689 +0,0 @@
-// PLASMA STORE: This is a simple object store server process
-//
-// It accepts incoming client connections on a unix domain socket
-// (name passed in via the -s option of the executable) and uses a
-// single thread to serve the clients. Each client establishes a
-// connection and can create objects, wait for objects and seal
-// objects through that connection.
-//
-// It keeps a hash table that maps object_ids (which are 20 byte long,
-// just enough to store and SHA1 hash) to memory mapped files.
-
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/ioctl.h>
-#include <sys/socket.h>
-#include <sys/statvfs.h>
-#include <sys/types.h>
-#include <sys/un.h>
-#include <getopt.h>
-#include <string.h>
-#include <signal.h>
-#include <limits.h>
-
-#include <deque>
-#include <unordered_map>
-#include <unordered_set>
-#include <vector>
-
-#include "plasma_common.h"
-#include "plasma_store.h"
-#include "format/common_generated.h"
-#include "plasma_io.h"
-#include "malloc.h"
-
-extern "C" {
-#include "fling.h"
-void *dlmalloc(size_t);
-void *dlmemalign(size_t alignment, size_t bytes);
-void dlfree(void *);
-size_t dlmalloc_set_footprint_limit(size_t bytes);
-}
-
-struct GetRequest {
-  GetRequest(Client *client, const std::vector<ObjectID> &object_ids);
-
-  /// The client that called get.
-  Client *client;
-  /// The ID of the timer that will time out and cause this wait to return to
-  ///  the client if it hasn't already returned.
-  int64_t timer;
-  /// The object IDs involved in this request. This is used in the reply.
-  std::vector<ObjectID> object_ids;
-  /// The object information for the objects in this request. This is used in
-  /// the reply.
-  std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> objects;
-  /// The minimum number of objects to wait for in this request.
-  int64_t num_objects_to_wait_for;
-  /// The number of object requests in this wait request that are already
-  /// satisfied.
-  int64_t num_satisfied;
-};
-
-GetRequest::GetRequest(Client *client, const std::vector<ObjectID> &object_ids)
-    : client(client),
-      timer(-1),
-      object_ids(object_ids.begin(), object_ids.end()),
-      objects(object_ids.size()),
-      num_satisfied(0) {
-  std::unordered_set<ObjectID, UniqueIDHasher> unique_ids(object_ids.begin(),
-                                                          object_ids.end());
-  num_objects_to_wait_for = unique_ids.size();
-}
-
-Client::Client(int fd) : fd(fd) {}
-
-PlasmaStore::PlasmaStore(EventLoop *loop, int64_t system_memory)
-    : loop_(loop), eviction_policy_(&store_info_) {
-  store_info_.memory_capacity = system_memory;
-}
-
-PlasmaStore::~PlasmaStore() {
-  for (const auto &element : pending_notifications_) {
-    auto object_notifications = element.second.object_notifications;
-    for (int i = 0; i < object_notifications.size(); ++i) {
-      uint8_t *notification = (uint8_t *) object_notifications.at(i);
-      uint8_t *data = notification;
-      free(data);
-    }
-  }
-}
-
-// If this client is not already using the object, add the client to the
-// object's list of clients, otherwise do nothing.
-void PlasmaStore::add_client_to_object_clients(ObjectTableEntry *entry,
-                                               Client *client) {
-  // Check if this client is already using the object.
-  if (entry->clients.find(client) != entry->clients.end()) {
-    return;
-  }
-  // If there are no other clients using this object, notify the eviction policy
-  // that the object is being used.
-  if (entry->clients.size() == 0) {
-    // Tell the eviction policy that this object is being used.
-    std::vector<ObjectID> objects_to_evict;
-    eviction_policy_.begin_object_access(entry->object_id, objects_to_evict);
-    delete_objects(objects_to_evict);
-  }
-  // Add the client pointer to the list of clients using this object.
-  entry->clients.insert(client);
-}
-
-// Create a new object buffer in the hash table.
-int PlasmaStore::create_object(ObjectID object_id,
-                               int64_t data_size,
-                               int64_t metadata_size,
-                               Client *client,
-                               PlasmaObject *result) {
-  ARROW_LOG(DEBUG) << "creating object " << object_id.hex();
-  if (store_info_.objects.count(object_id) != 0) {
-    // There is already an object with the same ID in the Plasma Store, so
-    // ignore this requst.
-    return PlasmaError_ObjectExists;
-  }
-  // Try to evict objects until there is enough space.
-  uint8_t *pointer;
-  do {
-    // Allocate space for the new object. We use dlmemalign instead of dlmalloc
-    // in order to align the allocated region to a 64-byte boundary. This is not
-    // strictly necessary, but it is an optimization that could speed up the
-    // computation of a hash of the data (see compute_object_hash_parallel in
-    // plasma_client.cc). Note that even though this pointer is 64-byte aligned,
-    // it is not guaranteed that the corresponding pointer in the client will be
-    // 64-byte aligned, but in practice it often will be.
-    pointer = (uint8_t *) dlmemalign(BLOCK_SIZE, data_size + metadata_size);
-    if (pointer == NULL) {
-      // Tell the eviction policy how much space we need to create this object.
-      std::vector<ObjectID> objects_to_evict;
-      bool success = eviction_policy_.require_space(data_size + metadata_size,
-                                                    objects_to_evict);
-      delete_objects(objects_to_evict);
-      // Return an error to the client if not enough space could be freed to
-      // create the object.
-      if (!success) {
-        return PlasmaError_OutOfMemory;
-      }
-    }
-  } while (pointer == NULL);
-  int fd;
-  int64_t map_size;
-  ptrdiff_t offset;
-  get_malloc_mapinfo(pointer, &fd, &map_size, &offset);
-  assert(fd != -1);
-
-  auto entry = std::unique_ptr<ObjectTableEntry>(new ObjectTableEntry());
-  entry->object_id = object_id;
-  entry->info.object_id = object_id.binary();
-  entry->info.data_size = data_size;
-  entry->info.metadata_size = metadata_size;
-  entry->pointer = pointer;
-  // TODO(pcm): Set the other fields.
-  entry->fd = fd;
-  entry->map_size = map_size;
-  entry->offset = offset;
-  entry->state = PLASMA_CREATED;
-
-  store_info_.objects[object_id] = std::move(entry);
-  result->handle.store_fd = fd;
-  result->handle.mmap_size = map_size;
-  result->data_offset = offset;
-  result->metadata_offset = offset + data_size;
-  result->data_size = data_size;
-  result->metadata_size = metadata_size;
-  // Notify the eviction policy that this object was created. This must be done
-  // immediately before the call to add_client_to_object_clients so that the
-  // eviction policy does not have an opportunity to evict the object.
-  eviction_policy_.object_created(object_id);
-  // Record that this client is using this object.
-  add_client_to_object_clients(store_info_.objects[object_id].get(), client);
-  return PlasmaError_OK;
-}
-
-void PlasmaObject_init(PlasmaObject *object, ObjectTableEntry *entry) {
-  DCHECK(object != NULL);
-  DCHECK(entry != NULL);
-  DCHECK(entry->state == PLASMA_SEALED);
-  object->handle.store_fd = entry->fd;
-  object->handle.mmap_size = entry->map_size;
-  object->data_offset = entry->offset;
-  object->metadata_offset = entry->offset + entry->info.data_size;
-  object->data_size = entry->info.data_size;
-  object->metadata_size = entry->info.metadata_size;
-}
-
-void PlasmaStore::return_from_get(GetRequest *get_req) {
-  // Send the get reply to the client.
-  Status s = SendGetReply(get_req->client->fd, &get_req->object_ids[0],
-                          get_req->objects, get_req->object_ids.size());
-  warn_if_sigpipe(s.ok() ? 0 : -1, get_req->client->fd);
-  // If we successfully sent the get reply message to the client, then also send
-  // the file descriptors.
-  if (s.ok()) {
-    // Send all of the file descriptors for the present objects.
-    for (const auto &object_id : get_req->object_ids) {
-      PlasmaObject &object = get_req->objects[object_id];
-      // We use the data size to indicate whether the object is present or not.
-      if (object.data_size != -1) {
-        int error_code = send_fd(get_req->client->fd, object.handle.store_fd);
-        // If we failed to send the file descriptor, loop until we have sent it
-        // successfully. TODO(rkn): This is problematic for two reasons. First
-        // of all, sending the file descriptor should just succeed without any
-        // errors, but sometimes I see a "Message too long" error number.
-        // Second, looping like this allows a client to potentially block the
-        // plasma store event loop which should never happen.
-        while (error_code < 0) {
-          if (errno == EMSGSIZE) {
-            ARROW_LOG(WARNING) << "Failed to send file descriptor, retrying.";
-            error_code = send_fd(get_req->client->fd, object.handle.store_fd);
-            continue;
-          }
-          warn_if_sigpipe(error_code, get_req->client->fd);
-          break;
-        }
-      }
-    }
-  }
-
-  // Remove the get request from each of the relevant object_get_requests hash
-  // tables if it is present there. It should only be present there if the get
-  // request timed out.
-  for (ObjectID &object_id : get_req->object_ids) {
-    auto &get_requests = object_get_requests_[object_id];
-    // Erase get_req from the vector.
-    auto it = std::find(get_requests.begin(), get_requests.end(), get_req);
-    if (it != get_requests.end()) {
-      get_requests.erase(it);
-    }
-  }
-  // Remove the get request.
-  if (get_req->timer != -1) {
-    ARROW_CHECK(loop_->remove_timer(get_req->timer) == AE_OK);
-  }
-  delete get_req;
-}
-
-void PlasmaStore::update_object_get_requests(ObjectID object_id) {
-  std::vector<GetRequest *> &get_requests = object_get_requests_[object_id];
-  int index = 0;
-  int num_requests = get_requests.size();
-  for (int i = 0; i < num_requests; ++i) {
-    GetRequest *get_req = get_requests[index];
-    auto entry = get_object_table_entry(&store_info_, object_id);
-    ARROW_CHECK(entry != NULL);
-
-    PlasmaObject_init(&get_req->objects[object_id], entry);
-    get_req->num_satisfied += 1;
-    // Record the fact that this client will be using this object and will
-    // be responsible for releasing this object.
-    add_client_to_object_clients(entry, get_req->client);
-
-    // If this get request is done, reply to the client.
-    if (get_req->num_satisfied == get_req->num_objects_to_wait_for) {
-      return_from_get(get_req);
-    } else {
-      // The call to return_from_get will remove the current element in the
-      // array, so we only increment the counter in the else branch.
-      index += 1;
-    }
-  }
-
-  DCHECK(index == get_requests.size());
-  // Remove the array of get requests for this object, since no one should be
-  // waiting for this object anymore.
-  object_get_requests_.erase(object_id);
-}
-
-void PlasmaStore::process_get_request(Client *client,
-                                      const std::vector<ObjectID> &object_ids,
-                                      uint64_t timeout_ms) {
-  // Create a get request for this object.
-  GetRequest *get_req = new GetRequest(client, object_ids);
-
-  for (auto object_id : object_ids) {
-    // Check if this object is already present locally. If so, record that the
-    // object is being used and mark it as accounted for.
-    auto entry = get_object_table_entry(&store_info_, object_id);
-    if (entry && entry->state == PLASMA_SEALED) {
-      // Update the get request to take into account the present object.
-      PlasmaObject_init(&get_req->objects[object_id], entry);
-      get_req->num_satisfied += 1;
-      // If necessary, record that this client is using this object. In the case
-      // where entry == NULL, this will be called from seal_object.
-      add_client_to_object_clients(entry, client);
-    } else {
-      // Add a placeholder plasma object to the get request to indicate that the
-      // object is not present. This will be parsed by the client. We set the
-      // data size to -1 to indicate that the object is not present.
-      get_req->objects[object_id].data_size = -1;
-      // Add the get request to the relevant data structures.
-      object_get_requests_[object_id].push_back(get_req);
-    }
-  }
-
-  // If all of the objects are present already or if the timeout is 0, return to
-  // the client.
-  if (get_req->num_satisfied == get_req->num_objects_to_wait_for ||
-      timeout_ms == 0) {
-    return_from_get(get_req);
-  } else if (timeout_ms != -1) {
-    // Set a timer that will cause the get request to return to the client. Note
-    // that a timeout of -1 is used to indicate that no timer should be set.
-    get_req->timer =
-        loop_->add_timer(timeout_ms, [this, get_req](int64_t timer_id) {
-          return_from_get(get_req);
-          return kEventLoopTimerDone;
-        });
-  }
-}
-
-int PlasmaStore::remove_client_from_object_clients(ObjectTableEntry *entry,
-                                                   Client *client) {
-  auto it = entry->clients.find(client);
-  if (it != entry->clients.end()) {
-    entry->clients.erase(it);
-    // If no more clients are using this object, notify the eviction policy
-    // that the object is no longer being used.
-    if (entry->clients.size() == 0) {
-      // Tell the eviction policy that this object is no longer being used.
-      std::vector<ObjectID> objects_to_evict;
-      eviction_policy_.end_object_access(entry->object_id, objects_to_evict);
-      delete_objects(objects_to_evict);
-    }
-    // Return 1 to indicate that the client was removed.
-    return 1;
-  } else {
-    // Return 0 to indicate that the client was not removed.
-    return 0;
-  }
-}
-
-void PlasmaStore::release_object(ObjectID object_id, Client *client) {
-  auto entry = get_object_table_entry(&store_info_, object_id);
-  ARROW_CHECK(entry != NULL);
-  // Remove the client from the object's array of clients.
-  ARROW_CHECK(remove_client_from_object_clients(entry, client) == 1);
-}
-
-// Check if an object is present.
-int PlasmaStore::contains_object(ObjectID object_id) {
-  auto entry = get_object_table_entry(&store_info_, object_id);
-  return entry && (entry->state == PLASMA_SEALED) ? OBJECT_FOUND
-                                                  : OBJECT_NOT_FOUND;
-}
-
-// Seal an object that has been created in the hash table.
-void PlasmaStore::seal_object(ObjectID object_id, unsigned char digest[]) {
-  ARROW_LOG(DEBUG) << "sealing object " << object_id.hex();
-  auto entry = get_object_table_entry(&store_info_, object_id);
-  ARROW_CHECK(entry != NULL);
-  ARROW_CHECK(entry->state == PLASMA_CREATED);
-  // Set the state of object to SEALED.
-  entry->state = PLASMA_SEALED;
-  // Set the object digest.
-  entry->info.digest = std::string((char *) &digest[0], kDigestSize);
-  // Inform all subscribers that a new object has been sealed.
-  push_notification(&entry->info);
-
-  // Update all get requests that involve this object.
-  update_object_get_requests(object_id);
-}
-
-void PlasmaStore::delete_objects(const std::vector<ObjectID> &object_ids) {
-  for (const auto &object_id : object_ids) {
-    ARROW_LOG(DEBUG) << "deleting object " << object_id.hex();
-    auto entry = get_object_table_entry(&store_info_, object_id);
-    // TODO(rkn): This should probably not fail, but should instead throw an
-    // error. Maybe we should also support deleting objects that have been
-    // created but not sealed.
-    ARROW_CHECK(entry != NULL)
-        << "To delete an object it must be in the object table.";
-    ARROW_CHECK(entry->state == PLASMA_SEALED)
-        << "To delete an object it must have been sealed.";
-    ARROW_CHECK(entry->clients.size() == 0)
-        << "To delete an object, there must be no clients currently using it.";
-    dlfree(entry->pointer);
-    store_info_.objects.erase(object_id);
-    // Inform all subscribers that the object has been deleted.
-    ObjectInfoT notification;
-    notification.object_id = object_id.binary();
-    notification.is_deletion = true;
-    push_notification(&notification);
-  }
-}
-
-void PlasmaStore::connect_client(int listener_sock) {
-  int client_fd = AcceptClient(listener_sock);
-  // This is freed in disconnect_client.
-  Client *client = new Client(client_fd);
-  // Add a callback to handle events on this socket.
-  // TODO(pcm): Check return value.
-  loop_->add_file_event(client_fd, kEventLoopRead, [this, client](int events) {
-    process_message(client);
-  });
-  ARROW_LOG(DEBUG) << "New connection with fd " << client_fd;
-}
-
-void PlasmaStore::disconnect_client(Client *client) {
-  ARROW_CHECK(client != NULL);
-  ARROW_CHECK(client->fd > 0);
-  loop_->remove_file_event(client->fd);
-  // Close the socket.
-  close(client->fd);
-  ARROW_LOG(INFO) << "Disconnecting client on fd " << client->fd;
-  // If this client was using any objects, remove it from the appropriate
-  // lists.
-  for (const auto &entry : store_info_.objects) {
-    remove_client_from_object_clients(entry.second.get(), client);
-  }
-  // Note, the store may still attempt to send a message to the disconnected
-  // client (for example, when an object ID that the client was waiting for
-  // is ready). In these cases, the attempt to send the message will fail, but
-  // the store should just ignore the failure.
-  delete client;
-}
-
-/// Send notifications about sealed objects to the subscribers. This is called
-/// in seal_object. If the socket's send buffer is full, the notification will
-/// be
-/// buffered, and this will be called again when the send buffer has room.
-///
-/// @param client The client to send the notification to.
-/// @return Void.
-void PlasmaStore::send_notifications(int client_fd) {
-  auto it = pending_notifications_.find(client_fd);
-
-  int num_processed = 0;
-  bool closed = false;
-  // Loop over the array of pending notifications and send as many of them as
-  // possible.
-  for (int i = 0; i < it->second.object_notifications.size(); ++i) {
-    uint8_t *notification = (uint8_t *) it->second.object_notifications.at(i);
-    // Decode the length, which is the first bytes of the message.
-    int64_t size = *((int64_t *) notification);
-
-    // Attempt to send a notification about this object ID.
-    int nbytes = send(client_fd, notification, sizeof(int64_t) + size, 0);
-    if (nbytes >= 0) {
-      ARROW_CHECK(nbytes == sizeof(int64_t) + size);
-    } else if (nbytes == -1 &&
-               (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR)) {
-      ARROW_LOG(DEBUG)
-          << "The socket's send buffer is full, so we are caching this "
-             "notification and will send it later.";
-      // Add a callback to the event loop to send queued notifications whenever
-      // there is room in the socket's send buffer. Callbacks can be added
-      // more than once here and will be overwritten. The callback is removed
-      // at the end of the method.
-      // TODO(pcm): Introduce status codes and check in case the file descriptor
-      // is added twice.
-      loop_->add_file_event(
-          client_fd, kEventLoopWrite,
-          [this, client_fd](int events) { send_notifications(client_fd); });
-      break;
-    } else {
-      ARROW_LOG(WARNING) << "Failed to send notification to client on fd "
-                         << client_fd;
-      if (errno == EPIPE) {
-        closed = true;
-        break;
-      }
-    }
-    num_processed += 1;
-    // The corresponding malloc happened in create_object_info_buffer
-    // within push_notification.
-    free(notification);
-  }
-  // Remove the sent notifications from the array.
-  it->second.object_notifications.erase(
-      it->second.object_notifications.begin(),
-      it->second.object_notifications.begin() + num_processed);
-
-  // Stop sending notifications if the pipe was broken.
-  if (closed) {
-    close(client_fd);
-    pending_notifications_.erase(client_fd);
-  }
-
-  // If we have sent all notifications, remove the fd from the event loop.
-  if (it->second.object_notifications.empty()) {
-    loop_->remove_file_event(client_fd);
-  }
-}
-
-void PlasmaStore::push_notification(ObjectInfoT *object_info) {
-  for (auto &element : pending_notifications_) {
-    uint8_t *notification = create_object_info_buffer(object_info);
-    element.second.object_notifications.push_back(notification);
-    send_notifications(element.first);
-    // The notification gets freed in send_notifications when the notification
-    // is sent over the socket.
-  }
-}
-
-// Subscribe to notifications about sealed objects.
-void PlasmaStore::subscribe_to_updates(Client *client) {
-  ARROW_LOG(DEBUG) << "subscribing to updates on fd " << client->fd;
-  // TODO(rkn): The store could block here if the client doesn't send a file
-  // descriptor.
-  int fd = recv_fd(client->fd);
-  if (fd < 0) {
-    // This may mean that the client died before sending the file descriptor.
-    ARROW_LOG(WARNING) << "Failed to receive file descriptor from client on fd "
-                       << client->fd << ".";
-    return;
-  }
-
-  // Create a new array to buffer notifications that can't be sent to the
-  // subscriber yet because the socket send buffer is full. TODO(rkn): the queue
-  // never gets freed.
-  NotificationQueue &queue = pending_notifications_[fd];
-
-  // Push notifications to the new subscriber about existing objects.
-  for (const auto &entry : store_info_.objects) {
-    push_notification(&entry.second->info);
-  }
-  send_notifications(fd);
-}
-
-Status PlasmaStore::process_message(Client *client) {
-  int64_t type;
-  Status s = ReadMessage(client->fd, &type, input_buffer_);
-  ARROW_CHECK(s.ok() || s.IsIOError());
-
-  uint8_t *input = input_buffer_.data();
-  ObjectID object_id;
-  PlasmaObject object;
-  // TODO(pcm): Get rid of the following.
-  memset(&object, 0, sizeof(object));
-
-  // Process the different types of requests.
-  switch (type) {
-  case MessageType_PlasmaCreateRequest: {
-    int64_t data_size;
-    int64_t metadata_size;
-    RETURN_NOT_OK(
-        ReadCreateRequest(input, &object_id, &data_size, &metadata_size));
-    int error_code =
-        create_object(object_id, data_size, metadata_size, client, &object);
-    HANDLE_SIGPIPE(SendCreateReply(client->fd, object_id, &object, error_code),
-                   client->fd);
-    if (error_code == PlasmaError_OK) {
-      warn_if_sigpipe(send_fd(client->fd, object.handle.store_fd), client->fd);
-    }
-  } break;
-  case MessageType_PlasmaGetRequest: {
-    std::vector<ObjectID> object_ids_to_get;
-    int64_t timeout_ms;
-    RETURN_NOT_OK(ReadGetRequest(input, object_ids_to_get, &timeout_ms));
-    process_get_request(client, object_ids_to_get, timeout_ms);
-  } break;
-  case MessageType_PlasmaReleaseRequest:
-    RETURN_NOT_OK(ReadReleaseRequest(input, &object_id));
-    release_object(object_id, client);
-    break;
-  case MessageType_PlasmaContainsRequest:
-    RETURN_NOT_OK(ReadContainsRequest(input, &object_id));
-    if (contains_object(object_id) == OBJECT_FOUND) {
-      HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 1), client->fd);
-    } else {
-      HANDLE_SIGPIPE(SendContainsReply(client->fd, object_id, 0), client->fd);
-    }
-    break;
-  case MessageType_PlasmaSealRequest: {
-    unsigned char digest[kDigestSize];
-    RETURN_NOT_OK(ReadSealRequest(input, &object_id, &digest[0]));
-    seal_object(object_id, &digest[0]);
-  } break;
-  case MessageType_PlasmaEvictRequest: {
-    // This code path should only be used for testing.
-    int64_t num_bytes;
-    RETURN_NOT_OK(ReadEvictRequest(input, &num_bytes));
-    std::vector<ObjectID> objects_to_evict;
-    int64_t num_bytes_evicted =
-        eviction_policy_.choose_objects_to_evict(num_bytes, objects_to_evict);
-    delete_objects(objects_to_evict);
-    HANDLE_SIGPIPE(SendEvictReply(client->fd, num_bytes_evicted), client->fd);
-  } break;
-  case MessageType_PlasmaSubscribeRequest:
-    subscribe_to_updates(client);
-    break;
-  case MessageType_PlasmaConnectRequest: {
-    HANDLE_SIGPIPE(SendConnectReply(client->fd, store_info_.memory_capacity),
-                   client->fd);
-  } break;
-  case DISCONNECT_CLIENT:
-    ARROW_LOG(DEBUG) << "Disconnecting client on fd " << client->fd;
-    disconnect_client(client);
-    break;
-  default:
-    // This code should be unreachable.
-    ARROW_CHECK(0);
-  }
-  return Status::OK();
-}
-
-// Report "success" to valgrind.
-void signal_handler(int signal) {
-  if (signal == SIGTERM) {
-    exit(0);
-  }
-}
-
-void start_server(char *socket_name, int64_t system_memory) {
-  // Ignore SIGPIPE signals. If we don't do this, then when we attempt to write
-  // to a client that has already died, the store could die.
-  signal(SIGPIPE, SIG_IGN);
-  // Create the event loop.
-  EventLoop loop;
-  PlasmaStore store(&loop, system_memory);
-  int socket = bind_ipc_sock(socket_name, true);
-  ARROW_CHECK(socket >= 0);
-  // TODO(pcm): Check return value.
-  loop.add_file_event(socket, kEventLoopRead, [&store, socket](int events) {
-    store.connect_client(socket);
-  });
-  loop.run();
-}
-
-int main(int argc, char *argv[]) {
-  signal(SIGTERM, signal_handler);
-  char *socket_name = NULL;
-  int64_t system_memory = -1;
-  int c;
-  while ((c = getopt(argc, argv, "s:m:")) != -1) {
-    switch (c) {
-    case 's':
-      socket_name = optarg;
-      break;
-    case 'm': {
-      char extra;
-      int scanned = sscanf(optarg, "%" SCNd64 "%c", &system_memory, &extra);
-      ARROW_CHECK(scanned == 1);
-      ARROW_LOG(INFO) << "Allowing the Plasma store to use up to "
-                      << ((double) system_memory) / 1000000000
-                      << "GB of memory.";
-      break;
-    }
-    default:
-      exit(-1);
-    }
-  }
-  if (!socket_name) {
-    ARROW_LOG(FATAL)
-        << "please specify socket for incoming connections with -s switch";
-  }
-  if (system_memory == -1) {
-    ARROW_LOG(FATAL)
-        << "please specify the amount of system memory with -m switch";
-  }
-#ifdef __linux__
-  // On Linux, check that the amount of memory available in /dev/shm is large
-  // enough to accommodate the request. If it isn't, then fail.
-  int shm_fd = open("/dev/shm", O_RDONLY);
-  struct statvfs shm_vfs_stats;
-  fstatvfs(shm_fd, &shm_vfs_stats);
-  // The value shm_vfs_stats.f_bsize is the block size, and the value
-  // shm_vfs_stats.f_bavail is the number of available blocks.
-  int64_t shm_mem_avail = shm_vfs_stats.f_bsize * shm_vfs_stats.f_bavail;
-  close(shm_fd);
-  if (system_memory > shm_mem_avail) {
-    ARROW_LOG(FATAL)
-        << "System memory request exceeds memory available in /dev/shm. The "
-           "request is for "
-        << system_memory << " bytes, and the amount available is "
-        << shm_mem_avail
-        << " bytes. You may be able to free up space by deleting files in "
-           "/dev/shm. If you are inside a Docker container, you may need to "
-           "pass "
-           "an argument with the flag '--shm-size' to 'docker run'.";
-  }
-#endif
-  // Make it so dlmalloc fails if we try to request more memory than is
-  // available.
-  dlmalloc_set_footprint_limit((size_t) system_memory);
-  ARROW_LOG(DEBUG) << "starting server listening on " << socket_name;
-  start_server(socket_name, system_memory);
-}

src/plasma/plasma_store.h

@@ -1,154 +0,0 @@
-#ifndef PLASMA_STORE_H
-#define PLASMA_STORE_H
-
-#include "eviction_policy.h"
-#include "plasma.h"
-#include "plasma_common.h"
-#include "plasma_events.h"
-#include "plasma_protocol.h"
-
-class GetRequest;
-
-struct NotificationQueue {
-  /// The object notifications for clients. We notify the client about the
-  /// objects in the order that the objects were sealed or deleted.
-  std::deque<uint8_t *> object_notifications;
-};
-
-/// Contains all information that is associated with a Plasma store client.
-struct Client {
-  Client(int fd);
-
-  /// The file descriptor used to communicate with the client.
-  int fd;
-};
-
-class PlasmaStore {
- public:
-  PlasmaStore(EventLoop *loop, int64_t system_memory);
-
-  ~PlasmaStore();
-
-  /// Create a new object. The client must do a call to release_object to tell
-  /// the store when it is done with the object.
-  ///
-  /// @param object_id Object ID of the object to be created.
-  /// @param data_size Size in bytes of the object to be created.
-  /// @param metadata_size Size in bytes of the object metadata.
-  /// @return One of the following error codes:
-  ///  - PlasmaError_OK, if the object was created successfully.
-  ///  - PlasmaError_ObjectExists, if an object with this ID is already
-  ///    present in the store. In this case, the client should not call
-  ///    plasma_release.
-  ///  - PlasmaError_OutOfMemory, if the store is out of memory and
-  ///    cannot create the object. In this case, the client should not call
-  ///    plasma_release.
-  int create_object(ObjectID object_id,
-                    int64_t data_size,
-                    int64_t metadata_size,
-                    Client *client,
-                    PlasmaObject *result);
-
-  /// Delete objects that have been created in the hash table. This should only
-  /// be called on objects that are returned by the eviction policy to evict.
-  ///
-  /// @param object_ids Object IDs of the objects to be deleted.
-  /// @return Void.
-  void delete_objects(const std::vector<ObjectID> &object_ids);
-
-  /// Process a get request from a client. This method assumes that we will
-  /// eventually have these objects sealed. If one of the objects has not yet
-  /// been sealed, the client that requested the object will be notified when it
-  /// is sealed.
-  ///
-  /// For each object, the client must do a call to release_object to tell the
-  /// store when it is done with the object.
-  ///
-  /// @param client The client making this request.
-  /// @param object_ids Object IDs of the objects to be gotten.
-  /// @param timeout_ms The timeout for the get request in milliseconds.
-  /// @return Void.
-  void process_get_request(Client *client,
-                           const std::vector<ObjectID> &object_ids,
-                           uint64_t timeout_ms);
-
-  /// Seal an object. The object is now immutable and can be accessed with get.
-  ///
-  /// @param object_id Object ID of the object to be sealed.
-  /// @param digest The digest of the object. This is used to tell if two
-  /// objects
-  ///        with the same object ID are the same.
-  /// @return Void.
-  void seal_object(ObjectID object_id, unsigned char digest[]);
-
-  /// Check if the plasma store contains an object:
-  ///
-  /// @param object_id Object ID that will be checked.
-  /// @return OBJECT_FOUND if the object is in the store, OBJECT_NOT_FOUND if
-  /// not
-  int contains_object(ObjectID object_id);
-
-  /// Record the fact that a particular client is no longer using an object.
-  ///
-  /// @param object_id The object ID of the object that is being released.
-  /// @param client The client making this request.
-  /// @param Void.
-  void release_object(ObjectID object_id, Client *client);
-
-  /// Subscribe a file descriptor to updates about new sealed objects.
-  ///
-  /// @param client The client making this request.
-  /// @return Void.
-  void subscribe_to_updates(Client *client);
-
-  /// Connect a new client to the PlasmaStore.
-  ///
-  /// @param listener_sock The socket that is listening to incoming connections.
-  /// @return Void.
-  void connect_client(int listener_sock);
-
-  /// Disconnect a client from the PlasmaStore.
-  ///
-  /// @param client The client that is disconnected.
-  /// @return Void.
-  void disconnect_client(Client *client);
-
-  void send_notifications(int client_fd);
-
-  Status process_message(Client *client);
-
- private:
-  void push_notification(ObjectInfoT *object_notification);
-
-  void add_client_to_object_clients(ObjectTableEntry *entry, Client *client);
-
-  void return_from_get(GetRequest *get_req);
-
-  void update_object_get_requests(ObjectID object_id);
-
-  int remove_client_from_object_clients(ObjectTableEntry *entry,
-                                        Client *client);
-
-  /// Event loop of the plasma store.
-  EventLoop *loop_;
-  /// The plasma store information, including the object tables, that is exposed
-  /// to the eviction policy.
-  PlasmaStoreInfo store_info_;
-  /// The state that is managed by the eviction policy.
-  EvictionPolicy eviction_policy_;
-  /// Input buffer. This is allocated only once to avoid mallocs for every
-  /// call to process_message.
-  std::vector<uint8_t> input_buffer_;
-  /// A hash table mapping object IDs to a vector of the get requests that are
-  /// waiting for the object to arrive.
-  std::unordered_map<ObjectID, std::vector<GetRequest *>, UniqueIDHasher>
-      object_get_requests_;
-  /// The pending notifications that have not been sent to subscribers because
-  /// the socket send buffers were full. This is a hash table from client file
-  /// descriptor to an array of object_ids to send to that client.
-  /// TODO(pcm): Consider putting this into the Client data structure and
-  /// reorganize the code slightly.
-  std::unordered_map<int, NotificationQueue> pending_notifications_;
-};
-
-#endif  // PLASMA_STORE_H

src/plasma/status.cc

@@ -1,90 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Status encapsulates the result of an operation.  It may indicate success,
-// or it may indicate an error with an associated error message.
-//
-// Multiple threads can invoke const methods on a Status without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Status must use
-// external synchronization.
-
-#include "status.h"
-
-#include <assert.h>
-
-namespace arrow {
-
-Status::Status(StatusCode code, const std::string &msg, int16_t posix_code) {
-  assert(code != StatusCode::OK);
-  const uint32_t size = static_cast<uint32_t>(msg.size());
-  char *result = new char[size + 7];
-  memcpy(result, &size, sizeof(size));
-  result[4] = static_cast<char>(code);
-  memcpy(result + 5, &posix_code, sizeof(posix_code));
-  memcpy(result + 7, msg.c_str(), msg.size());
-  state_ = result;
-}
-
-const char *Status::CopyState(const char *state) {
-  uint32_t size;
-  memcpy(&size, state, sizeof(size));
-  char *result = new char[size + 7];
-  memcpy(result, state, size + 7);
-  return result;
-}
-
-std::string Status::CodeAsString() const {
-  if (state_ == NULL) {
-    return "OK";
-  }
-
-  const char *type;
-  switch (code()) {
-  case StatusCode::OK:
-    type = "OK";
-    break;
-  case StatusCode::OutOfMemory:
-    type = "Out of memory";
-    break;
-  case StatusCode::KeyError:
-    type = "Key error";
-    break;
-  case StatusCode::TypeError:
-    type = "Type error";
-    break;
-  case StatusCode::Invalid:
-    type = "Invalid";
-    break;
-  case StatusCode::IOError:
-    type = "IOError";
-    break;
-  case StatusCode::UnknownError:
-    type = "Unknown error";
-    break;
-  case StatusCode::NotImplemented:
-    type = "NotImplemented";
-    break;
-  default:
-    type = "Unknown";
-    break;
-  }
-  return std::string(type);
-}
-
-std::string Status::ToString() const {
-  std::string result(CodeAsString());
-  if (state_ == NULL) {
-    return result;
-  }
-
-  result.append(": ");
-
-  uint32_t length;
-  memcpy(&length, state_, sizeof(length));
-  result.append(reinterpret_cast<const char *>(state_ + 7), length);
-  return result;
-}
-
-}  // namespace arrow

src/plasma/status.h

@@ -1,226 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-//
-// A Status encapsulates the result of an operation.  It may indicate success,
-// or it may indicate an error with an associated error message.
-//
-// Multiple threads can invoke const methods on a Status without
-// external synchronization, but if any of the threads may call a
-// non-const method, all threads accessing the same Status must use
-// external synchronization.
-
-// Adapted from Kudu github.com/cloudera/kudu
-
-#ifndef ARROW_STATUS_H_
-#define ARROW_STATUS_H_
-
-#include <cstdint>
-#include <cstring>
-#include <string>
-
-// Return the given status if it is not OK.
-#define ARROW_RETURN_NOT_OK(s) \
-  do {                         \
-    ::arrow::Status _s = (s);  \
-    if (!_s.ok()) {            \
-      return _s;               \
-    }                          \
-  } while (0);
-
-// Return the given status if it is not OK, but first clone it and
-// prepend the given message.
-#define ARROW_RETURN_NOT_OK_PREPEND(s, msg) \
-  do {                                      \
-    ::arrow::Status _s = (s);               \
-    if (::gutil::PREDICT_FALSE(!_s.ok()))   \
-      return _s.CloneAndPrepend(msg);       \
-  } while (0);
-
-// Return 'to_return' if 'to_call' returns a bad status.
-// The substitution for 'to_return' may reference the variable
-// 's' for the bad status.
-#define ARROW_RETURN_NOT_OK_RET(to_call, to_return) \
-  do {                                              \
-    ::arrow::Status s = (to_call);                  \
-    if (::gutil::PREDICT_FALSE(!s.ok()))            \
-      return (to_return);                           \
-  } while (0);
-
-// If 'to_call' returns a bad status, CHECK immediately with a logged message
-// of 'msg' followed by the status.
-#define ARROW_CHECK_OK_PREPEND(to_call, msg)                \
-  do {                                                      \
-    ::arrow::Status _s = (to_call);                         \
-    ARROW_CHECK(_s.ok()) << (msg) << ": " << _s.ToString(); \
-  } while (0);
-
-// If the status is bad, CHECK immediately, appending the status to the
-// logged message.
-#define ARROW_CHECK_OK(s) ARROW_CHECK_OK_PREPEND(s, "Bad status")
-
-namespace arrow {
-
-#define RETURN_NOT_OK(s) \
-  do {                   \
-    Status _s = (s);     \
-    if (!_s.ok()) {      \
-      return _s;         \
-    }                    \
-  } while (0);
-
-#define RETURN_NOT_OK_ELSE(s, else_) \
-  do {                               \
-    Status _s = (s);                 \
-    if (!_s.ok()) {                  \
-      else_;                         \
-      return _s;                     \
-    }                                \
-  } while (0);
-
-enum class StatusCode : char {
-  OK = 0,
-  OutOfMemory = 1,
-  KeyError = 2,
-  TypeError = 3,
-  Invalid = 4,
-  IOError = 5,
-  UnknownError = 9,
-  NotImplemented = 10,
-  PlasmaObjectExists = 20,
-  PlasmaObjectNonexistent = 21,
-  PlasmaStoreFull = 22
-};
-
-class Status {
- public:
-  // Create a success status.
-  Status() : state_(NULL) {}
-  ~Status() { delete[] state_; }
-
-  Status(StatusCode code, const std::string &msg) : Status(code, msg, -1) {}
-
-  // Copy the specified status.
-  Status(const Status &s);
-  void operator=(const Status &s);
-
-  // Return a success status.
-  static Status OK() { return Status(); }
-
-  // Return error status of an appropriate type.
-  static Status OutOfMemory(const std::string &msg, int16_t posix_code = -1) {
-    return Status(StatusCode::OutOfMemory, msg, posix_code);
-  }
-
-  static Status KeyError(const std::string &msg) {
-    return Status(StatusCode::KeyError, msg, -1);
-  }
-
-  static Status TypeError(const std::string &msg) {
-    return Status(StatusCode::TypeError, msg, -1);
-  }
-
-  static Status UnknownError(const std::string &msg) {
-    return Status(StatusCode::UnknownError, msg, -1);
-  }
-
-  static Status NotImplemented(const std::string &msg) {
-    return Status(StatusCode::NotImplemented, msg, -1);
-  }
-
-  static Status Invalid(const std::string &msg) {
-    return Status(StatusCode::Invalid, msg, -1);
-  }
-
-  static Status IOError(const std::string &msg) {
-    return Status(StatusCode::IOError, msg, -1);
-  }
-
-  static Status PlasmaObjectExists(const std::string &msg) {
-    return Status(StatusCode::PlasmaObjectExists, msg, -1);
-  }
-
-  static Status PlasmaObjectNonexistent(const std::string &msg) {
-    return Status(StatusCode::PlasmaObjectNonexistent, msg, -1);
-  }
-
-  static Status PlasmaStoreFull(const std::string &msg) {
-    return Status(StatusCode::PlasmaStoreFull, msg, -1);
-  }
-
-  // Returns true iff the status indicates success.
-  bool ok() const { return (state_ == NULL); }
-
-  bool IsOutOfMemory() const { return code() == StatusCode::OutOfMemory; }
-  bool IsKeyError() const { return code() == StatusCode::KeyError; }
-  bool IsInvalid() const { return code() == StatusCode::Invalid; }
-  bool IsIOError() const { return code() == StatusCode::IOError; }
-  bool IsTypeError() const { return code() == StatusCode::TypeError; }
-  bool IsUnknownError() const { return code() == StatusCode::UnknownError; }
-  bool IsNotImplemented() const { return code() == StatusCode::NotImplemented; }
-  // An object with this object ID already exists in the plasma store.
-  bool IsPlasmaObjectExists() const {
-    return code() == StatusCode::PlasmaObjectExists;
-  }
-  // An object was requested that doesn't exist in the plasma store.
-  bool IsPlasmaObjectNonexistent() const {
-    return code() == StatusCode::PlasmaObjectNonexistent;
-  }
-  // An object is too large to fit into the plasma store.
-  bool IsPlasmaStoreFull() const {
-    return code() == StatusCode::PlasmaStoreFull;
-  }
-
-  // Return a string representation of this status suitable for printing.
-  // Returns the string "OK" for success.
-  std::string ToString() const;
-
-  // Return a string representation of the status code, without the message
-  // text or posix code information.
-  std::string CodeAsString() const;
-
-  // Get the POSIX code associated with this Status, or -1 if there is none.
-  int16_t posix_code() const;
-
-  StatusCode code() const {
-    return ((state_ == NULL) ? StatusCode::OK
-                             : static_cast<StatusCode>(state_[4]));
-  }
-
-  std::string message() const {
-    uint32_t length;
-    memcpy(&length, state_, sizeof(length));
-    std::string msg;
-    msg.append((state_ + 7), length);
-    return msg;
-  }
-
- private:
-  // OK status has a NULL state_.  Otherwise, state_ is a new[] array
-  // of the following form:
-  //    state_[0..3] == length of message
-  //    state_[4]    == code
-  //    state_[5..6] == posix_code
-  //    state_[7..]  == message
-  const char *state_;
-
-  Status(StatusCode code, const std::string &msg, int16_t posix_code);
-  static const char *CopyState(const char *s);
-};
-
-inline Status::Status(const Status &s) {
-  state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-}
-
-inline void Status::operator=(const Status &s) {
-  // The following condition catches both aliasing (when this == &s),
-  // and the common case where both s and *this are ok.
-  if (state_ != s.state_) {
-    delete[] state_;
-    state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_);
-  }
-}
-
-}  // namespace arrow
-
-#endif  // ARROW_STATUS_H_

src/plasma/test/client_tests.cc

@@ -4,10 +4,12 @@
 #include <unistd.h>
 #include <sys/time.h>
 
-#include "plasma_common.h"
-#include "plasma.h"
-#include "plasma_protocol.h"
-#include "plasma_client.h"
+#include "plasma/common.h"
+#include "plasma/plasma.h"
+#include "plasma/protocol.h"
+#include "plasma/client.h"
+
+using namespace plasma;
 
 SUITE(plasma_client_tests);
 
@@ -177,7 +179,7 @@ TEST plasma_wait_for_objects_tests(void) {
   gettimeofday(&start, NULL);
   int n;
   ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
-                              WAIT_TIMEOUT_MS, n));
+                              WAIT_TIMEOUT_MS, &n));
   ASSERT(n == 0);
   gettimeofday(&end, NULL);
   float diff_ms = (end.tv_sec - start.tv_sec);
@@ -195,7 +197,7 @@ TEST plasma_wait_for_objects_tests(void) {
   ARROW_CHECK_OK(client1.Seal(oid1));
 
   ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
-                              WAIT_TIMEOUT_MS, n));
+                              WAIT_TIMEOUT_MS, &n));
   ASSERT(n == 1);
 
   /* Create and insert an object in client2. */
@@ -204,21 +206,21 @@ TEST plasma_wait_for_objects_tests(void) {
   ARROW_CHECK_OK(client2.Seal(oid2));
 
   ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
-                              WAIT_TIMEOUT_MS, n));
+                              WAIT_TIMEOUT_MS, &n));
   ASSERT(n == 2);
 
   ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
-                              WAIT_TIMEOUT_MS, n));
+                              WAIT_TIMEOUT_MS, &n));
   ASSERT(n == 2);
 
   obj_requests[0].type = PLASMA_QUERY_LOCAL;
   obj_requests[1].type = PLASMA_QUERY_LOCAL;
   ARROW_CHECK_OK(client1.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
-                              WAIT_TIMEOUT_MS, n));
+                              WAIT_TIMEOUT_MS, &n));
   ASSERT(n == 1);
 
   ARROW_CHECK_OK(client2.Wait(NUM_OBJ_REQUEST, obj_requests, NUM_OBJ_REQUEST,
-                              WAIT_TIMEOUT_MS, n));
+                              WAIT_TIMEOUT_MS, &n));
   ASSERT(n == 1);
 
   ARROW_CHECK_OK(client1.Disconnect());

src/plasma/test/manager_tests.cc

@@ -13,10 +13,10 @@
 #include "io.h"
 #include "utstring.h"
 
-#include "plasma.h"
-#include "plasma_client.h"
-#include "plasma_manager.h"
-#include "plasma_protocol.h"
+#include "plasma/plasma.h"
+#include "plasma/client.h"
+#include "../plasma_manager.h"
+#include "plasma/protocol.h"
 
 SUITE(plasma_manager_tests);
 
@@ -53,7 +53,7 @@ typedef struct {
   ClientConnection *read_conn;
   /* Connect a new client to the local plasma manager and mock a request to an
    * object. */
-  PlasmaClient *plasma_client;
+  plasma::PlasmaClient *plasma_client;
   ClientConnection *client_conn;
 } plasma_mock;
 
@@ -85,7 +85,7 @@ plasma_mock *init_plasma_mock(plasma_mock *remote_mock) {
   }
   /* Connect a new client to the local plasma manager and mock a request to an
    * object. */
-  mock->plasma_client = new PlasmaClient();
+  mock->plasma_client = new plasma::PlasmaClient();
   ARROW_CHECK_OK(mock->plasma_client->Connect(
       plasma_store_socket_name, utstring_body(manager_socket_name), 0));
   wait_for_pollin(mock->manager_local_fd);
@@ -130,13 +130,13 @@ TEST request_transfer_test(void) {
   event_loop_run(local_mock->loop);
   int read_fd = get_client_sock(remote_mock->read_conn);
   std::vector<uint8_t> request_data;
-  ARROW_CHECK_OK(
-      PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, request_data));
-  ObjectID object_id2;
+  ARROW_CHECK_OK(plasma::PlasmaReceive(read_fd, MessageType_PlasmaDataRequest,
+                                       &request_data));
+  plasma::ObjectID object_id2;
   char *address;
   int port;
-  ARROW_CHECK_OK(
-      ReadDataRequest(request_data.data(), &object_id2, &address, &port));
+  ARROW_CHECK_OK(plasma::ReadDataRequest(
+      request_data.data(), request_data.size(), &object_id2, &address, &port));
   ASSERT(ObjectID_equal(object_id, object_id2));
   free(address);
   /* Clean up. */
@@ -184,13 +184,13 @@ TEST request_transfer_retry_test(void) {
 
   int read_fd = get_client_sock(remote_mock2->read_conn);
   std::vector<uint8_t> request_data;
-  ARROW_CHECK_OK(
-      PlasmaReceive(read_fd, MessageType_PlasmaDataRequest, request_data));
-  ObjectID object_id2;
+  ARROW_CHECK_OK(plasma::PlasmaReceive(read_fd, MessageType_PlasmaDataRequest,
+                                       &request_data));
+  plasma::ObjectID object_id2;
   char *address;
   int port;
-  ARROW_CHECK_OK(
-      ReadDataRequest(request_data.data(), &object_id2, &address, &port));
+  ARROW_CHECK_OK(plasma::ReadDataRequest(
+      request_data.data(), request_data.size(), &object_id2, &address, &port));
   free(address);
   ASSERT(ObjectID_equal(object_id, object_id2));
   /* Clean up. */
@@ -271,24 +271,24 @@ TEST object_notifications_test(void) {
   ASSERT(!is_local);
 
   /* Check that the object is local after receiving an object notification. */
-  uint8_t *notification = create_object_info_buffer(&info);
+  uint8_t *notification = plasma::create_object_info_buffer(&info);
   int64_t size = *((int64_t *) notification);
   send(fd[1], notification, sizeof(int64_t) + size, 0);
   process_object_notification(local_mock->loop, fd[0], local_mock->state, 0);
   is_local = is_object_local(local_mock->state, object_id);
   ASSERT(is_local);
-  free(notification);
+  delete[] notification;
 
   /* Check that the object is not local after receiving a notification about
    * the object deletion. */
   info.is_deletion = true;
-  notification = create_object_info_buffer(&info);
+  notification = plasma::create_object_info_buffer(&info);
   size = *((int64_t *) notification);
   send(fd[1], notification, sizeof(int64_t) + size, 0);
   process_object_notification(local_mock->loop, fd[0], local_mock->state, 0);
   is_local = is_object_local(local_mock->state, object_id);
   ASSERT(!is_local);
-  free(notification);
+  delete[] notification;
 
   /* Clean up. */
   close(fd[0]);

src/plasma/test/run_tests.sh

@@ -7,7 +7,6 @@ set -e
 sleep 1
 ./src/plasma/manager_tests
 killall plasma_store
-./src/plasma/serialization_tests
 
 # Start the Redis shards.
 ./src/common/thirdparty/redis/src/redis-server --loglevel warning --loadmodule ./src/common/redis_module/libray_redis_module.so --port 6379 &

src/plasma/test/run_valgrind.sh

@@ -7,4 +7,3 @@ set -e
 sleep 1
 valgrind --leak-check=full --error-exitcode=1 ./src/plasma/manager_tests
 killall plasma_store
-valgrind --leak-check=full --error-exitcode=1 ./src/plasma/serialization_tests

src/plasma/test/serialization_tests.cc

@@ -1,439 +0,0 @@
-#include "greatest.h"
-
-#include <sys/types.h>
-#include <unistd.h>
-
-#include "plasma_common.h"
-#include "plasma.h"
-#include "plasma_io.h"
-#include "plasma_protocol.h"
-
-SUITE(plasma_serialization_tests);
-
-/**
- * Create a temporary file. Needs to be closed by the caller.
- *
- * @return File descriptor of the file.
- */
-int create_temp_file(void) {
-  static char temp[] = "/tmp/tempfileXXXXXX";
-  char file_name[32];
-  strncpy(file_name, temp, 32);
-  return mkstemp(file_name);
-}
-
-/**
- * Seek to the beginning of a file and read a message from it.
- *
- * @param fd File descriptor of the file.
- * @param message type Message type that we expect in the file.
- *
- * @return Pointer to the content of the message. Needs to be freed by the
- * caller.
- */
-std::vector<uint8_t> read_message_from_file(int fd, int message_type) {
-  /* Go to the beginning of the file. */
-  lseek(fd, 0, SEEK_SET);
-  int64_t type;
-  std::vector<uint8_t> data;
-  ARROW_CHECK_OK(ReadMessage(fd, &type, data));
-  ARROW_CHECK(type == message_type);
-  return data;
-}
-
-PlasmaObject random_plasma_object(void) {
-  int random = rand();
-  PlasmaObject object;
-  memset(&object, 0, sizeof(object));
-  object.handle.store_fd = random + 7;
-  object.handle.mmap_size = random + 42;
-  object.data_offset = random + 1;
-  object.metadata_offset = random + 2;
-  object.data_size = random + 3;
-  object.metadata_size = random + 4;
-  return object;
-}
-
-TEST plasma_create_request_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  int64_t data_size1 = 42;
-  int64_t metadata_size1 = 11;
-  ARROW_CHECK_OK(SendCreateRequest(fd, object_id1, data_size1, metadata_size1));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaCreateRequest);
-  ObjectID object_id2;
-  int64_t data_size2;
-  int64_t metadata_size2;
-  ARROW_CHECK_OK(ReadCreateRequest(data.data(), &object_id2, &data_size2,
-                                   &metadata_size2));
-  ASSERT_EQ(data_size1, data_size2);
-  ASSERT_EQ(metadata_size1, metadata_size2);
-  ASSERT(object_id1 == object_id2);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_create_reply_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  PlasmaObject object1 = random_plasma_object();
-  ARROW_CHECK_OK(SendCreateReply(fd, object_id1, &object1, 0));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaCreateReply);
-  ObjectID object_id2;
-  PlasmaObject object2;
-  memset(&object2, 0, sizeof(object2));
-  ARROW_CHECK_OK(ReadCreateReply(data.data(), &object_id2, &object2));
-  ASSERT(object_id1 == object_id2);
-  ASSERT(memcmp(&object1, &object2, sizeof(object1)) == 0);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_seal_request_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  unsigned char digest1[kDigestSize];
-  memset(&digest1[0], 7, kDigestSize);
-  ARROW_CHECK_OK(SendSealRequest(fd, object_id1, &digest1[0]));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaSealRequest);
-  ObjectID object_id2;
-  unsigned char digest2[kDigestSize];
-  ARROW_CHECK_OK(ReadSealRequest(data.data(), &object_id2, &digest2[0]));
-  ASSERT(object_id1 == object_id2);
-  ASSERT(memcmp(&digest1[0], &digest2[0], kDigestSize) == 0);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_seal_reply_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  ARROW_CHECK_OK(SendSealReply(fd, object_id1, PlasmaError_ObjectExists));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaSealReply);
-  ObjectID object_id2;
-  Status s = ReadSealReply(data.data(), &object_id2);
-  ASSERT(object_id1 == object_id2);
-  ASSERT(s.IsPlasmaObjectExists());
-  close(fd);
-  PASS();
-}
-
-TEST plasma_get_request_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_ids[2];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  int64_t timeout_ms = 1234;
-  ARROW_CHECK_OK(SendGetRequest(fd, object_ids, 2, timeout_ms));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaGetRequest);
-  std::vector<ObjectID> object_ids_return;
-  int64_t timeout_ms_return;
-  ARROW_CHECK_OK(
-      ReadGetRequest(data.data(), object_ids_return, &timeout_ms_return));
-  ASSERT(object_ids[0] == object_ids_return[0]);
-  ASSERT(object_ids[1] == object_ids_return[1]);
-  ASSERT(timeout_ms == timeout_ms_return);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_get_reply_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_ids[2];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  std::unordered_map<ObjectID, PlasmaObject, UniqueIDHasher> plasma_objects;
-  plasma_objects[object_ids[0]] = random_plasma_object();
-  plasma_objects[object_ids[1]] = random_plasma_object();
-  ARROW_CHECK_OK(SendGetReply(fd, object_ids, plasma_objects, 2));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaGetReply);
-  ObjectID object_ids_return[2];
-  PlasmaObject plasma_objects_return[2];
-  memset(&plasma_objects_return, 0, sizeof(plasma_objects_return));
-  ARROW_CHECK_OK(ReadGetReply(data.data(), object_ids_return,
-                              &plasma_objects_return[0], 2));
-  ASSERT(object_ids[0] == object_ids_return[0]);
-  ASSERT(object_ids[1] == object_ids_return[1]);
-  ASSERT(memcmp(&plasma_objects[object_ids[0]], &plasma_objects_return[0],
-                sizeof(PlasmaObject)) == 0);
-  ASSERT(memcmp(&plasma_objects[object_ids[1]], &plasma_objects_return[1],
-                sizeof(PlasmaObject)) == 0);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_release_request_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  ARROW_CHECK_OK(SendReleaseRequest(fd, object_id1));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaReleaseRequest);
-  ObjectID object_id2;
-  ARROW_CHECK_OK(ReadReleaseRequest(data.data(), &object_id2));
-  ASSERT(object_id1 == object_id2);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_release_reply_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  ARROW_CHECK_OK(SendReleaseReply(fd, object_id1, PlasmaError_ObjectExists));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaReleaseReply);
-  ObjectID object_id2;
-  Status s = ReadReleaseReply(data.data(), &object_id2);
-  ASSERT(object_id1 == object_id2);
-  ASSERT(s.IsPlasmaObjectExists());
-  close(fd);
-  PASS();
-}
-
-TEST plasma_delete_request_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  ARROW_CHECK_OK(SendDeleteRequest(fd, object_id1));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaDeleteRequest);
-  ObjectID object_id2;
-  ARROW_CHECK_OK(ReadDeleteRequest(data.data(), &object_id2));
-  ASSERT(object_id1 == object_id2);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_delete_reply_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  int error1 = PlasmaError_ObjectExists;
-  ARROW_CHECK_OK(SendDeleteReply(fd, object_id1, error1));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaDeleteReply);
-  ObjectID object_id2;
-  Status s = ReadDeleteReply(data.data(), &object_id2);
-  ASSERT(object_id1 == object_id2);
-  ASSERT(s.IsPlasmaObjectExists());
-  close(fd);
-  PASS();
-}
-
-TEST plasma_status_request_test(void) {
-  int fd = create_temp_file();
-  int64_t num_objects = 2;
-  ObjectID object_ids[num_objects];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  ARROW_CHECK_OK(SendStatusRequest(fd, object_ids, num_objects));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaStatusRequest);
-  ObjectID object_ids_read[num_objects];
-  ARROW_CHECK_OK(ReadStatusRequest(data.data(), object_ids_read, num_objects));
-  ASSERT(object_ids[0] == object_ids_read[0]);
-  ASSERT(object_ids[1] == object_ids_read[1]);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_status_reply_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_ids[2];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  int object_statuses[2] = {42, 43};
-  ARROW_CHECK_OK(SendStatusReply(fd, object_ids, object_statuses, 2));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaStatusReply);
-  int64_t num_objects = ReadStatusReply_num_objects(data.data());
-  ObjectID object_ids_read[num_objects];
-  int object_statuses_read[num_objects];
-  ARROW_CHECK_OK(ReadStatusReply(data.data(), object_ids_read,
-                                 object_statuses_read, num_objects));
-  ASSERT(object_ids[0] == object_ids_read[0]);
-  ASSERT(object_ids[1] == object_ids_read[1]);
-  ASSERT_EQ(object_statuses[0], object_statuses_read[0]);
-  ASSERT_EQ(object_statuses[1], object_statuses_read[1]);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_evict_request_test(void) {
-  int fd = create_temp_file();
-  int64_t num_bytes = 111;
-  ARROW_CHECK_OK(SendEvictRequest(fd, num_bytes));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaEvictRequest);
-  int64_t num_bytes_received;
-  ARROW_CHECK_OK(ReadEvictRequest(data.data(), &num_bytes_received));
-  ASSERT_EQ(num_bytes, num_bytes_received);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_evict_reply_test(void) {
-  int fd = create_temp_file();
-  int64_t num_bytes = 111;
-  ARROW_CHECK_OK(SendEvictReply(fd, num_bytes));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaEvictReply);
-  int64_t num_bytes_received;
-  ARROW_CHECK_OK(ReadEvictReply(data.data(), num_bytes_received));
-  ASSERT_EQ(num_bytes, num_bytes_received);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_fetch_request_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_ids[2];
-  object_ids[0] = ObjectID::from_random();
-  object_ids[1] = ObjectID::from_random();
-  ARROW_CHECK_OK(SendFetchRequest(fd, object_ids, 2));
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaFetchRequest);
-  std::vector<ObjectID> object_ids_read;
-  ARROW_CHECK_OK(ReadFetchRequest(data.data(), object_ids_read));
-  ASSERT(object_ids[0] == object_ids_read[0]);
-  ASSERT(object_ids[1] == object_ids_read[1]);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_wait_request_test(void) {
-  int fd = create_temp_file();
-  const int num_objects_in = 2;
-  ObjectRequest object_requests_in[num_objects_in] = {
-      ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_ANYWHERE, 0}),
-      ObjectRequest({ObjectID::from_random(), PLASMA_QUERY_LOCAL, 0})};
-  const int num_ready_objects_in = 1;
-  int64_t timeout_ms = 1000;
-
-  ARROW_CHECK_OK(SendWaitRequest(fd, &object_requests_in[0], num_objects_in,
-                                 num_ready_objects_in, timeout_ms));
-  /* Read message back. */
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaWaitRequest);
-  int num_ready_objects_out;
-  int64_t timeout_ms_read;
-  ObjectRequestMap object_requests_out;
-  ARROW_CHECK_OK(ReadWaitRequest(data.data(), object_requests_out,
-                                 &timeout_ms_read, &num_ready_objects_out));
-  ASSERT_EQ(num_objects_in, object_requests_out.size());
-  ASSERT_EQ(num_ready_objects_out, num_ready_objects_in);
-  for (int i = 0; i < num_objects_in; i++) {
-    const ObjectID &object_id = object_requests_in[i].object_id;
-    ASSERT_EQ(1, object_requests_out.count(object_id));
-    const auto &entry = object_requests_out.find(object_id);
-    ASSERT(entry != object_requests_out.end());
-    ASSERT(entry->second.object_id == object_requests_in[i].object_id);
-    ASSERT_EQ(entry->second.type, object_requests_in[i].type);
-  }
-  close(fd);
-  PASS();
-}
-
-TEST plasma_wait_reply_test(void) {
-  int fd = create_temp_file();
-  const int num_objects_in = 2;
-  /* Create a map with two ObjectRequests in it. */
-  ObjectRequestMap objects_in(num_objects_in);
-  ObjectID id1 = ObjectID::from_random();
-  objects_in[id1] = ObjectRequest({id1, 0, ObjectStatus_Local});
-  ObjectID id2 = ObjectID::from_random();
-  objects_in[id2] = ObjectRequest({id2, 0, ObjectStatus_Nonexistent});
-
-  ARROW_CHECK_OK(SendWaitReply(fd, objects_in, num_objects_in));
-  /* Read message back. */
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaWaitReply);
-  ObjectRequest objects_out[2];
-  int num_objects_out;
-  ARROW_CHECK_OK(ReadWaitReply(data.data(), &objects_out[0], &num_objects_out));
-  ASSERT(num_objects_in == num_objects_out);
-  for (int i = 0; i < num_objects_out; i++) {
-    /* Each object request must appear exactly once. */
-    ASSERT(1 == objects_in.count(objects_out[i].object_id));
-    const auto &entry = objects_in.find(objects_out[i].object_id);
-    ASSERT(entry != objects_in.end());
-    ASSERT(entry->second.object_id == objects_out[i].object_id);
-    ASSERT(entry->second.status == objects_out[i].status);
-  }
-  close(fd);
-  PASS();
-}
-
-TEST plasma_data_request_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  const char *address1 = "address1";
-  int port1 = 12345;
-  ARROW_CHECK_OK(SendDataRequest(fd, object_id1, address1, port1));
-  /* Reading message back. */
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaDataRequest);
-  ObjectID object_id2;
-  char *address2;
-  int port2;
-  ARROW_CHECK_OK(ReadDataRequest(data.data(), &object_id2, &address2, &port2));
-  ASSERT(object_id1 == object_id2);
-  ASSERT(strcmp(address1, address2) == 0);
-  ASSERT(port1 == port2);
-  free(address2);
-  close(fd);
-  PASS();
-}
-
-TEST plasma_data_reply_test(void) {
-  int fd = create_temp_file();
-  ObjectID object_id1 = ObjectID::from_random();
-  int64_t object_size1 = 146;
-  int64_t metadata_size1 = 198;
-  ARROW_CHECK_OK(SendDataReply(fd, object_id1, object_size1, metadata_size1));
-  /* Reading message back. */
-  std::vector<uint8_t> data =
-      read_message_from_file(fd, MessageType_PlasmaDataReply);
-  ObjectID object_id2;
-  int64_t object_size2;
-  int64_t metadata_size2;
-  ARROW_CHECK_OK(
-      ReadDataReply(data.data(), &object_id2, &object_size2, &metadata_size2));
-  ASSERT(object_id1 == object_id2);
-  ASSERT(object_size1 == object_size2);
-  ASSERT(metadata_size1 == metadata_size2);
-  PASS();
-}
-
-SUITE(plasma_serialization_tests) {
-  RUN_TEST(plasma_create_request_test);
-  RUN_TEST(plasma_create_reply_test);
-  RUN_TEST(plasma_seal_request_test);
-  RUN_TEST(plasma_seal_reply_test);
-  RUN_TEST(plasma_get_request_test);
-  RUN_TEST(plasma_get_reply_test);
-  RUN_TEST(plasma_release_request_test);
-  RUN_TEST(plasma_release_reply_test);
-  RUN_TEST(plasma_delete_request_test);
-  RUN_TEST(plasma_delete_reply_test);
-  RUN_TEST(plasma_status_request_test);
-  RUN_TEST(plasma_status_reply_test);
-  RUN_TEST(plasma_evict_request_test);
-  RUN_TEST(plasma_evict_reply_test);
-  RUN_TEST(plasma_fetch_request_test);
-  RUN_TEST(plasma_wait_request_test);
-  RUN_TEST(plasma_wait_reply_test);
-  RUN_TEST(plasma_data_request_test);
-  RUN_TEST(plasma_data_reply_test);
-}
-
-GREATEST_MAIN_DEFS();
-
-int main(int argc, char **argv) {
-  GREATEST_MAIN_BEGIN();
-  RUN_SUITE(plasma_serialization_tests);
-  GREATEST_MAIN_END();
-}

src/thirdparty/build_thirdparty.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+if [[ -z  "$1" ]]; then
+  PYTHON_EXECUTABLE=`which python`
+else
+  PYTHON_EXECUTABLE=$1
+fi
+echo "Using Python executable $PYTHON_EXECUTABLE."
+
+# Determine how many parallel jobs to use for make based on the number of cores
+unamestr="$(uname)"
+if [[ "$unamestr" == "Linux" ]]; then
+  PARALLEL=$(nproc)
+elif [[ "$unamestr" == "Darwin" ]]; then
+  PARALLEL=$(sysctl -n hw.ncpu)
+  echo "Platform is macosx."
+else
+  echo "Unrecognized platform."
+  exit 1
+fi
+
+echo "building arrow"
+cd $TP_DIR/arrow/cpp
+mkdir -p $TP_DIR/arrow/cpp/build
+cd $TP_DIR/arrow/cpp/build
+export ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
+cmake -DCMAKE_BUILD_TYPE=Release \
+      -DCMAKE_C_FLAGS="-g -O3" \
+      -DCMAKE_CXX_FLAGS="-g -O3" \
+      -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+      -DARROW_BUILD_TESTS=off \
+      -DARROW_HDFS=on \
+      -DARROW_PYTHON=on \
+      -DARROW_PLASMA=on \
+      -DPLASMA_PYTHON=on \
+      -DARROW_JEMALLOC=off \
+      -DARROW_WITH_BROTLI=off \
+      -DARROW_WITH_LZ4=off \
+      -DARROW_WITH_SNAPPY=off \
+      -DARROW_WITH_ZLIB=off \
+      -DARROW_WITH_ZSTD=off \
+      ..
+make VERBOSE=1 -j$PARALLEL
+make install
+
+echo "installing pyarrow"
+cd $TP_DIR/arrow/python
+# We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
+# find plasma.
+ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
+PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig PYARROW_WITH_PLASMA=1 PYARROW_BUNDLE_ARROW_CPP=1 $PYTHON_EXECUTABLE setup.py install

src/thirdparty/download_thirdparty.sh

@@ -11,6 +11,6 @@ if [ ! -d $TP_DIR/arrow ]; then
   git clone https://github.com/apache/arrow/ "$TP_DIR/arrow"
 fi
 cd $TP_DIR/arrow
-git pull origin master
+git fetch origin master
 
-git checkout 8a700ccdad745c250fe5d91a9104e7c2d6364c1b
+git checkout dca5d96c7a029c079183e2903db425e486e2deb9

test/component_failures_test.py

@@ -6,6 +6,8 @@ import ray
 import time
 import unittest
 
+import pyarrow as pa
+
 
 class ComponentFailureTest(unittest.TestCase):
 
@@ -38,8 +40,9 @@ class ComponentFailureTest(unittest.TestCase):
 
         # Seal the object so the store attempts to notify the worker that the
         # get has been fulfilled.
-        ray.worker.global_worker.plasma_client.create(obj_id, 100)
-        ray.worker.global_worker.plasma_client.seal(obj_id)
+        ray.worker.global_worker.plasma_client.create(
+             pa.plasma.ObjectID(obj_id), 100)
+        ray.worker.global_worker.plasma_client.seal(pa.plasma.ObjectID(obj_id))
         time.sleep(0.1)
 
         # Make sure that nothing has died.
@@ -72,8 +75,9 @@ class ComponentFailureTest(unittest.TestCase):
 
         # Seal the object so the store attempts to notify the worker that the
         # get has been fulfilled.
-        ray.worker.global_worker.plasma_client.create(obj_id, 100)
-        ray.worker.global_worker.plasma_client.seal(obj_id)
+        ray.worker.global_worker.plasma_client.create(
+            pa.plasma.ObjectID(obj_id), 100)
+        ray.worker.global_worker.plasma_client.seal(pa.plasma.ObjectID(obj_id))
         time.sleep(0.1)
 
         # Make sure that nothing has died.