new arrow serialization code (serialize python objects recursively) (#284)

src/ipc.cc

@@ -7,12 +7,16 @@
 #include <stdlib.h>
 #include "ray/ray.h"
 
-using namespace arrow;
+#ifndef __APPLE__
+  using namespace arrow;
+#endif
 
 ObjHandle::ObjHandle(SegmentId segmentid, size_t size, IpcPointer ipcpointer, size_t metadata_offset)
   : segmentid_(segmentid), size_(size), ipcpointer_(ipcpointer), metadata_offset_(metadata_offset)
 {}
 
+#ifndef __APPLE__
+
 Status BufferMemorySource::Write(int64_t position, const uint8_t* data, int64_t nbytes) {
   // TODO(pcm): error handling
   std::memcpy(data_ + position, data, nbytes);
@@ -33,6 +37,8 @@ int64_t BufferMemorySource::Size() const {
   return size_;
 }
 
+#endif
+
 MessageQueue<>::MessageQueue() : create_(false) { }
 
 MessageQueue<>::~MessageQueue() {

src/ipc.h

@@ -23,8 +23,10 @@ namespace boost {
 #include <boost/interprocess/managed_shared_memory.hpp>
 #include <boost/interprocess/ipc/message_queue.hpp>
 
-#include <arrow/api.h>
-#include <arrow/ipc/memory.h>
+#ifndef __APPLE__
+  #include <arrow/api.h>
+  #include <arrow/ipc/memory.h>
+#endif
 
 #include "ray/ray.h"
 
@@ -108,6 +110,8 @@ private:
   size_t metadata_offset_; // offset of the metadata that describes this object
 };
 
+#ifndef __APPLE__
+
 class BufferMemorySource: public arrow::ipc::MemorySource {
 public:
   BufferMemorySource(uint8_t* data, int64_t capacity) : data_(data), capacity_(capacity), size_(0) {}
@@ -121,6 +125,8 @@ public:
   int64_t size_;
 };
 
+#endif
+
 // Memory segment pool: A collection of shared memory segments
 // used in two modes:
 // \item on the object store it is used with create = true, in this case the

src/raylib.cc

@@ -4,9 +4,11 @@
 
 #include <Python.h>
 #include <structmember.h>
-#define PY_ARRAY_UNIQUE_SYMBOL NUMBUF_ARRAY_API
+#define PY_ARRAY_UNIQUE_SYMBOL RAYLIB_ARRAY_API
 #include <numpy/arrayobject.h>
-#include <arrow/api.h>
+#ifndef __APPLE__
+  #include <arrow/api.h>
+#endif
 #include <iostream>
 
 #include "types.pb.h"
@@ -480,6 +482,7 @@ static PyObject* serialize_object(PyObject* self, PyObject* args) {
   return t;
 }
 
+#ifndef __APPLE__
 static PyObject* put_arrow(PyObject* self, PyObject* args) {
   Worker* worker;
   ObjRef objref;
@@ -494,6 +497,7 @@ static PyObject* put_arrow(PyObject* self, PyObject* args) {
   Py_XDECREF(array); // GETCONTIGUOUS from above returned a new reference
   Py_RETURN_NONE;
 }
+#endif
 
 static PyObject* allocate_buffer(PyObject* self, PyObject* args) {
   Worker* worker;
@@ -527,18 +531,22 @@ static PyObject* get_buffer(PyObject* self, PyObject* args) {
   ObjRef objref;
   int64_t size;
   SegmentId segmentid;
+  int64_t metadata_offset;
   if (!PyArg_ParseTuple(args, "O&O&", &PyObjectToWorker, &worker, &PyObjectToObjRef, &objref)) {
     return NULL;
   }
-  void* address = reinterpret_cast<void*>(const_cast<char*>(worker->get_buffer(objref, size, segmentid)));
+  void* address = reinterpret_cast<void*>(const_cast<char*>(worker->get_buffer(objref, size, segmentid, metadata_offset)));
   std::vector<npy_intp> dim({size});
-  PyObject* t = PyTuple_New(2);
+  PyObject* t = PyTuple_New(3);
   PyTuple_SetItem(t, 0, PyArray_SimpleNewFromData(1, dim.data(), NPY_BYTE, address));
   PyTuple_SetItem(t, 1, PyInt_FromLong(segmentid));
+  PyTuple_SetItem(t, 2, PyInt_FromLong(metadata_offset));
   return t;
 }
 
 
+#ifndef __APPLE__
+
 static PyObject* get_arrow(PyObject* self, PyObject* args) {
   Worker* worker;
   ObjRef objref;
@@ -553,6 +561,8 @@ static PyObject* get_arrow(PyObject* self, PyObject* args) {
   return val_and_segmentid;
 }
 
+#endif
+
 static PyObject* is_arrow(PyObject* self, PyObject* args) {
   Worker* worker;
   ObjRef objref;
@@ -999,11 +1009,15 @@ static PyObject* kill_workers(PyObject* self, PyObject* args) {
 static PyMethodDef RayLibMethods[] = {
  { "serialize_object", serialize_object, METH_VARARGS, "serialize an object to protocol buffers" },
  { "deserialize_object", deserialize_object, METH_VARARGS, "deserialize an object from protocol buffers" },
+#ifndef __APPLE__
  { "put_arrow", put_arrow, METH_VARARGS, "put an arrow array on the local object store"},
+#endif
  { "allocate_buffer", allocate_buffer, METH_VARARGS, "Allocates and returns buffer for objref."},
  { "finish_buffer", finish_buffer, METH_VARARGS, "Makes the buffer immutable and closes memory segment of objref."},
  { "get_buffer", get_buffer, METH_VARARGS, "Gets buffer for objref"},
+#ifndef __APPLE__
  { "get_arrow", get_arrow, METH_VARARGS, "get an arrow array from the local object store"},
+#endif
  { "is_arrow", is_arrow, METH_VARARGS, "is the object in the local object store an arrow object?"},
  { "unmap_object", unmap_object, METH_VARARGS, "unmap the object from the client's shared memory pool"},
  { "serialize_task", serialize_task, METH_VARARGS, "serialize a task to protocol buffers" },

src/worker.cc

@@ -5,7 +5,9 @@
 
 #include "utils.h"
 
-#include <pynumbuf/serialize.h>
+#ifndef __APPLE__
+  #include <pynumbuf/serialize.h>
+#endif
 
 extern "C" {
   static PyObject *RayError;
@@ -194,6 +196,8 @@ void Worker::put_object(ObjRef objref, const Obj* obj, std::vector<ObjRef> &cont
     }                                                           \
   } while (0);
 
+#ifndef __APPLE__
+
 PyObject* Worker::put_arrow(ObjRef objref, PyObject* value) {
   RAY_CHECK(connected_, "Attempted to perform put_arrow but failed.");
   ObjRequest request;
@@ -221,6 +225,8 @@ PyObject* Worker::put_arrow(ObjRef objref, PyObject* value) {
   Py_RETURN_NONE;
 }
 
+#endif
+
 const char* Worker::allocate_buffer(ObjRef objref, int64_t size, SegmentId& segmentid) {
   RAY_CHECK(connected_, "Attempted to perform put_arrow but failed.");
   ObjRequest request;
@@ -247,7 +253,7 @@ PyObject* Worker::finish_buffer(ObjRef objref, SegmentId segmentid, int64_t meta
   Py_RETURN_NONE;
 }
 
-const char* Worker::get_buffer(ObjRef objref, int64_t &size, SegmentId& segmentid) {
+const char* Worker::get_buffer(ObjRef objref, int64_t &size, SegmentId& segmentid, int64_t& metadata_offset) {
   RAY_CHECK(connected_, "Attempted to perform get_arrow but failed.");
   ObjRequest request;
   request.workerid = workerid_;
@@ -259,9 +265,12 @@ const char* Worker::get_buffer(ObjRef objref, int64_t &size, SegmentId& segmenti
   const char* address = reinterpret_cast<const char*>(segmentpool_->get_address(result));
   size = result.size();
   segmentid = result.segmentid();
+  metadata_offset = result.metadata_offset();
   return address;
 }
 
+#ifndef __APPLE__
+
 // returns python list containing the value represented by objref and the
 // segmentid in which the object is stored
 PyObject* Worker::get_arrow(ObjRef objref, SegmentId& segmentid) {
@@ -281,6 +290,8 @@ PyObject* Worker::get_arrow(ObjRef objref, SegmentId& segmentid) {
   return value;
 }
 
+#endif
+
 bool Worker::is_arrow(ObjRef objref) {
   RAY_CHECK(connected_, "Attempted to perform is_arrow but failed.");
   ObjRequest request;

src/worker.h

@@ -67,16 +67,20 @@ class Worker {
   void put_object(ObjRef objref, const Obj* obj, std::vector<ObjRef> &contained_objrefs);
   // retrieve serialized object from local object store
   slice get_object(ObjRef objref);
+#ifndef __APPLE__
   // stores an arrow object to the local object store
   PyObject* put_arrow(ObjRef objref, PyObject* array);
+#endif
   // Allocates buffer for objref with size of size
   const char* allocate_buffer(ObjRef objref, int64_t size, SegmentId& segmentid);
   // Finishes buffer with segmentid and an offset of metadata_ofset
   PyObject* finish_buffer(ObjRef objref, SegmentId segmentid, int64_t metadata_offset);
   // Gets the buffer for objref
-  const char* get_buffer(ObjRef objref, int64_t& size, SegmentId& segmentid);
+  const char* get_buffer(ObjRef objref, int64_t& size, SegmentId& segmentid, int64_t& metadata_offset);
+#ifndef __APPLE__
   // gets an arrow object from the local object store
   PyObject* get_arrow(ObjRef objref, SegmentId& segmentid);
+#endif
   // determine if the object stored in objref is an arrow object // TODO(pcm): more general mechanism for this?
   bool is_arrow(ObjRef objref);
   // unmap the segment containing an object from the local address space