Add setup.py and travis. (#8)

* Add setup.py and travis.

* Add script to install dependencies for travis.

* Put Arrow into thirdparty.

* Add .gitmodules file.

* Install boost dependencies.

* Make tests verbose.

* Build arrow and numbuf in setup.py.

* Change build_ext to install.

* Switch develop to install in pip.

* Test

* fix numbuf build and installation

* make import numbuf work

* fix documentation

* fix tests

* quotes

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "thirdparty/arrow"]
+	path = thirdparty/arrow
+	url = https://github.com/ray-project/arrow.git

.travis.yml

@@ -0,0 +1,25 @@
+sudo: required
+
+language: generic
+
+matrix:
+  include:
+    - os: linux
+      dist: trusty
+      python: "2.7"
+    - os: linux
+      dist: trusty
+      python: "3.5"
+    - os: osx
+      osx_image: xcode7
+      python: "2.7"
+    - os: osx
+      osx_image: xcode7
+      python: "3.5"
+
+install:
+  - ./install-dependencies.sh
+  - python setup.py install --user
+
+script:
+  - python python/test/runtest.py

CMakeLists.txt

@@ -50,10 +50,10 @@ if (UNIX AND NOT APPLE)
    link_libraries(rt)
 endif()
 
-set(ARROW_DIR "${CMAKE_SOURCE_DIR}/../arrow/" CACHE STRING
+set(ARROW_DIR "${CMAKE_SOURCE_DIR}/thirdparty/arrow/" CACHE STRING
   "Path of the arrow source directory")
 
-set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/../arrow/cpp/build/release/libarrow.a" CACHE STRING
+set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a" CACHE STRING
   "Path to libarrow.a (needs to be changed if arrow is build in debug mode)")
 
 include_directories("${ARROW_DIR}/cpp/src/")
@@ -78,3 +78,5 @@ if(APPLE)
 endif(APPLE)
 
 target_link_libraries(numbuf ${ARROW_STATIC_LIB} ${PYTHON_LIBRARIES})
+
+install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR})

build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+# Determine how many parallel jobs to use for make based on the number of cores
+unamestr="$(uname)"
+if [[ "$unamestr" == "Linux" ]]; then
+  PARALLEL=$(nproc)
+elif [[ "$unamestr" == "Darwin" ]]; then
+  PARALLEL=$(sysctl -n hw.ncpu)
+else
+  echo "Unrecognized platform."
+  exit 1
+fi
+
+mkdir -p "$ROOT_DIR/build"
+pushd "$ROOT_DIR/build"
+  cmake ..
+  make install -j$PARALLEL
+popd

install-dependencies.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+platform="unknown"
+unamestr="$(uname)"
+if [[ "$unamestr" == "Linux" ]]; then
+  echo "Platform is linux."
+  platform="linux"
+elif [[ "$unamestr" == "Darwin" ]]; then
+  echo "Platform is macosx."
+  platform="macosx"
+else
+  echo "Unrecognized platform."
+  exit 1
+fi
+
+if [[ $platform == "macosx" ]]; then
+  # check that brew is installed
+  which -s brew
+  if [[ $? != 0 ]]; then
+    echo "Could not find brew, please install brew (see http://brew.sh/)."
+    exit 1
+  else
+    echo "Updating brew."
+    brew update
+  fi
+fi
+
+if [[ $platform == "linux" ]]; then
+  # These commands must be kept in sync with the installation instructions.
+  sudo apt-get update
+  sudo apt-get install -y cmake build-essential autoconf libtool python-dev python-numpy python-pip libboost-all-dev
+elif [[ $platform == "macosx" ]]; then
+  # These commands must be kept in sync with the installation instructions.
+  brew install cmake automake autoconf libtool boost
+  sudo easy_install pip
+  sudo pip install numpy --ignore-installed six
+fi

numbuf/__init__.py

@@ -0,0 +1 @@
+from libnumbuf import *

python/test/runtest.py

@@ -1,10 +1,10 @@
 import unittest
-import libnumbuf
+import numbuf
 import numpy as np
 from numpy.testing import assert_equal
 
-TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world", 
-                u"x", u"\u262F", 42.0, 
+TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world",
+                u"x", u"\u262F", 42.0,
                 1L << 62, (1.0, "hi"),
                 None, (None, None), ("hello", None),
                 True, False, (True, False), "hello",
@@ -17,8 +17,8 @@ TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world"
 class SerializationTests(unittest.TestCase):
 
   def roundTripTest(self, data):
-    schema, size, serialized = libnumbuf.serialize_list(data)
-    result = libnumbuf.deserialize_list(serialized)
+    schema, size, serialized = numbuf.serialize_list(data)
+    result = numbuf.deserialize_list(serialized)
     assert_equal(data, result)
 
   def testSimple(self):
@@ -78,10 +78,10 @@ class SerializationTests(unittest.TestCase):
     bar = Bar()
     bar.foo.x = 42
 
-    libnumbuf.register_callbacks(serialize, deserialize)
+    numbuf.register_callbacks(serialize, deserialize)
 
-    metadata, size, serialized = libnumbuf.serialize_list([bar])
-    self.assertEqual(libnumbuf.deserialize_list(serialized)[0].foo.x, 42)
+    metadata, size, serialized = numbuf.serialize_list([bar])
+    self.assertEqual(numbuf.deserialize_list(serialized)[0].foo.x, 42)
 
   def testObjectArray(self):
     x = np.array([1, 2, "hello"], dtype=object)
@@ -94,21 +94,21 @@ class SerializationTests(unittest.TestCase):
       if obj["_pytype_"] == "numpy.array":
         return np.array(obj["data"], dtype=object)
 
-    libnumbuf.register_callbacks(myserialize, mydeserialize)
+    numbuf.register_callbacks(myserialize, mydeserialize)
 
-    metadata, size, serialized = libnumbuf.serialize_list([x, y])
+    metadata, size, serialized = numbuf.serialize_list([x, y])
 
-    assert_equal(libnumbuf.deserialize_list(serialized), [x, y])
+    assert_equal(numbuf.deserialize_list(serialized), [x, y])
 
   def testBuffer(self):
     for (i, obj) in enumerate(TEST_OBJECTS):
-      schema, size, batch = libnumbuf.serialize_list([obj])
+      schema, size, batch = numbuf.serialize_list([obj])
       size = size + 4096 # INITIAL_METADATA_SIZE in arrow
       buff = np.zeros(size, dtype="uint8")
-      metadata_offset = libnumbuf.write_to_buffer(batch, memoryview(buff))
-      array = libnumbuf.read_from_buffer(memoryview(buff), schema, metadata_offset)
-      result = libnumbuf.deserialize_list(array)
+      metadata_offset = numbuf.write_to_buffer(batch, memoryview(buff))
+      array = numbuf.read_from_buffer(memoryview(buff), schema, metadata_offset)
+      result = numbuf.deserialize_list(array)
       assert_equal(result[0], obj)
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main(verbosity=2)

setup.py

@@ -0,0 +1,23 @@
+import subprocess
+from setuptools import setup, find_packages, Extension
+import setuptools.command.install as _install
+
+# Because of relative paths, this must be run from inside numbuf/.
+
+class install(_install.install):
+  def run(self):
+    subprocess.check_call(["./setup.sh"])
+    subprocess.check_call(["./build.sh"])
+    subprocess.check_call(["cp", "libnumbuf.so", "numbuf/"])
+    # Calling _install.install.run(self) does not fetch required packages and
+    # instead performs an old-style install. See command/install.py in
+    # setuptools. So, calling do_egg_install() manually here.
+    self.do_egg_install()
+
+setup(name="numbuf",
+      version="0.0.1",
+      packages=find_packages(),
+      package_data={"numbuf": ["libnumbuf.so"]},
+      cmdclass={"install": install},
+      include_package_data=True,
+      zip_safe=False)

setup.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+platform="unknown"
+unamestr="$(uname)"
+if [[ "$unamestr" == "Linux" ]]; then
+  echo "Platform is linux."
+  platform="linux"
+elif [[ "$unamestr" == "Darwin" ]]; then
+  echo "Platform is macosx."
+  platform="macosx"
+else
+  echo "Unrecognized platform."
+  exit 1
+fi
+
+pushd "$ROOT_DIR/thirdparty"
+  ./download_thirdparty.sh
+  ./build_thirdparty.sh
+popd

thirdparty/build_thirdparty.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -x
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+PREFIX=$TP_DIR/installed
+
+# Determine how many parallel jobs to use for make based on the number of cores
+unamestr="$(uname)"
+if [[ "$unamestr" == "Linux" ]]; then
+  PARALLEL=$(nproc)
+elif [[ "$unamestr" == "Darwin" ]]; then
+  PARALLEL=$(sysctl -n hw.ncpu)
+  echo "Platform is macosx."
+else
+  echo "Unrecognized platform."
+  exit 1
+fi
+
+echo "building arrow"
+cd $TP_DIR/arrow/cpp
+source setup_build_env.sh
+mkdir -p $TP_DIR/arrow/cpp/build
+cd $TP_DIR/arrow/cpp/build
+cmake -DLIBARROW_LINKAGE=STATIC -DCMAKE_BUILD_TYPE=Release ..
+make VERBOSE=1 -j$PARALLEL

thirdparty/download_thirdparty.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -x
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+git submodule update --init --recursive -- "$TP_DIR/arrow"