From d1974afbab9f0f1bcf8af15a8c476d868ad31aff Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Thu, 27 Oct 2016 23:40:10 -0700 Subject: [PATCH] Add setup.py and travis. (#8) * Add setup.py and travis. * Add script to install dependencies for travis. * Put Arrow into thirdparty. * Add .gitmodules file. * Install boost dependencies. * Make tests verbose. * Build arrow and numbuf in setup.py. * Change build_ext to install. * Switch develop to install in pip. * Test * fix numbuf build and installation * make import numbuf work * fix documentation * fix tests * quotes --- .gitmodules | 3 +++ .travis.yml | 25 ++++++++++++++++++++ CMakeLists.txt | 6 +++-- build.sh | 20 ++++++++++++++++ install-dependencies.sh | 39 +++++++++++++++++++++++++++++++ numbuf/__init__.py | 1 + python/test/runtest.py | 32 ++++++++++++------------- setup.py | 23 ++++++++++++++++++ setup.sh | 21 +++++++++++++++++ thirdparty/arrow | 1 + thirdparty/build_thirdparty.sh | 27 +++++++++++++++++++++ thirdparty/download_thirdparty.sh | 8 +++++++ 12 files changed, 188 insertions(+), 18 deletions(-) create mode 100644 .gitmodules create mode 100644 .travis.yml create mode 100755 build.sh create mode 100755 install-dependencies.sh create mode 100644 numbuf/__init__.py create mode 100644 setup.py create mode 100755 setup.sh create mode 160000 thirdparty/arrow create mode 100755 thirdparty/build_thirdparty.sh create mode 100755 thirdparty/download_thirdparty.sh diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..89d94b72e --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "thirdparty/arrow"] + path = thirdparty/arrow + url = https://github.com/ray-project/arrow.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..fe784d053 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,25 @@ +sudo: required + +language: generic + +matrix: + include: + - os: linux + dist: trusty + python: "2.7" + - os: linux + dist: trusty + python: "3.5" + - os: osx + osx_image: xcode7 + python: "2.7" + - os: osx + osx_image: xcode7 + python: "3.5" + +install: + - ./install-dependencies.sh + - python setup.py install --user + +script: + - python python/test/runtest.py diff --git a/CMakeLists.txt b/CMakeLists.txt index a46830891..760bdde11 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,10 +50,10 @@ if (UNIX AND NOT APPLE) link_libraries(rt) endif() -set(ARROW_DIR "${CMAKE_SOURCE_DIR}/../arrow/" CACHE STRING +set(ARROW_DIR "${CMAKE_SOURCE_DIR}/thirdparty/arrow/" CACHE STRING "Path of the arrow source directory") -set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/../arrow/cpp/build/release/libarrow.a" CACHE STRING +set(ARROW_STATIC_LIB "${CMAKE_SOURCE_DIR}/thirdparty/arrow/cpp/build/release/libarrow.a" CACHE STRING "Path to libarrow.a (needs to be changed if arrow is build in debug mode)") include_directories("${ARROW_DIR}/cpp/src/") @@ -78,3 +78,5 @@ if(APPLE) endif(APPLE) target_link_libraries(numbuf ${ARROW_STATIC_LIB} ${PYTHON_LIBRARIES}) + +install(TARGETS numbuf DESTINATION ${CMAKE_SOURCE_DIR}) diff --git a/build.sh b/build.sh new file mode 100755 index 000000000..33a1355c8 --- /dev/null +++ b/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +# Determine how many parallel jobs to use for make based on the number of cores +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + PARALLEL=$(nproc) +elif [[ "$unamestr" == "Darwin" ]]; then + PARALLEL=$(sysctl -n hw.ncpu) +else + echo "Unrecognized platform." + exit 1 +fi + +mkdir -p "$ROOT_DIR/build" +pushd "$ROOT_DIR/build" + cmake .. + make install -j$PARALLEL +popd diff --git a/install-dependencies.sh b/install-dependencies.sh new file mode 100755 index 000000000..8d7de5288 --- /dev/null +++ b/install-dependencies.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +platform="unknown" +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + echo "Platform is linux." + platform="linux" +elif [[ "$unamestr" == "Darwin" ]]; then + echo "Platform is macosx." + platform="macosx" +else + echo "Unrecognized platform." + exit 1 +fi + +if [[ $platform == "macosx" ]]; then + # check that brew is installed + which -s brew + if [[ $? != 0 ]]; then + echo "Could not find brew, please install brew (see http://brew.sh/)." + exit 1 + else + echo "Updating brew." + brew update + fi +fi + +if [[ $platform == "linux" ]]; then + # These commands must be kept in sync with the installation instructions. + sudo apt-get update + sudo apt-get install -y cmake build-essential autoconf libtool python-dev python-numpy python-pip libboost-all-dev +elif [[ $platform == "macosx" ]]; then + # These commands must be kept in sync with the installation instructions. + brew install cmake automake autoconf libtool boost + sudo easy_install pip + sudo pip install numpy --ignore-installed six +fi diff --git a/numbuf/__init__.py b/numbuf/__init__.py new file mode 100644 index 000000000..21cb259ba --- /dev/null +++ b/numbuf/__init__.py @@ -0,0 +1 @@ +from libnumbuf import * diff --git a/python/test/runtest.py b/python/test/runtest.py index c1b7d21e8..4c9b409f6 100644 --- a/python/test/runtest.py +++ b/python/test/runtest.py @@ -1,10 +1,10 @@ import unittest -import libnumbuf +import numbuf import numpy as np from numpy.testing import assert_equal -TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world", - u"x", u"\u262F", 42.0, +TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world", + u"x", u"\u262F", 42.0, 1L << 62, (1.0, "hi"), None, (None, None), ("hello", None), True, False, (True, False), "hello", @@ -17,8 +17,8 @@ TEST_OBJECTS = [{(1,2) : 1}, {() : 2}, [1, "hello", 3.0], 42, 43L, "hello world" class SerializationTests(unittest.TestCase): def roundTripTest(self, data): - schema, size, serialized = libnumbuf.serialize_list(data) - result = libnumbuf.deserialize_list(serialized) + schema, size, serialized = numbuf.serialize_list(data) + result = numbuf.deserialize_list(serialized) assert_equal(data, result) def testSimple(self): @@ -78,10 +78,10 @@ class SerializationTests(unittest.TestCase): bar = Bar() bar.foo.x = 42 - libnumbuf.register_callbacks(serialize, deserialize) + numbuf.register_callbacks(serialize, deserialize) - metadata, size, serialized = libnumbuf.serialize_list([bar]) - self.assertEqual(libnumbuf.deserialize_list(serialized)[0].foo.x, 42) + metadata, size, serialized = numbuf.serialize_list([bar]) + self.assertEqual(numbuf.deserialize_list(serialized)[0].foo.x, 42) def testObjectArray(self): x = np.array([1, 2, "hello"], dtype=object) @@ -94,21 +94,21 @@ class SerializationTests(unittest.TestCase): if obj["_pytype_"] == "numpy.array": return np.array(obj["data"], dtype=object) - libnumbuf.register_callbacks(myserialize, mydeserialize) + numbuf.register_callbacks(myserialize, mydeserialize) - metadata, size, serialized = libnumbuf.serialize_list([x, y]) + metadata, size, serialized = numbuf.serialize_list([x, y]) - assert_equal(libnumbuf.deserialize_list(serialized), [x, y]) + assert_equal(numbuf.deserialize_list(serialized), [x, y]) def testBuffer(self): for (i, obj) in enumerate(TEST_OBJECTS): - schema, size, batch = libnumbuf.serialize_list([obj]) + schema, size, batch = numbuf.serialize_list([obj]) size = size + 4096 # INITIAL_METADATA_SIZE in arrow buff = np.zeros(size, dtype="uint8") - metadata_offset = libnumbuf.write_to_buffer(batch, memoryview(buff)) - array = libnumbuf.read_from_buffer(memoryview(buff), schema, metadata_offset) - result = libnumbuf.deserialize_list(array) + metadata_offset = numbuf.write_to_buffer(batch, memoryview(buff)) + array = numbuf.read_from_buffer(memoryview(buff), schema, metadata_offset) + result = numbuf.deserialize_list(array) assert_equal(result[0], obj) if __name__ == "__main__": - unittest.main() + unittest.main(verbosity=2) diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..27b82a9de --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +import subprocess +from setuptools import setup, find_packages, Extension +import setuptools.command.install as _install + +# Because of relative paths, this must be run from inside numbuf/. + +class install(_install.install): + def run(self): + subprocess.check_call(["./setup.sh"]) + subprocess.check_call(["./build.sh"]) + subprocess.check_call(["cp", "libnumbuf.so", "numbuf/"]) + # Calling _install.install.run(self) does not fetch required packages and + # instead performs an old-style install. See command/install.py in + # setuptools. So, calling do_egg_install() manually here. + self.do_egg_install() + +setup(name="numbuf", + version="0.0.1", + packages=find_packages(), + package_data={"numbuf": ["libnumbuf.so"]}, + cmdclass={"install": install}, + include_package_data=True, + zip_safe=False) diff --git a/setup.sh b/setup.sh new file mode 100755 index 000000000..71482975e --- /dev/null +++ b/setup.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +platform="unknown" +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + echo "Platform is linux." + platform="linux" +elif [[ "$unamestr" == "Darwin" ]]; then + echo "Platform is macosx." + platform="macosx" +else + echo "Unrecognized platform." + exit 1 +fi + +pushd "$ROOT_DIR/thirdparty" + ./download_thirdparty.sh + ./build_thirdparty.sh +popd diff --git a/thirdparty/arrow b/thirdparty/arrow new file mode 160000 index 000000000..c781ef856 --- /dev/null +++ b/thirdparty/arrow @@ -0,0 +1 @@ +Subproject commit c781ef8564714bb5149e8b5a311673f9162232ac diff --git a/thirdparty/build_thirdparty.sh b/thirdparty/build_thirdparty.sh new file mode 100755 index 000000000..7c307f816 --- /dev/null +++ b/thirdparty/build_thirdparty.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -x +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +PREFIX=$TP_DIR/installed + +# Determine how many parallel jobs to use for make based on the number of cores +unamestr="$(uname)" +if [[ "$unamestr" == "Linux" ]]; then + PARALLEL=$(nproc) +elif [[ "$unamestr" == "Darwin" ]]; then + PARALLEL=$(sysctl -n hw.ncpu) + echo "Platform is macosx." +else + echo "Unrecognized platform." + exit 1 +fi + +echo "building arrow" +cd $TP_DIR/arrow/cpp +source setup_build_env.sh +mkdir -p $TP_DIR/arrow/cpp/build +cd $TP_DIR/arrow/cpp/build +cmake -DLIBARROW_LINKAGE=STATIC -DCMAKE_BUILD_TYPE=Release .. +make VERBOSE=1 -j$PARALLEL diff --git a/thirdparty/download_thirdparty.sh b/thirdparty/download_thirdparty.sh new file mode 100755 index 000000000..044565252 --- /dev/null +++ b/thirdparty/download_thirdparty.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -x +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +git submodule update --init --recursive -- "$TP_DIR/arrow"