Changes to build to fix creation of wheels. (#840)

* Pass DPYTHON_EXECUTABLE into cmake for arrow and for ray.

* Add cython to setup.py install_requires.

* Revert custom code for finding python in cmake.

* Correctly find arrow on CentOS.

* In cmake, don't find PythonLibs, just find PYTHON_INCLUDE_DIRS.

* Fix typo.

* Do not use boost shared libraries when building arrow.

* Add six to the setup.py install_requires because it is needed by pyarrow.

* Don't link numbuf against boost_system and boost_filesystem.

* Compile boost when we are on Linux.

* Make numbuf find the correct boost libraries.

* Only use find_package Boost on Linux, suppress output when building boost.

* Changes to wheel building scripts, install cython in mac script.

* Compile flatbuffers ourselves on Linux and pass it in when compiling Arrow.

* Clean up build_flatbuffers.sh and build_boost.sh scripts a little.

* Install cython when building linux wheel.

src/thirdparty/build_boost.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+# Download and compile boost if it isn't already present.
+if [ ! -d $TP_DIR/boost ]; then
+  wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.60.0/boost_1_60_0.tar.gz -O $TP_DIR/boost_1_60_0.tar.gz
+  tar xf $TP_DIR/boost_1_60_0.tar.gz -C $TP_DIR/
+  rm -rf $TP_DIR/boost_1_60_0.tar.gz
+
+  # Compile boost.
+  pushd $TP_DIR/boost_1_60_0
+    ./bootstrap.sh
+    ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=$TP_DIR/boost --with-filesystem --with-system install > /dev/null
+  popd
+fi

src/thirdparty/build_flatbuffers.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+
+FLATBUFFERS_VERSION=1.7.1
+
+# Download and compile flatbuffers if it isn't already present.
+if [ ! -d $TP_DIR/flatbuffers ]; then
+  echo "building flatbuffers"
+  wget https://github.com/google/flatbuffers/archive/v$FLATBUFFERS_VERSION.tar.gz -O flatbuffers-$FLATBUFFERS_VERSION.tar.gz
+  tar xf flatbuffers-$FLATBUFFERS_VERSION.tar.gz
+  rm -rf flatbuffers-$FLATBUFFERS_VERSION.tar.gz
+
+  # Compile flatbuffers.
+  pushd flatbuffers-$FLATBUFFERS_VERSION
+    cmake -DCMAKE_CXX_FLAGS=-fPIC \
+          -DCMAKE_INSTALL_PREFIX:PATH=$TP_DIR/flatbuffers \
+          -DFLATBUFFERS_BUILD_TESTS=OFF
+    make -j5
+    make install
+  popd
+fi

src/thirdparty/build_thirdparty.sh

@@ -26,24 +26,38 @@ else
   exit 1
 fi
 
+# If we're on Linux, then compile boost. This installs boost to $TP_DIR/boost.
+if [[ "$unamestr" == "Linux" ]]; then
+  echo "building boost"
+  bash "$TP_DIR/build_boost.sh"
+fi
+
+# If we're on Linux, then compile flatbuffers. This installs flatbuffers to
+# $TP_DIR/flatbuffers.
+if [[ "$unamestr" == "Linux" ]]; then
+  echo "building flatbuffers"
+  bash "$TP_DIR/build_flatbuffers.sh"
+  FLATBUFFERS_HOME=$TP_DIR/flatbuffers
+else
+  FLATBUFFERS_HOME=""
+fi
+
 echo "building arrow"
 cd $TP_DIR/arrow/cpp
 mkdir -p $TP_DIR/arrow/cpp/build
 cd $TP_DIR/arrow/cpp/build
-export ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
+ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
 
-# Get the directory of the Python executable.
-PYTHON_EXECUTABLE_DIR=$(dirname $PYTHON_EXECUTABLE)
-
-# Pass a slightly different path into this command so that cmake finds the right
-# Python interpreter and libraries.
-PATH=$PYTHON_EXECUTABLE_DIR:$PATH \
+BOOST_ROOT=$TP_DIR/boost \
+FLATBUFFERS_HOME=$FLATBUFFERS_HOME \
 cmake -DCMAKE_BUILD_TYPE=Release \
       -DCMAKE_C_FLAGS="-g -O3" \
       -DCMAKE_CXX_FLAGS="-g -O3" \
       -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
       -DARROW_BUILD_TESTS=off \
       -DARROW_HDFS=on \
+      -DARROW_BOOST_USE_SHARED=off \
+      -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
       -DARROW_PYTHON=on \
       -DARROW_PLASMA=on \
       -DPLASMA_PYTHON=on \
@@ -57,13 +71,25 @@ cmake -DCMAKE_BUILD_TYPE=Release \
 make VERBOSE=1 -j$PARALLEL
 make install
 
+if [[ -d $ARROW_HOME/lib64 ]]; then
+  # On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for
+  # now. TODO(rkn): A preferable solution would be to add both directories to
+  # the PKG_CONFIG_PATH, but that didn't seem to work.
+  cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib
+fi
+
 echo "installing pyarrow"
 cd $TP_DIR/arrow/python
 # We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
 # find plasma.
-ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
-PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig PYARROW_WITH_PLASMA=1 PYARROW_BUNDLE_ARROW_CPP=1 $PYTHON_EXECUTABLE setup.py build
-PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig PYARROW_WITH_PLASMA=1 PYARROW_BUNDLE_ARROW_CPP=1 $PYTHON_EXECUTABLE setup.py build_ext
+PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
+PYARROW_WITH_PLASMA=1 \
+PYARROW_BUNDLE_ARROW_CPP=1 \
+$PYTHON_EXECUTABLE setup.py build
+PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
+PYARROW_WITH_PLASMA=1 \
+PYARROW_BUNDLE_ARROW_CPP=1 \
+$PYTHON_EXECUTABLE setup.py build_ext
 # Find the pyarrow directory that was just built and copy it to ray/python/ray/
 # so that pyarrow can be packaged along with ray. TODO(rkn): This doesn't seem
 # very robust. Fix this.