diff --git a/.travis.yml b/.travis.yml index ee2e2c98c..0eed43fc3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -72,7 +72,7 @@ matrix: install: - ./.travis/install-dependencies.sh # This command should be kept in sync with ray/python/README-building-wheels.md. - - docker run --rm -w /ray -v `pwd`:/ray -ti quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-1024 /ray/python/build-wheel-manylinux1.sh + - docker run --rm -w /ray -v `pwd`:/ray -ti quay.io/xhochy/arrow_manylinux1_x86_64_base:latest /ray/python/build-wheel-manylinux1.sh script: - ./.travis/test-wheels.sh diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index f524fb914..e3c56cc3d 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -4,7 +4,7 @@ FROM ubuntu:xenial RUN apt-get update \ && apt-get install -y vim git wget \ - && apt-get install -y cmake pkg-config build-essential autoconf curl libtool unzip + && apt-get install -y cmake pkg-config build-essential autoconf curl libtool unzip flex bison RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \ && wget --quiet 'https://repo.continuum.io/archive/Anaconda2-4.2.0-Linux-x86_64.sh' -O /tmp/anaconda.sh \ && /bin/bash /tmp/anaconda.sh -b -p /opt/conda \ diff --git a/python/README-building-wheels.md b/python/README-building-wheels.md index 9ad0ceda7..41a73d22e 100644 --- a/python/README-building-wheels.md +++ b/python/README-building-wheels.md @@ -9,7 +9,7 @@ produce .whl files owned by root. Inside the root directory (i.e., one level above this python directory), run ``` -docker run --rm -w /ray -v `pwd`:/ray -ti quay.io/xhochy/arrow_manylinux1_x86_64_base:ARROW-1024 /ray/python/build-wheel-manylinux1.sh +docker run --rm -w /ray -v `pwd`:/ray -ti quay.io/xhochy/arrow_manylinux1_x86_64_base:latest /ray/python/build-wheel-manylinux1.sh ``` The wheel files will be placed in the .whl directory. diff --git a/src/thirdparty/build_boost.sh b/src/thirdparty/build_boost.sh index 1da2ca015..b57b64433 100755 --- a/src/thirdparty/build_boost.sh +++ b/src/thirdparty/build_boost.sh @@ -28,6 +28,6 @@ if [[ ! -d $TP_DIR/boost ]]; then # Compile boost. pushd $TP_DIR/boost_$BOOST_VERSION_UNDERSCORE ./bootstrap.sh - ./bjam cxxflags=-fPIC cflags=-fPIC --prefix=$TP_DIR/boost --with-filesystem --with-system install > /dev/null + ./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/boost --with-filesystem --with-system --with-regex install > /dev/null popd fi diff --git a/src/thirdparty/build_parquet.sh b/src/thirdparty/build_parquet.sh new file mode 100755 index 000000000..bcbf69631 --- /dev/null +++ b/src/thirdparty/build_parquet.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -x + +# Cause the script to exit if a single command fails. +set -e + +unamestr="$(uname)" +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +if [ "$unamestr" == "Darwin" ]; then + brew update > /dev/null + brew install boost && true + brew install openssl + brew install bison + export OPENSSL_ROOT_DIR=/usr/local/opt/openssl + export LD_LIBRARY_PATH=/usr/local/opt/openssl/lib:$LD_LIBRARY_PATH + export PATH="/usr/local/opt/bison/bin:$PATH" +else + export BOOST_ROOT=$TP_DIR/boost +fi + +cd $TP_DIR/parquet-cpp +ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install +cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ + -DPARQUET_BUILD_BENCHMARKS=off \ + -DPARQUET_BUILD_EXECUTABLES=off \ + -DPARQUET_BUILD_TESTS=off \ + . + +make -j4 +make install diff --git a/src/thirdparty/build_thirdparty.sh b/src/thirdparty/build_thirdparty.sh index 3742c8830..cbda7806a 100755 --- a/src/thirdparty/build_thirdparty.sh +++ b/src/thirdparty/build_thirdparty.sh @@ -67,7 +67,6 @@ cmake -DCMAKE_BUILD_TYPE=Release \ -DARROW_JEMALLOC=off \ -DARROW_WITH_BROTLI=off \ -DARROW_WITH_LZ4=off \ - -DARROW_WITH_SNAPPY=off \ -DARROW_WITH_ZLIB=off \ -DARROW_WITH_ZSTD=off \ .. @@ -81,6 +80,10 @@ if [[ -d $ARROW_HOME/lib64 ]]; then cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib fi +export PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install + +bash "$TP_DIR/build_parquet.sh" + echo "installing pyarrow" cd $TP_DIR/arrow/python # We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can @@ -92,7 +95,9 @@ $PYTHON_EXECUTABLE setup.py build PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ PYARROW_WITH_PLASMA=1 \ PYARROW_BUNDLE_ARROW_CPP=1 \ +PYARROW_WITH_PARQUET=1 \ $PYTHON_EXECUTABLE setup.py build_ext + # Find the pyarrow directory that was just built and copy it to ray/python/ray/ # so that pyarrow can be packaged along with ray. pushd . diff --git a/src/thirdparty/download_thirdparty.sh b/src/thirdparty/download_thirdparty.sh index 30e55fd21..7d79eb25b 100755 --- a/src/thirdparty/download_thirdparty.sh +++ b/src/thirdparty/download_thirdparty.sh @@ -12,5 +12,12 @@ if [ ! -d $TP_DIR/arrow ]; then fi cd $TP_DIR/arrow git fetch origin master - git checkout e26f3dad3675288564ef0c0330a5c9afcac652f1 + +cd $TP_DIR +if [ ! -d $TP_DIR/parquet-cpp ]; then + git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/parquet-cpp" + pushd $TP_DIR/parquet-cpp + git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b + popd +fi