diff --git a/.travis/install-dependencies.sh b/.travis/install-dependencies.sh index 91b0e5641..ed447f72d 100755 --- a/.travis/install-dependencies.sh +++ b/.travis/install-dependencies.sh @@ -41,9 +41,9 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then exit 1 else echo "Updating brew." - brew update >/dev/null + brew update > /dev/null fi - brew install cmake pkg-config automake autoconf libtool boost + brew install cmake pkg-config automake autoconf libtool boost openssl bison > /dev/null # Install miniconda. wget https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda @@ -57,9 +57,9 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then exit 1 else echo "Updating brew." - brew update >/dev/null + brew update > /dev/null fi - brew install cmake pkg-config automake autoconf libtool boost + brew install cmake pkg-config automake autoconf libtool boost openssl bison > /dev/null # Install miniconda. wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh -nv bash miniconda.sh -b -p $HOME/miniconda @@ -78,7 +78,17 @@ elif [[ "$LINUX_WHEELS" == "1" ]]; then sudo apt-get install docker sudo usermod -a -G docker travis elif [[ "$MAC_WHEELS" == "1" ]]; then - # Don't need to do anything here. + which -s brew + if [[ $? != 0 ]]; then + echo "Could not find brew, please install brew (see http://brew.sh/)." + exit 1 + else + echo "Updating brew." + brew update > /dev/null + fi + brew install cmake pkg-config automake autoconf libtool boost openssl bison > /dev/null + # We use true to avoid exiting with an error code because the brew install can + # fail if a package is already installed. true else echo "Unrecognized environment." diff --git a/doc/source/install-on-macosx.rst b/doc/source/install-on-macosx.rst index af1fbcc85..4934e31a4 100644 --- a/doc/source/install-on-macosx.rst +++ b/doc/source/install-on-macosx.rst @@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using .. code-block:: bash brew update - brew install cmake pkg-config automake autoconf libtool boost wget + brew install cmake pkg-config automake autoconf libtool boost openssl bison wget pip install numpy funcsigs click colorama psutil redis flatbuffers cython --ignore-installed six diff --git a/doc/source/install-on-ubuntu.rst b/doc/source/install-on-ubuntu.rst index 4a70e8408..800de2a87 100644 --- a/doc/source/install-on-ubuntu.rst +++ b/doc/source/install-on-ubuntu.rst @@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using .. code-block:: bash sudo apt-get update - sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool unzip python # we install python here because python2 is required to build the webui + sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool unzip flex bison python # we install python here because python2 is required to build the webui # If you are not using Anaconda, you need the following. sudo apt-get install python-dev # For Python 2. diff --git a/src/thirdparty/build_parquet.sh b/src/thirdparty/build_parquet.sh index bcbf69631..69a9d078c 100755 --- a/src/thirdparty/build_parquet.sh +++ b/src/thirdparty/build_parquet.sh @@ -8,26 +8,44 @@ set -e unamestr="$(uname)" TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -if [ "$unamestr" == "Darwin" ]; then - brew update > /dev/null - brew install boost && true - brew install openssl - brew install bison - export OPENSSL_ROOT_DIR=/usr/local/opt/openssl - export LD_LIBRARY_PATH=/usr/local/opt/openssl/lib:$LD_LIBRARY_PATH - export PATH="/usr/local/opt/bison/bin:$PATH" -else - export BOOST_ROOT=$TP_DIR/boost -fi - cd $TP_DIR/parquet-cpp -ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install -cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ - -DPARQUET_BUILD_BENCHMARKS=off \ - -DPARQUET_BUILD_EXECUTABLES=off \ - -DPARQUET_BUILD_TESTS=off \ - . -make -j4 -make install +PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install + +OPENSSL_DIR=/usr/local/opt/openssl +BISON_DIR=/usr/local/opt/bison/bin + +if [ "$unamestr" == "Darwin" ]; then + OPENSSL_ROOT_DIR=$OPENSSL_DIR \ + PATH="$BISON_DIR:$PATH" \ + ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ + -DPARQUET_BUILD_BENCHMARKS=off \ + -DPARQUET_BUILD_EXECUTABLES=off \ + -DPARQUET_BUILD_TESTS=off \ + . + + OPENSSL_ROOT_DIR=$OPENSSL_DIR \ + PATH="$BISON_DIR:$PATH" \ + make -j4 + + OPENSSL_ROOT_DIR=$OPENSSL_DIR \ + PATH="$BISON_DIR:$PATH" \ + make install +else + BOOST_ROOT=$TP_DIR/boost \ + ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ + -DPARQUET_BUILD_BENCHMARKS=off \ + -DPARQUET_BUILD_EXECUTABLES=off \ + -DPARQUET_BUILD_TESTS=off \ + . + + PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install \ + BOOST_ROOT=$TP_DIR/boost \ + make -j4 + + make install +fi diff --git a/src/thirdparty/build_thirdparty.sh b/src/thirdparty/build_thirdparty.sh index cbda7806a..31d0c5df2 100755 --- a/src/thirdparty/build_thirdparty.sh +++ b/src/thirdparty/build_thirdparty.sh @@ -80,8 +80,6 @@ if [[ -d $ARROW_HOME/lib64 ]]; then cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib fi -export PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install - bash "$TP_DIR/build_parquet.sh" echo "installing pyarrow" @@ -92,17 +90,18 @@ PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ PYARROW_WITH_PLASMA=1 \ PYARROW_BUNDLE_ARROW_CPP=1 \ $PYTHON_EXECUTABLE setup.py build + PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ PYARROW_WITH_PLASMA=1 \ PYARROW_BUNDLE_ARROW_CPP=1 \ +PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install \ PYARROW_WITH_PARQUET=1 \ $PYTHON_EXECUTABLE setup.py build_ext # Find the pyarrow directory that was just built and copy it to ray/python/ray/ # so that pyarrow can be packaged along with ray. -pushd . -cd $TP_DIR/arrow/python/build -PYARROW_BUILD_LIB_DIR="$TP_DIR/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')" +pushd $TP_DIR/arrow/python/build + PYARROW_BUILD_LIB_DIR="$TP_DIR/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')" popd echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow" cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../../python/ray/pyarrow_files/ diff --git a/src/thirdparty/download_thirdparty.sh b/src/thirdparty/download_thirdparty.sh index 7d79eb25b..5031f7147 100755 --- a/src/thirdparty/download_thirdparty.sh +++ b/src/thirdparty/download_thirdparty.sh @@ -10,14 +10,15 @@ TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) if [ ! -d $TP_DIR/arrow ]; then git clone https://github.com/apache/arrow.git "$TP_DIR/arrow" fi -cd $TP_DIR/arrow -git fetch origin master -git checkout e26f3dad3675288564ef0c0330a5c9afcac652f1 +pushd $TP_DIR/arrow + git fetch origin master + git checkout e26f3dad3675288564ef0c0330a5c9afcac652f1 +popd -cd $TP_DIR if [ ! -d $TP_DIR/parquet-cpp ]; then git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/parquet-cpp" - pushd $TP_DIR/parquet-cpp - git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b - popd fi +pushd $TP_DIR/parquet-cpp + git fetch origin master + git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b +popd