Cleanup parquet installation. (#1549)

* Cleanup parquet installation.

* Fix

* Small changes.

* Add brew installs

* Modify paths for compilation of parquet.

* Remove LD_LIBRARY_PATH

* Don't set unnecessary environment variables on Linux.

* Set environment variables for make.

* Brew installs for macos wheels.

* Update

* Pass PARQUET_HOME when building pyarrow.

* Don't exit with error code.

.travis/install-dependencies.sh

@@ -41,9 +41,9 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
     exit 1
   else
     echo "Updating brew."
-    brew update >/dev/null
+    brew update > /dev/null
   fi
-  brew install cmake pkg-config automake autoconf libtool boost
+  brew install cmake pkg-config automake autoconf libtool boost openssl bison > /dev/null
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
@@ -57,9 +57,9 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
     exit 1
   else
     echo "Updating brew."
-    brew update >/dev/null
+    brew update > /dev/null
   fi
-  brew install cmake pkg-config automake autoconf libtool boost
+  brew install cmake pkg-config automake autoconf libtool boost openssl bison > /dev/null
   # Install miniconda.
   wget https://repo.continuum.io/miniconda/Miniconda3-latest-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
@@ -78,7 +78,17 @@ elif [[ "$LINUX_WHEELS" == "1" ]]; then
   sudo apt-get install docker
   sudo usermod -a -G docker travis
 elif [[ "$MAC_WHEELS" == "1" ]]; then
-  # Don't need to do anything here.
+  which -s brew
+  if [[ $? != 0 ]]; then
+    echo "Could not find brew, please install brew (see http://brew.sh/)."
+    exit 1
+  else
+    echo "Updating brew."
+    brew update > /dev/null
+  fi
+  brew install cmake pkg-config automake autoconf libtool boost openssl bison > /dev/null
+  # We use true to avoid exiting with an error code because the brew install can
+  # fail if a package is already installed.
   true
 else
   echo "Unrecognized environment."

doc/source/install-on-macosx.rst

@@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using
 .. code-block:: bash
 
   brew update
-  brew install cmake pkg-config automake autoconf libtool boost wget
+  brew install cmake pkg-config automake autoconf libtool boost openssl bison wget
 
   pip install numpy funcsigs click colorama psutil redis flatbuffers cython --ignore-installed six
 

doc/source/install-on-ubuntu.rst

@@ -26,7 +26,7 @@ To build Ray, first install the following dependencies. We recommend using
 .. code-block:: bash
 
   sudo apt-get update
-  sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool unzip python # we install python here because python2 is required to build the webui
+  sudo apt-get install -y cmake pkg-config build-essential autoconf curl libtool unzip flex bison python # we install python here because python2 is required to build the webui
 
   # If you are not using Anaconda, you need the following.
   sudo apt-get install python-dev  # For Python 2.

src/thirdparty/build_parquet.sh

@@ -8,26 +8,44 @@ set -e
 unamestr="$(uname)"
 TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 
-if [ "$unamestr" == "Darwin" ]; then
-  brew update > /dev/null
-  brew install boost && true
-  brew install openssl
-  brew install bison
-  export OPENSSL_ROOT_DIR=/usr/local/opt/openssl
-  export LD_LIBRARY_PATH=/usr/local/opt/openssl/lib:$LD_LIBRARY_PATH
-  export PATH="/usr/local/opt/bison/bin:$PATH"
-else
-  export BOOST_ROOT=$TP_DIR/boost
-fi
-
 cd $TP_DIR/parquet-cpp
-ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
-cmake -DCMAKE_BUILD_TYPE=Release \
-      -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
-      -DPARQUET_BUILD_BENCHMARKS=off \
-      -DPARQUET_BUILD_EXECUTABLES=off \
-      -DPARQUET_BUILD_TESTS=off \
-      .
 
-make -j4
-make install
+PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install
+
+OPENSSL_DIR=/usr/local/opt/openssl
+BISON_DIR=/usr/local/opt/bison/bin
+
+if [ "$unamestr" == "Darwin" ]; then
+  OPENSSL_ROOT_DIR=$OPENSSL_DIR \
+  PATH="$BISON_DIR:$PATH" \
+  ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install \
+  cmake -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+        -DPARQUET_BUILD_BENCHMARKS=off \
+        -DPARQUET_BUILD_EXECUTABLES=off \
+        -DPARQUET_BUILD_TESTS=off \
+        .
+
+  OPENSSL_ROOT_DIR=$OPENSSL_DIR \
+  PATH="$BISON_DIR:$PATH" \
+  make -j4
+
+  OPENSSL_ROOT_DIR=$OPENSSL_DIR \
+  PATH="$BISON_DIR:$PATH" \
+  make install
+else
+  BOOST_ROOT=$TP_DIR/boost \
+  ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install \
+  cmake -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+        -DPARQUET_BUILD_BENCHMARKS=off \
+        -DPARQUET_BUILD_EXECUTABLES=off \
+        -DPARQUET_BUILD_TESTS=off \
+        .
+
+  PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install \
+  BOOST_ROOT=$TP_DIR/boost \
+  make -j4
+
+  make install
+fi

src/thirdparty/build_thirdparty.sh

@@ -80,8 +80,6 @@ if [[ -d $ARROW_HOME/lib64 ]]; then
   cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib
 fi
 
-export PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install
-
 bash "$TP_DIR/build_parquet.sh"
 
 echo "installing pyarrow"
@@ -92,17 +90,18 @@ PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
 PYARROW_WITH_PLASMA=1 \
 PYARROW_BUNDLE_ARROW_CPP=1 \
 $PYTHON_EXECUTABLE setup.py build
+
 PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
 PYARROW_WITH_PLASMA=1 \
 PYARROW_BUNDLE_ARROW_CPP=1 \
+PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install \
 PYARROW_WITH_PARQUET=1 \
 $PYTHON_EXECUTABLE setup.py build_ext
 
 # Find the pyarrow directory that was just built and copy it to ray/python/ray/
 # so that pyarrow can be packaged along with ray.
-pushd .
-cd $TP_DIR/arrow/python/build
-PYARROW_BUILD_LIB_DIR="$TP_DIR/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')"
+pushd $TP_DIR/arrow/python/build
+  PYARROW_BUILD_LIB_DIR="$TP_DIR/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')"
 popd
 echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow"
 cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../../python/ray/pyarrow_files/

src/thirdparty/download_thirdparty.sh

@@ -10,14 +10,15 @@ TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 if [ ! -d $TP_DIR/arrow ]; then
   git clone https://github.com/apache/arrow.git "$TP_DIR/arrow"
 fi
-cd $TP_DIR/arrow
-git fetch origin master
-git checkout e26f3dad3675288564ef0c0330a5c9afcac652f1
+pushd $TP_DIR/arrow
+  git fetch origin master
+  git checkout e26f3dad3675288564ef0c0330a5c9afcac652f1
+popd
 
-cd $TP_DIR
 if [ ! -d $TP_DIR/parquet-cpp ]; then
   git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/parquet-cpp"
-  pushd $TP_DIR/parquet-cpp
-    git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b
-  popd
 fi
+pushd $TP_DIR/parquet-cpp
+  git fetch origin master
+  git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b
+popd