From bbea73155afcdf3e95e64735ad267795a9ca7027 Mon Sep 17 00:00:00 2001 From: Yuhong Guo Date: Sun, 15 Jul 2018 07:08:13 +0800 Subject: [PATCH] Fix parquet missing error and improve arrow commit id changing (#2319) * Fix parquet missing error and improve arrow commit id changing * Remove build cache for arrow. * Update build_parquet.sh * Update build_ui.sh * Update build_arrow.sh --- thirdparty/scripts/build_arrow.sh | 208 +++++++++++++++------------- thirdparty/scripts/build_credis.sh | 6 +- thirdparty/scripts/build_parquet.sh | 28 +++- thirdparty/scripts/build_ui.sh | 18 ++- 4 files changed, 150 insertions(+), 110 deletions(-) diff --git a/thirdparty/scripts/build_arrow.sh b/thirdparty/scripts/build_arrow.sh index ef3209d6f..7a4437f4a 100755 --- a/thirdparty/scripts/build_arrow.sh +++ b/thirdparty/scripts/build_arrow.sh @@ -40,106 +40,122 @@ else exit 1 fi -# Download and compile arrow if it isn't already present. -if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow || \ - "$LANGUAGE" == "java" && ! -f $TP_DIR/../build/src/plasma/libplasma_java.dylib ]]; then - echo "building arrow" +# The PR for this commit is https://github.com/apache/arrow/pull/2235. We +# include the link here to make it easier to find the right commit because +# Arrow often rewrites git history and invalidates certain commits. +TARGET_COMMIT_ID=fa08ddfbe8ada173b2c621172e176f159ce3b728 +build_arrow() { + echo "building arrow" - # Make sure arrow will be built again when building ray for java later than python - if [[ "$LANGUAGE" == "java" ]]; then - rm -rf $TP_DIR/build/arrow - rm -rf $TP_DIR/build/parquet-cpp - rm -rf $TP_DIR/pkg/arrow - fi + # Make sure arrow will be built again when building ray for java later than python + if [[ "$LANGUAGE" == "java" ]]; then + rm -rf $TP_DIR/build/arrow/cpp/build/CMakeCache.txt + fi - if [[ ! -d $TP_DIR/build/arrow ]]; then - git clone https://github.com/apache/arrow.git "$TP_DIR/build/arrow" - fi + if [[ ! -d $TP_DIR/build/arrow ]]; then + git clone https://github.com/apache/arrow.git "$TP_DIR/build/arrow" + fi - if ! [ -x "$(command -v bison)" ]; then - echo 'Error: bison is not installed.' >&2 - exit 1 - fi + if ! [ -x "$(command -v bison)" ]; then + echo 'Error: bison is not installed.' >&2 + exit 1 + fi - if ! [ -x "$(command -v flex)" ]; then - echo 'Error: flex is not installed.' >&2 - exit 1 - fi + if ! [ -x "$(command -v flex)" ]; then + echo 'Error: flex is not installed.' >&2 + exit 1 + fi - pushd $TP_DIR/build/arrow + pushd $TP_DIR/build/arrow + git fetch origin master + + git checkout $TARGET_COMMIT_ID + + cd cpp + if [ ! -d "build" ]; then + mkdir build + fi + cd build + + BUILD_ARROW_PLASMA_JAVA_CLIENT=off + if [[ "$LANGUAGE" == "java" ]]; then + BUILD_ARROW_PLASMA_JAVA_CLIENT=on + fi + + # Clean the build cache for arrow and parquet, in case the error of "Cannot find Parquet" occurs. + rm -rf $TP_DIR/build/arrow/python/build/temp* + + ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install + BOOST_ROOT=$TP_DIR/pkg/boost \ + FLATBUFFERS_HOME=$FLATBUFFERS_HOME \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="-g -O3" \ + -DCMAKE_CXX_FLAGS="-g -O3" \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DARROW_BUILD_TESTS=off \ + -DARROW_HDFS=on \ + -DARROW_BOOST_USE_SHARED=off \ + -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \ + -DARROW_PYTHON=on \ + -DARROW_PLASMA=on \ + -DPLASMA_PYTHON=on \ + -DARROW_JEMALLOC=off \ + -DARROW_WITH_BROTLI=off \ + -DARROW_WITH_LZ4=off \ + -DARROW_WITH_ZLIB=off \ + -DARROW_WITH_ZSTD=off \ + -DARROW_PLASMA_JAVA_CLIENT=$BUILD_ARROW_PLASMA_JAVA_CLIENT \ + .. + make VERBOSE=1 -j$PARALLEL + make install + + if [[ -d $ARROW_HOME/lib64 ]]; then + # On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for + # now. TODO(rkn): A preferable solution would be to add both directories to + # the PKG_CONFIG_PATH, but that didn't seem to work. + cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib + fi + + bash "$TP_DIR/scripts/build_parquet.sh" + + echo "installing pyarrow" + cd $TP_DIR/build/arrow/python + # We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can + # find plasma. + PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ + PYARROW_WITH_PLASMA=1 \ + PYARROW_BUNDLE_ARROW_CPP=1 \ + $PYTHON_EXECUTABLE setup.py build + + PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ + PYARROW_WITH_PLASMA=1 \ + PYARROW_BUNDLE_ARROW_CPP=1 \ + PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \ + PYARROW_WITH_PARQUET=1 \ + $PYTHON_EXECUTABLE setup.py build_ext + + # Find the pyarrow directory that was just built and copy it to ray/python/ray/ + # so that pyarrow can be packaged along with ray. + pushd $TP_DIR/build/arrow/python/build + PYARROW_BUILD_LIB_DIR="$TP_DIR/build/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')" + popd + echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow" + cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../python/ray/pyarrow_files/ + + popd +} +# Download and compile arrow if it isn't already present or the commit-id mismatches. +if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]] || \ + [[ "$LANGUAGE" == "java" && ! -f $TP_DIR/build/arrow/cpp/build/release/libplasma_java.dylib ]]; then + build_arrow +else + pushd $TP_DIR/build/arrow + if [[ "$TARGET_COMMIT_ID" != `git rev-parse HEAD` ]]; then + # TARGET_COMMIT_ID may change to later commit. + echo "Commit ID mismatches." git fetch origin master - # The PR for this commit is https://github.com/apache/arrow/pull/2235. We - # include the link here to make it easier to find the right commit because - # Arrow often rewrites git history and invalidates certain commits. - git checkout fa08ddfbe8ada173b2c621172e176f159ce3b728 - - cd cpp - if [ ! -d "build" ]; then - mkdir build - fi - cd build - - BUILD_ARROW_PLASMA_JAVA_CLIENT=off - if [[ "$LANGUAGE" == "java" ]]; then - BUILD_ARROW_PLASMA_JAVA_CLIENT=on - fi - - ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install - BOOST_ROOT=$TP_DIR/pkg/boost \ - FLATBUFFERS_HOME=$FLATBUFFERS_HOME \ - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_FLAGS="-g -O3" \ - -DCMAKE_CXX_FLAGS="-g -O3" \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DARROW_BUILD_TESTS=off \ - -DARROW_HDFS=on \ - -DARROW_BOOST_USE_SHARED=off \ - -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \ - -DARROW_PYTHON=on \ - -DARROW_PLASMA=on \ - -DPLASMA_PYTHON=on \ - -DARROW_JEMALLOC=off \ - -DARROW_WITH_BROTLI=off \ - -DARROW_WITH_LZ4=off \ - -DARROW_WITH_ZLIB=off \ - -DARROW_WITH_ZSTD=off \ - -DARROW_PLASMA_JAVA_CLIENT=$BUILD_ARROW_PLASMA_JAVA_CLIENT \ - .. - make VERBOSE=1 -j$PARALLEL - make install - - if [[ -d $ARROW_HOME/lib64 ]]; then - # On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for - # now. TODO(rkn): A preferable solution would be to add both directories to - # the PKG_CONFIG_PATH, but that didn't seem to work. - cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib - fi - - bash "$TP_DIR/scripts/build_parquet.sh" - - echo "installing pyarrow" - cd $TP_DIR/build/arrow/python - # We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can - # find plasma. - PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ - PYARROW_WITH_PLASMA=1 \ - PYARROW_BUNDLE_ARROW_CPP=1 \ - $PYTHON_EXECUTABLE setup.py build - - PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ - PYARROW_WITH_PLASMA=1 \ - PYARROW_BUNDLE_ARROW_CPP=1 \ - PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \ - PYARROW_WITH_PARQUET=1 \ - $PYTHON_EXECUTABLE setup.py build_ext - - # Find the pyarrow directory that was just built and copy it to ray/python/ray/ - # so that pyarrow can be packaged along with ray. - pushd $TP_DIR/build/arrow/python/build - PYARROW_BUILD_LIB_DIR="$TP_DIR/build/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')" - popd - echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow" - cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../python/ray/pyarrow_files/ - - popd + git checkout $TARGET_COMMIT_ID + fi + popd + build_arrow fi diff --git a/thirdparty/scripts/build_credis.sh b/thirdparty/scripts/build_credis.sh index dc394fb11..004252610 100644 --- a/thirdparty/scripts/build_credis.sh +++ b/thirdparty/scripts/build_credis.sh @@ -28,11 +28,13 @@ fi if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then pushd "$TP_DIR/pkg/" - rm -rf credis - git clone --recursive https://github.com/ray-project/credis + if [[ ! -d "credis" ]]; then + git clone --recursive https://github.com/ray-project/credis + fi popd pushd "$TP_DIR/pkg/credis" + git fetch origin master git checkout 273d667e5126c246b45f5dcf030b651a653136c3 # If the above commit points to different submodules' commits than diff --git a/thirdparty/scripts/build_parquet.sh b/thirdparty/scripts/build_parquet.sh index e81213024..304aca3be 100755 --- a/thirdparty/scripts/build_parquet.sh +++ b/thirdparty/scripts/build_parquet.sh @@ -10,13 +10,10 @@ TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../ PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install OPENSSL_DIR=/usr/local/opt/openssl BISON_DIR=/usr/local/opt/bison/bin +TARGET_COMMIT_ID=0875e43010af485e1c0b506d77d7e0edc80c66cc -if [ ! -d $TP_DIR/build/parquet-cpp ]; then - git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/build/parquet-cpp" - pushd $TP_DIR/build/parquet-cpp - git fetch origin master - git checkout 0875e43010af485e1c0b506d77d7e0edc80c66cc - +build_parquet() { + echo "Building Parquet" if [ "$unamestr" == "Darwin" ]; then OPENSSL_ROOT_DIR=$OPENSSL_DIR \ PATH="$BISON_DIR:$PATH" \ @@ -51,6 +48,25 @@ if [ ! -d $TP_DIR/build/parquet-cpp ]; then make -j4 make install fi +} +if [ ! -d $TP_DIR/build/parquet-cpp ]; then + git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/build/parquet-cpp" + pushd $TP_DIR/build/parquet-cpp + git fetch origin master + git checkout $TARGET_COMMIT_ID + + build_parquet + popd +else + pushd $TP_DIR/build/parquet-cpp + if [[ "$TARGET_COMMIT_ID" != `git rev-parse HEAD` ]]; then + # TARGET_COMMIT_ID may change to later commit. + echo "Commit ID mismatches." + git fetch origin master + git checkout $TARGET_COMMIT_ID + fi + + build_parquet popd fi diff --git a/thirdparty/scripts/build_ui.sh b/thirdparty/scripts/build_ui.sh index aa2530d89..82e3d7802 100755 --- a/thirdparty/scripts/build_ui.sh +++ b/thirdparty/scripts/build_ui.sh @@ -45,15 +45,21 @@ if [[ ! -d $CATAPULT_HOME ]]; then exit 1 fi done - - # Check out the appropriate commit from catapult. - pushd $CATAPULT_HOME - git checkout $CATAPULT_COMMIT - popd fi +REBUILD=off + +# Check out the appropriate commit from catapult. +pushd $CATAPULT_HOME +if [ "$CATAPULT_COMMIT" != `git rev-parse HEAD` ]; then + git fetch origin master + git checkout $CATAPULT_COMMIT + REBUILD=on +fi +popd + # If the autogenerated catapult files aren't present, then generate them. -if [[ ! -f $CATAPULT_FILES/index.html ]]; then +if [[ ! -f $CATAPULT_FILES/index.html || "$REBUILD" == "on" ]]; then python2 $VULCANIZE_BIN --config chrome --output $CATAPULT_FILES/trace_viewer_full.html cp $CATAPULT_HOME/tracing/bin/index.html $CATAPULT_FILES/index.html fi