mirror of
https://github.com/wassname/ray.git
synced 2026-07-13 17:45:08 +08:00
Fix parquet missing error and improve arrow commit id changing (#2319)
* Fix parquet missing error and improve arrow commit id changing * Remove build cache for arrow. * Update build_parquet.sh * Update build_ui.sh * Update build_arrow.sh
This commit is contained in:
Yuhong Guo
committed by
Philipp Moritz
Philipp Moritz
parent
7865dbab84
commit
bbea73155a
Vendored
+112
-96
@@ -40,106 +40,122 @@ else
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Download and compile arrow if it isn't already present.
|
||||
if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow || \
|
||||
"$LANGUAGE" == "java" && ! -f $TP_DIR/../build/src/plasma/libplasma_java.dylib ]]; then
|
||||
echo "building arrow"
|
||||
# The PR for this commit is https://github.com/apache/arrow/pull/2235. We
|
||||
# include the link here to make it easier to find the right commit because
|
||||
# Arrow often rewrites git history and invalidates certain commits.
|
||||
TARGET_COMMIT_ID=fa08ddfbe8ada173b2c621172e176f159ce3b728
|
||||
build_arrow() {
|
||||
echo "building arrow"
|
||||
|
||||
# Make sure arrow will be built again when building ray for java later than python
|
||||
if [[ "$LANGUAGE" == "java" ]]; then
|
||||
rm -rf $TP_DIR/build/arrow
|
||||
rm -rf $TP_DIR/build/parquet-cpp
|
||||
rm -rf $TP_DIR/pkg/arrow
|
||||
fi
|
||||
# Make sure arrow will be built again when building ray for java later than python
|
||||
if [[ "$LANGUAGE" == "java" ]]; then
|
||||
rm -rf $TP_DIR/build/arrow/cpp/build/CMakeCache.txt
|
||||
fi
|
||||
|
||||
if [[ ! -d $TP_DIR/build/arrow ]]; then
|
||||
git clone https://github.com/apache/arrow.git "$TP_DIR/build/arrow"
|
||||
fi
|
||||
if [[ ! -d $TP_DIR/build/arrow ]]; then
|
||||
git clone https://github.com/apache/arrow.git "$TP_DIR/build/arrow"
|
||||
fi
|
||||
|
||||
if ! [ -x "$(command -v bison)" ]; then
|
||||
echo 'Error: bison is not installed.' >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! [ -x "$(command -v bison)" ]; then
|
||||
echo 'Error: bison is not installed.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! [ -x "$(command -v flex)" ]; then
|
||||
echo 'Error: flex is not installed.' >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! [ -x "$(command -v flex)" ]; then
|
||||
echo 'Error: flex is not installed.' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
pushd $TP_DIR/build/arrow
|
||||
pushd $TP_DIR/build/arrow
|
||||
git fetch origin master
|
||||
|
||||
git checkout $TARGET_COMMIT_ID
|
||||
|
||||
cd cpp
|
||||
if [ ! -d "build" ]; then
|
||||
mkdir build
|
||||
fi
|
||||
cd build
|
||||
|
||||
BUILD_ARROW_PLASMA_JAVA_CLIENT=off
|
||||
if [[ "$LANGUAGE" == "java" ]]; then
|
||||
BUILD_ARROW_PLASMA_JAVA_CLIENT=on
|
||||
fi
|
||||
|
||||
# Clean the build cache for arrow and parquet, in case the error of "Cannot find Parquet" occurs.
|
||||
rm -rf $TP_DIR/build/arrow/python/build/temp*
|
||||
|
||||
ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install
|
||||
BOOST_ROOT=$TP_DIR/pkg/boost \
|
||||
FLATBUFFERS_HOME=$FLATBUFFERS_HOME \
|
||||
cmake -DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_FLAGS="-g -O3" \
|
||||
-DCMAKE_CXX_FLAGS="-g -O3" \
|
||||
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
|
||||
-DARROW_BUILD_TESTS=off \
|
||||
-DARROW_HDFS=on \
|
||||
-DARROW_BOOST_USE_SHARED=off \
|
||||
-DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
|
||||
-DARROW_PYTHON=on \
|
||||
-DARROW_PLASMA=on \
|
||||
-DPLASMA_PYTHON=on \
|
||||
-DARROW_JEMALLOC=off \
|
||||
-DARROW_WITH_BROTLI=off \
|
||||
-DARROW_WITH_LZ4=off \
|
||||
-DARROW_WITH_ZLIB=off \
|
||||
-DARROW_WITH_ZSTD=off \
|
||||
-DARROW_PLASMA_JAVA_CLIENT=$BUILD_ARROW_PLASMA_JAVA_CLIENT \
|
||||
..
|
||||
make VERBOSE=1 -j$PARALLEL
|
||||
make install
|
||||
|
||||
if [[ -d $ARROW_HOME/lib64 ]]; then
|
||||
# On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for
|
||||
# now. TODO(rkn): A preferable solution would be to add both directories to
|
||||
# the PKG_CONFIG_PATH, but that didn't seem to work.
|
||||
cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib
|
||||
fi
|
||||
|
||||
bash "$TP_DIR/scripts/build_parquet.sh"
|
||||
|
||||
echo "installing pyarrow"
|
||||
cd $TP_DIR/build/arrow/python
|
||||
# We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
|
||||
# find plasma.
|
||||
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
|
||||
PYARROW_WITH_PLASMA=1 \
|
||||
PYARROW_BUNDLE_ARROW_CPP=1 \
|
||||
$PYTHON_EXECUTABLE setup.py build
|
||||
|
||||
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
|
||||
PYARROW_WITH_PLASMA=1 \
|
||||
PYARROW_BUNDLE_ARROW_CPP=1 \
|
||||
PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
|
||||
PYARROW_WITH_PARQUET=1 \
|
||||
$PYTHON_EXECUTABLE setup.py build_ext
|
||||
|
||||
# Find the pyarrow directory that was just built and copy it to ray/python/ray/
|
||||
# so that pyarrow can be packaged along with ray.
|
||||
pushd $TP_DIR/build/arrow/python/build
|
||||
PYARROW_BUILD_LIB_DIR="$TP_DIR/build/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')"
|
||||
popd
|
||||
echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow"
|
||||
cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../python/ray/pyarrow_files/
|
||||
|
||||
popd
|
||||
}
|
||||
# Download and compile arrow if it isn't already present or the commit-id mismatches.
|
||||
if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]] || \
|
||||
[[ "$LANGUAGE" == "java" && ! -f $TP_DIR/build/arrow/cpp/build/release/libplasma_java.dylib ]]; then
|
||||
build_arrow
|
||||
else
|
||||
pushd $TP_DIR/build/arrow
|
||||
if [[ "$TARGET_COMMIT_ID" != `git rev-parse HEAD` ]]; then
|
||||
# TARGET_COMMIT_ID may change to later commit.
|
||||
echo "Commit ID mismatches."
|
||||
git fetch origin master
|
||||
# The PR for this commit is https://github.com/apache/arrow/pull/2235. We
|
||||
# include the link here to make it easier to find the right commit because
|
||||
# Arrow often rewrites git history and invalidates certain commits.
|
||||
git checkout fa08ddfbe8ada173b2c621172e176f159ce3b728
|
||||
|
||||
cd cpp
|
||||
if [ ! -d "build" ]; then
|
||||
mkdir build
|
||||
fi
|
||||
cd build
|
||||
|
||||
BUILD_ARROW_PLASMA_JAVA_CLIENT=off
|
||||
if [[ "$LANGUAGE" == "java" ]]; then
|
||||
BUILD_ARROW_PLASMA_JAVA_CLIENT=on
|
||||
fi
|
||||
|
||||
ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install
|
||||
BOOST_ROOT=$TP_DIR/pkg/boost \
|
||||
FLATBUFFERS_HOME=$FLATBUFFERS_HOME \
|
||||
cmake -DCMAKE_BUILD_TYPE=Release \
|
||||
-DCMAKE_C_FLAGS="-g -O3" \
|
||||
-DCMAKE_CXX_FLAGS="-g -O3" \
|
||||
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
|
||||
-DARROW_BUILD_TESTS=off \
|
||||
-DARROW_HDFS=on \
|
||||
-DARROW_BOOST_USE_SHARED=off \
|
||||
-DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
|
||||
-DARROW_PYTHON=on \
|
||||
-DARROW_PLASMA=on \
|
||||
-DPLASMA_PYTHON=on \
|
||||
-DARROW_JEMALLOC=off \
|
||||
-DARROW_WITH_BROTLI=off \
|
||||
-DARROW_WITH_LZ4=off \
|
||||
-DARROW_WITH_ZLIB=off \
|
||||
-DARROW_WITH_ZSTD=off \
|
||||
-DARROW_PLASMA_JAVA_CLIENT=$BUILD_ARROW_PLASMA_JAVA_CLIENT \
|
||||
..
|
||||
make VERBOSE=1 -j$PARALLEL
|
||||
make install
|
||||
|
||||
if [[ -d $ARROW_HOME/lib64 ]]; then
|
||||
# On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for
|
||||
# now. TODO(rkn): A preferable solution would be to add both directories to
|
||||
# the PKG_CONFIG_PATH, but that didn't seem to work.
|
||||
cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib
|
||||
fi
|
||||
|
||||
bash "$TP_DIR/scripts/build_parquet.sh"
|
||||
|
||||
echo "installing pyarrow"
|
||||
cd $TP_DIR/build/arrow/python
|
||||
# We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
|
||||
# find plasma.
|
||||
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
|
||||
PYARROW_WITH_PLASMA=1 \
|
||||
PYARROW_BUNDLE_ARROW_CPP=1 \
|
||||
$PYTHON_EXECUTABLE setup.py build
|
||||
|
||||
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
|
||||
PYARROW_WITH_PLASMA=1 \
|
||||
PYARROW_BUNDLE_ARROW_CPP=1 \
|
||||
PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
|
||||
PYARROW_WITH_PARQUET=1 \
|
||||
$PYTHON_EXECUTABLE setup.py build_ext
|
||||
|
||||
# Find the pyarrow directory that was just built and copy it to ray/python/ray/
|
||||
# so that pyarrow can be packaged along with ray.
|
||||
pushd $TP_DIR/build/arrow/python/build
|
||||
PYARROW_BUILD_LIB_DIR="$TP_DIR/build/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')"
|
||||
popd
|
||||
echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow"
|
||||
cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../python/ray/pyarrow_files/
|
||||
|
||||
popd
|
||||
git checkout $TARGET_COMMIT_ID
|
||||
fi
|
||||
popd
|
||||
build_arrow
|
||||
fi
|
||||
|
||||
Vendored
+4
-2
@@ -28,11 +28,13 @@ fi
|
||||
|
||||
if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
|
||||
pushd "$TP_DIR/pkg/"
|
||||
rm -rf credis
|
||||
git clone --recursive https://github.com/ray-project/credis
|
||||
if [[ ! -d "credis" ]]; then
|
||||
git clone --recursive https://github.com/ray-project/credis
|
||||
fi
|
||||
popd
|
||||
|
||||
pushd "$TP_DIR/pkg/credis"
|
||||
git fetch origin master
|
||||
git checkout 273d667e5126c246b45f5dcf030b651a653136c3
|
||||
|
||||
# If the above commit points to different submodules' commits than
|
||||
|
||||
Vendored
+22
-6
@@ -10,13 +10,10 @@ TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
|
||||
PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install
|
||||
OPENSSL_DIR=/usr/local/opt/openssl
|
||||
BISON_DIR=/usr/local/opt/bison/bin
|
||||
TARGET_COMMIT_ID=0875e43010af485e1c0b506d77d7e0edc80c66cc
|
||||
|
||||
if [ ! -d $TP_DIR/build/parquet-cpp ]; then
|
||||
git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/build/parquet-cpp"
|
||||
pushd $TP_DIR/build/parquet-cpp
|
||||
git fetch origin master
|
||||
git checkout 0875e43010af485e1c0b506d77d7e0edc80c66cc
|
||||
|
||||
build_parquet() {
|
||||
echo "Building Parquet"
|
||||
if [ "$unamestr" == "Darwin" ]; then
|
||||
OPENSSL_ROOT_DIR=$OPENSSL_DIR \
|
||||
PATH="$BISON_DIR:$PATH" \
|
||||
@@ -51,6 +48,25 @@ if [ ! -d $TP_DIR/build/parquet-cpp ]; then
|
||||
make -j4
|
||||
make install
|
||||
fi
|
||||
}
|
||||
|
||||
if [ ! -d $TP_DIR/build/parquet-cpp ]; then
|
||||
git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/build/parquet-cpp"
|
||||
pushd $TP_DIR/build/parquet-cpp
|
||||
git fetch origin master
|
||||
git checkout $TARGET_COMMIT_ID
|
||||
|
||||
build_parquet
|
||||
popd
|
||||
else
|
||||
pushd $TP_DIR/build/parquet-cpp
|
||||
if [[ "$TARGET_COMMIT_ID" != `git rev-parse HEAD` ]]; then
|
||||
# TARGET_COMMIT_ID may change to later commit.
|
||||
echo "Commit ID mismatches."
|
||||
git fetch origin master
|
||||
git checkout $TARGET_COMMIT_ID
|
||||
fi
|
||||
|
||||
build_parquet
|
||||
popd
|
||||
fi
|
||||
|
||||
Vendored
+12
-6
@@ -45,15 +45,21 @@ if [[ ! -d $CATAPULT_HOME ]]; then
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Check out the appropriate commit from catapult.
|
||||
pushd $CATAPULT_HOME
|
||||
git checkout $CATAPULT_COMMIT
|
||||
popd
|
||||
fi
|
||||
|
||||
REBUILD=off
|
||||
|
||||
# Check out the appropriate commit from catapult.
|
||||
pushd $CATAPULT_HOME
|
||||
if [ "$CATAPULT_COMMIT" != `git rev-parse HEAD` ]; then
|
||||
git fetch origin master
|
||||
git checkout $CATAPULT_COMMIT
|
||||
REBUILD=on
|
||||
fi
|
||||
popd
|
||||
|
||||
# If the autogenerated catapult files aren't present, then generate them.
|
||||
if [[ ! -f $CATAPULT_FILES/index.html ]]; then
|
||||
if [[ ! -f $CATAPULT_FILES/index.html || "$REBUILD" == "on" ]]; then
|
||||
python2 $VULCANIZE_BIN --config chrome --output $CATAPULT_FILES/trace_viewer_full.html
|
||||
cp $CATAPULT_HOME/tracing/bin/index.html $CATAPULT_FILES/index.html
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user