mirror of
https://github.com/wassname/ray.git
synced 2026-08-03 13:10:57 +08:00
Robert NishiharaandPhilipp Moritz
03f2325780
* Rough pass at installing pyarrow along with Ray. * Remove hardcoded path and try to find correct path automatically. * Add print. * Fix linting. * Copy pyarrow files to a location that we manually add to python path in order to avoid interfering with pre-existing pyarrow installations. * Move call to build.sh back into build_ext in setup.py. * Ignore some linting errors. * Fix problem in which pyarrow files to copy were listed before they were built. * Fix tests by importing ray before pyarrow.
73 lines
2.3 KiB
Bash
Executable File
73 lines
2.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
set -x
|
|
|
|
# Cause the script to exit if a single command fails.
|
|
set -e
|
|
|
|
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
|
|
|
|
if [[ -z "$1" ]]; then
|
|
PYTHON_EXECUTABLE=`which python`
|
|
else
|
|
PYTHON_EXECUTABLE=$1
|
|
fi
|
|
echo "Using Python executable $PYTHON_EXECUTABLE."
|
|
|
|
# Determine how many parallel jobs to use for make based on the number of cores
|
|
unamestr="$(uname)"
|
|
if [[ "$unamestr" == "Linux" ]]; then
|
|
PARALLEL=$(nproc)
|
|
elif [[ "$unamestr" == "Darwin" ]]; then
|
|
PARALLEL=$(sysctl -n hw.ncpu)
|
|
echo "Platform is macosx."
|
|
else
|
|
echo "Unrecognized platform."
|
|
exit 1
|
|
fi
|
|
|
|
echo "building arrow"
|
|
cd $TP_DIR/arrow/cpp
|
|
mkdir -p $TP_DIR/arrow/cpp/build
|
|
cd $TP_DIR/arrow/cpp/build
|
|
export ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
|
|
|
|
# Get the directory of the Python executable.
|
|
PYTHON_EXECUTABLE_DIR=$(dirname $PYTHON_EXECUTABLE)
|
|
|
|
# Pass a slightly different path into this command so that cmake finds the right
|
|
# Python interpreter and libraries.
|
|
PATH=$PYTHON_EXECUTABLE_DIR:$PATH \
|
|
cmake -DCMAKE_BUILD_TYPE=Release \
|
|
-DCMAKE_C_FLAGS="-g -O3" \
|
|
-DCMAKE_CXX_FLAGS="-g -O3" \
|
|
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
|
|
-DARROW_BUILD_TESTS=off \
|
|
-DARROW_HDFS=on \
|
|
-DARROW_PYTHON=on \
|
|
-DARROW_PLASMA=on \
|
|
-DPLASMA_PYTHON=on \
|
|
-DARROW_JEMALLOC=off \
|
|
-DARROW_WITH_BROTLI=off \
|
|
-DARROW_WITH_LZ4=off \
|
|
-DARROW_WITH_SNAPPY=off \
|
|
-DARROW_WITH_ZLIB=off \
|
|
-DARROW_WITH_ZSTD=off \
|
|
..
|
|
make VERBOSE=1 -j$PARALLEL
|
|
make install
|
|
|
|
echo "installing pyarrow"
|
|
cd $TP_DIR/arrow/python
|
|
# We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
|
|
# find plasma.
|
|
ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
|
|
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig PYARROW_WITH_PLASMA=1 PYARROW_BUNDLE_ARROW_CPP=1 $PYTHON_EXECUTABLE setup.py build
|
|
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig PYARROW_WITH_PLASMA=1 PYARROW_BUNDLE_ARROW_CPP=1 $PYTHON_EXECUTABLE setup.py build_ext
|
|
# Find the pyarrow directory that was just built and copy it to ray/python/ray/
|
|
# so that pyarrow can be packaged along with ray. TODO(rkn): This doesn't seem
|
|
# very robust. Fix this.
|
|
PYARROW_BUILD_LIB_DIR=$(find $TP_DIR/arrow/python/build -type d -maxdepth 1 -print | grep -m1 'lib')
|
|
echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow"
|
|
cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../../python/ray/pyarrow_files/
|