wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-12 12:20:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

restructure how to organize 3rd party libs (#1630)

Browse Source 
* restructure how to organize 3rd party libs

* Minor whitespace changes.

* Fix compilation on Linux.

* Pass around Python executable so that the correct version of Python is used.
This commit is contained in:
Zhenyu Guo
2018-03-01 14:29:56 -08:00
committed by Robert Nishihara
parent ec9dfe7748
commit f1e5789c26
16 changed files with 283 additions and 225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+1 -1
View File
@@ -4,7 +4,7 @@ project(ray)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules")
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/src/thirdparty/arrow/python/cmake_modules)
include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/scripts/thirdparty.cmake)
find_package(Arrow)
find_package(Plasma)
build.sh
+7 -13
View File
@@ -14,6 +14,8 @@ else
fi
echo "Using Python executable $PYTHON_EXECUTABLE."
bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE
# Determine how many parallel jobs to use for make based on the number of cores
unamestr="$(uname)"
if [[ "$unamestr" == "Linux" ]]; then
@@ -25,28 +27,21 @@ else
exit 1
fi
pushd "$ROOT_DIR/src/common/thirdparty/"
bash build-redis.sh
popd
bash "$ROOT_DIR/src/thirdparty/download_thirdparty.sh"
bash "$ROOT_DIR/src/thirdparty/build_thirdparty.sh" $PYTHON_EXECUTABLE
# Now we build everything.
pushd "$ROOT_DIR/python/ray/core"
# We use these variables to set PKG_CONFIG_PATH, which is important so that
# in cmake, pkg-config can find plasma.
TP_DIR=$ROOT_DIR/src/thirdparty
ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
TP_PKG_DIR=$ROOT_DIR/thirdparty/pkg
ARROW_HOME=$TP_PKG_DIR/arrow/cpp/build/cpp-install
if [[ "$VALGRIND" = "1" ]]; then
BOOST_ROOT=$TP_DIR/boost \
BOOST_ROOT=$TP_PKG_DIR/boost \
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
cmake -DCMAKE_BUILD_TYPE=Debug \
-DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
-DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
../../..
else
BOOST_ROOT=$TP_DIR/boost \
BOOST_ROOT=$TP_PKG_DIR/boost \
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
cmake -DCMAKE_BUILD_TYPE=Release \
-DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
@@ -58,5 +53,4 @@ pushd "$ROOT_DIR/python/ray/core"
popd
# Move stuff from Arrow to Ray.
mv $ROOT_DIR/src/thirdparty/arrow/cpp/build/release/plasma_store $ROOT_DIR/python/ray/core/src/plasma/
cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $ROOT_DIR/python/ray/core/src/plasma/
setup_thirdparty.sh
Executable
+16
View File
@@ -0,0 +1,16 @@
#!/bin/bash
set -x
# Cause the script to exit if a single command fails.
set -e
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
if [[ -z "$1" ]]; then
PYTHON_EXECUTABLE=`which python`
else
PYTHON_EXECUTABLE=$1
fi
echo "Using Python executable $PYTHON_EXECUTABLE."
$TP_DIR/thirdparty/scripts/setup.sh $PYTHON_EXECUTABLE
src/common/CMakeLists.txt
+1 -7
View File
@@ -89,16 +89,10 @@ define_test(redis_tests "")
define_test(task_table_tests "")
define_test(object_table_tests "")
add_custom_target(
redis
COMMAND make
WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/thirdparty/redis)
add_custom_target(copy_redis ALL)
foreach(file "redis-cli" "redis-server")
add_custom_command(TARGET copy_redis POST_BUILD
COMMAND ${CMAKE_COMMAND} -E
copy ${CMAKE_CURRENT_LIST_DIR}/thirdparty/redis/src/${file}
copy ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/redis/src/${file}
${CMAKE_BINARY_DIR}/src/common/thirdparty/redis/src/${file})
endforeach()
add_dependencies(copy_redis redis)
src/local_scheduler/CMakeLists.txt
+1 -1
View File
@@ -6,7 +6,7 @@ project(local_scheduler)
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
# Include plasma
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../thirdparty/arrow/python/cmake_modules)
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/arrow/python/cmake_modules)
find_package(Arrow)
find_package(Plasma)
src/thirdparty/build_parquet.sh
Vendored
-51
View File
@@ -1,51 +0,0 @@
#!/bin/bash
set -x
# Cause the script to exit if a single command fails.
set -e
unamestr="$(uname)"
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
cd $TP_DIR/parquet-cpp
PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install
OPENSSL_DIR=/usr/local/opt/openssl
BISON_DIR=/usr/local/opt/bison/bin
if [ "$unamestr" == "Darwin" ]; then
OPENSSL_ROOT_DIR=$OPENSSL_DIR \
PATH="$BISON_DIR:$PATH" \
ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install \
cmake -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
-DPARQUET_BUILD_BENCHMARKS=off \
-DPARQUET_BUILD_EXECUTABLES=off \
-DPARQUET_BUILD_TESTS=off \
.
OPENSSL_ROOT_DIR=$OPENSSL_DIR \
PATH="$BISON_DIR:$PATH" \
make -j4
OPENSSL_ROOT_DIR=$OPENSSL_DIR \
PATH="$BISON_DIR:$PATH" \
make install
else
BOOST_ROOT=$TP_DIR/boost \
ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install \
cmake -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
-DPARQUET_BUILD_BENCHMARKS=off \
-DPARQUET_BUILD_EXECUTABLES=off \
-DPARQUET_BUILD_TESTS=off \
.
PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install \
BOOST_ROOT=$TP_DIR/boost \
make -j4
make install
fi
src/thirdparty/build_thirdparty.sh
Vendored
-107
View File
@@ -1,107 +0,0 @@
#!/bin/bash
set -x
# Cause the script to exit if a single command fails.
set -e
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
if [[ -z "$1" ]]; then
PYTHON_EXECUTABLE=`which python`
else
PYTHON_EXECUTABLE=$1
fi
echo "Using Python executable $PYTHON_EXECUTABLE."
# Determine how many parallel jobs to use for make based on the number of cores
unamestr="$(uname)"
if [[ "$unamestr" == "Linux" ]]; then
PARALLEL=$(nproc)
elif [[ "$unamestr" == "Darwin" ]]; then
PARALLEL=$(sysctl -n hw.ncpu)
echo "Platform is macosx."
else
echo "Unrecognized platform."
exit 1
fi
# If we're on Linux, then compile boost. This installs boost to $TP_DIR/boost.
if [[ "$unamestr" == "Linux" ]]; then
echo "building boost"
bash "$TP_DIR/build_boost.sh"
fi
# If we're on Linux, then compile flatbuffers. This installs flatbuffers to
# $TP_DIR/flatbuffers.
if [[ "$unamestr" == "Linux" ]]; then
echo "building flatbuffers"
bash "$TP_DIR/build_flatbuffers.sh"
FLATBUFFERS_HOME=$TP_DIR/flatbuffers
else
FLATBUFFERS_HOME=""
fi
# Clone catapult and build the static HTML needed for the UI.
bash "$TP_DIR/build_ui.sh"
echo "building arrow"
cd $TP_DIR/arrow/cpp
mkdir -p $TP_DIR/arrow/cpp/build
cd $TP_DIR/arrow/cpp/build
ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install
BOOST_ROOT=$TP_DIR/boost \
FLATBUFFERS_HOME=$FLATBUFFERS_HOME \
cmake -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_FLAGS="-g -O3" \
-DCMAKE_CXX_FLAGS="-g -O3" \
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-DARROW_BUILD_TESTS=off \
-DARROW_HDFS=on \
-DARROW_BOOST_USE_SHARED=off \
-DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
-DARROW_PYTHON=on \
-DARROW_PLASMA=on \
-DPLASMA_PYTHON=on \
-DARROW_JEMALLOC=off \
-DARROW_WITH_BROTLI=off \
-DARROW_WITH_LZ4=off \
-DARROW_WITH_ZLIB=off \
-DARROW_WITH_ZSTD=off \
..
make VERBOSE=1 -j$PARALLEL
make install
if [[ -d $ARROW_HOME/lib64 ]]; then
# On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for
# now. TODO(rkn): A preferable solution would be to add both directories to
# the PKG_CONFIG_PATH, but that didn't seem to work.
cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib
fi
bash "$TP_DIR/build_parquet.sh"
echo "installing pyarrow"
cd $TP_DIR/arrow/python
# We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
# find plasma.
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
PYARROW_WITH_PLASMA=1 \
PYARROW_BUNDLE_ARROW_CPP=1 \
$PYTHON_EXECUTABLE setup.py build
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
PYARROW_WITH_PLASMA=1 \
PYARROW_BUNDLE_ARROW_CPP=1 \
PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install \
PYARROW_WITH_PARQUET=1 \
$PYTHON_EXECUTABLE setup.py build_ext
# Find the pyarrow directory that was just built and copy it to ray/python/ray/
# so that pyarrow can be packaged along with ray.
pushd $TP_DIR/arrow/python/build
PYARROW_BUILD_LIB_DIR="$TP_DIR/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')"
popd
echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow"
cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../../python/ray/pyarrow_files/
src/thirdparty/download_thirdparty.sh
Vendored
-27
View File
@@ -1,27 +0,0 @@
#!/bin/bash
set -x
# Cause the script to exit if a single command fails.
set -e
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
if [ ! -d $TP_DIR/arrow ]; then
git clone https://github.com/apache/arrow.git "$TP_DIR/arrow"
fi
pushd $TP_DIR/arrow
git fetch origin master
# The PR for this commit is https://github.com/apache/arrow/pull/1581. We
# include the link here to make it easier to find the right commit because
# Arrow often rewrites git history and invalidates certain commits.
git checkout 46aa99e9843ac0148357bb36a9235cfd48903e73
popd
if [ ! -d $TP_DIR/parquet-cpp ]; then
git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/parquet-cpp"
fi
pushd $TP_DIR/parquet-cpp
git fetch origin master
git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b
popd
thirdparty/scripts/build_arrow.sh
Vendored Executable
+110
View File
@@ -0,0 +1,110 @@
#!/bin/bash
set -x
# Cause the script to exit if a single command fails.
set -e
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
# setup env
if [[ -z "$1" ]]; then
PYTHON_EXECUTABLE=`which python`
else
PYTHON_EXECUTABLE=$1
fi
echo "Using Python executable $PYTHON_EXECUTABLE."
unamestr="$(uname)"
if [[ "$unamestr" == "Linux" ]]; then
FLATBUFFERS_HOME=$TP_DIR/pkg/flatbuffers
else
FLATBUFFERS_HOME=""
fi
# Determine how many parallel jobs to use for make based on the number of cores
if [[ "$unamestr" == "Linux" ]]; then
PARALLEL=$(nproc)
elif [[ "$unamestr" == "Darwin" ]]; then
PARALLEL=$(sysctl -n hw.ncpu)
echo "Platform is macosx."
else
echo "Unrecognized platform."
exit 1
fi
# Download and compile arrow if it isn't already present.
if [[ ! -d $TP_DIR/pkg/arrow ]]; then
echo "building arrow"
git clone https://github.com/apache/arrow.git "$TP_DIR/build/arrow"
pushd $TP_DIR/build/arrow
git fetch origin master
# The PR for this commit is https://github.com/apache/arrow/pull/1581. We
# include the link here to make it easier to find the right commit because
# Arrow often rewrites git history and invalidates certain commits.
git checkout 46aa99e9843ac0148357bb36a9235cfd48903e73
cd cpp
mkdir build
cd build
ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install
BOOST_ROOT=$TP_DIR/pkg/boost \
FLATBUFFERS_HOME=$FLATBUFFERS_HOME \
cmake -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_C_FLAGS="-g -O3" \
-DCMAKE_CXX_FLAGS="-g -O3" \
-DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
-DARROW_BUILD_TESTS=off \
-DARROW_HDFS=on \
-DARROW_BOOST_USE_SHARED=off \
-DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
-DARROW_PYTHON=on \
-DARROW_PLASMA=on \
-DPLASMA_PYTHON=on \
-DARROW_JEMALLOC=off \
-DARROW_WITH_BROTLI=off \
-DARROW_WITH_LZ4=off \
-DARROW_WITH_ZLIB=off \
-DARROW_WITH_ZSTD=off \
..
make VERBOSE=1 -j$PARALLEL
make install
if [[ -d $ARROW_HOME/lib64 ]]; then
# On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for
# now. TODO(rkn): A preferable solution would be to add both directories to
# the PKG_CONFIG_PATH, but that didn't seem to work.
cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib
fi
bash "$TP_DIR/scripts/build_parquet.sh"
echo "installing pyarrow"
cd $TP_DIR/build/arrow/python
# We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
# find plasma.
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
PYARROW_WITH_PLASMA=1 \
PYARROW_BUNDLE_ARROW_CPP=1 \
$PYTHON_EXECUTABLE setup.py build
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
PYARROW_WITH_PLASMA=1 \
PYARROW_BUNDLE_ARROW_CPP=1 \
PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
PYARROW_WITH_PARQUET=1 \
$PYTHON_EXECUTABLE setup.py build_ext
# Find the pyarrow directory that was just built and copy it to ray/python/ray/
# so that pyarrow can be packaged along with ray.
pushd $TP_DIR/build/arrow/python/build
PYARROW_BUILD_LIB_DIR="$TP_DIR/build/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')"
popd
echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow"
cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../python/ray/pyarrow_files/
popd
fi
src/thirdparty/build_boost.sh → thirdparty/scripts/build_boost.sh
Vendored
+7 -7
View File
@@ -5,29 +5,29 @@ set -x
# Cause the script to exit if a single command fails.
set -e
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
BOOST_VERSION=1.65.1
BOOST_VERSION_UNDERSCORE=1_65_1
# Download and compile boost if it isn't already present.
if [[ ! -d $TP_DIR/boost ]]; then
if [[ ! -d $TP_DIR/pkg/boost ]]; then
# The wget command frequently fails, so retry up to 20 times.
for COUNT in {1..20}; do
# Attempt to wget boost and break from the retry loop if it succeeds.
wget --no-check-certificate https://dl.bintray.com/boostorg/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -O $TP_DIR/boost_$BOOST_VERSION_UNDERSCORE.tar.gz && break
wget --no-check-certificate https://dl.bintray.com/boostorg/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -O $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz && break
# If none of the retries succeeded at getting boost, then fail.
if [[ $COUNT == 20 ]]; then
exit 1
fi
done
tar xf $TP_DIR/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -C $TP_DIR/
rm -rf $TP_DIR/boost_$BOOST_VERSION_UNDERSCORE.tar.gz
tar xf $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -C $TP_DIR/build/
rm -rf $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz
# Compile boost.
pushd $TP_DIR/boost_$BOOST_VERSION_UNDERSCORE
pushd $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE
./bootstrap.sh
./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/boost --with-filesystem --with-system --with-regex install > /dev/null
./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/pkg/boost --with-filesystem --with-system --with-regex install > /dev/null
popd
fi
src/thirdparty/build_flatbuffers.sh → thirdparty/scripts/build_flatbuffers.sh
Vendored
+5 -3
View File
@@ -5,13 +5,14 @@ set -x
# Cause the script to exit if a single command fails.
set -e
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
FLATBUFFERS_VERSION=1.7.1
# Download and compile flatbuffers if it isn't already present.
if [ ! -d $TP_DIR/flatbuffers ]; then
if [ ! -d $TP_DIR/pkg/flatbuffers ]; then
echo "building flatbuffers"
pushd $TP_DIR/build
curl -sL https://github.com/google/flatbuffers/archive/v$FLATBUFFERS_VERSION.tar.gz -o flatbuffers-$FLATBUFFERS_VERSION.tar.gz
tar xf flatbuffers-$FLATBUFFERS_VERSION.tar.gz
rm -rf flatbuffers-$FLATBUFFERS_VERSION.tar.gz
@@ -19,9 +20,10 @@ if [ ! -d $TP_DIR/flatbuffers ]; then
# Compile flatbuffers.
pushd flatbuffers-$FLATBUFFERS_VERSION
cmake -DCMAKE_CXX_FLAGS=-fPIC \
-DCMAKE_INSTALL_PREFIX:PATH=$TP_DIR/flatbuffers \
-DCMAKE_INSTALL_PREFIX:PATH=$TP_DIR/pkg/flatbuffers \
-DFLATBUFFERS_BUILD_TESTS=OFF
make -j5
make install
popd
popd
fi
thirdparty/scripts/build_parquet.sh
Vendored Executable
+55
View File
@@ -0,0 +1,55 @@
#!/bin/bash
set -x
# Cause the script to exit if a single command fails.
set -e
unamestr="$(uname)"
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install
OPENSSL_DIR=/usr/local/opt/openssl
BISON_DIR=/usr/local/opt/bison/bin
if [ ! -d $TP_DIR/build/parquet-cpp ]; then
git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/build/parquet-cpp"
pushd $TP_DIR/build/parquet-cpp
git fetch origin master
git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b
if [ "$unamestr" == "Darwin" ]; then
OPENSSL_ROOT_DIR=$OPENSSL_DIR \
PATH="$BISON_DIR:$PATH" \
ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
cmake -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
-DPARQUET_BUILD_BENCHMARKS=off \
-DPARQUET_BUILD_EXECUTABLES=off \
-DPARQUET_BUILD_TESTS=off \
.
OPENSSL_ROOT_DIR=$OPENSSL_DIR \
PATH="$BISON_DIR:$PATH" \
make -j4
OPENSSL_ROOT_DIR=$OPENSSL_DIR \
PATH="$BISON_DIR:$PATH" \
make install
else
BOOST_ROOT=$TP_DIR/pkg/boost \
ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
cmake -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
-DPARQUET_BUILD_BENCHMARKS=off \
-DPARQUET_BUILD_EXECUTABLES=off \
-DPARQUET_BUILD_TESTS=off \
.
PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
BOOST_ROOT=$TP_DIR/pkg/boost \
make -j4
make install
fi
popd
fi
src/common/thirdparty/build-redis.sh → thirdparty/scripts/build_redis.sh
Vendored
+8 -5
View File
@@ -5,15 +5,18 @@ set -x
# Cause the script to exit if a single command fails.
set -e
if [ ! -f redis/src/redis-server ]; then
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
if [ ! -f $TP_DIR/pkg/redis/src/redis-server ]; then
redis_vname="4.0-rc2"
# This check is to make sure the tarball has been fully extracted. The only
# relevant bit about redis/utils/whatisdoing.sh is that it is one of the last
# files in the tarball.
if [ ! -f redis/utils/whatisdoing.sh ]; then
mkdir -p "./redis"
curl -sL "https://github.com/antirez/redis/archive/$redis_vname.tar.gz" | tar xz --strip-components=1 -C "./redis"
if [ ! -f $TP_DIR/pkg/redis/utils/whatisdoing.sh ]; then
mkdir -p "$TP_DIR/pkg/redis"
curl -sL "https://github.com/antirez/redis/archive/$redis_vname.tar.gz" | tar xz --strip-components=1 -C "$TP_DIR/pkg/redis"
fi
cd redis
pushd $TP_DIR/pkg/redis
make
popd
fi
src/thirdparty/build_ui.sh → thirdparty/scripts/build_ui.sh
Vendored
+3 -3
View File
@@ -5,13 +5,13 @@ set -x
# Cause the script to exit if a single command fails.
set -e
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
CATAPULT_COMMIT=18cd334755701cf0c3b90b7172126c686d2eb787
CATAPULT_HOME=$TP_DIR/catapult
CATAPULT_HOME=$TP_DIR/pkg/catapult
VULCANIZE_BIN=$CATAPULT_HOME/tracing/bin/vulcanize_trace_viewer
CATAPULT_FILES=$TP_DIR/../../python/ray/core/src/catapult_files
CATAPULT_FILES=$TP_DIR/../python/ray/core/src/catapult_files
# This is where we will copy the files that need to be packaged with the wheels.
mkdir -p $CATAPULT_FILES
thirdparty/scripts/setup.sh
Vendored Executable
+63
View File
@@ -0,0 +1,63 @@
#!/bin/bash
set -x
# Cause the script to exit if a single command fails.
set -e
TP_SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
TP_DIR=$TP_SCRIPT_DIR/..
mkdir -p $TP_DIR/build
mkdir -p $TP_DIR/pkg
if [[ -z "$1" ]]; then
PYTHON_EXECUTABLE=`which python`
else
PYTHON_EXECUTABLE=$1
fi
echo "Using Python executable $PYTHON_EXECUTABLE."
unamestr="$(uname)"
##############################################
# redis
##############################################
bash "$TP_SCRIPT_DIR/build_redis.sh"
##############################################
# boost if necessary
##############################################
if [[ "$unamestr" == "Linux" ]]; then
echo "building boost"
bash "$TP_SCRIPT_DIR/build_boost.sh"
fi
##############################################
# flatbuffers if necessary
##############################################
if [[ "$unamestr" == "Linux" ]]; then
echo "building flatbuffers"
bash "$TP_SCRIPT_DIR/build_flatbuffers.sh"
fi
##############################################
# arrow
##############################################
bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE
##############################################
# parquet (skipped as it is inlined in build_arrow.sh)
##############################################
# bash "$TP_SCRIPT_DIR/build_parquet.sh"
##############################################
# catapult
##############################################
# Clone catapult and build the static HTML needed for the UI.
bash "$TP_SCRIPT_DIR/build_ui.sh"
##############################################
# rDSN (optional)
##############################################
# bash "$TP_SCRIPT_DIR/build_rdsn.sh"
thirdparty/scripts/thirdparty.cmake
Vendored
+6
View File
@@ -0,0 +1,6 @@
# add all thirdparty related path definitions here
list(APPEND CMAKE_MODULE_PATH
${CMAKE_CURRENT_LIST_DIR}/../build/arrow/python/cmake_modules)
message (WARNING ${CMAKE_MODULE_PATH})