diff --git a/CMakeLists.txt b/CMakeLists.txt index 56cf81215..c1932ecbf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,7 +4,7 @@ project(ray) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules") -list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/src/thirdparty/arrow/python/cmake_modules) +include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/scripts/thirdparty.cmake) find_package(Arrow) find_package(Plasma) diff --git a/build.sh b/build.sh index e2cb46e0a..0931e9ab6 100755 --- a/build.sh +++ b/build.sh @@ -14,6 +14,8 @@ else fi echo "Using Python executable $PYTHON_EXECUTABLE." +bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE + # Determine how many parallel jobs to use for make based on the number of cores unamestr="$(uname)" if [[ "$unamestr" == "Linux" ]]; then @@ -25,28 +27,21 @@ else exit 1 fi -pushd "$ROOT_DIR/src/common/thirdparty/" - bash build-redis.sh -popd - -bash "$ROOT_DIR/src/thirdparty/download_thirdparty.sh" -bash "$ROOT_DIR/src/thirdparty/build_thirdparty.sh" $PYTHON_EXECUTABLE - # Now we build everything. pushd "$ROOT_DIR/python/ray/core" # We use these variables to set PKG_CONFIG_PATH, which is important so that # in cmake, pkg-config can find plasma. - TP_DIR=$ROOT_DIR/src/thirdparty - ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install + TP_PKG_DIR=$ROOT_DIR/thirdparty/pkg + ARROW_HOME=$TP_PKG_DIR/arrow/cpp/build/cpp-install if [[ "$VALGRIND" = "1" ]]; then - BOOST_ROOT=$TP_DIR/boost \ + BOOST_ROOT=$TP_PKG_DIR/boost \ PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ cmake -DCMAKE_BUILD_TYPE=Debug \ -DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \ -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \ ../../.. else - BOOST_ROOT=$TP_DIR/boost \ + BOOST_ROOT=$TP_PKG_DIR/boost \ PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ cmake -DCMAKE_BUILD_TYPE=Release \ -DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \ @@ -58,5 +53,4 @@ pushd "$ROOT_DIR/python/ray/core" popd # Move stuff from Arrow to Ray. - -mv $ROOT_DIR/src/thirdparty/arrow/cpp/build/release/plasma_store $ROOT_DIR/python/ray/core/src/plasma/ +cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $ROOT_DIR/python/ray/core/src/plasma/ diff --git a/setup_thirdparty.sh b/setup_thirdparty.sh new file mode 100755 index 000000000..77014226c --- /dev/null +++ b/setup_thirdparty.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -x + +# Cause the script to exit if a single command fails. +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +if [[ -z "$1" ]]; then + PYTHON_EXECUTABLE=`which python` +else + PYTHON_EXECUTABLE=$1 +fi +echo "Using Python executable $PYTHON_EXECUTABLE." + +$TP_DIR/thirdparty/scripts/setup.sh $PYTHON_EXECUTABLE diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 0d61673ee..29d5fc488 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -89,16 +89,10 @@ define_test(redis_tests "") define_test(task_table_tests "") define_test(object_table_tests "") -add_custom_target( - redis - COMMAND make - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/thirdparty/redis) - add_custom_target(copy_redis ALL) foreach(file "redis-cli" "redis-server") add_custom_command(TARGET copy_redis POST_BUILD COMMAND ${CMAKE_COMMAND} -E - copy ${CMAKE_CURRENT_LIST_DIR}/thirdparty/redis/src/${file} + copy ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/redis/src/${file} ${CMAKE_BINARY_DIR}/src/common/thirdparty/redis/src/${file}) endforeach() -add_dependencies(copy_redis redis) diff --git a/src/local_scheduler/CMakeLists.txt b/src/local_scheduler/CMakeLists.txt index c2b011557..12ee85c27 100644 --- a/src/local_scheduler/CMakeLists.txt +++ b/src/local_scheduler/CMakeLists.txt @@ -6,7 +6,7 @@ project(local_scheduler) include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake) # Include plasma -list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../thirdparty/arrow/python/cmake_modules) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/arrow/python/cmake_modules) find_package(Arrow) find_package(Plasma) diff --git a/src/thirdparty/build_parquet.sh b/src/thirdparty/build_parquet.sh deleted file mode 100755 index 69a9d078c..000000000 --- a/src/thirdparty/build_parquet.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -set -x - -# Cause the script to exit if a single command fails. -set -e - -unamestr="$(uname)" -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -cd $TP_DIR/parquet-cpp - -PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install - -OPENSSL_DIR=/usr/local/opt/openssl -BISON_DIR=/usr/local/opt/bison/bin - -if [ "$unamestr" == "Darwin" ]; then - OPENSSL_ROOT_DIR=$OPENSSL_DIR \ - PATH="$BISON_DIR:$PATH" \ - ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install \ - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ - -DPARQUET_BUILD_BENCHMARKS=off \ - -DPARQUET_BUILD_EXECUTABLES=off \ - -DPARQUET_BUILD_TESTS=off \ - . - - OPENSSL_ROOT_DIR=$OPENSSL_DIR \ - PATH="$BISON_DIR:$PATH" \ - make -j4 - - OPENSSL_ROOT_DIR=$OPENSSL_DIR \ - PATH="$BISON_DIR:$PATH" \ - make install -else - BOOST_ROOT=$TP_DIR/boost \ - ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install \ - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ - -DPARQUET_BUILD_BENCHMARKS=off \ - -DPARQUET_BUILD_EXECUTABLES=off \ - -DPARQUET_BUILD_TESTS=off \ - . - - PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install \ - BOOST_ROOT=$TP_DIR/boost \ - make -j4 - - make install -fi diff --git a/src/thirdparty/build_thirdparty.sh b/src/thirdparty/build_thirdparty.sh deleted file mode 100755 index 31d0c5df2..000000000 --- a/src/thirdparty/build_thirdparty.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash - -set -x - -# Cause the script to exit if a single command fails. -set -e - -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -if [[ -z "$1" ]]; then - PYTHON_EXECUTABLE=`which python` -else - PYTHON_EXECUTABLE=$1 -fi -echo "Using Python executable $PYTHON_EXECUTABLE." - -# Determine how many parallel jobs to use for make based on the number of cores -unamestr="$(uname)" -if [[ "$unamestr" == "Linux" ]]; then - PARALLEL=$(nproc) -elif [[ "$unamestr" == "Darwin" ]]; then - PARALLEL=$(sysctl -n hw.ncpu) - echo "Platform is macosx." -else - echo "Unrecognized platform." - exit 1 -fi - -# If we're on Linux, then compile boost. This installs boost to $TP_DIR/boost. -if [[ "$unamestr" == "Linux" ]]; then - echo "building boost" - bash "$TP_DIR/build_boost.sh" -fi - -# If we're on Linux, then compile flatbuffers. This installs flatbuffers to -# $TP_DIR/flatbuffers. -if [[ "$unamestr" == "Linux" ]]; then - echo "building flatbuffers" - bash "$TP_DIR/build_flatbuffers.sh" - FLATBUFFERS_HOME=$TP_DIR/flatbuffers -else - FLATBUFFERS_HOME="" -fi - -# Clone catapult and build the static HTML needed for the UI. -bash "$TP_DIR/build_ui.sh" - -echo "building arrow" -cd $TP_DIR/arrow/cpp -mkdir -p $TP_DIR/arrow/cpp/build -cd $TP_DIR/arrow/cpp/build -ARROW_HOME=$TP_DIR/arrow/cpp/build/cpp-install - -BOOST_ROOT=$TP_DIR/boost \ -FLATBUFFERS_HOME=$FLATBUFFERS_HOME \ -cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_C_FLAGS="-g -O3" \ - -DCMAKE_CXX_FLAGS="-g -O3" \ - -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ - -DARROW_BUILD_TESTS=off \ - -DARROW_HDFS=on \ - -DARROW_BOOST_USE_SHARED=off \ - -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \ - -DARROW_PYTHON=on \ - -DARROW_PLASMA=on \ - -DPLASMA_PYTHON=on \ - -DARROW_JEMALLOC=off \ - -DARROW_WITH_BROTLI=off \ - -DARROW_WITH_LZ4=off \ - -DARROW_WITH_ZLIB=off \ - -DARROW_WITH_ZSTD=off \ - .. -make VERBOSE=1 -j$PARALLEL -make install - -if [[ -d $ARROW_HOME/lib64 ]]; then - # On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for - # now. TODO(rkn): A preferable solution would be to add both directories to - # the PKG_CONFIG_PATH, but that didn't seem to work. - cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib -fi - -bash "$TP_DIR/build_parquet.sh" - -echo "installing pyarrow" -cd $TP_DIR/arrow/python -# We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can -# find plasma. -PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ -PYARROW_WITH_PLASMA=1 \ -PYARROW_BUNDLE_ARROW_CPP=1 \ -$PYTHON_EXECUTABLE setup.py build - -PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ -PYARROW_WITH_PLASMA=1 \ -PYARROW_BUNDLE_ARROW_CPP=1 \ -PARQUET_HOME=$TP_DIR/arrow/cpp/build/cpp-install \ -PYARROW_WITH_PARQUET=1 \ -$PYTHON_EXECUTABLE setup.py build_ext - -# Find the pyarrow directory that was just built and copy it to ray/python/ray/ -# so that pyarrow can be packaged along with ray. -pushd $TP_DIR/arrow/python/build - PYARROW_BUILD_LIB_DIR="$TP_DIR/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')" -popd -echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow" -cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../../python/ray/pyarrow_files/ diff --git a/src/thirdparty/download_thirdparty.sh b/src/thirdparty/download_thirdparty.sh deleted file mode 100755 index 96c41e93e..000000000 --- a/src/thirdparty/download_thirdparty.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -x - -# Cause the script to exit if a single command fails. -set -e - -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -if [ ! -d $TP_DIR/arrow ]; then - git clone https://github.com/apache/arrow.git "$TP_DIR/arrow" -fi -pushd $TP_DIR/arrow - git fetch origin master - # The PR for this commit is https://github.com/apache/arrow/pull/1581. We - # include the link here to make it easier to find the right commit because - # Arrow often rewrites git history and invalidates certain commits. - git checkout 46aa99e9843ac0148357bb36a9235cfd48903e73 -popd - -if [ ! -d $TP_DIR/parquet-cpp ]; then - git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/parquet-cpp" -fi -pushd $TP_DIR/parquet-cpp - git fetch origin master - git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b -popd diff --git a/thirdparty/scripts/build_arrow.sh b/thirdparty/scripts/build_arrow.sh new file mode 100755 index 000000000..3691fbdfa --- /dev/null +++ b/thirdparty/scripts/build_arrow.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +set -x + +# Cause the script to exit if a single command fails. +set -e + +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../ + +# setup env +if [[ -z "$1" ]]; then + PYTHON_EXECUTABLE=`which python` +else + PYTHON_EXECUTABLE=$1 +fi +echo "Using Python executable $PYTHON_EXECUTABLE." + +unamestr="$(uname)" + +if [[ "$unamestr" == "Linux" ]]; then + FLATBUFFERS_HOME=$TP_DIR/pkg/flatbuffers +else + FLATBUFFERS_HOME="" +fi + +# Determine how many parallel jobs to use for make based on the number of cores +if [[ "$unamestr" == "Linux" ]]; then + PARALLEL=$(nproc) +elif [[ "$unamestr" == "Darwin" ]]; then + PARALLEL=$(sysctl -n hw.ncpu) + echo "Platform is macosx." +else + echo "Unrecognized platform." + exit 1 +fi + +# Download and compile arrow if it isn't already present. +if [[ ! -d $TP_DIR/pkg/arrow ]]; then + echo "building arrow" + git clone https://github.com/apache/arrow.git "$TP_DIR/build/arrow" + + pushd $TP_DIR/build/arrow + git fetch origin master + # The PR for this commit is https://github.com/apache/arrow/pull/1581. We + # include the link here to make it easier to find the right commit because + # Arrow often rewrites git history and invalidates certain commits. + git checkout 46aa99e9843ac0148357bb36a9235cfd48903e73 + + cd cpp + mkdir build + cd build + + ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install + BOOST_ROOT=$TP_DIR/pkg/boost \ + FLATBUFFERS_HOME=$FLATBUFFERS_HOME \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_FLAGS="-g -O3" \ + -DCMAKE_CXX_FLAGS="-g -O3" \ + -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \ + -DARROW_BUILD_TESTS=off \ + -DARROW_HDFS=on \ + -DARROW_BOOST_USE_SHARED=off \ + -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \ + -DARROW_PYTHON=on \ + -DARROW_PLASMA=on \ + -DPLASMA_PYTHON=on \ + -DARROW_JEMALLOC=off \ + -DARROW_WITH_BROTLI=off \ + -DARROW_WITH_LZ4=off \ + -DARROW_WITH_ZLIB=off \ + -DARROW_WITH_ZSTD=off \ + .. + make VERBOSE=1 -j$PARALLEL + make install + + if [[ -d $ARROW_HOME/lib64 ]]; then + # On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for + # now. TODO(rkn): A preferable solution would be to add both directories to + # the PKG_CONFIG_PATH, but that didn't seem to work. + cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib + fi + + bash "$TP_DIR/scripts/build_parquet.sh" + + echo "installing pyarrow" + cd $TP_DIR/build/arrow/python + # We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can + # find plasma. + PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ + PYARROW_WITH_PLASMA=1 \ + PYARROW_BUNDLE_ARROW_CPP=1 \ + $PYTHON_EXECUTABLE setup.py build + + PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ + PYARROW_WITH_PLASMA=1 \ + PYARROW_BUNDLE_ARROW_CPP=1 \ + PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \ + PYARROW_WITH_PARQUET=1 \ + $PYTHON_EXECUTABLE setup.py build_ext + + # Find the pyarrow directory that was just built and copy it to ray/python/ray/ + # so that pyarrow can be packaged along with ray. + pushd $TP_DIR/build/arrow/python/build + PYARROW_BUILD_LIB_DIR="$TP_DIR/build/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')" + popd + echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow" + cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../python/ray/pyarrow_files/ + + popd +fi diff --git a/src/thirdparty/build_boost.sh b/thirdparty/scripts/build_boost.sh similarity index 61% rename from src/thirdparty/build_boost.sh rename to thirdparty/scripts/build_boost.sh index b57b64433..d6f047d4a 100755 --- a/src/thirdparty/build_boost.sh +++ b/thirdparty/scripts/build_boost.sh @@ -5,29 +5,29 @@ set -x # Cause the script to exit if a single command fails. set -e -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../ BOOST_VERSION=1.65.1 BOOST_VERSION_UNDERSCORE=1_65_1 # Download and compile boost if it isn't already present. -if [[ ! -d $TP_DIR/boost ]]; then +if [[ ! -d $TP_DIR/pkg/boost ]]; then # The wget command frequently fails, so retry up to 20 times. for COUNT in {1..20}; do # Attempt to wget boost and break from the retry loop if it succeeds. - wget --no-check-certificate https://dl.bintray.com/boostorg/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -O $TP_DIR/boost_$BOOST_VERSION_UNDERSCORE.tar.gz && break + wget --no-check-certificate https://dl.bintray.com/boostorg/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -O $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz && break # If none of the retries succeeded at getting boost, then fail. if [[ $COUNT == 20 ]]; then exit 1 fi done - tar xf $TP_DIR/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -C $TP_DIR/ - rm -rf $TP_DIR/boost_$BOOST_VERSION_UNDERSCORE.tar.gz + tar xf $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -C $TP_DIR/build/ + rm -rf $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz # Compile boost. - pushd $TP_DIR/boost_$BOOST_VERSION_UNDERSCORE + pushd $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE ./bootstrap.sh - ./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/boost --with-filesystem --with-system --with-regex install > /dev/null + ./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/pkg/boost --with-filesystem --with-system --with-regex install > /dev/null popd fi diff --git a/src/thirdparty/build_flatbuffers.sh b/thirdparty/scripts/build_flatbuffers.sh similarity index 76% rename from src/thirdparty/build_flatbuffers.sh rename to thirdparty/scripts/build_flatbuffers.sh index 918d955e5..59f14a05e 100755 --- a/src/thirdparty/build_flatbuffers.sh +++ b/thirdparty/scripts/build_flatbuffers.sh @@ -5,13 +5,14 @@ set -x # Cause the script to exit if a single command fails. set -e -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../ FLATBUFFERS_VERSION=1.7.1 # Download and compile flatbuffers if it isn't already present. -if [ ! -d $TP_DIR/flatbuffers ]; then +if [ ! -d $TP_DIR/pkg/flatbuffers ]; then echo "building flatbuffers" + pushd $TP_DIR/build curl -sL https://github.com/google/flatbuffers/archive/v$FLATBUFFERS_VERSION.tar.gz -o flatbuffers-$FLATBUFFERS_VERSION.tar.gz tar xf flatbuffers-$FLATBUFFERS_VERSION.tar.gz rm -rf flatbuffers-$FLATBUFFERS_VERSION.tar.gz @@ -19,9 +20,10 @@ if [ ! -d $TP_DIR/flatbuffers ]; then # Compile flatbuffers. pushd flatbuffers-$FLATBUFFERS_VERSION cmake -DCMAKE_CXX_FLAGS=-fPIC \ - -DCMAKE_INSTALL_PREFIX:PATH=$TP_DIR/flatbuffers \ + -DCMAKE_INSTALL_PREFIX:PATH=$TP_DIR/pkg/flatbuffers \ -DFLATBUFFERS_BUILD_TESTS=OFF make -j5 make install popd + popd fi diff --git a/thirdparty/scripts/build_parquet.sh b/thirdparty/scripts/build_parquet.sh new file mode 100755 index 000000000..fabf9efa0 --- /dev/null +++ b/thirdparty/scripts/build_parquet.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -x + +# Cause the script to exit if a single command fails. +set -e + +unamestr="$(uname)" +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../ +PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install +OPENSSL_DIR=/usr/local/opt/openssl +BISON_DIR=/usr/local/opt/bison/bin + +if [ ! -d $TP_DIR/build/parquet-cpp ]; then + git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/build/parquet-cpp" + pushd $TP_DIR/build/parquet-cpp + git fetch origin master + git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b + + if [ "$unamestr" == "Darwin" ]; then + OPENSSL_ROOT_DIR=$OPENSSL_DIR \ + PATH="$BISON_DIR:$PATH" \ + ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ + -DPARQUET_BUILD_BENCHMARKS=off \ + -DPARQUET_BUILD_EXECUTABLES=off \ + -DPARQUET_BUILD_TESTS=off \ + . + + OPENSSL_ROOT_DIR=$OPENSSL_DIR \ + PATH="$BISON_DIR:$PATH" \ + make -j4 + + OPENSSL_ROOT_DIR=$OPENSSL_DIR \ + PATH="$BISON_DIR:$PATH" \ + make install + else + BOOST_ROOT=$TP_DIR/pkg/boost \ + ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \ + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ + -DPARQUET_BUILD_BENCHMARKS=off \ + -DPARQUET_BUILD_EXECUTABLES=off \ + -DPARQUET_BUILD_TESTS=off \ + . + + PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \ + BOOST_ROOT=$TP_DIR/pkg/boost \ + make -j4 + make install + fi + + popd +fi diff --git a/src/common/thirdparty/build-redis.sh b/thirdparty/scripts/build_redis.sh similarity index 55% rename from src/common/thirdparty/build-redis.sh rename to thirdparty/scripts/build_redis.sh index ddce13513..7a90be8bf 100755 --- a/src/common/thirdparty/build-redis.sh +++ b/thirdparty/scripts/build_redis.sh @@ -5,15 +5,18 @@ set -x # Cause the script to exit if a single command fails. set -e -if [ ! -f redis/src/redis-server ]; then +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../ + +if [ ! -f $TP_DIR/pkg/redis/src/redis-server ]; then redis_vname="4.0-rc2" # This check is to make sure the tarball has been fully extracted. The only # relevant bit about redis/utils/whatisdoing.sh is that it is one of the last # files in the tarball. - if [ ! -f redis/utils/whatisdoing.sh ]; then - mkdir -p "./redis" - curl -sL "https://github.com/antirez/redis/archive/$redis_vname.tar.gz" | tar xz --strip-components=1 -C "./redis" + if [ ! -f $TP_DIR/pkg/redis/utils/whatisdoing.sh ]; then + mkdir -p "$TP_DIR/pkg/redis" + curl -sL "https://github.com/antirez/redis/archive/$redis_vname.tar.gz" | tar xz --strip-components=1 -C "$TP_DIR/pkg/redis" fi - cd redis + pushd $TP_DIR/pkg/redis make + popd fi diff --git a/src/thirdparty/build_ui.sh b/thirdparty/scripts/build_ui.sh similarity index 91% rename from src/thirdparty/build_ui.sh rename to thirdparty/scripts/build_ui.sh index cb906e87f..8b95db4dc 100755 --- a/src/thirdparty/build_ui.sh +++ b/thirdparty/scripts/build_ui.sh @@ -5,13 +5,13 @@ set -x # Cause the script to exit if a single command fails. set -e -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../ CATAPULT_COMMIT=18cd334755701cf0c3b90b7172126c686d2eb787 -CATAPULT_HOME=$TP_DIR/catapult +CATAPULT_HOME=$TP_DIR/pkg/catapult VULCANIZE_BIN=$CATAPULT_HOME/tracing/bin/vulcanize_trace_viewer -CATAPULT_FILES=$TP_DIR/../../python/ray/core/src/catapult_files +CATAPULT_FILES=$TP_DIR/../python/ray/core/src/catapult_files # This is where we will copy the files that need to be packaged with the wheels. mkdir -p $CATAPULT_FILES diff --git a/thirdparty/scripts/setup.sh b/thirdparty/scripts/setup.sh new file mode 100755 index 000000000..8cbdace04 --- /dev/null +++ b/thirdparty/scripts/setup.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +set -x + +# Cause the script to exit if a single command fails. +set -e + +TP_SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +TP_DIR=$TP_SCRIPT_DIR/.. + +mkdir -p $TP_DIR/build +mkdir -p $TP_DIR/pkg + +if [[ -z "$1" ]]; then + PYTHON_EXECUTABLE=`which python` +else + PYTHON_EXECUTABLE=$1 +fi +echo "Using Python executable $PYTHON_EXECUTABLE." + +unamestr="$(uname)" + +############################################## +# redis +############################################## +bash "$TP_SCRIPT_DIR/build_redis.sh" + +############################################## +# boost if necessary +############################################## +if [[ "$unamestr" == "Linux" ]]; then + echo "building boost" + bash "$TP_SCRIPT_DIR/build_boost.sh" +fi + +############################################## +# flatbuffers if necessary +############################################## +if [[ "$unamestr" == "Linux" ]]; then + echo "building flatbuffers" + bash "$TP_SCRIPT_DIR/build_flatbuffers.sh" +fi + +############################################## +# arrow +############################################## +bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE + +############################################## +# parquet (skipped as it is inlined in build_arrow.sh) +############################################## +# bash "$TP_SCRIPT_DIR/build_parquet.sh" + +############################################## +# catapult +############################################## +# Clone catapult and build the static HTML needed for the UI. +bash "$TP_SCRIPT_DIR/build_ui.sh" + +############################################## +# rDSN (optional) +############################################## +# bash "$TP_SCRIPT_DIR/build_rdsn.sh" diff --git a/thirdparty/scripts/thirdparty.cmake b/thirdparty/scripts/thirdparty.cmake new file mode 100644 index 000000000..75efd595e --- /dev/null +++ b/thirdparty/scripts/thirdparty.cmake @@ -0,0 +1,6 @@ + +# add all thirdparty related path definitions here + +list(APPEND CMAKE_MODULE_PATH + ${CMAKE_CURRENT_LIST_DIR}/../build/arrow/python/cmake_modules) +message (WARNING ${CMAKE_MODULE_PATH})