restructure how to organize 3rd party libs (#1630)

* restructure how to organize 3rd party libs * Minor whitespace changes. * Fix compilation on Linux. * Pass around Python executable so that the correct version of Python is used.
2026-09-12 12:51:15 +08:00 · 2018-03-01 14:29:56 -08:00
parent ec9dfe7748
commit f1e5789c26
16 changed files with 283 additions and 225 deletions
@@ -0,0 +1,110 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
+
+# setup env
+if [[ -z  "$1" ]]; then
+  PYTHON_EXECUTABLE=`which python`
+else
+  PYTHON_EXECUTABLE=$1
+fi
+echo "Using Python executable $PYTHON_EXECUTABLE."
+
+unamestr="$(uname)"
+
+if [[ "$unamestr" == "Linux" ]]; then
+  FLATBUFFERS_HOME=$TP_DIR/pkg/flatbuffers
+else
+  FLATBUFFERS_HOME=""
+fi
+
+# Determine how many parallel jobs to use for make based on the number of cores
+if [[ "$unamestr" == "Linux" ]]; then
+  PARALLEL=$(nproc)
+elif [[ "$unamestr" == "Darwin" ]]; then
+  PARALLEL=$(sysctl -n hw.ncpu)
+  echo "Platform is macosx."
+else
+  echo "Unrecognized platform."
+  exit 1
+fi
+
+# Download and compile arrow if it isn't already present.
+if [[ ! -d $TP_DIR/pkg/arrow ]]; then
+    echo "building arrow"
+    git clone https://github.com/apache/arrow.git "$TP_DIR/build/arrow"
+
+    pushd $TP_DIR/build/arrow
+    git fetch origin master
+    # The PR for this commit is https://github.com/apache/arrow/pull/1581. We
+    # include the link here to make it easier to find the right commit because
+    # Arrow often rewrites git history and invalidates certain commits.
+    git checkout 46aa99e9843ac0148357bb36a9235cfd48903e73
+
+    cd cpp
+    mkdir build
+    cd build
+
+    ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install
+    BOOST_ROOT=$TP_DIR/pkg/boost \
+    FLATBUFFERS_HOME=$FLATBUFFERS_HOME \
+    cmake -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_C_FLAGS="-g -O3" \
+        -DCMAKE_CXX_FLAGS="-g -O3" \
+        -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+        -DARROW_BUILD_TESTS=off \
+        -DARROW_HDFS=on \
+        -DARROW_BOOST_USE_SHARED=off \
+        -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
+        -DARROW_PYTHON=on \
+        -DARROW_PLASMA=on \
+        -DPLASMA_PYTHON=on \
+        -DARROW_JEMALLOC=off \
+        -DARROW_WITH_BROTLI=off \
+        -DARROW_WITH_LZ4=off \
+        -DARROW_WITH_ZLIB=off \
+        -DARROW_WITH_ZSTD=off \
+        ..
+    make VERBOSE=1 -j$PARALLEL
+    make install
+
+    if [[ -d $ARROW_HOME/lib64 ]]; then
+        # On CentOS, Arrow gets installed under lib64 instead of lib, so copy it for
+        # now. TODO(rkn): A preferable solution would be to add both directories to
+        # the PKG_CONFIG_PATH, but that didn't seem to work.
+        cp -r $ARROW_HOME/lib64 $ARROW_HOME/lib
+    fi
+
+    bash "$TP_DIR/scripts/build_parquet.sh"
+
+    echo "installing pyarrow"
+    cd $TP_DIR/build/arrow/python
+    # We set PKG_CONFIG_PATH, which is important so that in cmake, pkg-config can
+    # find plasma.
+    PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
+    PYARROW_WITH_PLASMA=1 \
+    PYARROW_BUNDLE_ARROW_CPP=1 \
+    $PYTHON_EXECUTABLE setup.py build
+
+    PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
+    PYARROW_WITH_PLASMA=1 \
+    PYARROW_BUNDLE_ARROW_CPP=1 \
+    PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
+    PYARROW_WITH_PARQUET=1 \
+    $PYTHON_EXECUTABLE setup.py build_ext
+
+    # Find the pyarrow directory that was just built and copy it to ray/python/ray/
+    # so that pyarrow can be packaged along with ray.
+    pushd $TP_DIR/build/arrow/python/build
+    PYARROW_BUILD_LIB_DIR="$TP_DIR/build/arrow/python/build/$(find ./ -maxdepth 1 -type d -print | grep -m1 'lib')"
+    popd
+    echo "copying pyarrow files from $PYARROW_BUILD_LIB_DIR/pyarrow"
+    cp -r $PYARROW_BUILD_LIB_DIR/pyarrow $TP_DIR/../python/ray/pyarrow_files/
+
+    popd
+fi
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
+
+BOOST_VERSION=1.65.1
+BOOST_VERSION_UNDERSCORE=1_65_1
+
+# Download and compile boost if it isn't already present.
+if [[ ! -d $TP_DIR/pkg/boost ]]; then
+  # The wget command frequently fails, so retry up to 20 times.
+  for COUNT in {1..20}; do
+    # Attempt to wget boost and break from the retry loop if it succeeds.
+    wget --no-check-certificate https://dl.bintray.com/boostorg/release/$BOOST_VERSION/source/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -O $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz && break
+    # If none of the retries succeeded at getting boost, then fail.
+    if [[ $COUNT == 20 ]]; then
+      exit 1
+    fi
+  done
+
+  tar xf $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz -C $TP_DIR/build/
+  rm -rf $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE.tar.gz
+
+  # Compile boost.
+  pushd $TP_DIR/build/boost_$BOOST_VERSION_UNDERSCORE
+    ./bootstrap.sh
+    ./bjam cxxflags=-fPIC cflags=-fPIC variant=release link=static --prefix=$TP_DIR/pkg/boost --with-filesystem --with-system --with-regex install > /dev/null
+  popd
+fi
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
+
+FLATBUFFERS_VERSION=1.7.1
+
+# Download and compile flatbuffers if it isn't already present.
+if [ ! -d $TP_DIR/pkg/flatbuffers ]; then
+  echo "building flatbuffers"
+  pushd $TP_DIR/build
+  curl -sL https://github.com/google/flatbuffers/archive/v$FLATBUFFERS_VERSION.tar.gz -o flatbuffers-$FLATBUFFERS_VERSION.tar.gz
+  tar xf flatbuffers-$FLATBUFFERS_VERSION.tar.gz
+  rm -rf flatbuffers-$FLATBUFFERS_VERSION.tar.gz
+
+  # Compile flatbuffers.
+  pushd flatbuffers-$FLATBUFFERS_VERSION
+    cmake -DCMAKE_CXX_FLAGS=-fPIC \
+          -DCMAKE_INSTALL_PREFIX:PATH=$TP_DIR/pkg/flatbuffers \
+          -DFLATBUFFERS_BUILD_TESTS=OFF
+    make -j5
+    make install
+  popd
+  popd
+fi
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+unamestr="$(uname)"
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
+PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install
+OPENSSL_DIR=/usr/local/opt/openssl
+BISON_DIR=/usr/local/opt/bison/bin
+
+if [ ! -d $TP_DIR/build/parquet-cpp ]; then
+  git clone https://github.com/apache/parquet-cpp.git "$TP_DIR/build/parquet-cpp"
+  pushd $TP_DIR/build/parquet-cpp
+  git fetch origin master
+  git checkout 76388ea4eb8b23656283116bc656b0c8f5db093b
+
+  if [ "$unamestr" == "Darwin" ]; then
+    OPENSSL_ROOT_DIR=$OPENSSL_DIR \
+    PATH="$BISON_DIR:$PATH" \
+    ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
+    cmake -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+          -DPARQUET_BUILD_BENCHMARKS=off \
+          -DPARQUET_BUILD_EXECUTABLES=off \
+          -DPARQUET_BUILD_TESTS=off \
+          .
+
+    OPENSSL_ROOT_DIR=$OPENSSL_DIR \
+    PATH="$BISON_DIR:$PATH" \
+    make -j4
+
+    OPENSSL_ROOT_DIR=$OPENSSL_DIR \
+    PATH="$BISON_DIR:$PATH" \
+    make install
+  else
+    BOOST_ROOT=$TP_DIR/pkg/boost \
+    ARROW_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
+    cmake -DCMAKE_BUILD_TYPE=Release \
+          -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \
+          -DPARQUET_BUILD_BENCHMARKS=off \
+          -DPARQUET_BUILD_EXECUTABLES=off \
+          -DPARQUET_BUILD_TESTS=off \
+          .
+
+    PARQUET_HOME=$TP_DIR/pkg/arrow/cpp/build/cpp-install \
+    BOOST_ROOT=$TP_DIR/pkg/boost \
+    make -j4
+    make install
+  fi
+
+  popd
+fi
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
+
+if [ ! -f $TP_DIR/pkg/redis/src/redis-server ]; then
+  redis_vname="4.0-rc2"
+  # This check is to make sure the tarball has been fully extracted. The only
+  # relevant bit about redis/utils/whatisdoing.sh is that it is one of the last
+  # files in the tarball.
+  if [ ! -f $TP_DIR/pkg/redis/utils/whatisdoing.sh ]; then
+    mkdir -p "$TP_DIR/pkg/redis"
+    curl -sL "https://github.com/antirez/redis/archive/$redis_vname.tar.gz" | tar xz --strip-components=1 -C "$TP_DIR/pkg/redis"
+  fi
+  pushd $TP_DIR/pkg/redis
+  make
+  popd
+fi
@@ -0,0 +1,59 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)/../
+
+CATAPULT_COMMIT=18cd334755701cf0c3b90b7172126c686d2eb787
+CATAPULT_HOME=$TP_DIR/pkg/catapult
+VULCANIZE_BIN=$CATAPULT_HOME/tracing/bin/vulcanize_trace_viewer
+
+CATAPULT_FILES=$TP_DIR/../python/ray/core/src/catapult_files
+
+# This is where we will copy the files that need to be packaged with the wheels.
+mkdir -p $CATAPULT_FILES
+
+if [[ "$INCLUDE_UI" == "0" ]]; then
+  # Let installation continue without building the UI.
+  exit 0
+fi
+
+if ! type python2 > /dev/null; then
+  echo "cannot properly set up UI without a python2 executable"
+  if [[ "$INCLUDE_UI" == "1" ]]; then
+    # Since the UI is explicitly supposed to be included, fail here.
+    exit 1
+  else
+    # Let installation continue without building the UI.
+    exit 0
+  fi
+fi
+
+# Download catapult and use it to autogenerate some static html if it isn't
+# already present.
+if [[ ! -d $CATAPULT_HOME ]]; then
+  echo "setting up catapult"
+  # The git clone command seems to fail in Travis, so retry up to 20 times.
+  for COUNT in {1..20}; do
+    # Attempt to git clone catapult and break from the retry loop if it succeeds.
+    git clone https://github.com/ray-project/catapult.git $CATAPULT_HOME && break
+    # If none of the retries succeeded at getting boost, then fail.
+    if [[ $COUNT == 20 ]]; then
+      exit 1
+    fi
+  done
+
+  # Check out the appropriate commit from catapult.
+  pushd $CATAPULT_HOME
+    git checkout $CATAPULT_COMMIT
+  popd
+fi
+
+# If the autogenerated catapult files aren't present, then generate them.
+if [[ ! -f $CATAPULT_FILES/index.html ]]; then
+  python2 $VULCANIZE_BIN --config chrome --output $CATAPULT_FILES/trace_viewer_full.html
+  cp $CATAPULT_HOME/tracing/bin/index.html $CATAPULT_FILES/index.html
+fi
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+set -x
+
+# Cause the script to exit if a single command fails.
+set -e
+
+TP_SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+TP_DIR=$TP_SCRIPT_DIR/..
+
+mkdir -p $TP_DIR/build
+mkdir -p $TP_DIR/pkg
+
+if [[ -z  "$1" ]]; then
+  PYTHON_EXECUTABLE=`which python`
+else
+  PYTHON_EXECUTABLE=$1
+fi
+echo "Using Python executable $PYTHON_EXECUTABLE."
+
+unamestr="$(uname)"
+
+##############################################
+# redis
+##############################################
+bash "$TP_SCRIPT_DIR/build_redis.sh"
+
+##############################################
+# boost if necessary
+##############################################
+if [[ "$unamestr" == "Linux" ]]; then
+  echo "building boost"
+  bash "$TP_SCRIPT_DIR/build_boost.sh"
+fi
+
+##############################################
+# flatbuffers if necessary
+##############################################
+if [[ "$unamestr" == "Linux" ]]; then
+  echo "building flatbuffers"
+  bash "$TP_SCRIPT_DIR/build_flatbuffers.sh"
+fi
+
+##############################################
+# arrow
+##############################################
+bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE
+
+##############################################
+# parquet (skipped as it is inlined in build_arrow.sh)
+##############################################
+# bash "$TP_SCRIPT_DIR/build_parquet.sh"
+
+##############################################
+# catapult
+##############################################
+# Clone catapult and build the static HTML needed for the UI.
+bash "$TP_SCRIPT_DIR/build_ui.sh"
+
+##############################################
+# rDSN (optional)
+##############################################
+# bash "$TP_SCRIPT_DIR/build_rdsn.sh"
@@ -0,0 +1,6 @@
+
+# add all thirdparty related path definitions here
+
+list(APPEND CMAKE_MODULE_PATH
+            ${CMAKE_CURRENT_LIST_DIR}/../build/arrow/python/cmake_modules)
+message (WARNING ${CMAKE_MODULE_PATH})