From 4bd98eed452deb05f84ba49fb128444e5f290025 Mon Sep 17 00:00:00 2001 From: Yuhong Guo Date: Wed, 15 Aug 2018 02:33:51 +0800 Subject: [PATCH] Support building Java and Python version at the same time. (#2640) * Support building Java and Python version at the same time. * Remove duplicated definition. * Refine the building process of local_scheduler * Refine * Add comment for languages * Modify instruction and add python,jave building to CI. * change according to comment --- CMakeLists.txt | 21 ++++--------------- build.sh | 33 +++++++++++++++++++++--------- java/test.sh | 2 +- setup_thirdparty.sh | 11 ++++------ src/local_scheduler/CMakeLists.txt | 9 ++++---- thirdparty/scripts/build_arrow.sh | 15 ++++---------- thirdparty/scripts/setup.sh | 13 +++++++----- 7 files changed, 49 insertions(+), 55 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c3c4de0c1..72ef6a91d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,19 +2,6 @@ cmake_minimum_required(VERSION 3.4) project(ray) -set(CMAKE_RAY_LANG_PYTHON "NO") -set(CMAKE_RAY_LANG_JAVA "NO") -if ("${CMAKE_RAY_LANGUAGE}" STREQUAL "python") - set(CMAKE_RAY_LANG_PYTHON "YES") -elseif ("${CMAKE_RAY_LANGUAGE}" STREQUAL "java") - set(CMAKE_RAY_LANG_JAVA "YES") -elseif ("${CMAKE_RAY_LANGUAGE}" STREQUAL "") - message(WARNING "Language is not set, choose Python as default.") - set(CMAKE_RAY_LANG_PYTHON "YES") -else() - message(FATAL_ERROR "Unrecognized language, use -DCMAKE_RAY_LANGUAGE=java|python. Abort.") -endif() - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules") include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/scripts/thirdparty.cmake) @@ -116,12 +103,12 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") # Make sure that the Python extensions are built before copying the files. if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") - get_local_scheduler_library("python" LOCAL_SCHEDULER_LIBRARY_LANG) - add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_LANG}) + get_local_scheduler_library("python" LOCAL_SCHEDULER_LIBRARY_PYTHON) + add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_PYTHON}) endif() if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES") - get_local_scheduler_library("java" LOCAL_SCHEDULER_LIBRARY_LANG) - add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_LANG}) + get_local_scheduler_library("java" LOCAL_SCHEDULER_LIBRARY_JAVA) + add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_JAVA}) endif() add_dependencies(copy_ray ray_redis_module) diff --git a/build.sh b/build.sh index 6087023b7..e3b458b10 100755 --- a/build.sh +++ b/build.sh @@ -14,9 +14,10 @@ function usage() echo "Options:" echo " -h|--help print the help info" echo " -d|--debug CMAKE_BUILD_TYPE=Debug (default is RelWithDebInfo)" - echo " -l|--language python(default) " - echo " build native library for python" - echo " java build native library for java" + echo " -l|--language language1[,language2]" + echo " a list of languages to build native libraries." + echo " Supported languages include \"python\" and \"java\"." + echo " If not specified, only python library will be built." echo " -p|--python which python executable (default from which python)" echo } @@ -32,7 +33,8 @@ else exit 1 fi -LANGUAGE="python" +RAY_BUILD_PYTHON="YES" +RAY_BUILD_JAVA="NO" PYTHON_EXECUTABLE="" BUILD_DIR="" if [ "$VALGRIND" = "1" ]; then @@ -54,8 +56,16 @@ while [[ $# > 0 ]]; do ;; -l|--languags) LANGUAGE="$2" - if [ "$LANGUAGE" != "python" ] && [ "$LANGUAGE" != "java" ]; then - echo "Unrecognized language." + RAY_BUILD_PYTHON="NO" + RAY_BUILD_JAVA="NO" + if [[ "$LANGUAGE" == *"python"* ]]; then + RAY_BUILD_PYTHON="YES" + fi + if [[ "$LANGUAGE" == *"java"* ]]; then + RAY_BUILD_JAVA="YES" + fi + if [ "$RAY_BUILD_PYTHON" == "NO" ] && [ "$RAY_BUILD_JAVA" == "NO" ]; then + echo "Unrecognized language: $LANGUAGE" exit -1 fi shift @@ -79,7 +89,9 @@ if [[ -z "$PYTHON_EXECUTABLE" ]]; then fi echo "Using Python executable $PYTHON_EXECUTABLE." -bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE $LANGUAGE +RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \ +RAY_BUILD_JAVA=$RAY_BUILD_JAVA \ +bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE # Now we build everything. BUILD_DIR="$ROOT_DIR/build/" @@ -95,7 +107,8 @@ ARROW_HOME=$TP_PKG_DIR/arrow/cpp/build/cpp-install BOOST_ROOT=$TP_PKG_DIR/boost \ PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \ cmake -DCMAKE_BUILD_TYPE=$CBUILD_TYPE \ - -DCMAKE_RAY_LANGUAGE=$LANGUAGE \ + -DCMAKE_RAY_LANG_JAVA=$RAY_BUILD_JAVA \ + -DCMAKE_RAY_LANG_PYTHON=$RAY_BUILD_PYTHON \ -DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \ -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE $ROOT_DIR @@ -105,9 +118,9 @@ popd # Move stuff from Arrow to Ray. cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/src/plasma/ -if [[ "$LANGUAGE" == "python" ]]; then +if [[ "$RAY_BUILD_PYTHON" == "YES" ]]; then cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/../python/ray/core/src/plasma/ fi -if [[ "$LANGUAGE" == "java" ]]; then +if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/ fi diff --git a/java/test.sh b/java/test.sh index b2c99af29..3e81151fa 100755 --- a/java/test.sh +++ b/java/test.sh @@ -7,7 +7,7 @@ set -e set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -$ROOT_DIR/../build.sh -l java +$ROOT_DIR/../build.sh -l java,python pushd $ROOT_DIR/../thirdparty/build/arrow/java mvn clean install -pl plasma -am -Dmaven.test.skip diff --git a/setup_thirdparty.sh b/setup_thirdparty.sh index c6f00930d..b9d22637e 100755 --- a/setup_thirdparty.sh +++ b/setup_thirdparty.sh @@ -13,14 +13,11 @@ else fi echo "Using Python executable $PYTHON_EXECUTABLE." -LANGUAGE="python" -if [[ -n "$2" ]]; then - LANGUAGE=$2 -fi +RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \ +RAY_BUILD_JAVA=$RAY_BUILD_JAVA \ +$ROOT_DIR/thirdparty/scripts/setup.sh $PYTHON_EXECUTABLE -$ROOT_DIR/thirdparty/scripts/setup.sh $PYTHON_EXECUTABLE $LANGUAGE - -if [[ "$LANGUAGE" == "java" ]]; then +if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then pushd $ROOT_DIR/thirdparty/build/arrow/java mvn clean install -pl plasma -am -Dmaven.test.skip popd diff --git a/src/local_scheduler/CMakeLists.txt b/src/local_scheduler/CMakeLists.txt index a2ddb0b2f..2634599ed 100644 --- a/src/local_scheduler/CMakeLists.txt +++ b/src/local_scheduler/CMakeLists.txt @@ -17,9 +17,6 @@ endif() add_definitions(-fPIC) if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") - if(APPLE) - SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so") - endif(APPLE) include_directories("${PYTHON_INCLUDE_DIRS}") include_directories("${NUMPY_INCLUDE_DIR}") endif() @@ -76,7 +73,8 @@ macro(get_local_scheduler_library LANG VAR) endmacro() macro(set_local_scheduler_library LANG) - get_local_scheduler_library(${LANG} LOCAL_SCHEDULER_LIBRARY_LANG) + get_local_scheduler_library(${LANG} LOCAL_SCHEDULER_LIBRARY_${LANG}) + set(LOCAL_SCHEDULER_LIBRARY_LANG ${LOCAL_SCHEDULER_LIBRARY_${LANG}}) include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/") file(GLOB LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC @@ -86,6 +84,9 @@ macro(set_local_scheduler_library LANG) ${LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC}) if(APPLE) + if ("${LANG}" STREQUAL "python") + SET_TARGET_PROPERTIES(${LOCAL_SCHEDULER_LIBRARY_LANG} PROPERTIES SUFFIX .so) + endif() target_link_libraries(${LOCAL_SCHEDULER_LIBRARY_LANG} "-undefined dynamic_lookup" local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY}) else(APPLE) target_link_libraries(${LOCAL_SCHEDULER_LIBRARY_LANG} local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY}) diff --git a/thirdparty/scripts/build_arrow.sh b/thirdparty/scripts/build_arrow.sh index 165e9fded..48ea0e780 100755 --- a/thirdparty/scripts/build_arrow.sh +++ b/thirdparty/scripts/build_arrow.sh @@ -15,12 +15,6 @@ else fi echo "Using Python executable $PYTHON_EXECUTABLE." -LANGUAGE="python" -if [[ -n "$2" ]]; then - LANGUAGE=$2 -fi -echo "Build language is $LANGUAGE." - unamestr="$(uname)" if [[ "$unamestr" == "Linux" ]]; then @@ -46,9 +40,8 @@ fi TARGET_COMMIT_ID=d48dce2cfebdbd044a8260d0a77f5fe3d89a4a2d build_arrow() { echo "building arrow" - # Make sure arrow will be built again when building ray for java later than python - if [[ "$LANGUAGE" == "java" ]]; then + if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then rm -rf $TP_DIR/build/arrow/cpp/build/CMakeCache.txt fi @@ -78,7 +71,7 @@ build_arrow() { cd build BUILD_ARROW_PLASMA_JAVA_CLIENT=off - if [[ "$LANGUAGE" == "java" ]]; then + if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then BUILD_ARROW_PLASMA_JAVA_CLIENT=on fi @@ -147,8 +140,8 @@ build_arrow() { popd } # Download and compile arrow if it isn't already present or the commit-id mismatches. -if [[ ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]] || \ - [[ "$LANGUAGE" == "java" && ! -f $TP_DIR/build/arrow/cpp/build/release/libplasma_java.dylib ]]; then +if [[ "$RAY_BUILD_PYTHON" == "YES" && ! -d $TP_DIR/../python/ray/pyarrow_files/pyarrow ]] || \ + [[ "$RAY_BUILD_JAVA" == "YES" && ! -f $TP_DIR/build/arrow/cpp/build/release/libplasma_java.dylib ]]; then build_arrow else REBUILD=off diff --git a/thirdparty/scripts/setup.sh b/thirdparty/scripts/setup.sh index 7a268aba6..125713f4e 100755 --- a/thirdparty/scripts/setup.sh +++ b/thirdparty/scripts/setup.sh @@ -18,11 +18,12 @@ else fi echo "Using Python executable $PYTHON_EXECUTABLE." -LANGUAGE="python" -if [[ -n "$2" ]]; then - LANGUAGE=$2 +if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then +echo "Java library will be built." +fi +if [[ "$RAY_BUILD_PYTHON" == "YES" ]]; then +echo "Python library will be built." fi -echo "Build language is $LANGUAGE." unamestr="$(uname)" @@ -52,7 +53,9 @@ fi ############################################## # arrow ############################################## -bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE $LANGUAGE +RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \ +RAY_BUILD_JAVA=$RAY_BUILD_JAVA \ +bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE ############################################## # parquet (skipped as it is inlined in build_arrow.sh)