From 060891a9c94b81a346f8ecb347132bcc3656924e Mon Sep 17 00:00:00 2001 From: Hanwei Jin Date: Thu, 11 Oct 2018 05:33:15 +0800 Subject: [PATCH] [cmake] avoid to re-build pyarrow (#2963) * bugfix: env exists check error * support to avoid re-build pyarrow in project * bugfix: adapt gtest for centos lib64 * bugfix: check gtest lib exists in the directory * bugfix: find gtest with checking all libs exists * prefix RAY_ to thirdparty env variables to avoid conflicts with other module * arrow use glog from ray * change the glog and gtest install dir --- cmake/Modules/ArrowExternalProject.cmake | 10 +-- cmake/Modules/BoostExternalProject.cmake | 4 +- .../Modules/FlatBuffersExternalProject.cmake | 9 +-- cmake/Modules/GlogExternalProject.cmake | 6 +- cmake/Modules/GtestExternalProject.cmake | 37 ++++++---- cmake/Modules/ThirdpartyToolchain.cmake | 72 +++++++++++-------- 6 files changed, 76 insertions(+), 62 deletions(-) diff --git a/cmake/Modules/ArrowExternalProject.cmake b/cmake/Modules/ArrowExternalProject.cmake index 827673c35..da3f27ba9 100644 --- a/cmake/Modules/ArrowExternalProject.cmake +++ b/cmake/Modules/ArrowExternalProject.cmake @@ -23,11 +23,6 @@ set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install) set(ARROW_HOME ${ARROW_INSTALL_PREFIX}) set(ARROW_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/arrow/src/arrow_ep) -# The following is needed because in CentOS, the lib directory is named lib64 -if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8) - set(LIB_SUFFIX 64) -endif() - set(ARROW_INCLUDE_DIR ${ARROW_HOME}/include) set(ARROW_LIBRARY_DIR ${ARROW_HOME}/lib${LIB_SUFFIX}) set(ARROW_SHARED_LIB ${ARROW_LIBRARY_DIR}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX}) @@ -58,7 +53,8 @@ set(ARROW_CMAKE_ARGS -DARROW_WITH_LZ4=off -DARROW_WITH_ZSTD=off -DFLATBUFFERS_HOME=${FLATBUFFERS_HOME} - -DBOOST_ROOT=${BOOST_ROOT}) + -DBOOST_ROOT=${BOOST_ROOT} + -DGLOG_HOME=${GLOG_HOME}) if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") # PyArrow needs following settings. @@ -92,7 +88,7 @@ endif() ExternalProject_Add(arrow_ep PREFIX external/arrow - DEPENDS flatbuffers boost + DEPENDS flatbuffers boost glog GIT_REPOSITORY ${arrow_URL} GIT_TAG ${arrow_TAG} ${ARROW_CONFIGURE} diff --git a/cmake/Modules/BoostExternalProject.cmake b/cmake/Modules/BoostExternalProject.cmake index bab016a02..1fbbb0c0b 100644 --- a/cmake/Modules/BoostExternalProject.cmake +++ b/cmake/Modules/BoostExternalProject.cmake @@ -9,9 +9,9 @@ # boost is a stable library in ray, and it supports to find # the boost pre-built in environment to speed up build process -if (DEFINED ENV{BOOST_ROOT} AND EXISTS ENV{BOOST_ROOT}) +if (DEFINED ENV{RAY_BOOST_ROOT} AND EXISTS $ENV{RAY_BOOST_ROOT}) set(Boost_USE_STATIC_LIBS ON) - set(BOOST_ROOT "$ENV{BOOST_ROOT}") + set(BOOST_ROOT "$ENV{RAY_BOOST_ROOT}") message(STATUS "Find BOOST_ROOT: ${BOOST_ROOT}") # find_package(Boost COMPONENTS system filesystem REQUIRED) set(Boost_INCLUDE_DIR ${BOOST_ROOT}/include) diff --git a/cmake/Modules/FlatBuffersExternalProject.cmake b/cmake/Modules/FlatBuffersExternalProject.cmake index 57c2216ce..508010afc 100644 --- a/cmake/Modules/FlatBuffersExternalProject.cmake +++ b/cmake/Modules/FlatBuffersExternalProject.cmake @@ -10,13 +10,8 @@ # - FLATBUFFERS_COMPILER # - FBS_DEPENDS, to keep compatible -# The following is needed because in CentOS, the lib directory is named lib64 -if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8) - set(LIB_SUFFIX 64) -endif() - -if(DEFINED ENV{FLATBUFFERS_HOME} AND EXISTS ENV{FLATBUFFERS_HOME}) - set(FLATBUFFERS_HOME "$ENV{FLATBUFFERS_HOME}") +if(DEFINED ENV{RAY_FLATBUFFERS_HOME} AND EXISTS $ENV{RAY_FLATBUFFERS_HOME}) + set(FLATBUFFERS_HOME "$ENV{RAY_FLATBUFFERS_HOME}") set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_HOME}/include") set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_HOME}/lib${LIB_SUFFIX}/libflatbuffers.a") set(FLATBUFFERS_COMPILER "${FLATBUFFERS_HOME}/bin/flatc") diff --git a/cmake/Modules/GlogExternalProject.cmake b/cmake/Modules/GlogExternalProject.cmake index 47f11fbdb..2900bae4d 100644 --- a/cmake/Modules/GlogExternalProject.cmake +++ b/cmake/Modules/GlogExternalProject.cmake @@ -6,8 +6,8 @@ # - GLOG_INCLUDE_DIR # - GLOG_STATIC_LIB -if(DEFINED ENV{GLOG_HOME} AND EXISTS ENV{GLOG_HOME}) - set(GLOG_HOME "$ENV{GLOG_HOME}") +if(DEFINED ENV{RAY_GLOG_HOME} AND EXISTS $ENV{RAY_GLOG_HOME}) + set(GLOG_HOME "$ENV{RAY_GLOG_HOME}") set(GLOG_INCLUDE_DIR "${GLOG_HOME}/include") set(GLOG_STATIC_LIB "${GLOG_HOME}/lib/libglog.a") @@ -23,7 +23,7 @@ else() endif() set(GLOG_URL "https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz") - set(GLOG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/glog/src/glog_ep") + set(GLOG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/glog-install") set(GLOG_HOME "${GLOG_PREFIX}") set(GLOG_INCLUDE_DIR "${GLOG_PREFIX}/include") set(GLOG_STATIC_LIB "${GLOG_PREFIX}/lib/libglog.a") diff --git a/cmake/Modules/GtestExternalProject.cmake b/cmake/Modules/GtestExternalProject.cmake index 5570066c6..66e5a76f1 100644 --- a/cmake/Modules/GtestExternalProject.cmake +++ b/cmake/Modules/GtestExternalProject.cmake @@ -7,18 +7,31 @@ # - GTEST_MAIN_STATIC_LIB # - GMOCK_MAIN_STATIC_LIB -if(DEFINED ENV{GTEST_HOME} AND EXISTS ENV{GTEST_HOME}) - set(GTEST_HOME "$ENV{GTEST_HOME}") - set(GTEST_INCLUDE_DIR "${GTEST_HOME}/include") - set(GTEST_STATIC_LIB - "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GTEST_MAIN_STATIC_LIB - "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GMOCK_MAIN_STATIC_LIB - "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_STATIC_LIBRARY_SUFFIX}") +set(GTEST_FOUND FALSE) - add_custom_target(googletest_ep) -else() +if(DEFINED ENV{RAY_GTEST_HOME} AND EXISTS $ENV{RAY_GTEST_HOME}) + set(GTEST_HOME "$ENV{RAY_GTEST_HOME}") + find_path(GTEST_INCLUDE_DIR NAMES gtest/gtest.h + PATHS ${GTEST_HOME} NO_DEFAULT_PATH + PATH_SUFFIXES "include") + find_library(GTEST_LIBRARIES NAMES gtest gtest_main gmock_main + PATHS ${GTEST_HOME} NO_DEFAULT_PATH + PATH_SUFFIXES "lib") + if(GTEST_INCLUDE_DIR AND GTEST_LIBRARIES) + set(GTEST_FOUND TRUE) + set(GTEST_STATIC_LIB + "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GTEST_MAIN_STATIC_LIB + "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GMOCK_MAIN_STATIC_LIB + "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_STATIC_LIBRARY_SUFFIX}") + + add_custom_target(googletest_ep) + endif() + +endif() + +if(NOT GTEST_FOUND) set(GTEST_VERSION "1.8.0") if(APPLE) @@ -31,7 +44,7 @@ else() endif() set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}") - set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/googletest/src/googletest_ep") + set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/googletest-install") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GTEST_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") diff --git a/cmake/Modules/ThirdpartyToolchain.cmake b/cmake/Modules/ThirdpartyToolchain.cmake index 0e0553483..de06b6b7f 100644 --- a/cmake/Modules/ThirdpartyToolchain.cmake +++ b/cmake/Modules/ThirdpartyToolchain.cmake @@ -4,6 +4,11 @@ # we have to turn it on for dependencies too set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") +# The following is needed because in CentOS, the lib directory is named lib64 +if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8) + set(LIB_SUFFIX 64) +endif() + if(RAY_BUILD_TESTS OR RAY_BUILD_BENCHMARKS) add_custom_target(unittest ctest -L unittest) @@ -25,18 +30,16 @@ if(RAY_BUILD_TESTS OR RAY_BUILD_BENCHMARKS) add_dependencies(gmock_main googletest_ep) endif() -if(RAY_USE_GLOG) - include(GlogExternalProject) - message(STATUS "Glog home: ${GLOG_HOME}") - message(STATUS "Glog include dir: ${GLOG_INCLUDE_DIR}") - message(STATUS "Glog static lib: ${GLOG_STATIC_LIB}") +include(GlogExternalProject) +message(STATUS "Glog home: ${GLOG_HOME}") +message(STATUS "Glog include dir: ${GLOG_INCLUDE_DIR}") +message(STATUS "Glog static lib: ${GLOG_STATIC_LIB}") - include_directories(${GLOG_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(glog - STATIC_LIB ${GLOG_STATIC_LIB}) +include_directories(${GLOG_INCLUDE_DIR}) +ADD_THIRDPARTY_LIB(glog + STATIC_LIB ${GLOG_STATIC_LIB}) - add_dependencies(glog glog_ep) -endif() +add_dependencies(glog glog_ep) # boost include(BoostExternalProject) @@ -95,19 +98,6 @@ ADD_THIRDPARTY_LIB(plasma STATIC_LIB ${PLASMA_STATIC_LIB}) add_dependencies(plasma plasma_ep) if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") - # pyarrow - find_package(PythonInterp REQUIRED) - message(STATUS "PYTHON_EXECUTABLE for pyarrow: ${PYTHON_EXECUTABLE}") - - set(pyarrow_ENV - "PKG_CONFIG_PATH=${ARROW_LIBRARY_DIR}/pkgconfig" - "PYARROW_WITH_PLASMA=1" - "PYARROW_WITH_TENSORFLOW=1" - "PYARROW_BUNDLE_ARROW_CPP=1" - "PARQUET_HOME=${PARQUET_HOME}" - "PYARROW_WITH_PARQUET=1" - ) - # clean the arrow_ep/python/build/lib.xxxxx directory, # or when you build with another python version, it creates multiple lib.xxxx directories set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${ARROW_SOURCE_DIR}/python/build/") @@ -115,13 +105,33 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") # here we use externalProject to process pyarrow building # add_custom_command would have problem with setup.py - ExternalProject_Add(pyarrow_ext - PREFIX external/pyarrow - DEPENDS arrow_ep - DOWNLOAD_COMMAND "" - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build - BUILD_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build_ext - INSTALL_COMMAND bash -c "cp -rf \$(find ${ARROW_SOURCE_DIR}/python/build/ -maxdepth 1 -type d -print | grep -m1 'lib')/pyarrow ${CMAKE_SOURCE_DIR}/python/ray/pyarrow_files/") + if(EXISTS ${ARROW_SOURCE_DIR}/python/build/) + # if we did not run `make clean`, skip the rebuild of pyarrow + add_custom_target(pyarrow_ext) + else() + # pyarrow + find_package(PythonInterp REQUIRED) + message(STATUS "PYTHON_EXECUTABLE for pyarrow: ${PYTHON_EXECUTABLE}") + + # PYARROW_PARALLEL= , so it will add -j to pyarrow build + set(pyarrow_ENV + "PKG_CONFIG_PATH=${ARROW_LIBRARY_DIR}/pkgconfig" + "PYARROW_WITH_PLASMA=1" + "PYARROW_WITH_TENSORFLOW=1" + "PYARROW_BUNDLE_ARROW_CPP=1" + "PARQUET_HOME=${PARQUET_HOME}" + "PYARROW_WITH_PARQUET=1" + "PYARROW_PARALLEL=") + + ExternalProject_Add(pyarrow_ext + PREFIX external/pyarrow + DEPENDS arrow_ep + DOWNLOAD_COMMAND "" + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build + BUILD_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build_ext + INSTALL_COMMAND bash -c "cp -rf \$(find ${ARROW_SOURCE_DIR}/python/build/ -maxdepth 1 -type d -print | grep -m1 'lib')/pyarrow ${CMAKE_SOURCE_DIR}/python/ray/pyarrow_files/") + + endif() endif ()