[cmake] avoid to re-build pyarrow (#2963)

* bugfix: env exists check error

* support to avoid re-build pyarrow in project

* bugfix: adapt gtest for centos lib64

* bugfix: check gtest lib exists in the directory

* bugfix: find gtest with checking all libs exists

* prefix RAY_ to thirdparty env variables to avoid conflicts with other module

* arrow use glog from ray

* change the glog and gtest install dir

cmake/Modules/ArrowExternalProject.cmake

@@ -23,11 +23,6 @@ set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
 set(ARROW_HOME ${ARROW_INSTALL_PREFIX})
 set(ARROW_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/arrow/src/arrow_ep)
 
-# The following is needed because in CentOS, the lib directory is named lib64
-if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
-  set(LIB_SUFFIX 64)
-endif()
-
 set(ARROW_INCLUDE_DIR ${ARROW_HOME}/include)
 set(ARROW_LIBRARY_DIR ${ARROW_HOME}/lib${LIB_SUFFIX})
 set(ARROW_SHARED_LIB ${ARROW_LIBRARY_DIR}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
@@ -58,7 +53,8 @@ set(ARROW_CMAKE_ARGS
   -DARROW_WITH_LZ4=off
   -DARROW_WITH_ZSTD=off
   -DFLATBUFFERS_HOME=${FLATBUFFERS_HOME}
-  -DBOOST_ROOT=${BOOST_ROOT})
+  -DBOOST_ROOT=${BOOST_ROOT}
+  -DGLOG_HOME=${GLOG_HOME})
 
 if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   # PyArrow needs following settings.
@@ -92,7 +88,7 @@ endif()
 
 ExternalProject_Add(arrow_ep
   PREFIX external/arrow
-  DEPENDS flatbuffers boost
+  DEPENDS flatbuffers boost glog
   GIT_REPOSITORY ${arrow_URL}
   GIT_TAG ${arrow_TAG}
   ${ARROW_CONFIGURE}

cmake/Modules/BoostExternalProject.cmake

@@ -9,9 +9,9 @@
 
 # boost is a stable library in ray, and it supports to find
 # the boost pre-built in environment to speed up build process
-if (DEFINED ENV{BOOST_ROOT} AND EXISTS ENV{BOOST_ROOT})
+if (DEFINED ENV{RAY_BOOST_ROOT} AND EXISTS $ENV{RAY_BOOST_ROOT})
   set(Boost_USE_STATIC_LIBS ON)
-  set(BOOST_ROOT "$ENV{BOOST_ROOT}")
+  set(BOOST_ROOT "$ENV{RAY_BOOST_ROOT}")
   message(STATUS "Find BOOST_ROOT: ${BOOST_ROOT}")
 #  find_package(Boost COMPONENTS system filesystem REQUIRED)
   set(Boost_INCLUDE_DIR ${BOOST_ROOT}/include)

cmake/Modules/FlatBuffersExternalProject.cmake

@@ -10,13 +10,8 @@
 #  - FLATBUFFERS_COMPILER
 #  - FBS_DEPENDS, to keep compatible
 
-# The following is needed because in CentOS, the lib directory is named lib64
-if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
-  set(LIB_SUFFIX 64)
-endif()
-
-if(DEFINED ENV{FLATBUFFERS_HOME} AND EXISTS ENV{FLATBUFFERS_HOME})
-  set(FLATBUFFERS_HOME "$ENV{FLATBUFFERS_HOME}")
+if(DEFINED ENV{RAY_FLATBUFFERS_HOME} AND EXISTS $ENV{RAY_FLATBUFFERS_HOME})
+  set(FLATBUFFERS_HOME "$ENV{RAY_FLATBUFFERS_HOME}")
   set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_HOME}/include")
   set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_HOME}/lib${LIB_SUFFIX}/libflatbuffers.a")
   set(FLATBUFFERS_COMPILER "${FLATBUFFERS_HOME}/bin/flatc")

cmake/Modules/GlogExternalProject.cmake

@@ -6,8 +6,8 @@
 #  - GLOG_INCLUDE_DIR
 #  - GLOG_STATIC_LIB
 
-if(DEFINED ENV{GLOG_HOME} AND EXISTS ENV{GLOG_HOME})
-  set(GLOG_HOME "$ENV{GLOG_HOME}")
+if(DEFINED ENV{RAY_GLOG_HOME} AND EXISTS $ENV{RAY_GLOG_HOME})
+  set(GLOG_HOME "$ENV{RAY_GLOG_HOME}")
   set(GLOG_INCLUDE_DIR "${GLOG_HOME}/include")
   set(GLOG_STATIC_LIB "${GLOG_HOME}/lib/libglog.a")
 
@@ -23,7 +23,7 @@ else()
   endif()
 
   set(GLOG_URL "https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz")
-  set(GLOG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/glog/src/glog_ep")
+  set(GLOG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/glog-install")
   set(GLOG_HOME "${GLOG_PREFIX}")
   set(GLOG_INCLUDE_DIR "${GLOG_PREFIX}/include")
   set(GLOG_STATIC_LIB "${GLOG_PREFIX}/lib/libglog.a")

cmake/Modules/GtestExternalProject.cmake

@@ -7,18 +7,31 @@
 #  - GTEST_MAIN_STATIC_LIB
 #  - GMOCK_MAIN_STATIC_LIB
 
-if(DEFINED ENV{GTEST_HOME} AND EXISTS ENV{GTEST_HOME})
-  set(GTEST_HOME "$ENV{GTEST_HOME}")
-  set(GTEST_INCLUDE_DIR "${GTEST_HOME}/include")
-  set(GTEST_STATIC_LIB
-    "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
-  set(GTEST_MAIN_STATIC_LIB
-    "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
-  set(GMOCK_MAIN_STATIC_LIB
-    "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
+set(GTEST_FOUND FALSE)
 
-  add_custom_target(googletest_ep)
-else()
+if(DEFINED ENV{RAY_GTEST_HOME} AND EXISTS $ENV{RAY_GTEST_HOME})
+  set(GTEST_HOME "$ENV{RAY_GTEST_HOME}")
+  find_path(GTEST_INCLUDE_DIR NAMES gtest/gtest.h
+    PATHS ${GTEST_HOME} NO_DEFAULT_PATH
+    PATH_SUFFIXES "include")
+  find_library(GTEST_LIBRARIES NAMES gtest gtest_main gmock_main
+    PATHS ${GTEST_HOME} NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib")
+  if(GTEST_INCLUDE_DIR AND GTEST_LIBRARIES)
+    set(GTEST_FOUND TRUE)
+    set(GTEST_STATIC_LIB
+      "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GTEST_MAIN_STATIC_LIB
+      "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GMOCK_MAIN_STATIC_LIB
+      "${GTEST_HOME}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
+
+    add_custom_target(googletest_ep)
+  endif()
+
+endif()
+
+if(NOT GTEST_FOUND)
   set(GTEST_VERSION "1.8.0")
 
   if(APPLE)
@@ -31,7 +44,7 @@ else()
   endif()
   set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}")
 
-  set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/googletest/src/googletest_ep")
+  set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/googletest-install")
   set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
   set(GTEST_STATIC_LIB
     "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")

cmake/Modules/ThirdpartyToolchain.cmake

@@ -4,6 +4,11 @@
 # we have to turn it on for dependencies too
 set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
 
+# The following is needed because in CentOS, the lib directory is named lib64
+if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
+  set(LIB_SUFFIX 64)
+endif()
+
 if(RAY_BUILD_TESTS OR RAY_BUILD_BENCHMARKS)
   add_custom_target(unittest ctest -L unittest)
 
@@ -25,18 +30,16 @@ if(RAY_BUILD_TESTS OR RAY_BUILD_BENCHMARKS)
   add_dependencies(gmock_main googletest_ep)
 endif()
 
-if(RAY_USE_GLOG)
-  include(GlogExternalProject)
-  message(STATUS "Glog home: ${GLOG_HOME}")
-  message(STATUS "Glog include dir: ${GLOG_INCLUDE_DIR}")
-  message(STATUS "Glog static lib: ${GLOG_STATIC_LIB}")
+include(GlogExternalProject)
+message(STATUS "Glog home: ${GLOG_HOME}")
+message(STATUS "Glog include dir: ${GLOG_INCLUDE_DIR}")
+message(STATUS "Glog static lib: ${GLOG_STATIC_LIB}")
 
-  include_directories(${GLOG_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(glog
-    STATIC_LIB ${GLOG_STATIC_LIB})
+include_directories(${GLOG_INCLUDE_DIR})
+ADD_THIRDPARTY_LIB(glog
+  STATIC_LIB ${GLOG_STATIC_LIB})
 
-  add_dependencies(glog glog_ep)
-endif()
+add_dependencies(glog glog_ep)
 
 # boost
 include(BoostExternalProject)
@@ -95,19 +98,6 @@ ADD_THIRDPARTY_LIB(plasma STATIC_LIB ${PLASMA_STATIC_LIB})
 add_dependencies(plasma plasma_ep)
 
 if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
-  # pyarrow
-  find_package(PythonInterp REQUIRED)
-  message(STATUS "PYTHON_EXECUTABLE for pyarrow: ${PYTHON_EXECUTABLE}")
-
-  set(pyarrow_ENV
-    "PKG_CONFIG_PATH=${ARROW_LIBRARY_DIR}/pkgconfig"
-    "PYARROW_WITH_PLASMA=1"
-    "PYARROW_WITH_TENSORFLOW=1"
-    "PYARROW_BUNDLE_ARROW_CPP=1"
-    "PARQUET_HOME=${PARQUET_HOME}"
-    "PYARROW_WITH_PARQUET=1"
-  )
-
   # clean the arrow_ep/python/build/lib.xxxxx directory,
   # or when you build with another python version, it creates multiple lib.xxxx directories
   set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${ARROW_SOURCE_DIR}/python/build/")
@@ -115,13 +105,33 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
 
   # here we use externalProject to process pyarrow building
   # add_custom_command would have problem with setup.py
-  ExternalProject_Add(pyarrow_ext
-    PREFIX external/pyarrow
-    DEPENDS arrow_ep
-    DOWNLOAD_COMMAND ""
-    BUILD_IN_SOURCE 1
-    CONFIGURE_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build
-    BUILD_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build_ext
-    INSTALL_COMMAND bash -c "cp -rf \$(find ${ARROW_SOURCE_DIR}/python/build/ -maxdepth 1 -type d -print | grep -m1 'lib')/pyarrow ${CMAKE_SOURCE_DIR}/python/ray/pyarrow_files/")
+  if(EXISTS ${ARROW_SOURCE_DIR}/python/build/)
+    # if we did not run `make clean`, skip the rebuild of pyarrow
+    add_custom_target(pyarrow_ext)
+  else()
+    # pyarrow
+    find_package(PythonInterp REQUIRED)
+    message(STATUS "PYTHON_EXECUTABLE for pyarrow: ${PYTHON_EXECUTABLE}")
+
+    # PYARROW_PARALLEL= , so it will add -j to pyarrow build
+    set(pyarrow_ENV
+      "PKG_CONFIG_PATH=${ARROW_LIBRARY_DIR}/pkgconfig"
+      "PYARROW_WITH_PLASMA=1"
+      "PYARROW_WITH_TENSORFLOW=1"
+      "PYARROW_BUNDLE_ARROW_CPP=1"
+      "PARQUET_HOME=${PARQUET_HOME}"
+      "PYARROW_WITH_PARQUET=1"
+      "PYARROW_PARALLEL=")
+
+    ExternalProject_Add(pyarrow_ext
+      PREFIX external/pyarrow
+      DEPENDS arrow_ep
+      DOWNLOAD_COMMAND ""
+      BUILD_IN_SOURCE 1
+      CONFIGURE_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build
+      BUILD_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build_ext
+      INSTALL_COMMAND bash -c "cp -rf \$(find ${ARROW_SOURCE_DIR}/python/build/ -maxdepth 1 -type d -print | grep -m1 'lib')/pyarrow ${CMAKE_SOURCE_DIR}/python/ray/pyarrow_files/")
+
+  endif()
 
 endif ()