update ray cmake build process (#2853)

* use cmake to build ray project, no need to appply build.sh before cmake, fix some abuse of cmake, improve the build performance

* support boost external project, avoid using the system or build.sh boost

* keep compatible with build.sh, remove boost and arrow build from it.

* bugfix: parquet bison version control, plasma_java lib install problem

* bugfix: cmake, do not compile plasma java client if no need

* bugfix: component failures test timeout machenism has problem for plasma manager failed case

* bugfix: arrow use lib64 in centos, travis check-git-clang-format-output.sh does not support other branches except master

* revert some fix

* set arrow python executable, fix format error in component_failures_test.py

* make clean arrow python build directory

* update cmake code style, back to support cmake minimum version 3.4

.gitignore

@@ -158,3 +158,5 @@ java/**/.settings
 java/**/.classpath
 java/**/.project
 
+# python virtual env
+venv

CMakeLists.txt

@@ -4,11 +4,6 @@ project(ray)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules")
 
-include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/scripts/thirdparty.cmake)
-
-find_package(Arrow)
-find_package(Plasma)
-
 # This ensures that things like gnu++11 get passed correctly
 set(CMAKE_CXX_STANDARD 11)
 
@@ -18,6 +13,15 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
 # We require a C++11 compliant compiler
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
+# ray build options
+option(RAY_BUILD_PYTHON
+    "build python library"
+    OFF)
+
+option(RAY_BUILD_JAVA
+    "build java library"
+    OFF)
+
 option(RAY_BUILD_STATIC
   "Build the libray static libraries"
   ON)
@@ -38,6 +42,19 @@ option(RAY_USE_GLOG
   "Build the logging system using glog"
   ON)
 
+if (RAY_BUILD_PYTHON)
+  set(CMAKE_RAY_LANG_PYTHON YES)
+endif ()
+
+if (RAY_BUILD_JAVA)
+  set(CMAKE_RAY_LANG_JAVA YES)
+endif ()
+
+if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "NO" AND "${CMAKE_RAY_LANG_PYTHON}" STREQUAL "NO")
+  message(FATAL_ERROR "Please specify which language support you want to build by passing \
+   -DRAY_BUILD_PYTHON=on and/or -DRAY_BUILD_JAVA=on to CMake")
+endif ()
+
 if (RAY_USE_NEW_GCS)
   add_definitions(-DRAY_USE_NEW_GCS)
 endif()
@@ -48,7 +65,7 @@ include(BuildUtils)
 enable_testing()
 
 include(ThirdpartyToolchain)
-
+include(Common)
 
 # TODO(rkn): Fix all of this. This include is needed for the following
 # reason. The local scheduler depends on tables.cc which depends on
@@ -70,7 +87,22 @@ add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/)
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/local_scheduler/)
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/global_scheduler/)
 
+# final target copy_ray
+add_custom_target(copy_ray ALL)
+
+# Make sure redis-server is ready before copying.
+add_dependencies(copy_ray copy_redis)
+add_dependencies(copy_ray ray_redis_module)
+
+# copy plasma_store_server
+add_custom_command(TARGET copy_ray POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E
+    copy ${ARROW_HOME}/bin/plasma_store_server ${CMAKE_CURRENT_BINARY_DIR}/src/plasma)
+
 if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
+  # add pyarrow as the dependency
+  add_dependencies(copy_ray pyarrow_ext)
+
   # NOTE: The lists below must be kept in sync with ray/python/setup.py.
 
   set(ray_file_list
@@ -99,23 +131,12 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
     list(APPEND build_ray_file_list ${CMAKE_BINARY_DIR}/${file})
   endforeach()
 
-  add_custom_target(copy_ray ALL
-      DEPENDS ${build_ray_file_list})
-
-  # Make sure redis-server is ready before copying.
-  add_dependencies(copy_ray copy_redis)
+  add_custom_target(copy_ray_files DEPENDS ${build_ray_file_list})
+  add_dependencies(copy_ray copy_ray_files)
 
   # Make sure that the Python extensions are built before copying the files.
-  if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
-    get_local_scheduler_library("python" LOCAL_SCHEDULER_LIBRARY_PYTHON)
-    add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_PYTHON})
-  endif()
-  if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
-    get_local_scheduler_library("java" LOCAL_SCHEDULER_LIBRARY_JAVA)
-    add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_JAVA})
-  endif()
-
-  add_dependencies(copy_ray ray_redis_module)
+  get_local_scheduler_library("python" LOCAL_SCHEDULER_LIBRARY_PYTHON)
+  add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_PYTHON})
 
   foreach(file ${ray_file_list})
   add_custom_command(TARGET copy_ray POST_BUILD
@@ -123,4 +144,20 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
                         copy ${CMAKE_BINARY_DIR}/${file}
                             ${CMAKE_BINARY_DIR}/../python/ray/core/${file})
   endforeach()
+
+  # copy plasma_store_server to python
+  add_custom_command(TARGET copy_ray POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E
+      copy ${ARROW_HOME}/bin/plasma_store_server ${CMAKE_SOURCE_DIR}/python/ray/core/src/plasma/)
+
 endif()
+
+if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
+  get_local_scheduler_library("java" LOCAL_SCHEDULER_LIBRARY_JAVA)
+  add_dependencies(copy_ray ${LOCAL_SCHEDULER_LIBRARY_JAVA})
+
+  # copy libplasma_java files
+  add_custom_command(TARGET copy_ray POST_BUILD
+    COMMAND bash -c "cp ${ARROW_HOME}/lib/libplasma_java.* ${CMAKE_CURRENT_BINARY_DIR}/src/plasma")
+endif()
+

build.sh

@@ -100,12 +100,7 @@ mkdir -p ${BUILD_DIR}
 fi
 
 pushd "$BUILD_DIR"
-TP_PKG_DIR=$ROOT_DIR/thirdparty/pkg
-# We use these variables to set PKG_CONFIG_PATH, which is important so that
-# in cmake, pkg-config can find plasma.
-ARROW_HOME=$TP_PKG_DIR/arrow/cpp/build/cpp-install
-BOOST_ROOT=$TP_PKG_DIR/boost \
-PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
+
 cmake -DCMAKE_BUILD_TYPE=$CBUILD_TYPE \
       -DCMAKE_RAY_LANG_JAVA=$RAY_BUILD_JAVA \
       -DCMAKE_RAY_LANG_PYTHON=$RAY_BUILD_PYTHON \
@@ -115,12 +110,3 @@ cmake -DCMAKE_BUILD_TYPE=$CBUILD_TYPE \
 make clean
 make -j${PARALLEL}
 popd
-
-# Move stuff from Arrow to Ray.
-cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store_server $BUILD_DIR/src/plasma/
-if [[ "$RAY_BUILD_PYTHON" == "YES" ]]; then
-  cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store_server $BUILD_DIR/../python/ray/core/src/plasma/
-fi
-if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
-  cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/
-fi

cmake/Modules/ArrowExternalProject.cmake

@@ -0,0 +1,91 @@
+# arrow external project
+# target:
+#  - arrow_ep
+# depends:
+#
+# this module defines:
+#  - ARROW_HOME
+#  - ARROW_SOURCE_DIR
+#  - ARROW_INCLUDE_DIR
+#  - ARROW_SHARED_LIB
+#  - ARROW_STATIC_LIB
+#  - PLASMA_INCLUDE_DIR
+#  - PLASMA_STATIC_LIB
+#  - PLASMA_SHARED_LIB
+
+set(arrow_URL https://github.com/apache/arrow.git)
+set(arrow_TAG 927bd34aaad875e82beca2584d5d777839fa8bb0)
+
+set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)
+set(ARROW_HOME ${ARROW_INSTALL_PREFIX})
+set(ARROW_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/arrow/src/arrow_ep)
+
+# The following is needed because in CentOS, the lib directory is named lib64
+if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
+  set(LIB_SUFFIX 64)
+endif()
+
+set(ARROW_INCLUDE_DIR ${ARROW_HOME}/include)
+set(ARROW_LIBRARY_DIR ${ARROW_HOME}/lib${LIB_SUFFIX})
+set(ARROW_SHARED_LIB ${ARROW_LIBRARY_DIR}/libarrow${CMAKE_SHARED_LIBRARY_SUFFIX})
+set(ARROW_STATIC_LIB ${ARROW_LIBRARY_DIR}/libarrow.a)
+
+# plasma in arrow
+set(PLASMA_INCLUDE_DIR ${ARROW_HOME}/include)
+set(PLASMA_SHARED_LIB ${ARROW_LIBRARY_DIR}/libplasma${CMAKE_SHARED_LIBRARY_SUFFIX})
+set(PLASMA_STATIC_LIB ${ARROW_LIBRARY_DIR}/libplasma.a)
+
+find_package(PythonInterp REQUIRED)
+message(STATUS "PYTHON_EXECUTABLE for arrow: ${PYTHON_EXECUTABLE}")
+
+set(ARROW_CMAKE_ARGS
+  -DCMAKE_BUILD_TYPE:STRING=Release
+  -DCMAKE_INSTALL_PREFIX=${ARROW_INSTALL_PREFIX}
+  -DCMAKE_C_FLAGS=-g -O3 ${EP_C_FLAGS}
+  -DCMAKE_CXX_FLAGS=-g -O3 ${EP_CXX_FLAGS}
+  -DARROW_BUILD_TESTS=off
+  -DARROW_HDFS=on
+  -DARROW_BOOST_USE_SHARED=off
+  -DPYTHON_EXECUTABLE:FILEPATH=${PYTHON_EXECUTABLE}
+  -DARROW_PYTHON=on
+  -DARROW_PLASMA=on
+  -DARROW_TENSORFLOW=on
+  -DARROW_JEMALLOC=off
+  -DARROW_WITH_BROTLI=off
+  -DARROW_WITH_LZ4=off
+  -DARROW_WITH_ZLIB=off
+  -DARROW_WITH_ZSTD=off
+  -DFLATBUFFERS_HOME=${FLATBUFFERS_HOME}
+  -DBOOST_ROOT=${BOOST_ROOT})
+
+if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
+  set(ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS} -DARROW_PLASMA_JAVA_CLIENT=ON)
+endif ()
+
+message(STATUS "ARROW_CMAKE_ARGS: ${ARROW_CMAKE_ARGS}")
+
+if (CMAKE_VERSION VERSION_GREATER "3.7")
+  set(ARROW_CONFIGURE SOURCE_SUBDIR "cpp" CMAKE_ARGS ${ARROW_CMAKE_ARGS})
+else()
+  set(ARROW_CONFIGURE CONFIGURE_COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}"
+    ${ARROW_CMAKE_ARGS} "${ARROW_SOURCE_DIR}/cpp")
+endif()
+
+ExternalProject_Add(arrow_ep
+  PREFIX external/arrow
+  DEPENDS flatbuffers_ep boost_ep
+  GIT_REPOSITORY ${arrow_URL}
+  GIT_TAG ${arrow_TAG}
+  ${ARROW_CONFIGURE}
+  BUILD_BYPRODUCTS "${ARROW_SHARED_LIB}" "${ARROW_STATIC_LIB}")
+
+if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
+  ExternalProject_Add_Step(arrow_ep arrow_ep_install_java_lib
+    COMMAND bash -c "cd ${ARROW_SOURCE_DIR}/java && mvn clean install -pl plasma -am -Dmaven.test.skip > /dev/null"
+    DEPENDEES build)
+
+  # add install of library plasma_java, it is not configured in plasma CMakeLists.txt
+  ExternalProject_Add_Step(arrow_ep arrow_ep_install_plasma_java
+    COMMAND bash -c "cp ${CMAKE_CURRENT_BINARY_DIR}/external/arrow/src/arrow_ep-build/release/libplasma_java.* ${ARROW_LIBRARY_DIR}/"
+    DEPENDEES install)
+endif ()

cmake/Modules/BoostExternalProject.cmake

@@ -0,0 +1,38 @@
+# Boost external project
+# target:
+#  - boost_ep
+# defines:
+#  - BOOST_ROOT
+#  - Boost_INCLUDE_DIR
+#  - Boost_SYSTEM_LIBRARY
+#  - Boost_FILESYSTEM_LIBRARY
+
+set(Boost_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/boost-install)
+
+set(Boost_INCLUDE_DIR ${Boost_INSTALL_PREFIX}/include)
+set(BOOST_ROOT ${Boost_INSTALL_PREFIX})
+set(Boost_LIBRARY_DIR ${Boost_INSTALL_PREFIX}/lib)
+set(Boost_SYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX})
+set(Boost_FILESYSTEM_LIBRARY ${Boost_LIBRARY_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX})
+
+#set(boost_URL https://github.com/boostorg/boost.git)
+#set(boost_TAG boost-1.65.1)
+
+set(Boost_TAR_GZ_URL http://dl.bintray.com/boostorg/release/1.65.1/source/boost_1_65_1.tar.gz)
+set(Boost_BUILD_PRODUCTS ${Boost_SYSTEM_LIBRARY} ${Boost_FILESYSTEM_LIBRARY})
+set(Boost_URL_MD5 "ee64fd29a3fe42232c6ac3c419e523cf")
+
+set(Boost_USE_STATIC_LIBS ON)
+
+ExternalProject_Add(boost_ep
+  PREFIX external/boost
+  URL ${Boost_TAR_GZ_URL}
+  URL_MD5 ${Boost_URL_MD5}
+  #    GIT_REPOSITORY ${boost_URL}
+  #    GIT_TAG ${boost_TAG}
+  #    GIT_SUBMODULES ""
+  BUILD_IN_SOURCE 1
+  BUILD_BYPRODUCTS ${Boost_BUILD_PRODUCTS}
+  CONFIGURE_COMMAND ./bootstrap.sh
+  BUILD_COMMAND bash -c "./b2 cxxflags=-fPIC cflags=-fPIC variant=release link=static --with-filesystem --with-system --with-regex -j8 install --prefix=${Boost_INSTALL_PREFIX} > /dev/null"
+  INSTALL_COMMAND "")

cmake/Modules/Common.cmake

@@ -10,35 +10,6 @@ if(UNIX AND NOT APPLE)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -rdynamic")
 endif()
 
-# The following is needed because in CentOS, the lib directory is named lib64
-if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
-  set(LIB_SUFFIX 64)
-endif()
-
-set(FLATBUFFERS_VERSION "1.9.0")
-
-set(FLATBUFFERS_PREFIX "${CMAKE_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
-if (NOT TARGET flatbuffers_ep)
-  ExternalProject_Add(flatbuffers_ep
-    URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
-    CMAKE_ARGS
-      "-DCMAKE_CXX_FLAGS=-fPIC"
-      "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
-      "-DFLATBUFFERS_BUILD_TESTS=OFF"
-      "-DCMAKE_BUILD_TYPE=RELEASE")
-endif()
-
-set(FBS_DEPENDS flatbuffers_ep)
-
-set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
-set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_PREFIX}/lib${LIB_SUFFIX}/libflatbuffers.a")
-set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
-
-message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
-message(STATUS "Flatbuffers static library: ${FLATBUFFERS_STATIC_LIB}")
-message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
-include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
-
 # Custom CFLAGS
 
 set(CMAKE_C_FLAGS "-g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -fPIC -std=c99")
@@ -71,13 +42,5 @@ if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
   endif()
 endif()
 
-# Common libraries
-
-set(COMMON_LIB "${CMAKE_BINARY_DIR}/src/common/libcommon.a"
-    CACHE STRING "Path to libcommon.a")
-
-include_directories("${CMAKE_CURRENT_LIST_DIR}/..")
-include_directories("${CMAKE_CURRENT_LIST_DIR}/../thirdparty/")
-if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
-  include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python")
-endif ()
+include_directories(${CMAKE_SOURCE_DIR}/src/common)
+include_directories(${CMAKE_SOURCE_DIR}/src/common/thirdparty)

cmake/Modules/FlatBuffersExternalProject.cmake

@@ -0,0 +1,38 @@
+# flatbuffers external project
+# target:
+#  - flatbuffer_ep
+# depends:
+#
+# defines:
+#  - FLATBUFFERS_HOME
+#  - FLATBUFFERS_INCLUDE_DIR
+#  - FLATBUFFERS_STATIC_LIB
+#  - FLATBUFFERS_COMPILER
+#  - FBS_DEPENDS, to keep compatible
+
+set(flatbuffers_VERSION "1.9.0")
+set(flatbuffers_URL "https://github.com/google/flatbuffers/archive/v${flatbuffers_VERSION}.tar.gz")
+set(flatbuffers_URL_MD5 "8be7513bf960034f6873326d09521a4b")
+
+set(FLATBUFFERS_INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/flatbuffers-install")
+set(FBS_DEPENDS flatbuffers_ep)
+
+# The following is needed because in CentOS, the lib directory is named lib64
+if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
+  set(LIB_SUFFIX 64)
+endif()
+
+set(FLATBUFFERS_HOME "${FLATBUFFERS_INSTALL_PREFIX}")
+set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_INSTALL_PREFIX}/include")
+set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_INSTALL_PREFIX}/lib${LIB_SUFFIX}/libflatbuffers.a")
+set(FLATBUFFERS_COMPILER "${FLATBUFFERS_INSTALL_PREFIX}/bin/flatc")
+
+ExternalProject_Add(flatbuffers_ep
+  PREFIX external/flatbuffers
+  URL ${flatbuffers_URL}
+  URL_MD5 ${flatbuffers_URL_MD5}
+  CMAKE_ARGS
+  "-DCMAKE_CXX_FLAGS=-fPIC"
+  "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_INSTALL_PREFIX}"
+  "-DFLATBUFFERS_BUILD_TESTS=OFF"
+  "-DCMAKE_BUILD_TYPE=RELEASE")

cmake/Modules/ParquetExternalProject.cmake

@@ -0,0 +1,45 @@
+# parquet external project
+# target:
+#  - parquet_ep
+# depends:
+#  - arrow_ep
+# defines:
+#  - PARQUET_HOME
+#  - PARQUET_INCLUDE_DIR
+#  - PARQUET_STATIC_LIB
+#  - PARQUET_SHARED_LIB
+
+include(ExternalProject)
+
+set(parquet_URL https://github.com/apache/parquet-cpp.git)
+set(parquet_TAG 63f41b00bddecb172bd5b3aa0366b4653f498811)
+
+# install parquet into arrow_home...
+set(PARQUET_INSTALL_PREFIX ${ARROW_HOME})
+set(PARQUET_HOME ${PARQUET_INSTALL_PREFIX})
+set(PARQUET_INCLUDE_DIR ${PARQUET_INSTALL_PREFIX}/include)
+set(PARQUET_STATIC_LIB ${PARQUET_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}parquet.a)
+set(PARQUET_SHARED_LIB ${PARQUET_INSTALL_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}parquet${CMAKE_SHARED_LIBRARY_SUFFIX})
+
+set(parquet_CMAKE_ARGS
+  -DCMAKE_BUILD_TYPE:STRING=Release
+  -DCMAKE_INSTALL_PREFIX=${PARQUET_INSTALL_PREFIX}
+  -DARROW_HOME=${ARROW_HOME}
+  -DBOOST_ROOT=${BOOST_ROOT}
+  -DPARQUET_BUILD_BENCHMARKS=off
+  -DPARQUET_BUILD_EXECUTABLES=off
+  -DPARQUET_BUILD_TESTS=off)
+
+set(parquet_ENV "")
+if (APPLE)
+  set(parquet_ENV "PATH=/usr/local/opt/bison/bin:$ENV{PATH}"
+    "OPENSSL_ROOT_DIR=/usr/local/opt/openssl")
+endif ()
+
+ExternalProject_Add(parquet_ep
+  PREFIX external/parquet
+  DEPENDS arrow_ep
+  GIT_REPOSITORY ${parquet_URL}
+  GIT_TAG ${parquet_TAG}
+  BUILD_COMMAND ${CMAKE_COMMAND} -E env ${parquet_ENV} $(MAKE)
+  CMAKE_ARGS ${parquet_CMAKE_ARGS})

cmake/Modules/ThirdpartyToolchain.cmake

@@ -14,10 +14,12 @@ if(RAY_BUILD_TESTS OR RAY_BUILD_BENCHMARKS)
   elseif(NOT MSVC)
     set(GTEST_CMAKE_CXX_FLAGS "-fPIC")
   endif()
-  string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
+  if(CMAKE_BUILD_TYPE)
+    string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE)
+  endif()
   set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}} ${GTEST_CMAKE_CXX_FLAGS}")
 
-  set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
+  set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/googletest/src/googletest_ep")
   set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
   set(GTEST_STATIC_LIB
     "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
@@ -32,8 +34,12 @@ if(RAY_BUILD_TESTS OR RAY_BUILD_BENCHMARKS)
     set(GTEST_CMAKE_ARGS ${GTEST_CMAKE_ARGS} -Dgtest_force_shared_crt=ON)
   endif()
 
+  set(GTEST_URL_MD5 "16877098823401d1bf2ed7891d7dce36")
+
   ExternalProject_Add(googletest_ep
+    PREFIX external/googletest
     URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz"
+    URL_MD5 ${GTEST_URL_MD5}
     BUILD_BYPRODUCTS ${GTEST_STATIC_LIB} ${GTEST_MAIN_STATIC_LIB} ${GMOCK_MAIN_STATIC_LIB}
     CMAKE_ARGS ${GTEST_CMAKE_ARGS}
     ${EP_LOG_OPTIONS})
@@ -53,24 +59,23 @@ if(RAY_BUILD_TESTS OR RAY_BUILD_BENCHMARKS)
   add_dependencies(gmock_main googletest_ep)
 endif()
 
-set(Boost_USE_STATIC_LIBS ON)
-find_package(Boost COMPONENTS system filesystem REQUIRED)
-include_directories(${Boost_INCLUDE_DIR})
-
 if(RAY_USE_GLOG)
   message(STATUS "Starting to build glog")
   set(GLOG_VERSION "0.3.5")
+  # keep the url md5 equals with the version, `md5 v0.3.5.tar.gz`
+  set(GLOG_URL_MD5 "5df6d78b81e51b90ac0ecd7ed932b0d4")
   set(GLOG_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC")
   if(APPLE)
     set(GLOG_CMAKE_CXX_FLAGS "${GLOG_CMAKE_CXX_FLAGS} -mmacosx-version-min=10.12")
   endif()
 
   set(GLOG_URL "https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz")
-  set(GLOG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/glog_ep-prefix/src/glog_ep")
+  set(GLOG_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/external/glog/src/glog_ep")
   set(GLOG_HOME "${GLOG_PREFIX}")
   set(GLOG_INCLUDE_DIR "${GLOG_PREFIX}/include")
   set(GLOG_STATIC_LIB "${GLOG_PREFIX}/lib/libglog.a")
 
+
   set(GLOG_CMAKE_ARGS -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
                         -DCMAKE_INSTALL_PREFIX=${GLOG_PREFIX}
                         -DBUILD_SHARED_LIBS=OFF
@@ -81,7 +86,9 @@ if(RAY_USE_GLOG)
                         -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS})
 
   ExternalProject_Add(glog_ep
+    PREFIX external/glog
     URL ${GLOG_URL}
+    URL_MD5 ${GLOG_URL_MD5}
     ${EP_LOG_OPTIONS}
     BUILD_IN_SOURCE 1
     BUILD_BYPRODUCTS "${GLOG_STATIC_LIB}"
@@ -95,3 +102,93 @@ if(RAY_USE_GLOG)
 
   add_dependencies(glog glog_ep)
 endif()
+
+# boost
+include(BoostExternalProject)
+
+message(STATUS "Boost root: ${BOOST_ROOT}")
+message(STATUS "Boost include dir: ${Boost_INCLUDE_DIR}")
+message(STATUS "Boost system library: ${Boost_SYSTEM_LIBRARY}")
+message(STATUS "Boost filesystem library: ${Boost_FILESYSTEM_LIBRARY}")
+include_directories(${Boost_INCLUDE_DIR})
+
+# flatbuffers
+include(FlatBuffersExternalProject)
+
+message(STATUS "Flatbuffers home: ${FLATBUFFERS_HOME}")
+message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
+message(STATUS "Flatbuffers static library: ${FLATBUFFERS_STATIC_LIB}")
+message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
+include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
+
+ADD_THIRDPARTY_LIB(flatbuffers STATIC_LIB ${FLATBUFFERS_STATIC_LIB})
+
+add_dependencies(flatbuffers flatbuffers_ep)
+
+# Apache Arrow, use FLATBUFFERS_HOME and BOOST_ROOT
+include(ArrowExternalProject)
+
+message(STATUS "Arrow home: ${ARROW_HOME}")
+message(STATUS "Arrow source dir: ${ARROW_SOURCE_DIR}")
+message(STATUS "Arrow include dir: ${ARROW_INCLUDE_DIR}")
+message(STATUS "Arrow static library: ${ARROW_STATIC_LIB}")
+message(STATUS "Arrow shared library: ${ARROW_SHARED_LIB}")
+include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
+
+ADD_THIRDPARTY_LIB(arrow STATIC_LIB ${ARROW_STATIC_LIB})
+
+add_dependencies(arrow arrow_ep)
+
+# Plasma, it is already built in arrow
+message(STATUS "Plasma include dir: ${PLASMA_INCLUDE_DIR}")
+message(STATUS "Plasma static library: ${PLASMA_STATIC_LIB}")
+message(STATUS "Plasma shared library: ${PLASMA_SHARED_LIB}")
+include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
+
+ADD_THIRDPARTY_LIB(plasma STATIC_LIB ${PLASMA_STATIC_LIB})
+
+add_dependencies(plasma plasma_ep)
+
+if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
+  # Apache parquet cpp
+  include(ParquetExternalProject)
+
+  message(STATUS "Parquet home: ${PARQUET_HOME}")
+  message(STATUS "Parquet include dir: ${PARQUET_INCLUDE_DIR}")
+  message(STATUS "Parquet static library: ${PARQUET_STATIC_LIB}")
+  message(STATUS "Parquet shared library: ${PARQUET_SHARED_LIB}")
+  include_directories(SYSTEM ${PARQUET_INCLUDE_DIR})
+
+  ADD_THIRDPARTY_LIB(parquet STATIC_LIB ${PARQUET_STATIC_LIB})
+
+  add_dependencies(parquet parquet_ep)
+
+  # pyarrow
+  find_package(PythonInterp REQUIRED)
+  message(STATUS "PYTHON_EXECUTABLE for pyarrow: ${PYTHON_EXECUTABLE}")
+
+  set(pyarrow_ENV
+    "PKG_CONFIG_PATH=${ARROW_LIBRARY_DIR}/pkgconfig"
+    "PYARROW_WITH_PLASMA=1"
+    "PYARROW_WITH_TENSORFLOW=1"
+    "PYARROW_BUNDLE_ARROW_CPP=1"
+    "PARQUET_HOME=${PARQUET_HOME}"
+    "PYARROW_WITH_PARQUET=1"
+  )
+
+  # clean the arrow_ep/python/build/lib.xxxxx directory,
+  # or when you build with another python version, it creates multiple lib.xxxx directories
+  set_property(DIRECTORY APPEND PROPERTY ADDITIONAL_MAKE_CLEAN_FILES "${ARROW_SOURCE_DIR}/python/build/")
+
+  # here we use externalProject to process pyarrow building
+  # add_custom_command would have problem with setup.py
+  ExternalProject_Add(pyarrow_ext
+    PREFIX external/pyarrow
+    DEPENDS parquet_ep
+    DOWNLOAD_COMMAND ""
+    BUILD_IN_SOURCE 1
+    CONFIGURE_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build
+    BUILD_COMMAND cd ${ARROW_SOURCE_DIR}/python && ${CMAKE_COMMAND} -E env ${pyarrow_ENV} ${PYTHON_EXECUTABLE} setup.py build_ext
+    INSTALL_COMMAND bash -c "cp -r \$(find ${ARROW_SOURCE_DIR}/python/build/ -maxdepth 1 -type d -print | grep -m1 'lib')/pyarrow ${CMAKE_SOURCE_DIR}/python/ray/pyarrow_files/")
+
+endif ()

java/build.sh

@@ -1,6 +1,3 @@
 #!/usr/bin/env bash
 
-pushd ../thirdparty/build/arrow/java/plasma
-mvn clean install -Dmaven.test.skip
-popd
 mvn clean install -Dmaven.test.skip

java/test.sh

@@ -7,17 +7,15 @@ set -e
 set -x
 
 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
-$ROOT_DIR/../build.sh -l java,python
+$ROOT_DIR/../build.sh -l java
 
-pushd $ROOT_DIR/../thirdparty/build/arrow/java
-mvn clean install -pl plasma -am -Dmaven.test.skip
-popd
 pushd $ROOT_DIR/../java
 mvn clean install -Dmaven.test.skip
 check_style=$(mvn checkstyle:check)
 echo "${check_style}"
 [[ ${check_style} =~ "BUILD FAILURE" ]] && exit 1
 
+# test raylet
 mvn_test=$(mvn test)
 echo "${mvn_test}"
 [[ ${mvn_test} =~ "BUILD SUCCESS" ]] || exit 1

python/ray/services.py

@@ -1544,7 +1544,7 @@ def start_ray_processes(address_info=None,
             local_scheduler_stdout_file, local_scheduler_stderr_file = (
                 new_log_files(
                     "local_scheduler_{}".format(i),
-                    redirect_output=redirect_worker_output))
+                    redirect_output=redirect_output))
             local_scheduler_name = start_local_scheduler(
                 redis_address,
                 node_ip_address,

setup_thirdparty.sh

@@ -17,8 +17,3 @@ RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \
 RAY_BUILD_JAVA=$RAY_BUILD_JAVA \
 $ROOT_DIR/thirdparty/scripts/setup.sh $PYTHON_EXECUTABLE
 
-if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
-    pushd $ROOT_DIR/thirdparty/build/arrow/java
-    mvn clean install -pl plasma -am -Dmaven.test.skip
-    popd
-fi

src/common/CMakeLists.txt

@@ -2,7 +2,9 @@ cmake_minimum_required(VERSION 3.4)
 
 project(common)
 
-include(${CMAKE_CURRENT_LIST_DIR}/cmake/Common.cmake)
+if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
+  include_directories("${CMAKE_CURRENT_LIST_DIR}/lib/python")
+endif ()
 
 add_subdirectory(redis_module)
 
@@ -92,6 +94,8 @@ add_library(common STATIC
   thirdparty/ae/ae.c
   thirdparty/sha256.c)
 
+add_dependencies(common arrow_ep)
+
 if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   add_dependencies(common gen_common_python_fbs)
 endif()

src/global_scheduler/CMakeLists.txt

@@ -2,13 +2,13 @@ cmake_minimum_required(VERSION 3.4)
 
 project(global_scheduler)
 
-include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
+include_directories(${CMAKE_CURRENT_LIST_DIR})
 
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall")
 
 add_executable(global_scheduler global_scheduler.cc global_scheduler_algorithm.cc)
 
 # Make sure ${HIREDIS_LIB} is ready before linking.
-add_dependencies(global_scheduler hiredis)
+add_dependencies(global_scheduler hiredis common)
 
 target_link_libraries(global_scheduler common ${HIREDIS_LIB} ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} pthread)

src/local_scheduler/CMakeLists.txt

@@ -2,18 +2,6 @@ cmake_minimum_required(VERSION 3.4)
 
 project(local_scheduler)
 
-# Recursively include common
-include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
-
-if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
-  # Include plasma
-  list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/arrow/python/cmake_modules)
-  find_package(Arrow)
-  find_package(Plasma)
-  include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
-endif()
-
-
 add_definitions(-fPIC)
 
 if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
@@ -34,7 +22,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/")
 endif()
 
-include_directories("${ARROW_DIR}/cpp/src/")
+include_directories("${ARROW_INCLUDE_DIR}")
 include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/format/")
 
 # Compile flatbuffers
@@ -54,7 +42,7 @@ add_custom_command(
 
 add_custom_target(gen_local_scheduler_fbs DEPENDS ${LOCAL_SCHEDULER_FBS_OUTPUT_FILES})
 
-add_dependencies(gen_local_scheduler_fbs flatbuffers_ep)
+add_dependencies(gen_local_scheduler_fbs arrow_ep)
 
 add_library(local_scheduler_client STATIC local_scheduler_client.cc)
 

src/plasma/CMakeLists.txt

@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.4)
 
 project(plasma)
 
-# Recursively include common
-include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
+include_directories(${CMAKE_CURRENT_LIST_DIR})
+include_directories(${CMAKE_CURRENT_LIST_DIR}/thirdparty)
 
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --std=c99 -O3")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++11 -O3 -Werror -Wall")
@@ -12,8 +12,7 @@ if(UNIX AND NOT APPLE)
   link_libraries(rt)
 endif()
 
-include_directories("${ARROW_DIR}/cpp/src/")
-# include_directories("${CMAKE_CURRENT_LIST_DIR}/../")
+include_directories("${ARROW_INCLUDE_DIR}")
 
 set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
 set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
@@ -23,14 +22,14 @@ set(PLASMA_FBS_OUTPUT_FILES
   "${OUTPUT_DIR}/common_generated.h")
 
 add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
-add_dependencies(gen_plasma_fbs flatbuffers_ep)
+add_dependencies(gen_plasma_fbs arrow_ep)
 
 # Copy the fbs files from Arrow project to local directory.
 add_custom_command(
   OUTPUT ${PLASMA_FBS_SRC}
   COMMAND mkdir -p ${CMAKE_CURRENT_LIST_DIR}/format/
-  COMMAND cp ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/build/arrow/cpp/src/plasma/format/plasma.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
-  COMMAND cp ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/build/arrow/cpp/src/plasma/format/common.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
+  COMMAND cp ${ARROW_SOURCE_DIR}/cpp/src/plasma/format/plasma.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
+  COMMAND cp ${ARROW_SOURCE_DIR}/cpp/src/plasma/format/common.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
   COMMENT "Copying ${PLASMA_FBS_SRC} to local"
   VERBATIM)
 

src/ray/CMakeLists.txt

@@ -1,14 +1,12 @@
 cmake_minimum_required(VERSION 3.2)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Werror -std=c++11")
 
-include_directories(${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty/ae)
-
 add_subdirectory(util)
 add_subdirectory(gcs)
 add_subdirectory(object_manager)
 add_subdirectory(raylet)
 
-include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
+include_directories(${CMAKE_SOURCE_DIR}/src/common/thirdparty/ae)
 
 set(AE_SRCS
   ${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty/ae/ae.c
@@ -63,7 +61,13 @@ set(RAY_SRCS
 )
 
 set(RAY_LIB_STATIC_LINK_LIBS ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB})
-set(RAY_LIB_DEPENDENCIES gen_gcs_fbs gen_object_manager_fbs gen_node_manager_fbs gen_local_scheduler_fbs ${COMMON_FBS_OUTPUT_FILES})
+set(RAY_LIB_DEPENDENCIES
+    arrow_ep
+    gen_gcs_fbs
+    gen_object_manager_fbs
+    gen_node_manager_fbs
+    gen_local_scheduler_fbs
+    ${COMMON_FBS_OUTPUT_FILES})
 
 if(RAY_USE_GLOG)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DRAY_USE_GLOG")

src/ray/gcs/CMakeLists.txt

@@ -1,4 +1,6 @@
-include(${CMAKE_CURRENT_LIST_DIR}/../../common/cmake/Common.cmake)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=unused-function")
+
+include_directories(${CMAKE_SOURCE_DIR}/src/common/thirdparty/ae)
 
 set(GCS_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/gcs.fbs")
 set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)

src/ray/object_manager/CMakeLists.txt

@@ -1,5 +1,3 @@
-include(${CMAKE_CURRENT_LIST_DIR}/../../common/cmake/Common.cmake)
-
 set(OBJECT_MANAGER_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/object_manager.fbs")
 set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
 

src/ray/raylet/CMakeLists.txt

@@ -1,5 +1,3 @@
-include(${CMAKE_CURRENT_LIST_DIR}/../../common/cmake/Common.cmake)
-
 set(NODE_MANAGER_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/node_manager.fbs")
 set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
 

test/component_failures_test.py

@@ -215,7 +215,12 @@ def _test_component_failed(component_type):
         components = ray.services.all_processes[component_type]
         for process in components[1:]:
             process.terminate()
-            time.sleep(1)
+
+        # while the local_scheduler is fetching object_ids,
+        # and would trigger `fetch_object_timeout_handler`,
+        # which leads to find the plasma manager or plasma
+        # store socket is broken, so local_scheduler failed.
+        time.sleep(1)
 
         for process in components[1:]:
             process.kill()

thirdparty/scripts/setup.sh

@@ -30,7 +30,7 @@ unamestr="$(uname)"
 ##############################################
 # boost
 ##############################################
-bash "$TP_SCRIPT_DIR/build_boost.sh"
+#bash "$TP_SCRIPT_DIR/build_boost.sh"
 
 ##############################################
 # redis
@@ -45,17 +45,17 @@ bash "$TP_SCRIPT_DIR/build_credis.sh"
 ##############################################
 # flatbuffers if necessary
 ##############################################
-if [[ "$unamestr" == "Linux" ]]; then
-  echo "building flatbuffers"
-  bash "$TP_SCRIPT_DIR/build_flatbuffers.sh"
-fi
+#if [[ "$unamestr" == "Linux" ]]; then
+#  echo "building flatbuffers"
+#  bash "$TP_SCRIPT_DIR/build_flatbuffers.sh"
+#fi
 
 ##############################################
 # arrow
 ##############################################
-RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \
-RAY_BUILD_JAVA=$RAY_BUILD_JAVA \
-bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE
+#RAY_BUILD_PYTHON=$RAY_BUILD_PYTHON \
+#RAY_BUILD_JAVA=$RAY_BUILD_JAVA \
+#bash "$TP_SCRIPT_DIR/build_arrow.sh" $PYTHON_EXECUTABLE
 
 ##############################################
 # parquet (skipped as it is inlined in build_arrow.sh)