wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-09 11:32:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

update ray cmake build process (#2853)

Browse Source 
* use cmake to build ray project, no need to appply build.sh before cmake, fix some abuse of cmake, improve the build performance

* support boost external project, avoid using the system or build.sh boost

* keep compatible with build.sh, remove boost and arrow build from it.

* bugfix: parquet bison version control, plasma_java lib install problem

* bugfix: cmake, do not compile plasma java client if no need

* bugfix: component failures test timeout machenism has problem for plasma manager failed case

* bugfix: arrow use lib64 in centos, travis check-git-clang-format-output.sh does not support other branches except master

* revert some fix

* set arrow python executable, fix format error in component_failures_test.py

* make clean arrow python build directory

* update cmake code style, back to support cmake minimum version 3.4
This commit is contained in:
Hanwei Jin
2018-09-12 11:19:33 -07:00
committed by Philipp Moritz
parent d9eeaaf00a
commit fbf214e408
23 changed files with 422 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/CMakeLists.txt
+5 -1
View File
@@ -2,7 +2,9 @@ cmake_minimum_required(VERSION 3.4)
project(common)
include(${CMAKE_CURRENT_LIST_DIR}/cmake/Common.cmake)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${CMAKE_CURRENT_LIST_DIR}/lib/python")
endif ()
add_subdirectory(redis_module)
@@ -92,6 +94,8 @@ add_library(common STATIC
thirdparty/ae/ae.c
thirdparty/sha256.c)
add_dependencies(common arrow_ep)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
add_dependencies(common gen_common_python_fbs)
endif()
src/common/cmake/Common.cmake
-83
View File
@@ -1,83 +0,0 @@
# Code for compiling flatbuffers
include(ExternalProject)
include(CMakeParseArguments)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -g -Werror -Wall -Wno-error=unused-function -Wno-error=strict-aliasing")
# The rdynamic flag is needed to produce better backtraces on Linux.
if(UNIX AND NOT APPLE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -rdynamic")
endif()
# The following is needed because in CentOS, the lib directory is named lib64
if(EXISTS "/etc/redhat-release" AND CMAKE_SIZEOF_VOID_P EQUAL 8)
set(LIB_SUFFIX 64)
endif()
set(FLATBUFFERS_VERSION "1.9.0")
set(FLATBUFFERS_PREFIX "${CMAKE_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install")
if (NOT TARGET flatbuffers_ep)
ExternalProject_Add(flatbuffers_ep
URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz"
CMAKE_ARGS
"-DCMAKE_CXX_FLAGS=-fPIC"
"-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}"
"-DFLATBUFFERS_BUILD_TESTS=OFF"
"-DCMAKE_BUILD_TYPE=RELEASE")
endif()
set(FBS_DEPENDS flatbuffers_ep)
set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include")
set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_PREFIX}/lib${LIB_SUFFIX}/libflatbuffers.a")
set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc")
message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
message(STATUS "Flatbuffers static library: ${FLATBUFFERS_STATIC_LIB}")
message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}")
include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
# Custom CFLAGS
set(CMAKE_C_FLAGS "-g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -fPIC -std=c99")
# language-specific
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
# Code for finding Python
find_package(PythonInterp REQUIRED)
find_package(NumPy REQUIRED)
# Now find the Python include directories.
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())"
OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE)
message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
message(STATUS "Using PYTHON_EXECUTABLE: " ${PYTHON_EXECUTABLE})
message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
endif ()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
find_package(JNI REQUIRED)
# add jni support
include_directories(${JAVA_INCLUDE_PATH})
include_directories(${JAVA_INCLUDE_PATH2})
if (JNI_FOUND)
message (STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
message (STATUS "JNI_LIBRARIES=${JNI_LIBRARIES}")
else()
message (WARNING "NOT FIND JNI")
endif()
endif()
# Common libraries
set(COMMON_LIB "${CMAKE_BINARY_DIR}/src/common/libcommon.a"
CACHE STRING "Path to libcommon.a")
include_directories("${CMAKE_CURRENT_LIST_DIR}/..")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../thirdparty/")
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python")
endif ()
src/global_scheduler/CMakeLists.txt
+2 -2
View File
@@ -2,13 +2,13 @@ cmake_minimum_required(VERSION 3.4)
project(global_scheduler)
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
include_directories(${CMAKE_CURRENT_LIST_DIR})
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall")
add_executable(global_scheduler global_scheduler.cc global_scheduler_algorithm.cc)
# Make sure ${HIREDIS_LIB} is ready before linking.
add_dependencies(global_scheduler hiredis)
add_dependencies(global_scheduler hiredis common)
target_link_libraries(global_scheduler common ${HIREDIS_LIB} ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY} pthread)
src/local_scheduler/CMakeLists.txt
+2 -14
View File
@@ -2,18 +2,6 @@ cmake_minimum_required(VERSION 3.4)
project(local_scheduler)
# Recursively include common
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
# Include plasma
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/arrow/python/cmake_modules)
find_package(Arrow)
find_package(Plasma)
include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
endif()
add_definitions(-fPIC)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
@@ -34,7 +22,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/")
endif()
include_directories("${ARROW_DIR}/cpp/src/")
include_directories("${ARROW_INCLUDE_DIR}")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/format/")
# Compile flatbuffers
@@ -54,7 +42,7 @@ add_custom_command(
add_custom_target(gen_local_scheduler_fbs DEPENDS ${LOCAL_SCHEDULER_FBS_OUTPUT_FILES})
add_dependencies(gen_local_scheduler_fbs flatbuffers_ep)
add_dependencies(gen_local_scheduler_fbs arrow_ep)
add_library(local_scheduler_client STATIC local_scheduler_client.cc)
src/plasma/CMakeLists.txt
+6 -7
View File
@@ -2,8 +2,8 @@ cmake_minimum_required(VERSION 3.4)
project(plasma)
# Recursively include common
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
include_directories(${CMAKE_CURRENT_LIST_DIR})
include_directories(${CMAKE_CURRENT_LIST_DIR}/thirdparty)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --std=c99 -O3")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --std=c++11 -O3 -Werror -Wall")
@@ -12,8 +12,7 @@ if(UNIX AND NOT APPLE)
link_libraries(rt)
endif()
include_directories("${ARROW_DIR}/cpp/src/")
# include_directories("${CMAKE_CURRENT_LIST_DIR}/../")
include_directories("${ARROW_INCLUDE_DIR}")
set(PLASMA_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/plasma.fbs" "${CMAKE_CURRENT_LIST_DIR}/format/common.fbs")
set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
@@ -23,14 +22,14 @@ set(PLASMA_FBS_OUTPUT_FILES
"${OUTPUT_DIR}/common_generated.h")
add_custom_target(gen_plasma_fbs DEPENDS ${PLASMA_FBS_OUTPUT_FILES})
add_dependencies(gen_plasma_fbs flatbuffers_ep)
add_dependencies(gen_plasma_fbs arrow_ep)
# Copy the fbs files from Arrow project to local directory.
add_custom_command(
OUTPUT ${PLASMA_FBS_SRC}
COMMAND mkdir -p ${CMAKE_CURRENT_LIST_DIR}/format/
COMMAND cp ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/build/arrow/cpp/src/plasma/format/plasma.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
COMMAND cp ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/build/arrow/cpp/src/plasma/format/common.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
COMMAND cp ${ARROW_SOURCE_DIR}/cpp/src/plasma/format/plasma.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
COMMAND cp ${ARROW_SOURCE_DIR}/cpp/src/plasma/format/common.fbs ${CMAKE_CURRENT_LIST_DIR}/format/
COMMENT "Copying ${PLASMA_FBS_SRC} to local"
VERBATIM)
src/ray/CMakeLists.txt
+8 -4
View File
@@ -1,14 +1,12 @@
cmake_minimum_required(VERSION 3.2)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -Wall -Werror -std=c++11")
include_directories(${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty/ae)
add_subdirectory(util)
add_subdirectory(gcs)
add_subdirectory(object_manager)
add_subdirectory(raylet)
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
include_directories(${CMAKE_SOURCE_DIR}/src/common/thirdparty/ae)
set(AE_SRCS
${CMAKE_CURRENT_LIST_DIR}/../common/thirdparty/ae/ae.c
@@ -63,7 +61,13 @@ set(RAY_SRCS
)
set(RAY_LIB_STATIC_LINK_LIBS ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB})
set(RAY_LIB_DEPENDENCIES gen_gcs_fbs gen_object_manager_fbs gen_node_manager_fbs gen_local_scheduler_fbs ${COMMON_FBS_OUTPUT_FILES})
set(RAY_LIB_DEPENDENCIES
arrow_ep
gen_gcs_fbs
gen_object_manager_fbs
gen_node_manager_fbs
gen_local_scheduler_fbs
${COMMON_FBS_OUTPUT_FILES})
if(RAY_USE_GLOG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DRAY_USE_GLOG")
src/ray/gcs/CMakeLists.txt
+3 -1
View File
@@ -1,4 +1,6 @@
include(${CMAKE_CURRENT_LIST_DIR}/../../common/cmake/Common.cmake)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-error=unused-function")
include_directories(${CMAKE_SOURCE_DIR}/src/common/thirdparty/ae)
set(GCS_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/gcs.fbs")
set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
src/ray/object_manager/CMakeLists.txt
-2
View File
@@ -1,5 +1,3 @@
include(${CMAKE_CURRENT_LIST_DIR}/../../common/cmake/Common.cmake)
set(OBJECT_MANAGER_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/object_manager.fbs")
set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)
src/ray/raylet/CMakeLists.txt
-2
View File
@@ -1,5 +1,3 @@
include(${CMAKE_CURRENT_LIST_DIR}/../../common/cmake/Common.cmake)
set(NODE_MANAGER_FBS_SRC "${CMAKE_CURRENT_LIST_DIR}/format/node_manager.fbs")
set(OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/format/)