wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-13 12:30:18 +08:00
Code Issues Packages Projects Releases Wiki Activity

[JavaWorker] Changes to the build system for support java worker (#2092)

Browse Source 
* Changes to the build system for support java worker
--------------------------
This commit includes changes to the build system, which is part of the java worker support of Ray.
It consists of the following changes:
 - the changes of CMakeLists.txt files
 - the changes of the python setup.py and init files for the adaptation of the changed build system
 - move the location of local_scheduler_extension.cc for the adaptation of the changed build system which maybe better support multi-language worker

* minor whitespace

* Linting
This commit is contained in:
Yujie Liu
2018-05-18 19:09:23 -07:00
committed by Philipp Moritz
parent 71e5cca59f
commit 5918776dd4
8 changed files with 231 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/common/CMakeLists.txt
+41 -14
View File
@@ -29,22 +29,43 @@ add_custom_command(
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
add_custom_target(gen_common_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
add_custom_target(gen_common_python_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
# Generate Python bindings for the flatbuffers objects.
set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
add_custom_command(
TARGET gen_common_fbs
COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
# Generate Python bindings for the flatbuffers objects.
set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
add_custom_command(
TARGET gen_common_python_fbs
COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
# Encode the fact that the ray redis module requires the autogenerated
# flatbuffer files to compile.
add_dependencies(ray_redis_module gen_common_fbs)
# Encode the fact that the ray redis module requires the autogenerated
# flatbuffer files to compile.
add_dependencies(ray_redis_module gen_common_python_fbs)
add_dependencies(gen_common_fbs flatbuffers_ep)
add_dependencies(gen_common_python_fbs flatbuffers_ep)
endif()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
add_custom_target(gen_common_java_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
# Generate Java bindings for the flatbuffers objects.
set(JAVA_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/java)
add_custom_command(
TARGET gen_common_java_fbs
COMMAND ${FLATBUFFERS_COMPILER} -j -o ${JAVA_OUTPUT_DIR} ${COMMON_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
# Encode the fact that the ray redis module requires the autogenerated
# flatbuffer files to compile.
add_dependencies(ray_redis_module gen_common_java_fbs)
add_dependencies(gen_common_java_fbs flatbuffers_ep)
endif()
add_custom_target(
hiredis
@@ -71,7 +92,13 @@ add_library(common STATIC
thirdparty/ae/ae.c
thirdparty/sha256.c)
add_dependencies(common gen_common_fbs)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
add_dependencies(common gen_common_python_fbs)
endif()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
add_dependencies(common gen_common_java_fbs)
endif()
target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhiredis.a")
src/common/cmake/Common.cmake
+28 -10
View File
@@ -43,17 +43,33 @@ include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
set(CMAKE_C_FLAGS "-g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -fPIC -std=c99")
# Code for finding Python
find_package(PythonInterp REQUIRED)
find_package(NumPy REQUIRED)
# language-specific
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
# Code for finding Python
find_package(PythonInterp REQUIRED)
find_package(NumPy REQUIRED)
# Now find the Python include directories.
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())"
OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE)
message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
# Now find the Python include directories.
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())"
OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE)
message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
message(STATUS "Using PYTHON_EXECUTABLE: " ${PYTHON_EXECUTABLE})
message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
message(STATUS "Using PYTHON_EXECUTABLE: " ${PYTHON_EXECUTABLE})
message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
endif ()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
find_package(JNI REQUIRED)
# add jni support
include_directories(${JAVA_INCLUDE_PATH})
include_directories(${JAVA_INCLUDE_PATH2})
if (JNI_FOUND)
message (STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
message (STATUS "JNI_LIBRARIES=${JNI_LIBRARIES}")
else()
message (WARNING "NOT FIND JNI")
endif()
endif()
# Common libraries
@@ -62,4 +78,6 @@ set(COMMON_LIB "${CMAKE_BINARY_DIR}/src/common/libcommon.a"
include_directories("${CMAKE_CURRENT_LIST_DIR}/..")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../thirdparty/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python")
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python")
endif ()
src/local_scheduler/CMakeLists.txt
+54 -27
View File
@@ -5,21 +5,24 @@ project(local_scheduler)
# Recursively include common
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
# Include plasma
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/arrow/python/cmake_modules)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
# Include plasma
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/arrow/python/cmake_modules)
find_package(Arrow)
find_package(Plasma)
include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
endif()
find_package(Arrow)
find_package(Plasma)
include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
add_definitions(-fPIC)
if(APPLE)
SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
endif(APPLE)
include_directories("${PYTHON_INCLUDE_DIRS}")
include_directories("${NUMPY_INCLUDE_DIR}")
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
if(APPLE)
SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
endif(APPLE)
include_directories("${PYTHON_INCLUDE_DIRS}")
include_directories("${NUMPY_INCLUDE_DIR}")
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall")
@@ -30,7 +33,10 @@ endif()
include_directories("${CMAKE_CURRENT_LIST_DIR}/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../")
# TODO(pcm): get rid of this:
include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/")
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/")
endif()
include_directories("${ARROW_DIR}/cpp/src/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/format/")
@@ -44,7 +50,7 @@ set(LOCAL_SCHEDULER_FBS_OUTPUT_FILES
add_custom_command(
OUTPUT ${LOCAL_SCHEDULER_FBS_OUTPUT_FILES}
COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${LOCAL_SCHEDULER_FBS_SRC}
COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${LOCAL_SCHEDULER_FBS_SRC} --gen-object-api
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${LOCAL_SCHEDULER_FBS_SRC}"
VERBATIM)
@@ -53,23 +59,10 @@ add_custom_target(gen_local_scheduler_fbs DEPENDS ${LOCAL_SCHEDULER_FBS_OUTPUT_F
add_dependencies(gen_local_scheduler_fbs flatbuffers_ep)
add_library(local_scheduler_library SHARED
local_scheduler_extension.cc
../common/lib/python/common_extension.cc
../common/lib/python/config_extension.cc)
add_library(local_scheduler_client STATIC local_scheduler_client.cc)
add_dependencies(local_scheduler_client gen_local_scheduler_fbs)
if(APPLE)
target_link_libraries(local_scheduler_library "-undefined dynamic_lookup" local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
else(APPLE)
target_link_libraries(local_scheduler_library local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
endif(APPLE)
add_dependencies(local_scheduler_library gen_local_scheduler_fbs)
add_executable(local_scheduler local_scheduler.cc local_scheduler_algorithm.cc)
target_link_libraries(local_scheduler local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY})
@@ -77,4 +70,38 @@ add_executable(local_scheduler_tests test/local_scheduler_tests.cc local_schedul
target_link_libraries(local_scheduler_tests local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY})
target_compile_options(local_scheduler_tests PUBLIC "-DLOCAL_SCHEDULER_TEST")
install(TARGETS local_scheduler_library DESTINATION ${CMAKE_SOURCE_DIR}/local_scheduler)
macro(set_local_scheduler_library LANG)
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/")
file(GLOB LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC
lib/${LANG}/*.cc
${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/*.cc)
add_library(local_scheduler_library_${LANG} SHARED
${LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC})
if(APPLE)
target_link_libraries(local_scheduler_library_${LANG} "-undefined dynamic_lookup" local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
else(APPLE)
target_link_libraries(local_scheduler_library_${LANG} local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
endif(APPLE)
add_dependencies(local_scheduler_library_${LANG} gen_local_scheduler_fbs)
install(TARGETS local_scheduler_library_${LANG} DESTINATION ${CMAKE_SOURCE_DIR}/local_scheduler)
endmacro()
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
set_local_scheduler_library("python")
endif()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
add_compile_options("-I$ENV{JAVA_HOME}/include/")
if(WIN32)
add_compile_options("-I$ENV{JAVA_HOME}/include/win32")
elseif(APPLE)
add_compile_options("-I$ENV{JAVA_HOME}/include/darwin")
else() # linux
add_compile_options("-I$ENV{JAVA_HOME}/include/linux")
endif()
set_local_scheduler_library("java")
endif()
src/local_scheduler/local_scheduler_extension.cc → src/local_scheduler/lib/python/local_scheduler_extension.cc
+4 -4
View File
@@ -286,7 +286,7 @@ static struct PyModuleDef moduledef = {
#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
#endif
MOD_INIT(liblocal_scheduler_library) {
MOD_INIT(liblocal_scheduler_library_python) {
if (PyType_Ready(&PyTaskType) < 0) {
INITERROR;
}
@@ -306,9 +306,9 @@ MOD_INIT(liblocal_scheduler_library) {
#if PY_MAJOR_VERSION >= 3
PyObject *m = PyModule_Create(&moduledef);
#else
PyObject *m =
Py_InitModule3("liblocal_scheduler_library", local_scheduler_methods,
"A module for the local scheduler.");
PyObject *m = Py_InitModule3("liblocal_scheduler_library_python",
local_scheduler_methods,
"A module for the local scheduler.");
#endif
init_numpy_module();