wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-13 17:45:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

[JavaWorker] Changes to the build system for support java worker (#2092)

Browse Source 
* Changes to the build system for support java worker
--------------------------
This commit includes changes to the build system, which is part of the java worker support of Ray.
It consists of the following changes:
 - the changes of CMakeLists.txt files
 - the changes of the python setup.py and init files for the adaptation of the changed build system
 - move the location of local_scheduler_extension.cc for the adaptation of the changed build system which maybe better support multi-language worker

* minor whitespace

* Linting
This commit is contained in:
Yujie Liu
2018-05-19 10:09:23 +08:00
committed by Philipp Moritz
parent 71e5cca59f
commit 5918776dd4
8 changed files with 231 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+13
View File
@@ -2,6 +2,19 @@ cmake_minimum_required(VERSION 3.4)
project(ray)
set(CMAKE_RAY_LANG_PYTHON "NO")
set(CMAKE_RAY_LANG_JAVA "NO")
if ("${CMAKE_RAY_LANGUAGE}" STREQUAL "python")
set(CMAKE_RAY_LANG_PYTHON "YES")
elseif ("${CMAKE_RAY_LANGUAGE}" STREQUAL "java")
set(CMAKE_RAY_LANG_JAVA "YES")
elseif ("${CMAKE_RAY_LANGUAGE}" STREQUAL "")
message(WARNING "Language is not set, choose Python as default.")
set(CMAKE_RAY_LANG_PYTHON "YES")
else()
message(FATAL_ERROR "Unrecognized language, use -DCMAKE_RAY_LANGUAGE=java|python. Abort.")
endif()
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules")
include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/scripts/thirdparty.cmake)
build.sh
+88 -30
View File
@@ -7,14 +7,19 @@ set -e
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
if [[ -z "$1" ]]; then
PYTHON_EXECUTABLE=`which python`
else
PYTHON_EXECUTABLE=$1
fi
echo "Using Python executable $PYTHON_EXECUTABLE."
bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE
function usage()
{
echo "Usage: build.sh [<args>]"
echo
echo "Options:"
echo " -h|--help print the help info"
echo " -d|--debug CMAKE_BUILD_TYPE=Debug (default is RelWithDebInfo)"
echo " -l|--language python(default) "
echo " build native library for python"
echo " java build native library for java"
echo " -p|--python which python executable (default from which python)"
echo
}
# Determine how many parallel jobs to use for make based on the number of cores
unamestr="$(uname)"
@@ -27,30 +32,83 @@ else
exit 1
fi
LANGUAGE="python"
PYTHON_EXECUTABLE=""
BUILD_DIR=""
if [ "$VALGRIND" = "1" ]; then
CBUILD_TYPE="Debug"
else
CBUILD_TYPE="RelWithDebInfo"
fi
# Parse options
while [[ $# > 0 ]]; do
key="$1"
case $key in
-h|--help)
usage
exit 0
;;
-d|--debug)
CBUILD_TYPE=Debug
;;
-l|--languags)
LANGUAGE="$2"
if [ "$LANGUAGE" != "python" ] && [ "$LANGUAGE" != "java" ]; then
echo "Unrecognized language."
exit -1
fi
shift
;;
-p|--python)
PYTHON_EXECUTABLE="$2"
shift
;;
*)
echo "ERROR: unknown option \"$key\""
echo
usage
exit -1
;;
esac
shift
done
if [[ -z "$PYTHON_EXECUTABLE" ]]; then
PYTHON_EXECUTABLE=`which python`
fi
echo "Using Python executable $PYTHON_EXECUTABLE."
bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE
# Now we build everything.
pushd "$ROOT_DIR/python/ray/core"
# We use these variables to set PKG_CONFIG_PATH, which is important so that
# in cmake, pkg-config can find plasma.
TP_PKG_DIR=$ROOT_DIR/thirdparty/pkg
ARROW_HOME=$TP_PKG_DIR/arrow/cpp/build/cpp-install
if [[ "$VALGRIND" = "1" ]]; then
BOOST_ROOT=$TP_PKG_DIR/boost \
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
cmake -DCMAKE_BUILD_TYPE=Debug \
-DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
-DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
../../..
else
BOOST_ROOT=$TP_PKG_DIR/boost \
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
cmake -DCMAKE_BUILD_TYPE=Release \
-DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
-DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE \
../../..
if [[ "$LANGUAGE" == "java" ]]; then
BUILD_DIR="$ROOT_DIR/build/"
if [ ! -d "${BUILD_DIR}" ]; then
mkdir -p ${BUILD_DIR}
fi
make clean
make -j${PARALLEL}
else
BUILD_DIR="$ROOT_DIR/python/ray/core"
fi
pushd "$BUILD_DIR"
TP_PKG_DIR=$ROOT_DIR/thirdparty/pkg
# We use these variables to set PKG_CONFIG_PATH, which is important so that
# in cmake, pkg-config can find plasma.
ARROW_HOME=$TP_PKG_DIR/arrow/cpp/build/cpp-install
BOOST_ROOT=$TP_PKG_DIR/boost \
PKG_CONFIG_PATH=$ARROW_HOME/lib/pkgconfig \
cmake -DCMAKE_BUILD_TYPE=$CBUILD_TYPE \
-DCMAKE_RAY_LANGUAGE=$LANGUAGE \
-DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
-DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE $ROOT_DIR
make clean
make -j${PARALLEL}
popd
# Move stuff from Arrow to Ray.
cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $ROOT_DIR/python/ray/core/src/plasma/
cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/src/plasma/
if [[ "$LANGUAGE" == "java" ]]; then
cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/
fi
python/ray/local_scheduler/__init__.py
+1 -1
View File
@@ -2,7 +2,7 @@ from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from ray.core.src.local_scheduler.liblocal_scheduler_library import (
from ray.core.src.local_scheduler.liblocal_scheduler_library_python import (
Task, LocalSchedulerClient, ObjectID, check_simple_value, task_from_string,
task_to_string, _config, common_error)
from .local_scheduler_services import start_local_scheduler
python/setup.py
+2 -2
View File
@@ -20,7 +20,7 @@ ray_files = [
"ray/core/src/common/redis_module/libray_redis_module.so",
"ray/core/src/plasma/plasma_store", "ray/core/src/plasma/plasma_manager",
"ray/core/src/local_scheduler/local_scheduler",
"ray/core/src/local_scheduler/liblocal_scheduler_library.so",
"ray/core/src/local_scheduler/liblocal_scheduler_library_python.so",
"ray/core/src/global_scheduler/global_scheduler",
"ray/core/src/ray/raylet/raylet_monitor", "ray/core/src/ray/raylet/raylet",
"ray/WebUI.ipynb"
@@ -63,7 +63,7 @@ class build_ext(_build_ext.build_ext):
# version of Python to build pyarrow inside the build.sh script. Note
# that certain flags will not be passed along such as --user or sudo.
# TODO(rkn): Fix this.
subprocess.check_call(["../build.sh", sys.executable])
subprocess.check_call(["../build.sh", "-p", sys.executable])
# We also need to install pyarrow along with Ray, so make sure that the
# relevant non-Python pyarrow files get copied.
src/common/CMakeLists.txt
+41 -14
View File
@@ -29,22 +29,43 @@ add_custom_command(
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
add_custom_target(gen_common_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
add_custom_target(gen_common_python_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
# Generate Python bindings for the flatbuffers objects.
set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
add_custom_command(
TARGET gen_common_fbs
COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
# Generate Python bindings for the flatbuffers objects.
set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
add_custom_command(
TARGET gen_common_python_fbs
COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
# Encode the fact that the ray redis module requires the autogenerated
# flatbuffer files to compile.
add_dependencies(ray_redis_module gen_common_fbs)
# Encode the fact that the ray redis module requires the autogenerated
# flatbuffer files to compile.
add_dependencies(ray_redis_module gen_common_python_fbs)
add_dependencies(gen_common_fbs flatbuffers_ep)
add_dependencies(gen_common_python_fbs flatbuffers_ep)
endif()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
add_custom_target(gen_common_java_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
# Generate Java bindings for the flatbuffers objects.
set(JAVA_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/java)
add_custom_command(
TARGET gen_common_java_fbs
COMMAND ${FLATBUFFERS_COMPILER} -j -o ${JAVA_OUTPUT_DIR} ${COMMON_FBS_SRC}
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${COMMON_FBS_SRC}"
VERBATIM)
# Encode the fact that the ray redis module requires the autogenerated
# flatbuffer files to compile.
add_dependencies(ray_redis_module gen_common_java_fbs)
add_dependencies(gen_common_java_fbs flatbuffers_ep)
endif()
add_custom_target(
hiredis
@@ -71,7 +92,13 @@ add_library(common STATIC
thirdparty/ae/ae.c
thirdparty/sha256.c)
add_dependencies(common gen_common_fbs)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
add_dependencies(common gen_common_python_fbs)
endif()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
add_dependencies(common gen_common_java_fbs)
endif()
target_link_libraries(common "${CMAKE_CURRENT_LIST_DIR}/thirdparty/hiredis/libhiredis.a")
src/common/cmake/Common.cmake
+28 -10
View File
@@ -43,17 +43,33 @@ include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
set(CMAKE_C_FLAGS "-g -Wall -Wextra -Werror=implicit-function-declaration -Wno-sign-compare -Wno-unused-parameter -Wno-type-limits -Wno-missing-field-initializers --std=c99 -fPIC -std=c99")
# Code for finding Python
find_package(PythonInterp REQUIRED)
find_package(NumPy REQUIRED)
# language-specific
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
# Code for finding Python
find_package(PythonInterp REQUIRED)
find_package(NumPy REQUIRED)
# Now find the Python include directories.
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())"
OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE)
message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
# Now find the Python include directories.
execute_process(COMMAND ${PYTHON_EXECUTABLE} -c "from distutils.sysconfig import *; print(get_python_inc())"
OUTPUT_VARIABLE PYTHON_INCLUDE_DIRS OUTPUT_STRIP_TRAILING_WHITESPACE)
message(STATUS "PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
message(STATUS "Using PYTHON_EXECUTABLE: " ${PYTHON_EXECUTABLE})
message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
message(STATUS "Using PYTHON_EXECUTABLE: " ${PYTHON_EXECUTABLE})
message(STATUS "Using PYTHON_INCLUDE_DIRS: " ${PYTHON_INCLUDE_DIRS})
endif ()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
find_package(JNI REQUIRED)
# add jni support
include_directories(${JAVA_INCLUDE_PATH})
include_directories(${JAVA_INCLUDE_PATH2})
if (JNI_FOUND)
message (STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
message (STATUS "JNI_LIBRARIES=${JNI_LIBRARIES}")
else()
message (WARNING "NOT FIND JNI")
endif()
endif()
# Common libraries
@@ -62,4 +78,6 @@ set(COMMON_LIB "${CMAKE_BINARY_DIR}/src/common/libcommon.a"
include_directories("${CMAKE_CURRENT_LIST_DIR}/..")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../thirdparty/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python")
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../lib/python")
endif ()
src/local_scheduler/CMakeLists.txt
+54 -27
View File
@@ -5,21 +5,24 @@ project(local_scheduler)
# Recursively include common
include(${CMAKE_CURRENT_LIST_DIR}/../common/cmake/Common.cmake)
# Include plasma
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/arrow/python/cmake_modules)
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
# Include plasma
list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/../../thirdparty/pkg/arrow/python/cmake_modules)
find_package(Arrow)
find_package(Plasma)
include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
endif()
find_package(Arrow)
find_package(Plasma)
include_directories(SYSTEM ${PLASMA_INCLUDE_DIR})
add_definitions(-fPIC)
if(APPLE)
SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
endif(APPLE)
include_directories("${PYTHON_INCLUDE_DIRS}")
include_directories("${NUMPY_INCLUDE_DIR}")
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
if(APPLE)
SET(CMAKE_SHARED_LIBRARY_SUFFIX ".so")
endif(APPLE)
include_directories("${PYTHON_INCLUDE_DIRS}")
include_directories("${NUMPY_INCLUDE_DIR}")
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror -Wall")
@@ -30,7 +33,10 @@ endif()
include_directories("${CMAKE_CURRENT_LIST_DIR}/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../")
# TODO(pcm): get rid of this:
include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/")
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../plasma/")
endif()
include_directories("${ARROW_DIR}/cpp/src/")
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/format/")
@@ -44,7 +50,7 @@ set(LOCAL_SCHEDULER_FBS_OUTPUT_FILES
add_custom_command(
OUTPUT ${LOCAL_SCHEDULER_FBS_OUTPUT_FILES}
COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${LOCAL_SCHEDULER_FBS_SRC}
COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${LOCAL_SCHEDULER_FBS_SRC} --gen-object-api
DEPENDS ${FBS_DEPENDS}
COMMENT "Running flatc compiler on ${LOCAL_SCHEDULER_FBS_SRC}"
VERBATIM)
@@ -53,23 +59,10 @@ add_custom_target(gen_local_scheduler_fbs DEPENDS ${LOCAL_SCHEDULER_FBS_OUTPUT_F
add_dependencies(gen_local_scheduler_fbs flatbuffers_ep)
add_library(local_scheduler_library SHARED
local_scheduler_extension.cc
../common/lib/python/common_extension.cc
../common/lib/python/config_extension.cc)
add_library(local_scheduler_client STATIC local_scheduler_client.cc)
add_dependencies(local_scheduler_client gen_local_scheduler_fbs)
if(APPLE)
target_link_libraries(local_scheduler_library "-undefined dynamic_lookup" local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
else(APPLE)
target_link_libraries(local_scheduler_library local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
endif(APPLE)
add_dependencies(local_scheduler_library gen_local_scheduler_fbs)
add_executable(local_scheduler local_scheduler.cc local_scheduler_algorithm.cc)
target_link_libraries(local_scheduler local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY})
@@ -77,4 +70,38 @@ add_executable(local_scheduler_tests test/local_scheduler_tests.cc local_schedul
target_link_libraries(local_scheduler_tests local_scheduler_client common ${HIREDIS_LIB} ${PLASMA_STATIC_LIB} ray_static ${ARROW_STATIC_LIB} -lpthread ${Boost_SYSTEM_LIBRARY})
target_compile_options(local_scheduler_tests PUBLIC "-DLOCAL_SCHEDULER_TEST")
install(TARGETS local_scheduler_library DESTINATION ${CMAKE_SOURCE_DIR}/local_scheduler)
macro(set_local_scheduler_library LANG)
include_directories("${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/")
file(GLOB LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC
lib/${LANG}/*.cc
${CMAKE_CURRENT_LIST_DIR}/../common/lib/${LANG}/*.cc)
add_library(local_scheduler_library_${LANG} SHARED
${LOCAL_SCHEDULER_LIBRARY_${LANG}_SRC})
if(APPLE)
target_link_libraries(local_scheduler_library_${LANG} "-undefined dynamic_lookup" local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
else(APPLE)
target_link_libraries(local_scheduler_library_${LANG} local_scheduler_client common ray_static ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${Boost_SYSTEM_LIBRARY})
endif(APPLE)
add_dependencies(local_scheduler_library_${LANG} gen_local_scheduler_fbs)
install(TARGETS local_scheduler_library_${LANG} DESTINATION ${CMAKE_SOURCE_DIR}/local_scheduler)
endmacro()
if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
set_local_scheduler_library("python")
endif()
if ("${CMAKE_RAY_LANG_JAVA}" STREQUAL "YES")
add_compile_options("-I$ENV{JAVA_HOME}/include/")
if(WIN32)
add_compile_options("-I$ENV{JAVA_HOME}/include/win32")
elseif(APPLE)
add_compile_options("-I$ENV{JAVA_HOME}/include/darwin")
else() # linux
add_compile_options("-I$ENV{JAVA_HOME}/include/linux")
endif()
set_local_scheduler_library("java")
endif()
src/local_scheduler/local_scheduler_extension.cc → src/local_scheduler/lib/python/local_scheduler_extension.cc
+4 -4
View File
@@ -286,7 +286,7 @@ static struct PyModuleDef moduledef = {
#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
#endif
MOD_INIT(liblocal_scheduler_library) {
MOD_INIT(liblocal_scheduler_library_python) {
if (PyType_Ready(&PyTaskType) < 0) {
INITERROR;
}
@@ -306,9 +306,9 @@ MOD_INIT(liblocal_scheduler_library) {
#if PY_MAJOR_VERSION >= 3
PyObject *m = PyModule_Create(&moduledef);
#else
PyObject *m =
Py_InitModule3("liblocal_scheduler_library", local_scheduler_methods,
"A module for the local scheduler.");
PyObject *m = Py_InitModule3("liblocal_scheduler_library_python",
local_scheduler_methods,
"A module for the local scheduler.");
#endif
init_numpy_module();