From 4dd46985645ad89609991a55fbcf7d79b38b7012 Mon Sep 17 00:00:00 2001 From: songqing Date: Sat, 2 Jun 2018 07:28:27 +0800 Subject: [PATCH] unify build dir for Python and Java (#2171) * unify build dir for Python and Java * enable executables auto installed when just running 'make' * fix plasma_store copy error * fix cmake error about copying executables * lint fix * recover python/setup.py * enable to copy optional file automatically * a small fix of path * lint fix * lint fix * lint fix * Add comment. --- .travis.yml | 26 ++++++------ CMakeLists.txt | 40 +++++++++++++++++++ build.sh | 13 +++--- doc/source/development.rst | 2 +- python/setup.py | 3 ++ src/common/CMakeLists.txt | 2 +- .../test/local_scheduler_tests.cc | 2 +- src/ray/gcs/CMakeLists.txt | 2 +- src/ray/raylet/CMakeLists.txt | 2 +- .../raylet/object_manager_integration_test.cc | 2 +- src/ray/test/run_gcs_tests.sh | 2 +- src/ray/test/run_object_manager_tests.sh | 4 +- src/ray/test/run_object_manager_valgrind.sh | 4 +- src/ray/test/start_raylet.sh | 4 +- src/ray/test/start_raylets.sh | 4 +- thirdparty/scripts/build_credis.sh | 10 ++--- thirdparty/scripts/build_ui.sh | 2 +- 17 files changed, 83 insertions(+), 41 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4ab963a31..cbb7a7719 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,12 +68,12 @@ matrix: - ./.travis/install-ray.sh script: - - cd python/ray/core - - bash ../../../src/common/test/run_valgrind.sh - - bash ../../../src/plasma/test/run_valgrind.sh - - bash ../../../src/local_scheduler/test/run_valgrind.sh - - bash ../../../src/ray/test/run_object_manager_valgrind.sh - - cd ../../.. + - cd build + - bash ../src/common/test/run_valgrind.sh + - bash ../src/plasma/test/run_valgrind.sh + - bash ../src/local_scheduler/test/run_valgrind.sh + - bash ../src/ray/test/run_object_manager_valgrind.sh + - cd .. - python ./python/ray/plasma/test/test.py valgrind - python ./python/ray/local_scheduler/test/test.py valgrind @@ -167,19 +167,19 @@ install: - ./.travis/install-ray.sh - ./.travis/install-cython-examples.sh - - cd python/ray/core - - bash ../../../src/ray/test/run_gcs_tests.sh + - cd build + - bash ../src/ray/test/run_gcs_tests.sh # Raylet tests. - - bash ../../../src/ray/test/run_object_manager_tests.sh + - bash ../src/ray/test/run_object_manager_tests.sh - ./src/ray/raylet/task_test - ./src/ray/raylet/worker_pool_test - ./src/ray/raylet/lineage_cache_test - ./src/ray/raylet/task_dependency_manager_test - - bash ../../../src/common/test/run_tests.sh - - bash ../../../src/plasma/test/run_tests.sh - - bash ../../../src/local_scheduler/test/run_tests.sh - - cd ../../.. + - bash ../src/common/test/run_tests.sh + - bash ../src/plasma/test/run_tests.sh + - bash ../src/local_scheduler/test/run_tests.sh + - cd .. script: - export PATH="$HOME/miniconda/bin:$PATH" diff --git a/CMakeLists.txt b/CMakeLists.txt index 80f222d4f..784e029cf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,3 +64,43 @@ add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/) add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/) add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/local_scheduler/) add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/global_scheduler/) + +if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") + # NOTE: The lists below must be kept in sync with ray/python/setup.py. + + set(ray_file_list + "src/common/thirdparty/redis/src/redis-server" + "src/common/redis_module/libray_redis_module.so" + "src/plasma/plasma_manager" + "src/local_scheduler/local_scheduler" + "src/local_scheduler/liblocal_scheduler_library_python.so" + "src/global_scheduler/global_scheduler" + "src/ray/raylet/raylet_monitor" + "src/ray/raylet/raylet") + + if (RAY_USE_NEW_GCS) + list(APPEND ray_file_list "src/credis/build/src/libmember.so") + list(APPEND ray_file_list "src/credis/build/src/libmaster.so") + list(APPEND ray_file_list "src/credis/redis/src/redis-server") + endif() + + if (DEFINED ENV{INCLUDE_UI} AND "$ENV{INCLUDE_UI}" STREQUAL "1") + list(APPEND ray_file_list "src/catapult_files/index.html") + list(APPEND ray_file_list "src/catapult_files/trace_viewer_full.html") + endif() + + set(build_ray_file_list) + foreach(file ${ray_file_list}) + list(APPEND build_ray_file_list ${CMAKE_BINARY_DIR}/${file}) + endforeach() + + add_custom_target(copy_ray ALL + DEPENDS ${build_ray_file_list}) + + foreach(file ${ray_file_list}) + add_custom_command(TARGET copy_ray POST_BUILD + COMMAND ${CMAKE_COMMAND} -E + copy ${CMAKE_BINARY_DIR}/${file} + ${CMAKE_BINARY_DIR}/../python/ray/core/${file}) + endforeach() +endif() diff --git a/build.sh b/build.sh index 7ab0ba3b1..6087023b7 100755 --- a/build.sh +++ b/build.sh @@ -82,13 +82,9 @@ echo "Using Python executable $PYTHON_EXECUTABLE." bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE $LANGUAGE # Now we build everything. -if [[ "$LANGUAGE" == "java" ]]; then - BUILD_DIR="$ROOT_DIR/build/" - if [ ! -d "${BUILD_DIR}" ]; then - mkdir -p ${BUILD_DIR} - fi -else - BUILD_DIR="$ROOT_DIR/python/ray/core" +BUILD_DIR="$ROOT_DIR/build/" +if [ ! -d "${BUILD_DIR}" ]; then +mkdir -p ${BUILD_DIR} fi pushd "$BUILD_DIR" @@ -109,6 +105,9 @@ popd # Move stuff from Arrow to Ray. cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/src/plasma/ +if [[ "$LANGUAGE" == "python" ]]; then + cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/../python/ray/core/src/plasma/ +fi if [[ "$LANGUAGE" == "java" ]]; then cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/ fi diff --git a/doc/source/development.rst b/doc/source/development.rst index 96e100b38..7c86ef1a0 100644 --- a/doc/source/development.rst +++ b/doc/source/development.rst @@ -29,7 +29,7 @@ helpful. .. code-block:: shell - cd ray/python/ray/core + cd ray/build make -j8 2. **Starting processes in a debugger:** When processes are crashing, it is diff --git a/python/setup.py b/python/setup.py index 94a62666e..a81ca9da6 100644 --- a/python/setup.py +++ b/python/setup.py @@ -15,6 +15,9 @@ import setuptools.command.build_ext as _build_ext # MANIFEST.in gets applied at the very beginning when setup.py runs # before these files have been created, so we have to move the files # manually. + +# NOTE: The lists below must be kept in sync with ray/CMakeLists.txt. + ray_files = [ "ray/core/src/common/thirdparty/redis/src/redis-server", "ray/core/src/common/redis_module/libray_redis_module.so", diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 0431cb6cb..0e9080d43 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -33,7 +33,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES") add_custom_target(gen_common_python_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES}) # Generate Python bindings for the flatbuffers objects. - set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/) + set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../python/ray/core/generated/) add_custom_command( TARGET gen_common_python_fbs COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC} diff --git a/src/local_scheduler/test/local_scheduler_tests.cc b/src/local_scheduler/test/local_scheduler_tests.cc index ba10e2a35..65467d289 100644 --- a/src/local_scheduler/test/local_scheduler_tests.cc +++ b/src/local_scheduler/test/local_scheduler_tests.cc @@ -95,7 +95,7 @@ LocalSchedulerMock *LocalSchedulerMock_init(int num_workers, /* Construct worker command */ std::stringstream worker_command_ss; - worker_command_ss << "python ../../../python/ray/workers/default_worker.py" + worker_command_ss << "python ../python/ray/workers/default_worker.py" << " --node-ip-address=" << node_ip_address << " --object-store-name=" << plasma_store_socket_name << " --object-store-manager-name=" diff --git a/src/ray/gcs/CMakeLists.txt b/src/ray/gcs/CMakeLists.txt index 015051971..0150bbb79 100644 --- a/src/ray/gcs/CMakeLists.txt +++ b/src/ray/gcs/CMakeLists.txt @@ -20,7 +20,7 @@ add_custom_command( add_custom_target(gen_gcs_fbs DEPENDS ${GCS_FBS_OUTPUT_FILES}) # Generate Python bindings for the flatbuffers objects. -set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/) +set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../python/ray/core/generated/) add_custom_command( TARGET gen_gcs_fbs COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${GCS_FBS_SRC} diff --git a/src/ray/raylet/CMakeLists.txt b/src/ray/raylet/CMakeLists.txt index aca592de0..b144e3211 100644 --- a/src/ray/raylet/CMakeLists.txt +++ b/src/ray/raylet/CMakeLists.txt @@ -20,7 +20,7 @@ add_custom_command( add_custom_target(gen_node_manager_fbs DEPENDS ${NODE_MANAGER_FBS_OUTPUT_FILES}) # Generate Python bindings for the flatbuffers objects. -set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/) +set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../python/ray/core/generated/) add_custom_command( TARGET gen_node_manager_fbs COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${NODE_MANAGER_FBS_SRC} diff --git a/src/ray/raylet/object_manager_integration_test.cc b/src/ray/raylet/object_manager_integration_test.cc index 23c5e2aac..451125581 100644 --- a/src/ray/raylet/object_manager_integration_test.cc +++ b/src/ray/raylet/object_manager_integration_test.cc @@ -40,7 +40,7 @@ class TestObjectManagerBase : public ::testing::Test { // Use a default worker that can execute empty tasks with dependencies. node_manager_config.worker_command.push_back("python"); node_manager_config.worker_command.push_back( - "../../../src/ray/python/default_worker.py"); + "../python/ray/workers/default_worker.py"); node_manager_config.worker_command.push_back(raylet_socket_name.c_str()); node_manager_config.worker_command.push_back(store_socket_name.c_str()); return node_manager_config; diff --git a/src/ray/test/run_gcs_tests.sh b/src/ray/test/run_gcs_tests.sh index 2808f188e..f9bd85a9c 100644 --- a/src/ray/test/run_gcs_tests.sh +++ b/src/ray/test/run_gcs_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This needs to be run in the build tree, which is normally ray/python/ray/core +# This needs to be run in the build tree, which is normally ray/build # Cause the script to exit if a single command fails. set -e diff --git a/src/ray/test/run_object_manager_tests.sh b/src/ray/test/run_object_manager_tests.sh index a9cd5d2b0..4bcd5e7a2 100644 --- a/src/ray/test/run_object_manager_tests.sh +++ b/src/ray/test/run_object_manager_tests.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This needs to be run in the build tree, which is normally ray/python/ray/core +# This needs to be run in the build tree, which is normally ray/build # Cause the script to exit if a single command fails. set -e @@ -20,7 +20,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then exit 1 fi -CORE_DIR="$RAY_ROOT/python/ray/core" +CORE_DIR="$RAY_ROOT/build" REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so" REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src" diff --git a/src/ray/test/run_object_manager_valgrind.sh b/src/ray/test/run_object_manager_valgrind.sh index f7024939b..29b3fe5f1 100644 --- a/src/ray/test/run_object_manager_valgrind.sh +++ b/src/ray/test/run_object_manager_valgrind.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This needs to be run in the build tree, which is normally ray/python/ray/core +# This needs to be run in the build tree, which is normally ray/build # Cause the script to exit if a single command fails. set -e @@ -20,7 +20,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then exit 1 fi -CORE_DIR="$RAY_ROOT/python/ray/core" +CORE_DIR="$RAY_ROOT/build" REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src" REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so" STORE_EXEC="$CORE_DIR/src/plasma/plasma_store" diff --git a/src/ray/test/start_raylet.sh b/src/ray/test/start_raylet.sh index a70c68e57..874ea4ae8 100644 --- a/src/ray/test/start_raylet.sh +++ b/src/ray/test/start_raylet.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This needs to be run in the build tree, which is normally ray/python/ray/core +# This needs to be run in the build tree, which is normally ray/build # Cause the script to exit if a single command fails. set -e @@ -25,5 +25,5 @@ fi ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 & echo -echo "WORKER COMMAND: python ../../../src/ray/python/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME" +echo "WORKER COMMAND: python ../python/ray/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME" echo diff --git a/src/ray/test/start_raylets.sh b/src/ray/test/start_raylets.sh index 7a608b266..135ea2653 100644 --- a/src/ray/test/start_raylets.sh +++ b/src/ray/test/start_raylets.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This needs to be run in the build tree, which is normally ray/python/ray/core +# This needs to be run in the build tree, which is normally ray/build # Cause the script to exit if a single command fails. set -e @@ -47,6 +47,6 @@ for i in `seq 1 $NUM_RAYLETS`; do ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 & echo - echo "WORKER COMMAND: python ../../../src/ray/python/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME" + echo "WORKER COMMAND: python ../python/ray/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME" echo done diff --git a/thirdparty/scripts/build_credis.sh b/thirdparty/scripts/build_credis.sh index 66cd83b74..1c05097bf 100644 --- a/thirdparty/scripts/build_credis.sh +++ b/thirdparty/scripts/build_credis.sh @@ -53,10 +53,10 @@ if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then make -j popd - mkdir -p $ROOT_DIR/python/ray/core/src/credis/redis/src/ - cp redis/src/redis-server $ROOT_DIR/python/ray/core/src/credis/redis/src/redis-server - mkdir -p $ROOT_DIR/python/ray/core/src/credis/build/src/ - cp build/src/libmaster.so $ROOT_DIR/python/ray/core/src/credis/build/src/libmaster.so - cp build/src/libmember.so $ROOT_DIR/python/ray/core/src/credis/build/src/libmember.so + mkdir -p $ROOT_DIR/build/src/credis/redis/src/ + cp redis/src/redis-server $ROOT_DIR/build/src/credis/redis/src/redis-server + mkdir -p $ROOT_DIR/build/src/credis/build/src/ + cp build/src/libmaster.so $ROOT_DIR/build/src/credis/build/src/libmaster.so + cp build/src/libmember.so $ROOT_DIR/build/src/credis/build/src/libmember.so popd fi diff --git a/thirdparty/scripts/build_ui.sh b/thirdparty/scripts/build_ui.sh index 8b95db4dc..aa2530d89 100755 --- a/thirdparty/scripts/build_ui.sh +++ b/thirdparty/scripts/build_ui.sh @@ -11,7 +11,7 @@ CATAPULT_COMMIT=18cd334755701cf0c3b90b7172126c686d2eb787 CATAPULT_HOME=$TP_DIR/pkg/catapult VULCANIZE_BIN=$CATAPULT_HOME/tracing/bin/vulcanize_trace_viewer -CATAPULT_FILES=$TP_DIR/../python/ray/core/src/catapult_files +CATAPULT_FILES=$TP_DIR/../build/src/catapult_files # This is where we will copy the files that need to be packaged with the wheels. mkdir -p $CATAPULT_FILES