unify build dir for Python and Java (#2171)

* unify build dir for Python and Java

* enable executables auto installed when just running 'make'

* fix plasma_store copy error

* fix cmake error about copying executables

* lint fix

* recover python/setup.py

* enable to copy optional file automatically

* a small fix of path

* lint fix

* lint fix

* lint fix

* Add comment.

.travis.yml

@@ -68,12 +68,12 @@ matrix:
         - ./.travis/install-ray.sh
 
       script:
-        - cd python/ray/core
-        - bash ../../../src/common/test/run_valgrind.sh
-        - bash ../../../src/plasma/test/run_valgrind.sh
-        - bash ../../../src/local_scheduler/test/run_valgrind.sh
-        - bash ../../../src/ray/test/run_object_manager_valgrind.sh
-        - cd ../../..
+        - cd build
+        - bash ../src/common/test/run_valgrind.sh
+        - bash ../src/plasma/test/run_valgrind.sh
+        - bash ../src/local_scheduler/test/run_valgrind.sh
+        - bash ../src/ray/test/run_object_manager_valgrind.sh
+        - cd ..
 
         - python ./python/ray/plasma/test/test.py valgrind
         - python ./python/ray/local_scheduler/test/test.py valgrind
@@ -167,19 +167,19 @@ install:
   - ./.travis/install-ray.sh
   - ./.travis/install-cython-examples.sh
 
-  - cd python/ray/core
-  - bash ../../../src/ray/test/run_gcs_tests.sh
+  - cd build
+  - bash ../src/ray/test/run_gcs_tests.sh
   # Raylet tests.
-  - bash ../../../src/ray/test/run_object_manager_tests.sh
+  - bash ../src/ray/test/run_object_manager_tests.sh
   - ./src/ray/raylet/task_test
   - ./src/ray/raylet/worker_pool_test
   - ./src/ray/raylet/lineage_cache_test
   - ./src/ray/raylet/task_dependency_manager_test
 
-  - bash ../../../src/common/test/run_tests.sh
-  - bash ../../../src/plasma/test/run_tests.sh
-  - bash ../../../src/local_scheduler/test/run_tests.sh
-  - cd ../../..
+  - bash ../src/common/test/run_tests.sh
+  - bash ../src/plasma/test/run_tests.sh
+  - bash ../src/local_scheduler/test/run_tests.sh
+  - cd ..
 
 script:
   - export PATH="$HOME/miniconda/bin:$PATH"

CMakeLists.txt

@@ -64,3 +64,43 @@ add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/common/)
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/plasma/)
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/local_scheduler/)
 add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/src/global_scheduler/)
+
+if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
+  # NOTE: The lists below must be kept in sync with ray/python/setup.py.
+
+  set(ray_file_list
+        "src/common/thirdparty/redis/src/redis-server"
+        "src/common/redis_module/libray_redis_module.so"
+        "src/plasma/plasma_manager"
+        "src/local_scheduler/local_scheduler"
+        "src/local_scheduler/liblocal_scheduler_library_python.so"
+        "src/global_scheduler/global_scheduler"
+        "src/ray/raylet/raylet_monitor"
+        "src/ray/raylet/raylet")
+
+  if (RAY_USE_NEW_GCS)
+    list(APPEND ray_file_list "src/credis/build/src/libmember.so")
+    list(APPEND ray_file_list "src/credis/build/src/libmaster.so")
+    list(APPEND ray_file_list "src/credis/redis/src/redis-server")
+  endif()
+
+  if (DEFINED ENV{INCLUDE_UI} AND "$ENV{INCLUDE_UI}" STREQUAL "1")
+    list(APPEND ray_file_list "src/catapult_files/index.html")
+    list(APPEND ray_file_list "src/catapult_files/trace_viewer_full.html")
+  endif()
+
+  set(build_ray_file_list)
+  foreach(file ${ray_file_list})
+    list(APPEND build_ray_file_list ${CMAKE_BINARY_DIR}/${file})
+  endforeach()
+
+  add_custom_target(copy_ray ALL
+      DEPENDS ${build_ray_file_list})
+
+  foreach(file ${ray_file_list})
+  add_custom_command(TARGET copy_ray POST_BUILD
+                    COMMAND ${CMAKE_COMMAND} -E
+                        copy ${CMAKE_BINARY_DIR}/${file}
+                            ${CMAKE_BINARY_DIR}/../python/ray/core/${file})
+  endforeach()
+endif()

build.sh

@@ -82,13 +82,9 @@ echo "Using Python executable $PYTHON_EXECUTABLE."
 bash $ROOT_DIR/setup_thirdparty.sh $PYTHON_EXECUTABLE $LANGUAGE
 
 # Now we build everything.
-if [[ "$LANGUAGE" == "java" ]]; then
-  BUILD_DIR="$ROOT_DIR/build/"
-  if [ ! -d "${BUILD_DIR}" ]; then
-  mkdir -p ${BUILD_DIR}
-  fi
-else
-  BUILD_DIR="$ROOT_DIR/python/ray/core"
+BUILD_DIR="$ROOT_DIR/build/"
+if [ ! -d "${BUILD_DIR}" ]; then
+mkdir -p ${BUILD_DIR}
 fi
 
 pushd "$BUILD_DIR"
@@ -109,6 +105,9 @@ popd
 
 # Move stuff from Arrow to Ray.
 cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/src/plasma/
+if [[ "$LANGUAGE" == "python" ]]; then
+  cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/../python/ray/core/src/plasma/
+fi
 if [[ "$LANGUAGE" == "java" ]]; then
   cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/
 fi

doc/source/development.rst

@@ -29,7 +29,7 @@ helpful.
 
    .. code-block:: shell
 
-     cd ray/python/ray/core
+     cd ray/build
      make -j8
 
 2. **Starting processes in a debugger:** When processes are crashing, it is

python/setup.py

@@ -15,6 +15,9 @@ import setuptools.command.build_ext as _build_ext
 # MANIFEST.in gets applied at the very beginning when setup.py runs
 # before these files have been created, so we have to move the files
 # manually.
+
+# NOTE: The lists below must be kept in sync with ray/CMakeLists.txt.
+
 ray_files = [
     "ray/core/src/common/thirdparty/redis/src/redis-server",
     "ray/core/src/common/redis_module/libray_redis_module.so",

src/common/CMakeLists.txt

@@ -33,7 +33,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   add_custom_target(gen_common_python_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
 
   # Generate Python bindings for the flatbuffers objects.
-  set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
+  set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../python/ray/core/generated/)
   add_custom_command(
     TARGET gen_common_python_fbs
     COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}

src/local_scheduler/test/local_scheduler_tests.cc

@@ -95,7 +95,7 @@ LocalSchedulerMock *LocalSchedulerMock_init(int num_workers,
 
   /* Construct worker command */
   std::stringstream worker_command_ss;
-  worker_command_ss << "python ../../../python/ray/workers/default_worker.py"
+  worker_command_ss << "python ../python/ray/workers/default_worker.py"
                     << " --node-ip-address=" << node_ip_address
                     << " --object-store-name=" << plasma_store_socket_name
                     << " --object-store-manager-name="

src/ray/gcs/CMakeLists.txt

@@ -20,7 +20,7 @@ add_custom_command(
 add_custom_target(gen_gcs_fbs DEPENDS ${GCS_FBS_OUTPUT_FILES})
 
 # Generate Python bindings for the flatbuffers objects.
-set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
+set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../python/ray/core/generated/)
 add_custom_command(
   TARGET gen_gcs_fbs
   COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${GCS_FBS_SRC}

src/ray/raylet/CMakeLists.txt

@@ -20,7 +20,7 @@ add_custom_command(
 add_custom_target(gen_node_manager_fbs DEPENDS ${NODE_MANAGER_FBS_OUTPUT_FILES})
 
 # Generate Python bindings for the flatbuffers objects.
-set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
+set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../python/ray/core/generated/)
 add_custom_command(
     TARGET gen_node_manager_fbs
     COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${NODE_MANAGER_FBS_SRC}

src/ray/raylet/object_manager_integration_test.cc

@@ -40,7 +40,7 @@ class TestObjectManagerBase : public ::testing::Test {
     // Use a default worker that can execute empty tasks with dependencies.
     node_manager_config.worker_command.push_back("python");
     node_manager_config.worker_command.push_back(
-        "../../../src/ray/python/default_worker.py");
+        "../python/ray/workers/default_worker.py");
     node_manager_config.worker_command.push_back(raylet_socket_name.c_str());
     node_manager_config.worker_command.push_back(store_socket_name.c_str());
     return node_manager_config;

src/ray/test/run_gcs_tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e

src/ray/test/run_object_manager_tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -20,7 +20,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then
   exit 1
 fi
 
-CORE_DIR="$RAY_ROOT/python/ray/core"
+CORE_DIR="$RAY_ROOT/build"
 REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so"
 REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src"
 

src/ray/test/run_object_manager_valgrind.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -20,7 +20,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then
   exit 1
 fi
 
-CORE_DIR="$RAY_ROOT/python/ray/core"
+CORE_DIR="$RAY_ROOT/build"
 REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src"
 REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so"
 STORE_EXEC="$CORE_DIR/src/plasma/plasma_store"

src/ray/test/start_raylet.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -25,5 +25,5 @@ fi
 ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 &
 
 echo
-echo "WORKER COMMAND: python ../../../src/ray/python/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME"
+echo "WORKER COMMAND: python ../python/ray/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME"
 echo

src/ray/test/start_raylets.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -47,6 +47,6 @@ for i in `seq 1 $NUM_RAYLETS`; do
   ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 &
 
   echo
-  echo "WORKER COMMAND: python ../../../src/ray/python/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME"
+  echo "WORKER COMMAND: python ../python/ray/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME"
   echo
 done

thirdparty/scripts/build_credis.sh

@@ -53,10 +53,10 @@ if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
       make -j
     popd
 
-    mkdir -p $ROOT_DIR/python/ray/core/src/credis/redis/src/
-    cp redis/src/redis-server $ROOT_DIR/python/ray/core/src/credis/redis/src/redis-server
-    mkdir -p $ROOT_DIR/python/ray/core/src/credis/build/src/
-    cp build/src/libmaster.so $ROOT_DIR/python/ray/core/src/credis/build/src/libmaster.so
-    cp build/src/libmember.so $ROOT_DIR/python/ray/core/src/credis/build/src/libmember.so
+    mkdir -p $ROOT_DIR/build/src/credis/redis/src/
+    cp redis/src/redis-server $ROOT_DIR/build/src/credis/redis/src/redis-server
+    mkdir -p $ROOT_DIR/build/src/credis/build/src/
+    cp build/src/libmaster.so $ROOT_DIR/build/src/credis/build/src/libmaster.so
+    cp build/src/libmember.so $ROOT_DIR/build/src/credis/build/src/libmember.so
   popd
 fi

thirdparty/scripts/build_ui.sh

@@ -11,7 +11,7 @@ CATAPULT_COMMIT=18cd334755701cf0c3b90b7172126c686d2eb787
 CATAPULT_HOME=$TP_DIR/pkg/catapult
 VULCANIZE_BIN=$CATAPULT_HOME/tracing/bin/vulcanize_trace_viewer
 
-CATAPULT_FILES=$TP_DIR/../python/ray/core/src/catapult_files
+CATAPULT_FILES=$TP_DIR/../build/src/catapult_files
 
 # This is where we will copy the files that need to be packaged with the wheels.
 mkdir -p $CATAPULT_FILES