unify build dir for Python and Java (#2171)

* unify build dir for Python and Java

* enable executables auto installed when just running 'make'

* fix plasma_store copy error

* fix cmake error about copying executables

* lint fix

* recover python/setup.py

* enable to copy optional file automatically

* a small fix of path

* lint fix

* lint fix

* lint fix

* Add comment.

src/common/CMakeLists.txt

@@ -33,7 +33,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
   add_custom_target(gen_common_python_fbs DEPENDS ${COMMON_FBS_OUTPUT_FILES})
 
   # Generate Python bindings for the flatbuffers objects.
-  set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
+  set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../python/ray/core/generated/)
   add_custom_command(
     TARGET gen_common_python_fbs
     COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${COMMON_FBS_SRC}

src/local_scheduler/test/local_scheduler_tests.cc

@@ -95,7 +95,7 @@ LocalSchedulerMock *LocalSchedulerMock_init(int num_workers,
 
   /* Construct worker command */
   std::stringstream worker_command_ss;
-  worker_command_ss << "python ../../../python/ray/workers/default_worker.py"
+  worker_command_ss << "python ../python/ray/workers/default_worker.py"
                     << " --node-ip-address=" << node_ip_address
                     << " --object-store-name=" << plasma_store_socket_name
                     << " --object-store-manager-name="

src/ray/gcs/CMakeLists.txt

@@ -20,7 +20,7 @@ add_custom_command(
 add_custom_target(gen_gcs_fbs DEPENDS ${GCS_FBS_OUTPUT_FILES})
 
 # Generate Python bindings for the flatbuffers objects.
-set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
+set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../python/ray/core/generated/)
 add_custom_command(
   TARGET gen_gcs_fbs
   COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${GCS_FBS_SRC}

src/ray/raylet/CMakeLists.txt

@@ -20,7 +20,7 @@ add_custom_command(
 add_custom_target(gen_node_manager_fbs DEPENDS ${NODE_MANAGER_FBS_OUTPUT_FILES})
 
 # Generate Python bindings for the flatbuffers objects.
-set(PYTHON_OUTPUT_DIR ${CMAKE_BINARY_DIR}/generated/)
+set(PYTHON_OUTPUT_DIR ${CMAKE_CURRENT_LIST_DIR}/../../../python/ray/core/generated/)
 add_custom_command(
     TARGET gen_node_manager_fbs
     COMMAND ${FLATBUFFERS_COMPILER} -p -o ${PYTHON_OUTPUT_DIR} ${NODE_MANAGER_FBS_SRC}

src/ray/raylet/object_manager_integration_test.cc

@@ -40,7 +40,7 @@ class TestObjectManagerBase : public ::testing::Test {
     // Use a default worker that can execute empty tasks with dependencies.
     node_manager_config.worker_command.push_back("python");
     node_manager_config.worker_command.push_back(
-        "../../../src/ray/python/default_worker.py");
+        "../python/ray/workers/default_worker.py");
     node_manager_config.worker_command.push_back(raylet_socket_name.c_str());
     node_manager_config.worker_command.push_back(store_socket_name.c_str());
     return node_manager_config;

src/ray/test/run_gcs_tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e

src/ray/test/run_object_manager_tests.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -20,7 +20,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then
   exit 1
 fi
 
-CORE_DIR="$RAY_ROOT/python/ray/core"
+CORE_DIR="$RAY_ROOT/build"
 REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so"
 REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src"
 

src/ray/test/run_object_manager_valgrind.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -20,7 +20,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then
   exit 1
 fi
 
-CORE_DIR="$RAY_ROOT/python/ray/core"
+CORE_DIR="$RAY_ROOT/build"
 REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src"
 REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so"
 STORE_EXEC="$CORE_DIR/src/plasma/plasma_store"

src/ray/test/start_raylet.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -25,5 +25,5 @@ fi
 ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 &
 
 echo
-echo "WORKER COMMAND: python ../../../src/ray/python/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME"
+echo "WORKER COMMAND: python ../python/ray/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME"
 echo

src/ray/test/start_raylets.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/python/ray/core
+# This needs to be run in the build tree, which is normally ray/build
 
 # Cause the script to exit if a single command fails.
 set -e
@@ -47,6 +47,6 @@ for i in `seq 1 $NUM_RAYLETS`; do
   ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 &
 
   echo
-  echo "WORKER COMMAND: python ../../../src/ray/python/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME"
+  echo "WORKER COMMAND: python ../python/ray/worker.py $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME"
   echo
 done