From b4c47a586170825a72b59b257581bf6014bd0b5e Mon Sep 17 00:00:00 2001 From: Philipp Moritz Date: Fri, 24 Aug 2018 11:44:04 -0700 Subject: [PATCH] Upgrade arrow to include more detailed flushing message (#2706) --- build.sh | 4 ++-- java/cleanup.sh | 2 +- java/prepare.sh | 2 +- java/ray.config.ini | 7 +++---- python/ray/plasma/plasma.py | 2 +- python/ray/plasma/test/test.py | 3 ++- python/setup.py | 3 ++- src/local_scheduler/test/run_tests.sh | 4 ++-- src/local_scheduler/test/run_valgrind.sh | 4 ++-- src/plasma/test/run_tests.sh | 8 ++++---- src/plasma/test/run_valgrind.sh | 4 ++-- src/ray/raylet/object_manager_integration_test.cc | 2 +- src/ray/test/run_object_manager_tests.sh | 2 +- src/ray/test/run_object_manager_valgrind.sh | 2 +- src/ray/test/start_raylet.sh | 2 +- src/ray/test/start_raylets.sh | 2 +- thirdparty/scripts/build_arrow.sh | 4 ++-- 17 files changed, 29 insertions(+), 28 deletions(-) diff --git a/build.sh b/build.sh index e3b458b10..2af951d1d 100755 --- a/build.sh +++ b/build.sh @@ -117,9 +117,9 @@ make -j${PARALLEL} popd # Move stuff from Arrow to Ray. -cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/src/plasma/ +cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store_server $BUILD_DIR/src/plasma/ if [[ "$RAY_BUILD_PYTHON" == "YES" ]]; then - cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/../python/ray/core/src/plasma/ + cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store_server $BUILD_DIR/../python/ray/core/src/plasma/ fi if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/ diff --git a/java/cleanup.sh b/java/cleanup.sh index 84b2110bd..ccfe4dd3f 100755 --- a/java/cleanup.sh +++ b/java/cleanup.sh @@ -1,7 +1,7 @@ pkill -9 python pkill -9 local_scheduler pkill -9 plasma_manager -pkill -9 plasma_store +pkill -9 plasma_store_server pkill -9 global_scheduler pkill -9 redis-server pkill -9 redis diff --git a/java/prepare.sh b/java/prepare.sh index b699b4a5f..bbea0fdca 100755 --- a/java/prepare.sh +++ b/java/prepare.sh @@ -43,7 +43,7 @@ fi declare -a nativeBinaries=( "./src/common/thirdparty/redis/src/redis-server" - "./src/plasma/plasma_store" + "./src/plasma/plasma_store_server" "./src/plasma/plasma_manager" "./src/local_scheduler/local_scheduler" "./src/global_scheduler/global_scheduler" diff --git a/java/ray.config.ini b/java/ray.config.ini index 5c7f9b696..bb9a1fd1b 100644 --- a/java/ray.config.ini +++ b/java/ray.config.ini @@ -121,7 +121,7 @@ driver_args = [ray.java.path.source] redis_server = %CONFIG_FILE_DIR%/../build/src/common/thirdparty/redis/src/redis-server redis_module = %CONFIG_FILE_DIR%/../build/src/common/redis_module/libray_redis_module.so -store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store +store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store_server store_manager = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_manager local_scheduler = %CONFIG_FILE_DIR%/../build/src/local_scheduler/local_scheduler global_scheduler = %CONFIG_FILE_DIR%/../build/src/global_scheduler/global_scheduler @@ -134,7 +134,7 @@ java_jnilib_paths = ray.java.path.jni.package [ray.java.path.package] redis_server = %CONFIG_FILE_DIR%/../build/src/common/thirdparty/redis/src/redis-server redis_module = %CONFIG_FILE_DIR%/../build/src/common/redis_module/libray_redis_module.so -store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store +store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store_server store_manager = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_manager local_scheduler = %CONFIG_FILE_DIR%/../build/src/local_scheduler/local_scheduler global_scheduler = %CONFIG_FILE_DIR%/../build/src/global_scheduler/global_scheduler @@ -147,7 +147,7 @@ java_jnilib_paths = ray.java.path.jni.package [ray.java.path.deploy] redis_server = %CONFIG_FILE_DIR%/native/bin/redis-server redis_module = %CONFIG_FILE_DIR%/native/lib/libray_redis_module.so -store = %CONFIG_FILE_DIR%/native/bin/plasma_store +store = %CONFIG_FILE_DIR%/native/bin/plasma_store_server store_manager = %CONFIG_FILE_DIR%/native/bin/plasma_manager local_scheduler = %CONFIG_FILE_DIR%/native/bin/local_scheduler global_scheduler = %CONFIG_FILE_DIR%/native/bin/global_scheduler @@ -156,4 +156,3 @@ python_dir = %CONFIG_FILE_DIR%/python java_runtime_rewritten_jars_dir = %CONFIG_FILE_DIR%/java/lib/ java_class_paths = ray.java.path.classes.deploy java_jnilib_paths = ray.java.path.jni.deploy - diff --git a/python/ray/plasma/plasma.py b/python/ray/plasma/plasma.py index f6beeef14..60870c2b2 100644 --- a/python/ray/plasma/plasma.py +++ b/python/ray/plasma/plasma.py @@ -65,7 +65,7 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, plasma_store_executable = os.path.join( os.path.abspath(os.path.dirname(__file__)), - "../core/src/plasma/plasma_store") + "../core/src/plasma/plasma_store_server") plasma_store_name = "/tmp/plasma_store{}".format(random_name()) command = [ plasma_store_executable, "-s", plasma_store_name, "-m", diff --git a/python/ray/plasma/test/test.py b/python/ray/plasma/test/test.py index e5a2d27ea..a67f2d255 100644 --- a/python/ray/plasma/test/test.py +++ b/python/ray/plasma/test/test.py @@ -87,7 +87,8 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY, """ if use_valgrind and use_profiler: raise Exception("Cannot use valgrind and profiler at the same time.") - plasma_store_executable = os.path.join(pa.__path__[0], "plasma_store") + plasma_store_executable = os.path.join(pa.__path__[0], + "plasma_store_server") plasma_store_name = "/tmp/plasma_store{}".format(random_name()) command = [ plasma_store_executable, "-s", plasma_store_name, "-m", diff --git a/python/setup.py b/python/setup.py index 050b90dd7..0bfddd6b5 100644 --- a/python/setup.py +++ b/python/setup.py @@ -21,7 +21,8 @@ import setuptools.command.build_ext as _build_ext ray_files = [ "ray/core/src/common/thirdparty/redis/src/redis-server", "ray/core/src/common/redis_module/libray_redis_module.so", - "ray/core/src/plasma/plasma_store", "ray/core/src/plasma/plasma_manager", + "ray/core/src/plasma/plasma_store_server", + "ray/core/src/plasma/plasma_manager", "ray/core/src/local_scheduler/local_scheduler", "ray/core/src/local_scheduler/liblocal_scheduler_library_python.so", "ray/core/src/global_scheduler/global_scheduler", diff --git a/src/local_scheduler/test/run_tests.sh b/src/local_scheduler/test/run_tests.sh index 6be7215c0..9c1d7be79 100644 --- a/src/local_scheduler/test/run_tests.sh +++ b/src/local_scheduler/test/run_tests.sh @@ -30,9 +30,9 @@ sleep 1s ./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1 ./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380 -./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 & +./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 & sleep 0.5s ./src/local_scheduler/local_scheduler_tests ./src/common/thirdparty/redis/src/redis-cli shutdown ./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown -killall plasma_store +killall plasma_store_server diff --git a/src/local_scheduler/test/run_valgrind.sh b/src/local_scheduler/test/run_valgrind.sh index 4e45c4238..6ff1dbe33 100644 --- a/src/local_scheduler/test/run_valgrind.sh +++ b/src/local_scheduler/test/run_valgrind.sh @@ -33,9 +33,9 @@ sleep 1s ./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1 ./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380 -./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 & +./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 & sleep 0.5s valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/local_scheduler/local_scheduler_tests ./src/common/thirdparty/redis/src/redis-cli shutdown ./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown -killall plasma_store +killall plasma_store_server diff --git a/src/plasma/test/run_tests.sh b/src/plasma/test/run_tests.sh index 48cdcf353..07001a65d 100644 --- a/src/plasma/test/run_tests.sh +++ b/src/plasma/test/run_tests.sh @@ -3,10 +3,10 @@ # Cause the script to exit if a single command fails. set -e -./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 & +./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 0 & sleep 1 ./src/plasma/manager_tests -killall plasma_store +killall plasma_store_server LaunchRedis() { port=$1 @@ -37,11 +37,11 @@ sleep 1s ./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1 ./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380 sleep 1 -./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 & +./src/plasma/plasma_store_server -s /tmp/store1 -m 1000000000 & plasma1_pid=$! ./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 & plasma2_pid=$! -./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 & +./src/plasma/plasma_store_server -s /tmp/store2 -m 1000000000 & plasma3_pid=$! ./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 & plasma4_pid=$! diff --git a/src/plasma/test/run_valgrind.sh b/src/plasma/test/run_valgrind.sh index 8addb4354..a3dfb1574 100644 --- a/src/plasma/test/run_valgrind.sh +++ b/src/plasma/test/run_valgrind.sh @@ -5,7 +5,7 @@ set -x # Cause the script to exit if a single command fails. set -e -./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 & +./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 0 & sleep 1 valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/plasma/manager_tests -killall plasma_store +killall plasma_store_server diff --git a/src/ray/raylet/object_manager_integration_test.cc b/src/ray/raylet/object_manager_integration_test.cc index b9af292c1..7d2ad6c7c 100644 --- a/src/ray/raylet/object_manager_integration_test.cc +++ b/src/ray/raylet/object_manager_integration_test.cc @@ -83,7 +83,7 @@ class TestObjectManagerBase : public ::testing::Test { this->server1.reset(); this->server2.reset(); - int s = system("killall plasma_store &"); + int s = system("killall plasma_store_server &"); ASSERT_TRUE(!s); std::string cmd_str = test_executable.substr(0, test_executable.find_last_of("/")); diff --git a/src/ray/test/run_object_manager_tests.sh b/src/ray/test/run_object_manager_tests.sh index 4bcd5e7a2..04da7e29a 100644 --- a/src/ray/test/run_object_manager_tests.sh +++ b/src/ray/test/run_object_manager_tests.sh @@ -34,7 +34,7 @@ else LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" fi -STORE_EXEC="$CORE_DIR/src/plasma/plasma_store" +STORE_EXEC="$CORE_DIR/src/plasma/plasma_store_server" # Allow cleanup commands to fail. $REDIS_DIR/redis-cli -p 6379 shutdown || true diff --git a/src/ray/test/run_object_manager_valgrind.sh b/src/ray/test/run_object_manager_valgrind.sh index 29b3fe5f1..23474c26d 100644 --- a/src/ray/test/run_object_manager_valgrind.sh +++ b/src/ray/test/run_object_manager_valgrind.sh @@ -23,7 +23,7 @@ fi CORE_DIR="$RAY_ROOT/build" REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src" REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so" -STORE_EXEC="$CORE_DIR/src/plasma/plasma_store" +STORE_EXEC="$CORE_DIR/src/plasma/plasma_store_server" VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1" if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then diff --git a/src/ray/test/start_raylet.sh b/src/ray/test/start_raylet.sh index 874ea4ae8..45c1f5d6d 100644 --- a/src/ray/test/start_raylet.sh +++ b/src/ray/test/start_raylet.sh @@ -21,7 +21,7 @@ if [[ `stat $STORE_SOCKET_NAME` ]]; then rm $STORE_SOCKET_NAME fi -./src/plasma/plasma_store -m 1000000000 -s $STORE_SOCKET_NAME & +./src/plasma/plasma_store_server -m 1000000000 -s $STORE_SOCKET_NAME & ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 & echo diff --git a/src/ray/test/start_raylets.sh b/src/ray/test/start_raylets.sh index 135ea2653..9b1d9b842 100644 --- a/src/ray/test/start_raylets.sh +++ b/src/ray/test/start_raylets.sh @@ -43,7 +43,7 @@ for i in `seq 1 $NUM_RAYLETS`; do rm $STORE_SOCKET_NAME fi - ./src/plasma/plasma_store -m 1000000000 -s $STORE_SOCKET_NAME & + ./src/plasma/plasma_store_server -m 1000000000 -s $STORE_SOCKET_NAME & ./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 & echo diff --git a/thirdparty/scripts/build_arrow.sh b/thirdparty/scripts/build_arrow.sh index 0992526d5..4d5eff268 100755 --- a/thirdparty/scripts/build_arrow.sh +++ b/thirdparty/scripts/build_arrow.sh @@ -34,10 +34,10 @@ else exit 1 fi -# The PR for this commit is https://github.com/apache/arrow/pull/2368. We +# The PR for this commit is https://github.com/apache/arrow/pull/2458. We # include the link here to make it easier to find the right commit because # Arrow often rewrites git history and invalidates certain commits. -TARGET_COMMIT_ID=4660833b2c5ef63a97445e304b8f72a2e0170f9c +TARGET_COMMIT_ID=fda4b3dcfc773612b12973df5053193f236fc696 build_arrow() { echo "building arrow" # Make sure arrow will be built again when building ray for java later than python