wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 19:32:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Upgrade arrow to include more detailed flushing message (#2706)

Browse Source
This commit is contained in:
Philipp Moritz
2018-08-24 11:44:04 -07:00
committed by Robert Nishihara
parent e467f546b5
commit b4c47a5861
17 changed files with 29 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
build.sh
+2 -2
View File
@@ -117,9 +117,9 @@ make -j${PARALLEL}
popd
# Move stuff from Arrow to Ray.
cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/src/plasma/
cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store_server $BUILD_DIR/src/plasma/
if [[ "$RAY_BUILD_PYTHON" == "YES" ]]; then
cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store $BUILD_DIR/../python/ray/core/src/plasma/
cp $ROOT_DIR/thirdparty/pkg/arrow/cpp/build/cpp-install/bin/plasma_store_server $BUILD_DIR/../python/ray/core/src/plasma/
fi
if [[ "$RAY_BUILD_JAVA" == "YES" ]]; then
cp $ROOT_DIR/thirdparty/build/arrow/cpp/build/release/libplasma_java.* $BUILD_DIR/src/plasma/
java/cleanup.sh
+1 -1
View File
@@ -1,7 +1,7 @@
pkill -9 python
pkill -9 local_scheduler
pkill -9 plasma_manager
pkill -9 plasma_store
pkill -9 plasma_store_server
pkill -9 global_scheduler
pkill -9 redis-server
pkill -9 redis
java/prepare.sh
+1 -1
View File
@@ -43,7 +43,7 @@ fi
declare -a nativeBinaries=(
"./src/common/thirdparty/redis/src/redis-server"
"./src/plasma/plasma_store"
"./src/plasma/plasma_store_server"
"./src/plasma/plasma_manager"
"./src/local_scheduler/local_scheduler"
"./src/global_scheduler/global_scheduler"
java/ray.config.ini
+3 -4
View File
@@ -121,7 +121,7 @@ driver_args =
[ray.java.path.source]
redis_server = %CONFIG_FILE_DIR%/../build/src/common/thirdparty/redis/src/redis-server
redis_module = %CONFIG_FILE_DIR%/../build/src/common/redis_module/libray_redis_module.so
store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store
store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store_server
store_manager = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_manager
local_scheduler = %CONFIG_FILE_DIR%/../build/src/local_scheduler/local_scheduler
global_scheduler = %CONFIG_FILE_DIR%/../build/src/global_scheduler/global_scheduler
@@ -134,7 +134,7 @@ java_jnilib_paths = ray.java.path.jni.package
[ray.java.path.package]
redis_server = %CONFIG_FILE_DIR%/../build/src/common/thirdparty/redis/src/redis-server
redis_module = %CONFIG_FILE_DIR%/../build/src/common/redis_module/libray_redis_module.so
store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store
store = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_store_server
store_manager = %CONFIG_FILE_DIR%/../build/src/plasma/plasma_manager
local_scheduler = %CONFIG_FILE_DIR%/../build/src/local_scheduler/local_scheduler
global_scheduler = %CONFIG_FILE_DIR%/../build/src/global_scheduler/global_scheduler
@@ -147,7 +147,7 @@ java_jnilib_paths = ray.java.path.jni.package
[ray.java.path.deploy]
redis_server = %CONFIG_FILE_DIR%/native/bin/redis-server
redis_module = %CONFIG_FILE_DIR%/native/lib/libray_redis_module.so
store = %CONFIG_FILE_DIR%/native/bin/plasma_store
store = %CONFIG_FILE_DIR%/native/bin/plasma_store_server
store_manager = %CONFIG_FILE_DIR%/native/bin/plasma_manager
local_scheduler = %CONFIG_FILE_DIR%/native/bin/local_scheduler
global_scheduler = %CONFIG_FILE_DIR%/native/bin/global_scheduler
@@ -156,4 +156,3 @@ python_dir = %CONFIG_FILE_DIR%/python
java_runtime_rewritten_jars_dir = %CONFIG_FILE_DIR%/java/lib/
java_class_paths = ray.java.path.classes.deploy
java_jnilib_paths = ray.java.path.jni.deploy
python/ray/plasma/plasma.py
+1 -1
View File
@@ -65,7 +65,7 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
plasma_store_executable = os.path.join(
os.path.abspath(os.path.dirname(__file__)),
"../core/src/plasma/plasma_store")
"../core/src/plasma/plasma_store_server")
plasma_store_name = "/tmp/plasma_store{}".format(random_name())
command = [
plasma_store_executable, "-s", plasma_store_name, "-m",
python/ray/plasma/test/test.py
+2 -1
View File
@@ -87,7 +87,8 @@ def start_plasma_store(plasma_store_memory=DEFAULT_PLASMA_STORE_MEMORY,
"""
if use_valgrind and use_profiler:
raise Exception("Cannot use valgrind and profiler at the same time.")
plasma_store_executable = os.path.join(pa.__path__[0], "plasma_store")
plasma_store_executable = os.path.join(pa.__path__[0],
"plasma_store_server")
plasma_store_name = "/tmp/plasma_store{}".format(random_name())
command = [
plasma_store_executable, "-s", plasma_store_name, "-m",
python/setup.py
+2 -1
View File
@@ -21,7 +21,8 @@ import setuptools.command.build_ext as _build_ext
ray_files = [
"ray/core/src/common/thirdparty/redis/src/redis-server",
"ray/core/src/common/redis_module/libray_redis_module.so",
"ray/core/src/plasma/plasma_store", "ray/core/src/plasma/plasma_manager",
"ray/core/src/plasma/plasma_store_server",
"ray/core/src/plasma/plasma_manager",
"ray/core/src/local_scheduler/local_scheduler",
"ray/core/src/local_scheduler/liblocal_scheduler_library_python.so",
"ray/core/src/global_scheduler/global_scheduler",
src/local_scheduler/test/run_tests.sh
+2 -2
View File
@@ -30,9 +30,9 @@ sleep 1s
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 &
./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 &
sleep 0.5s
./src/local_scheduler/local_scheduler_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
killall plasma_store
killall plasma_store_server
src/local_scheduler/test/run_valgrind.sh
+2 -2
View File
@@ -33,9 +33,9 @@ sleep 1s
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 100000000 &
./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 100000000 &
sleep 0.5s
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/local_scheduler/local_scheduler_tests
./src/common/thirdparty/redis/src/redis-cli shutdown
./src/common/thirdparty/redis/src/redis-cli -p 6380 shutdown
killall plasma_store
killall plasma_store_server
src/plasma/test/run_tests.sh
+4 -4
View File
@@ -3,10 +3,10 @@
# Cause the script to exit if a single command fails.
set -e
./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 0 &
sleep 1
./src/plasma/manager_tests
killall plasma_store
killall plasma_store_server
LaunchRedis() {
port=$1
@@ -37,11 +37,11 @@ sleep 1s
./src/common/thirdparty/redis/src/redis-cli set NumRedisShards 1
./src/common/thirdparty/redis/src/redis-cli rpush RedisShards 127.0.0.1:6380
sleep 1
./src/plasma/plasma_store -s /tmp/store1 -m 1000000000 &
./src/plasma/plasma_store_server -s /tmp/store1 -m 1000000000 &
plasma1_pid=$!
./src/plasma/plasma_manager -m /tmp/manager1 -s /tmp/store1 -h 127.0.0.1 -p 11111 -r 127.0.0.1:6379 &
plasma2_pid=$!
./src/plasma/plasma_store -s /tmp/store2 -m 1000000000 &
./src/plasma/plasma_store_server -s /tmp/store2 -m 1000000000 &
plasma3_pid=$!
./src/plasma/plasma_manager -m /tmp/manager2 -s /tmp/store2 -h 127.0.0.1 -p 22222 -r 127.0.0.1:6379 &
plasma4_pid=$!
src/plasma/test/run_valgrind.sh
+2 -2
View File
@@ -5,7 +5,7 @@ set -x
# Cause the script to exit if a single command fails.
set -e
./src/plasma/plasma_store -s /tmp/plasma_store_socket_1 -m 0 &
./src/plasma/plasma_store_server -s /tmp/plasma_store_socket_1 -m 0 &
sleep 1
valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1 ./src/plasma/manager_tests
killall plasma_store
killall plasma_store_server
src/ray/raylet/object_manager_integration_test.cc
+1 -1
View File
@@ -83,7 +83,7 @@ class TestObjectManagerBase : public ::testing::Test {
this->server1.reset();
this->server2.reset();
int s = system("killall plasma_store &");
int s = system("killall plasma_store_server &");
ASSERT_TRUE(!s);
std::string cmd_str = test_executable.substr(0, test_executable.find_last_of("/"));
src/ray/test/run_object_manager_tests.sh
+1 -1
View File
@@ -34,7 +34,7 @@ else
LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}"
fi
STORE_EXEC="$CORE_DIR/src/plasma/plasma_store"
STORE_EXEC="$CORE_DIR/src/plasma/plasma_store_server"
# Allow cleanup commands to fail.
$REDIS_DIR/redis-cli -p 6379 shutdown || true
src/ray/test/run_object_manager_valgrind.sh
+1 -1
View File
@@ -23,7 +23,7 @@ fi
CORE_DIR="$RAY_ROOT/build"
REDIS_DIR="$CORE_DIR/src/common/thirdparty/redis/src"
REDIS_MODULE="$CORE_DIR/src/common/redis_module/libray_redis_module.so"
STORE_EXEC="$CORE_DIR/src/plasma/plasma_store"
STORE_EXEC="$CORE_DIR/src/plasma/plasma_store_server"
VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1"
if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
src/ray/test/start_raylet.sh
+1 -1
View File
@@ -21,7 +21,7 @@ if [[ `stat $STORE_SOCKET_NAME` ]]; then
rm $STORE_SOCKET_NAME
fi
./src/plasma/plasma_store -m 1000000000 -s $STORE_SOCKET_NAME &
./src/plasma/plasma_store_server -m 1000000000 -s $STORE_SOCKET_NAME &
./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 &
echo
src/ray/test/start_raylets.sh
+1 -1
View File
@@ -43,7 +43,7 @@ for i in `seq 1 $NUM_RAYLETS`; do
rm $STORE_SOCKET_NAME
fi
./src/plasma/plasma_store -m 1000000000 -s $STORE_SOCKET_NAME &
./src/plasma/plasma_store_server -m 1000000000 -s $STORE_SOCKET_NAME &
./src/ray/raylet/raylet $RAYLET_SOCKET_NAME $STORE_SOCKET_NAME 127.0.0.1 127.0.0.1 6379 &
echo
thirdparty/scripts/build_arrow.sh
Vendored
+2 -2
View File
@@ -34,10 +34,10 @@ else
exit 1
fi
# The PR for this commit is https://github.com/apache/arrow/pull/2368. We
# The PR for this commit is https://github.com/apache/arrow/pull/2458. We
# include the link here to make it easier to find the right commit because
# Arrow often rewrites git history and invalidates certain commits.
TARGET_COMMIT_ID=4660833b2c5ef63a97445e304b8f72a2e0170f9c
TARGET_COMMIT_ID=fda4b3dcfc773612b12973df5053193f236fc696
build_arrow() {
echo "building arrow"
# Make sure arrow will be built again when building ray for java later than python