wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 17:01:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

Shellcheck quoting (#9596)

Browse Source 
* Fix SC2006: Use $(...) notation instead of legacy backticked `...`.

* Fix SC2016: Expressions don't expand in single quotes, use double quotes for that.

* Fix SC2046: Quote this to prevent word splitting.

* Fix SC2053: Quote the right-hand side of == in [[ ]] to prevent glob matching.

* Fix SC2068: Double quote array expansions to avoid re-splitting elements.

* Fix SC2086: Double quote to prevent globbing and word splitting.

* Fix SC2102: Ranges can only match single chars (mentioned due to duplicates).

* Fix SC2140: Word is of the form "A"B"C" (B indicated). Did you mean "ABC" or "A\"B\"C"?

* Fix SC2145: Argument mixes string and array. Use * or separate argument.

* Fix SC2209: warning: Use var=$(command) to assign output (or quote to assign string).

Co-authored-by: Mehrdad <noreply@github.com>
This commit is contained in:
mehrdadn
2020-07-21 19:56:41 -07:00
committed by GitHub
parent 382b314241
commit b14728d999
28 changed files with 166 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci/asan_tests/run_asan_tests.sh
+1 -1
View File
@@ -52,7 +52,7 @@ asan_recompile() {
} }
if [ 0 -lt "$#" ]; then if [ 0 -lt "$#" ]; then
asan_"$@" "asan_$1" "${@:2}"
else else
echo "Available commands: setup, run, recompile" echo "Available commands: setup, run, recompile"
fi fi
ci/jenkins_tests/entry_point.sh
+3 -3
View File
@@ -32,13 +32,13 @@ function retry {
} }
if [[ ! -z "$RUN_TUNE_TESTS" ]]; then if [[ ! -z "$RUN_TUNE_TESTS" ]]; then
retry bash $ROOT_DIR/run_tune_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} retry bash "$ROOT_DIR"/run_tune_tests.sh ${MEMORY_SIZE} ${SHM_SIZE}
fi fi
if [[ ! -z "$RUN_DOC_TESTS" ]]; then if [[ ! -z "$RUN_DOC_TESTS" ]]; then
retry bash $ROOT_DIR/run_doc_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} retry bash "$ROOT_DIR"/run_doc_tests.sh ${MEMORY_SIZE} ${SHM_SIZE}
fi fi
if [[ ! -z "$RUN_SGD_TESTS" ]]; then if [[ ! -z "$RUN_SGD_TESTS" ]]; then
retry bash $ROOT_DIR/run_sgd_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} retry bash "$ROOT_DIR"/run_sgd_tests.sh ${MEMORY_SIZE} ${SHM_SIZE}
fi fi
ci/jenkins_tests/run_doc_tests.sh
+7 -7
View File
@@ -25,7 +25,7 @@ if [ "$DOCKER_SHA" == "" ]; then
# Add Ray source # Add Ray source
git rev-parse HEAD > ./docker/tune_test/git-rev git rev-parse HEAD > ./docker/tune_test/git-rev
git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD) git archive -o ./docker/tune_test/ray.tar "$(git rev-parse HEAD)"
if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then
DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile) DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile)
else else
@@ -33,23 +33,23 @@ if [ "$DOCKER_SHA" == "" ]; then
fi fi
fi fi
echo "Using Docker image" $DOCKER_SHA echo "Using Docker image" "$DOCKER_SHA"
######################## EXAMPLE TESTS ################################# ######################## EXAMPLE TESTS #################################
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/doc/examples/plot_pong_example.py python /ray/doc/examples/plot_pong_example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/doc/examples/plot_parameter_server.py python /ray/doc/examples/plot_parameter_server.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/doc/examples/plot_hyperparameter.py python /ray/doc/examples/plot_hyperparameter.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/doc/examples/doc_code/torch_example.py python /ray/doc/examples/doc_code/torch_example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/doc/examples/doc_code/tf_example.py python /ray/doc/examples/doc_code/tf_example.py
ci/jenkins_tests/run_multi_node_tests.sh
+1 -1
View File
@@ -9,4 +9,4 @@ export RUN_DOC_TESTS=1
export RUN_SGD_TESTS=1 export RUN_SGD_TESTS=1
export CI_BUILD_FROM_SOURCE=1 export CI_BUILD_FROM_SOURCE=1
bash $ROOT_DIR/entry_point.sh bash "$ROOT_DIR"/entry_point.sh
ci/jenkins_tests/run_sgd_tests.sh
+20 -20
View File
@@ -25,7 +25,7 @@ if [ "$DOCKER_SHA" == "" ]; then
# Add Ray source # Add Ray source
git rev-parse HEAD > ./docker/tune_test/git-rev git rev-parse HEAD > ./docker/tune_test/git-rev
git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD) git archive -o ./docker/tune_test/ray.tar "$(git rev-parse HEAD)"
if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then
DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile) DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile)
else else
@@ -33,62 +33,62 @@ if [ "$DOCKER_SHA" == "" ]; then
fi fi
fi fi
echo "Using Docker image" $DOCKER_SHA echo "Using Docker image" "$DOCKER_SHA"
######################## SGD TESTS ################################# ######################## SGD TESTS #################################
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python -m pytest /ray/python/ray/util/sgd/tests python -m pytest /ray/python/ray/util/sgd/tests
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/raysgd_torch_signatures.py python /ray/python/ray/util/sgd/torch/examples/raysgd_torch_signatures.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/image_models/train.py --no-gpu --mock-data --smoke-test --ray-num-workers=2 --model mobilenetv3_small_075 data python /ray/python/ray/util/sgd/torch/examples/image_models/train.py --no-gpu --mock-data --smoke-test --ray-num-workers=2 --model mobilenetv3_small_075 data
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/train_example.py python /ray/python/ray/util/sgd/torch/examples/train_example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/train_example.py --num-workers=2 python /ray/python/ray/util/sgd/torch/examples/train_example.py --num-workers=2
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/tune_example.py python /ray/python/ray/util/sgd/torch/examples/tune_example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/tune_example.py --num-workers=2 python /ray/python/ray/util/sgd/torch/examples/tune_example.py --num-workers=2
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test --num-workers=2 python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test --num-workers=2
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_pbt.py --smoke-test python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_pbt.py --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/dcgan.py --smoke-test --num-workers=2 python /ray/python/ray/util/sgd/torch/examples/dcgan.py --smoke-test --num-workers=2
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/torch/examples/benchmarks/benchmark.py --smoke-test python /ray/python/ray/util/sgd/torch/examples/benchmarks/benchmark.py --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --num-replicas=2 python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --num-replicas=2
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --tune python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --tune
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --smoke-test python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test --augment-data python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test --augment-data
ci/jenkins_tests/run_tune_tests.sh
+39 -40
View File
@@ -25,8 +25,7 @@ if [ "$DOCKER_SHA" == "" ]; then
# Add Ray source # Add Ray source
git rev-parse HEAD > ./docker/tune_test/git-rev git rev-parse HEAD > ./docker/tune_test/git-rev
git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD) git archive -o ./docker/tune_test/ray.tar "$(git rev-parse HEAD)"
if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then
DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile) DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile)
else else
@@ -34,142 +33,142 @@ if [ "$DOCKER_SHA" == "" ]; then
fi fi
fi fi
echo "Using Docker image" $DOCKER_SHA echo "Using Docker image" "$DOCKER_SHA"
######################## TUNE TESTS ################################# ######################## TUNE TESTS #################################
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
pytest /ray/python/ray/tune/tests/test_actor_reuse.py pytest /ray/python/ray/tune/tests/test_actor_reuse.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
pytest /ray/python/ray/tune/tests/test_tune_restore.py pytest /ray/python/ray/tune/tests/test_tune_restore.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/tests/example.py python /ray/python/ray/tune/tests/example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
bash -c 'pip install -U tensorflow && python /ray/python/ray/tune/tests/test_logger.py' bash -c 'pip install -U tensorflow && python /ray/python/ray/tune/tests/test_logger.py'
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
bash -c 'pip install -U tensorflow==1.15 && python /ray/python/ray/tune/tests/test_logger.py' bash -c 'pip install -U tensorflow==1.15 && python /ray/python/ray/tune/tests/test_logger.py'
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
bash -c 'pip install -U tensorflow==1.14 && python /ray/python/ray/tune/tests/test_logger.py' bash -c 'pip install -U tensorflow==1.14 && python /ray/python/ray/tune/tests/test_logger.py'
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e MPLBACKEND=Agg $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e MPLBACKEND=Agg "$DOCKER_SHA" \
python /ray/python/ray/tune/tests/tutorial.py python /ray/python/ray/tune/tests/tutorial.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/pbt_example.py \ python /ray/python/ray/tune/examples/pbt_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/hyperband_example.py python /ray/python/ray/tune/examples/hyperband_example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/async_hyperband_example.py \ python /ray/python/ray/tune/examples/async_hyperband_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/tf_mnist_example.py --smoke-test python /ray/python/ray/tune/examples/tf_mnist_example.py --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/lightgbm_example.py python /ray/python/ray/tune/examples/lightgbm_example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/xgboost_example.py python /ray/python/ray/tune/examples/xgboost_example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/logging_example.py \ python /ray/python/ray/tune/examples/logging_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/mlflow_example.py python /ray/python/ray/tune/examples/mlflow_example.py
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/bayesopt_example.py \ python /ray/python/ray/tune/examples/bayesopt_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/cifar10_pytorch.py --smoke-test python /ray/python/ray/tune/examples/cifar10_pytorch.py --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/hyperopt_example.py \ python /ray/python/ray/tune/examples/hyperopt_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/doc/source/tune/_tutorials/tune-sklearn.py python /ray/doc/source/tune/_tutorials/tune-sklearn.py
# if [[ ! -z "$SIGOPT_KEY" ]]; then # if [[ ! -z "$SIGOPT_KEY" ]]; then
# $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e SIGOPT_KEY $DOCKER_SHA \ # $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e SIGOPT_KEY "$DOCKER_SHA" \
# python /ray/python/ray/tune/examples/sigopt_example.py \ # python /ray/python/ray/tune/examples/sigopt_example.py \
# --smoke-test # --smoke-test
# fi # fi
# Runs only on Python3 # Runs only on Python3
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/nevergrad_example.py \ python /ray/python/ray/tune/examples/nevergrad_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/tune_mnist_keras.py \ python /ray/python/ray/tune/examples/tune_mnist_keras.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/mnist_pytorch.py --smoke-test python /ray/python/ray/tune/examples/mnist_pytorch.py --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/mnist_pytorch_lightning.py --smoke-test python /ray/python/ray/tune/examples/mnist_pytorch_lightning.py --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/mnist_pytorch_trainable.py \ python /ray/python/ray/tune/examples/mnist_pytorch_trainable.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/genetic_example.py \ python /ray/python/ray/tune/examples/genetic_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/skopt_example.py \ python /ray/python/ray/tune/examples/skopt_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/dragonfly_example.py \ python /ray/python/ray/tune/examples/dragonfly_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/zoopt_example.py \ python /ray/python/ray/tune/examples/zoopt_example.py \
--smoke-test --smoke-test
# Commenting out because flaky # Commenting out because flaky
# $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ # $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
# python /ray/python/ray/tune/examples/pbt_memnn_example.py \ # python /ray/python/ray/tune/examples/pbt_memnn_example.py \
# --smoke-test # --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/pbt_convnet_example.py \ python /ray/python/ray/tune/examples/pbt_convnet_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/hyperband_function_example.py \ python /ray/python/ray/tune/examples/hyperband_function_example.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/pbt_function.py \ python /ray/python/ray/tune/examples/pbt_function.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/pbt_dcgan_mnist/pbt_dcgan_mnist.py \ python /ray/python/ray/tune/examples/pbt_dcgan_mnist/pbt_dcgan_mnist.py \
--smoke-test --smoke-test
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/ci/long_running_distributed_tests/workloads/pytorch_pbt_failure.py \ python /ray/ci/long_running_distributed_tests/workloads/pytorch_pbt_failure.py \
--smoke-test --smoke-test
# uncomment once statsmodels is updated. # uncomment once statsmodels is updated.
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
python /ray/python/ray/tune/examples/bohb_example.py python /ray/python/ray/tune/examples/bohb_example.py
# Moved to bottom because flaky # Moved to bottom because flaky
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \
pytest /ray/python/ray/tune/tests/test_cluster.py pytest /ray/python/ray/tune/tests/test_cluster.py
ci/keep_alive
+1 -1
View File
@@ -5,7 +5,7 @@ PID=$$
# Print output to avoid travis killing us # Print output to avoid travis killing us
watchdog() { watchdog() {
for i in `seq 2 2 150`; do for i in $(seq 2 2 150); do
sleep 120 sleep 120
echo "(running, ${i}m total)" echo "(running, ${i}m total)"
done done
ci/suppress_output
+5 -5
View File
@@ -1,17 +1,17 @@
#!/bin/bash #!/bin/bash
# Run a command, suppressing output unless it hangs or crashes. # Run a command, suppressing output unless it hangs or crashes.
TMPFILE=`mktemp` TMPFILE="$(mktemp)"
PID=$$ PID=$$
# Print output to avoid travis killing us # Print output to avoid travis killing us
watchdog() { watchdog() {
for i in `seq 5 5 150`; do for i in $(seq 5 5 150); do
sleep 300 sleep 300
echo "This command has been running for more than $i minutes..." echo "This command has been running for more than $i minutes..."
done done
echo "Command timed out after 2.5h, dumping logs:" echo "Command timed out after 2.5h, dumping logs:"
cat $TMPFILE cat "$TMPFILE"
echo "TIMED OUT" echo "TIMED OUT"
kill -SIGKILL $PID kill -SIGKILL $PID
} }
@@ -19,11 +19,11 @@ watchdog() {
watchdog & 2>/dev/null watchdog & 2>/dev/null
WATCHDOG_PID=$! WATCHDOG_PID=$!
time "$@" >$TMPFILE 2>&1 time "$@" >"$TMPFILE" 2>&1
CODE=$? CODE=$?
if [ $CODE != 0 ]; then if [ $CODE != 0 ]; then
tail -n 2000 $TMPFILE tail -n 2000 "$TMPFILE"
echo "FAILED $CODE" echo "FAILED $CODE"
kill $WATCHDOG_PID kill $WATCHDOG_PID
exit $CODE exit $CODE
ci/travis/bazel-format.sh
+6 -6
View File
@@ -17,7 +17,7 @@ function usage()
echo echo
} }
RUN_TYPE=diff RUN_TYPE="diff"
# Parse options # Parse options
while [[ $# > 0 ]]; do while [[ $# > 0 ]]; do
@@ -28,7 +28,7 @@ while [[ $# > 0 ]]; do
exit 0 exit 0
;; ;;
-c|--check) -c|--check)
RUN_TYPE=diff RUN_TYPE="diff"
;; ;;
-f|--fix) -f|--fix)
RUN_TYPE=fix RUN_TYPE=fix
@@ -43,8 +43,8 @@ while [[ $# > 0 ]]; do
shift shift
done done
pushd $ROOT_DIR/../.. pushd "$ROOT_DIR"/../..
BAZEL_FILES="bazel/BUILD bazel/BUILD.arrow bazel/ray.bzl BUILD.bazel java/BUILD.bazel BAZEL_FILES=(bazel/BUILD bazel/BUILD.arrow bazel/ray.bzl BUILD.bazel java/BUILD.bazel \
cpp/BUILD.bazel streaming/BUILD.bazel streaming/java/BUILD.bazel WORKSPACE" cpp/BUILD.bazel streaming/BUILD.bazel streaming/java/BUILD.bazel WORKSPACE)
buildifier -mode=$RUN_TYPE -diff_command="diff -u" $BAZEL_FILES buildifier -mode=$RUN_TYPE -diff_command="diff -u" "${BAZEL_FILES[@]}"
popd popd
ci/travis/build-autoscaler-images.sh
+15 -15
View File
@@ -6,7 +6,7 @@ set -e
set -x set -x
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
ROOT_DIR=$(cd $SCRIPT_DIR/../../; pwd) ROOT_DIR=$(cd "$SCRIPT_DIR"/../../; pwd)
DOCKER_USERNAME="raytravisbot" DOCKER_USERNAME="raytravisbot"
# We will only build and push when we are building branch build. # We will only build and push when we are building branch build.
@@ -14,33 +14,33 @@ if [[ "$TRAVIS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
wheel=$(cd $ROOT_DIR/.whl; ls | grep cp36m-manylinux) wheel=$(cd $ROOT_DIR/.whl; ls | grep cp36m-manylinux)
commit_sha=$(echo $TRAVIS_COMMIT | head -c 6) commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6)
cp -r $ROOT_DIR/.whl $ROOT_DIR/docker/autoscaler/.whl cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/autoscaler/.whl
docker build -q -t rayproject/base-deps docker/base-deps docker build -q -t rayproject/base-deps docker/base-deps
docker build \ docker build \
--build-arg WHEEL_PATH=".whl/$wheel" \ --build-arg WHEEL_PATH=".whl/$wheel" \
--build-arg WHEEL_NAME=$wheel \ --build-arg WHEEL_NAME=$wheel \
-t rayproject/autoscaler:$commit_sha \ -t rayproject/autoscaler:"$commit_sha" \
$ROOT_DIR/docker/autoscaler "$ROOT_DIR"/docker/autoscaler
docker tag rayproject/base-deps rayproject/base-deps:$commit_sha docker tag rayproject/base-deps rayproject/base-deps:"$commit_sha"
docker push rayproject/base-deps:$commit_sha docker push rayproject/base-deps:"$commit_sha"
docker push rayproject/autoscaler:$commit_sha docker push rayproject/autoscaler:"$commit_sha"
# We have a branch build, e.g. release/v0.7.0 # We have a branch build, e.g. release/v0.7.0
if [[ "$TRAVIS_BRANCH" != "master" ]]; then if [[ "$TRAVIS_BRANCH" != "master" ]]; then
# Replace / in branch name to - so it is legal tag name # Replace / in branch name to - so it is legal tag name
normalized_branch_name=$(echo $TRAVIS_BRANCH | sed -e "s/\//-/") normalized_branch_name=$(echo "$TRAVIS_BRANCH" | sed -e "s/\//-/")
docker tag rayproject/autoscaler:$commit_sha rayproject/autoscaler:$normalized_branch_name docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:"$normalized_branch_name"
docker tag rayproject/base-deps:$commit_sha rayproject/base-deps:$normalized_branch_name docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:"$normalized_branch_name"
docker push rayproject/autoscaler:$normalized_branch_name docker push rayproject/autoscaler:"$normalized_branch_name"
docker push rayproject/base-deps:$normalized_branch_name docker push rayproject/base-deps:"$normalized_branch_name"
else else
docker tag rayproject/autoscaler:$commit_sha rayproject/autoscaler:latest docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:latest
docker tag rayproject/base-deps:$commit_sha rayproject/base-deps:latest docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:latest
docker push rayproject/autoscaler:latest docker push rayproject/autoscaler:latest
docker push rayproject/base-deps:latest docker push rayproject/base-deps:latest
fi fi
ci/travis/ci.sh
+6 -3
View File
@@ -312,7 +312,7 @@ lint_bazel() {
# Run buildifier without affecting external environment variables # Run buildifier without affecting external environment variables
( (
mkdir -p -- "${GOPATH}" mkdir -p -- "${GOPATH}"
export PATH="${GOPATH}/bin":"${GOROOT}/bin":"${PATH}" export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}"
# Build buildifier # Build buildifier
go get github.com/bazelbuild/buildtools/buildifier go get github.com/bazelbuild/buildtools/buildifier
@@ -329,8 +329,11 @@ lint_web() {
. "${HOME}/.nvm/nvm.sh" . "${HOME}/.nvm/nvm.sh"
install_npm_project install_npm_project
nvm use --silent node nvm use --silent node
node_modules/.bin/eslint --max-warnings 0 $(find src -name "*.ts" -or -name "*.tsx") local filenames
node_modules/.bin/prettier --check $(find src -name "*.ts" -or -name "*.tsx") # shellcheck disable=SC2207
filenames=($(find src -name "*.ts" -or -name "*.tsx"))
node_modules/.bin/eslint --max-warnings 0 "${filenames[@]}"
node_modules/.bin/prettier --check "${filenames[@]}"
node_modules/.bin/prettier --check public/index.html node_modules/.bin/prettier --check public/index.html
) )
} }
ci/travis/format.sh
+4 -4
View File
@@ -21,7 +21,7 @@ check_command_exist() {
echo "$1 is not a required dependency" echo "$1 is not a required dependency"
exit 1 exit 1
esac esac
if ! [ -x "$(command -v $1)" ]; then if ! [ -x "$(command -v "$1")" ]; then
echo "$1 not installed. pip install $1==$VERSION" echo "$1 not installed. pip install $1==$VERSION"
exit 1 exit 1
fi fi
@@ -31,7 +31,7 @@ check_command_exist yapf
check_command_exist flake8 check_command_exist flake8
ver=$(yapf --version) ver=$(yapf --version)
if ! echo $ver | grep -q 0.23.0; then if ! echo "$ver" | grep -q 0.23.0; then
echo "Wrong YAPF version installed: 0.23.0 is required, not $ver. $YAPF_DOWNLOAD_COMMAND_MSG" echo "Wrong YAPF version installed: 0.23.0 is required, not $ver. $YAPF_DOWNLOAD_COMMAND_MSG"
exit 1 exit 1
fi fi
@@ -52,7 +52,7 @@ YAPF_VERSION=$(yapf --version | awk '{print $2}')
# params: tool name, tool version, required version # params: tool name, tool version, required version
tool_version_check() { tool_version_check() {
if [[ $2 != $3 ]]; then if [ "$2" != "$3" ]; then
echo "WARNING: Ray uses $1 $3, You currently are using $2. This might generate different results." echo "WARNING: Ray uses $1 $3, You currently are using $2. This might generate different results."
fi fi
} }
@@ -62,7 +62,7 @@ tool_version_check "yapf" $YAPF_VERSION $YAPF_VERSION_REQUIRED
if which clang-format >/dev/null; then if which clang-format >/dev/null; then
CLANG_FORMAT_VERSION=$(clang-format --version | awk '{print $3}') CLANG_FORMAT_VERSION=$(clang-format --version | awk '{print $3}')
tool_version_check "clang-format" $CLANG_FORMAT_VERSION "7.0.0" tool_version_check "clang-format" "$CLANG_FORMAT_VERSION" "7.0.0"
else else
echo "WARNING: clang-format is not installed!" echo "WARNING: clang-format is not installed!"
fi fi
ci/travis/install-bazel.sh
+1 -1
View File
@@ -38,7 +38,7 @@ esac
fi fi
done done
if [ ! 0 -eq "${#missing_symlinks[@]}" ]; then if [ ! 0 -eq "${#missing_symlinks[@]}" ]; then
echo "error: expected symlink: ${missing_symlinks[@]}" 1>&2 echo "error: expected symlink: ${missing_symlinks[*]}" 1>&2
echo "For a correct build, please run 'git config --local core.symlinks true' and re-run git checkout." 1>&2 echo "For a correct build, please run 'git config --local core.symlinks true' and re-run git checkout." 1>&2
false false
fi fi
ci/travis/install-dependencies.sh
+2 -2
View File
@@ -159,7 +159,7 @@ install_nvm() {
) )
printf "%s\n" \ printf "%s\n" \
"export NVM_HOME=\"$(cygpath -w -- "${NVM_HOME}")\"" \ "export NVM_HOME=\"$(cygpath -w -- "${NVM_HOME}")\"" \
'nvm() { "${NVM_HOME}/nvm.exe" "$@"; }' \ "nvm() { \"\${NVM_HOME}/nvm.exe\" \"\$@\"; }" \
> "${NVM_HOME}/nvm.sh" > "${NVM_HOME}/nvm.sh"
fi fi
else else
@@ -277,7 +277,7 @@ install_dependencies() {
# Additional RLlib dependencies. # Additional RLlib dependencies.
if [ "${RLLIB_TESTING-}" = 1 ]; then if [ "${RLLIB_TESTING-}" = 1 ]; then
pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 \ pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 \
torch=="${TORCH_VERSION-1.4}" torchvision atari_py gym[atari] lz4 smart_open torch=="${TORCH_VERSION-1.4}" torchvision atari_py "gym[atari]" lz4 smart_open
fi fi
# Additional streaming dependencies. # Additional streaming dependencies.
doc/azure/azure-init.sh
+8 -8
View File
@@ -7,11 +7,11 @@ RAY_HEAD_IP=$4
TYPE=$5 TYPE=$5
echo "Installing wheel..." echo "Installing wheel..."
sudo -u $USERNAME -i /bin/bash -l -c "conda init bash" sudo -u "$USERNAME" -i /bin/bash -l -c "conda init bash"
sudo -u $USERNAME -i /bin/bash -l -c "conda activate $CONDA_ENV; pip install $WHEEL" sudo -u "$USERNAME" -i /bin/bash -l -c "conda activate $CONDA_ENV; pip install $WHEEL"
echo "Setting up service scripts..." echo "Setting up service scripts..."
cat > /home/$USERNAME/ray-head.sh << EOM cat > /home/"$USERNAME"/ray-head.sh << EOM
#!/bin/bash #!/bin/bash
conda activate $CONDA_ENV conda activate $CONDA_ENV
@@ -22,7 +22,7 @@ ulimit -n 65536
ray start --head --redis-port=6379 --object-manager-port=8076 --num-gpus=\$NUM_GPUS --block --webui-host 0.0.0.0 ray start --head --redis-port=6379 --object-manager-port=8076 --num-gpus=\$NUM_GPUS --block --webui-host 0.0.0.0
EOM EOM
cat > /home/$USERNAME/ray-worker.sh << EOM cat > /home/"$USERNAME"/ray-worker.sh << EOM
#!/bin/bash #!/bin/bash
conda activate $CONDA_ENV conda activate $CONDA_ENV
@@ -39,7 +39,7 @@ do
done done
EOM EOM
cat > /home/$USERNAME/tensorboard.sh << EOM cat > /home/"$USERNAME"/tensorboard.sh << EOM
#!/bin/bash #!/bin/bash
conda activate $CONDA_ENV conda activate $CONDA_ENV
@@ -48,9 +48,9 @@ mkdir -p /home/$USERNAME/ray_results
tensorboard --bind_all --logdir=/home/$USERNAME/ray_results tensorboard --bind_all --logdir=/home/$USERNAME/ray_results
EOM EOM
chmod +x /home/$USERNAME/ray-head.sh chmod +x /home/"$USERNAME"/ray-head.sh
chmod +x /home/$USERNAME/ray-worker.sh chmod +x /home/"$USERNAME"/ray-worker.sh
chmod +x /home/$USERNAME/tensorboard.sh chmod +x /home/"$USERNAME"/tensorboard.sh
cat > /lib/systemd/system/ray.service << EOM cat > /lib/systemd/system/ray.service << EOM
[Unit] [Unit]
doc/dev/bin/pip_download_test.sh
+1 -1
View File
@@ -54,7 +54,7 @@ do
echo "=========================================================\n\n\n" echo "=========================================================\n\n\n"
conda deactivate conda deactivate
conda remove -y --name ${env_name} --all conda remove -y --name "${env_name}" --all
if [ "$failed" = true ]; then if [ "$failed" = true ]; then
echo "PYTHON ${PYTHON_VERSION} failed sanity check." echo "PYTHON ${PYTHON_VERSION} failed sanity check."
exit 1 exit 1
doc/dev/download_wheels.sh
+8 -8
View File
@@ -11,13 +11,13 @@ if [[ -z "$RAY_VERSION" ]]; then
fi fi
# Make sure Linux wheels are downloadable without errors. # Make sure Linux wheels are downloadable without errors.
wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-manylinux1_x86_64.whl wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-manylinux1_x86_64.whl"
wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl"
wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl"
wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl"
# Make sure macos wheels are downloadable without errors. # Make sure macos wheels are downloadable without errors.
wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-macosx_10_13_intel.whl wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-macosx_10_13_intel.whl"
wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl"
wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl"
# Wheel name convention has been changed from Python 3.8. # Wheel name convention has been changed from Python 3.8.
wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-macosx_10_13_x86_64.whl wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-macosx_10_13_x86_64.whl"
doc/examples/lm/ray_train.sh
+3 -3
View File
@@ -11,9 +11,9 @@ SAVE_INTERVAL_UPDATES=1000 # save a checkpoint every N updates
LOG_DIR=$HOME/efs/lm/log/ LOG_DIR=$HOME/efs/lm/log/
DATA_DIR=$HOME/efs/lm/data-bin/wikitext-103/ DATA_DIR=$HOME/efs/lm/data-bin/wikitext-103/
mkdir -p $LOG_DIR mkdir -p "$LOG_DIR"
python $HOME/efs/lm/ray_train.py --fp16 $DATA_DIR \ python "$HOME"/efs/lm/ray_train.py --fp16 "$DATA_DIR" \
--task masked_lm --criterion masked_lm \ --task masked_lm --criterion masked_lm \
--arch roberta_base --sample-break-mode complete --tokens-per-sample $TOKENS_PER_SAMPLE \ --arch roberta_base --sample-break-mode complete --tokens-per-sample $TOKENS_PER_SAMPLE \
--optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-6 --clip-norm 0.0 \ --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-6 --clip-norm 0.0 \
@@ -23,4 +23,4 @@ python $HOME/efs/lm/ray_train.py --fp16 $DATA_DIR \
--fix-batch-size $FIX_BATCH_SIZE \ --fix-batch-size $FIX_BATCH_SIZE \
--max-update $TOTAL_UPDATES --log-format simple --log-interval 1 \ --max-update $TOTAL_UPDATES --log-format simple --log-interval 1 \
--save-interval-updates $SAVE_INTERVAL_UPDATES \ --save-interval-updates $SAVE_INTERVAL_UPDATES \
--save-dir $LOG_DIR --ddp-backend=no_c10d --save-dir "$LOG_DIR" --ddp-backend=no_c10d
doc/tools/install-prometheus-server.sh
+1 -1
View File
@@ -5,7 +5,7 @@ set -e
TOOLS_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) TOOLS_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
pushd $TOOLS_DIR pushd "$TOOLS_DIR"
# Download Prometheus server. # Download Prometheus server.
unamestr="$(uname)" unamestr="$(uname)"
java/generate_jni_header_files.sh
+5 -5
View File
@@ -10,10 +10,10 @@ cd "$(dirname "$0")"
function generate_one() function generate_one()
{ {
file=${1//./_}.h file=${1//./_}.h
javah -classpath ../bazel-bin/java/all_tests_deploy.jar $1 javah -classpath ../bazel-bin/java/all_tests_deploy.jar "$1"
clang-format -i $file clang-format -i "$file"
cat <<EOF > ../src/ray/core_worker/lib/java/$file cat <<EOF > ../src/ray/core_worker/lib/java/"$file"
// Copyright 2017 The Ray Authors. // Copyright 2017 The Ray Authors.
// //
// Licensed under the Apache License, Version 2.0 (the "License"); // Licensed under the Apache License, Version 2.0 (the "License");
@@ -29,8 +29,8 @@ function generate_one()
// limitations under the License. // limitations under the License.
EOF EOF
cat $file >> ../src/ray/core_worker/lib/java/$file cat "$file" >> ../src/ray/core_worker/lib/java/"$file"
rm -f $file rm -f "$file"
} }
generate_one io.ray.runtime.RayNativeRuntime generate_one io.ray.runtime.RayNativeRuntime
java/test.sh
+4 -4
View File
@@ -20,7 +20,7 @@ run_testng() {
fi fi
} }
pushd $ROOT_DIR/.. pushd "$ROOT_DIR"/..
echo "Linting Java code with checkstyle." echo "Linting Java code with checkstyle."
# NOTE(hchen): The `test_tag_filters` option causes bazel to ignore caches. # NOTE(hchen): The `test_tag_filters` option causes bazel to ignore caches.
# Thus, we add the `build_tests_only` option to avoid re-building everything. # Thus, we add the `build_tests_only` option to avoid re-building everything.
@@ -36,15 +36,15 @@ echo "Running tests under cluster mode."
# TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests, # TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests,
# TestNG will exit with code 2. And bazel treats it as test failure. # TestNG will exit with code 2. And bazel treats it as test failure.
# bazel test //java:all_tests --action_env=ENABLE_MULTI_LANGUAGE_TESTS=1 --test_output="errors" || cluster_exit_code=$? # bazel test //java:all_tests --action_env=ENABLE_MULTI_LANGUAGE_TESTS=1 --test_output="errors" || cluster_exit_code=$?
ENABLE_MULTI_LANGUAGE_TESTS=1 run_testng java -cp $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output $ROOT_DIR/testng.xml ENABLE_MULTI_LANGUAGE_TESTS=1 run_testng java -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml
echo "Running tests under single-process mode." echo "Running tests under single-process mode."
# bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --test_output="errors" || single_exit_code=$? # bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --test_output="errors" || single_exit_code=$?
run_testng java -Dray.run-mode="SINGLE_PROCESS" -cp $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output $ROOT_DIR/testng.xml run_testng java -Dray.run-mode="SINGLE_PROCESS" -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml
popd popd
pushd $ROOT_DIR pushd "$ROOT_DIR"
echo "Testing maven install." echo "Testing maven install."
mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests
# Ensure mvn test works # Ensure mvn test works
python/build-wheel-macos.sh
+5 -5
View File
@@ -39,7 +39,7 @@ mkdir -p $DOWNLOAD_DIR
mkdir -p .whl mkdir -p .whl
# Use the latest version of Node.js in order to build the dashboard. # Use the latest version of Node.js in order to build the dashboard.
source $HOME/.nvm/nvm.sh source "$HOME"/.nvm/nvm.sh
nvm use node nvm use node
# Build the dashboard so its static assets can be included in the wheel. # Build the dashboard so its static assets can be included in the wheel.
@@ -61,11 +61,11 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
# Install Python. # Install Python.
INST_PATH=python_downloads/$PY_INST INST_PATH=python_downloads/$PY_INST
curl $MACPYTHON_URL/$PY_VERSION/$PY_INST > $INST_PATH curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH"
sudo installer -pkg $INST_PATH -target / sudo installer -pkg "$INST_PATH" -target /
PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM
PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM" PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM"
pushd /tmp pushd /tmp
# Install latest version of pip to avoid brownouts. # Install latest version of pip to avoid brownouts.
@@ -80,7 +80,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
$PIP_CMD install -q setuptools_scm==3.1.0 $PIP_CMD install -q setuptools_scm==3.1.0
# Fix the numpy version because this will be the oldest numpy version we can # Fix the numpy version because this will be the oldest numpy version we can
# support. # support.
$PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.15 $PIP_CMD install -q numpy=="$NUMPY_VERSION" cython==0.29.15
# Install wheel to avoid the error "invalid command 'bdist_wheel'". # Install wheel to avoid the error "invalid command 'bdist_wheel'".
$PIP_CMD install -q wheel $PIP_CMD install -q wheel
# Set the commit SHA in __init__.py. # Set the commit SHA in __init__.py.
python/build-wheel-manylinux1.sh
+3 -3
View File
@@ -32,7 +32,7 @@ export PATH=$PATH:/root/bin
# Install and use the latest version of Node.js in order to build the dashboard. # Install and use the latest version of Node.js in order to build the dashboard.
set +x set +x
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
source $HOME/.nvm/nvm.sh source "$HOME"/.nvm/nvm.sh
nvm install node nvm install node
nvm use node nvm use node
@@ -57,7 +57,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do
pushd python pushd python
# Fix the numpy version because this will be the oldest numpy version we can # Fix the numpy version because this will be the oldest numpy version we can
# support. # support.
/opt/python/${PYTHON}/bin/pip install -q numpy==${NUMPY_VERSION} cython==0.29.15 /opt/python/"${PYTHON}"/bin/pip install -q numpy=="${NUMPY_VERSION}" cython==0.29.15
# Set the commit SHA in __init__.py. # Set the commit SHA in __init__.py.
if [ -n "$TRAVIS_COMMIT" ]; then if [ -n "$TRAVIS_COMMIT" ]; then
sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak
@@ -66,7 +66,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do
exit 1 exit 1
fi fi
PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/"${PYTHON}"/bin/python setup.py bdist_wheel
# In the future, run auditwheel here. # In the future, run auditwheel here.
mv dist/*.whl ../.whl/ mv dist/*.whl ../.whl/
popd popd
python/ray/autoscaler/kubernetes/kubectl-rsync.sh
+2 -2
View File
@@ -6,7 +6,7 @@
if [ -z "$KRSYNC_STARTED" ]; then if [ -z "$KRSYNC_STARTED" ]; then
export KRSYNC_STARTED=true export KRSYNC_STARTED=true
exec rsync --blocking-io --rsh "$0" $@ exec rsync --blocking-io --rsh "$0" "$@"
fi fi
# Running as --rsh # Running as --rsh
@@ -22,4 +22,4 @@ if [ "X$pod" = "X-l" ]; then
shift shift
fi fi
exec kubectl $namespace exec -i $pod -- "$@" exec kubectl "$namespace" exec -i "$pod" -- "$@"
setup_hooks.sh
+2 -2
View File
@@ -1,3 +1,3 @@
#!/bin/bash #!/bin/bash
chmod +x $PWD/scripts/pre-push chmod +x "$PWD"/scripts/pre-push
ln -s $PWD/scripts/pre-push $PWD/.git/hooks/pre-push ln -s "$PWD"/scripts/pre-push "$PWD"/.git/hooks/pre-push
src/ray/test/run_object_manager_tests.sh
+4 -4
View File
@@ -9,10 +9,10 @@ set -x
bazel build "//:object_manager_stress_test" "//:object_manager_test" "//:plasma_store_server" bazel build "//:object_manager_stress_test" "//:object_manager_test" "//:plasma_store_server"
# Get the directory in which this script is executing. # Get the directory in which this script is executing.
SCRIPT_DIR="`dirname \"$0\"`" SCRIPT_DIR="$(dirname "$0")"
RAY_ROOT="$SCRIPT_DIR/../../.." RAY_ROOT="$SCRIPT_DIR/../../.."
# Makes $RAY_ROOT an absolute path. # Makes $RAY_ROOT an absolute path.
RAY_ROOT="`( cd \"$RAY_ROOT\" && pwd )`" RAY_ROOT="$(cd "$RAY_ROOT" && pwd)"
if [ -z "$RAY_ROOT" ] ; then if [ -z "$RAY_ROOT" ] ; then
exit 1 exit 1
fi fi
@@ -23,13 +23,13 @@ if [ ! -d "$RAY_ROOT/python" ]; then
fi fi
REDIS_MODULE="./bazel-bin/libray_redis_module.so" REDIS_MODULE="./bazel-bin/libray_redis_module.so"
LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" LOAD_MODULE_ARGS=(--loadmodule "${REDIS_MODULE}")
STORE_EXEC="./bazel-bin/plasma_store_server" STORE_EXEC="./bazel-bin/plasma_store_server"
# Allow cleanup commands to fail. # Allow cleanup commands to fail.
bazel run //:redis-cli -- -p 6379 shutdown || true bazel run //:redis-cli -- -p 6379 shutdown || true
sleep 1s sleep 1s
bazel run //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & bazel run //:redis-server -- --loglevel warning "${LOAD_MODULE_ARGS[@]}" --port 6379 &
sleep 1s sleep 1s
# Run tests. # Run tests.
./bazel-bin/object_manager_stress_test $STORE_EXEC ./bazel-bin/object_manager_stress_test $STORE_EXEC
src/ray/test/run_object_manager_valgrind.sh
+4 -4
View File
@@ -9,10 +9,10 @@ set -x
bazel build "//:object_manager_stress_test" "//:object_manager_test" "//:plasma_store_server" bazel build "//:object_manager_stress_test" "//:object_manager_test" "//:plasma_store_server"
# Get the directory in which this script is executing. # Get the directory in which this script is executing.
SCRIPT_DIR="`dirname \"$0\"`" SCRIPT_DIR="$(dirname "$0")"
RAY_ROOT="$SCRIPT_DIR/../../.." RAY_ROOT="$SCRIPT_DIR/../../.."
# Makes $RAY_ROOT an absolute path. # Makes $RAY_ROOT an absolute path.
RAY_ROOT="`( cd \"$RAY_ROOT\" && pwd )`" RAY_ROOT="$(cd "$RAY_ROOT" && pwd)"
if [ -z "$RAY_ROOT" ] ; then if [ -z "$RAY_ROOT" ] ; then
exit 1 exit 1
fi fi
@@ -23,7 +23,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then
fi fi
REDIS_MODULE="./bazel-bin/libray_redis_module.so" REDIS_MODULE="./bazel-bin/libray_redis_module.so"
LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" LOAD_MODULE_ARGS=(--loadmodule "${REDIS_MODULE}")
STORE_EXEC="./bazel-bin/plasma_store_server" STORE_EXEC="./bazel-bin/plasma_store_server"
VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1" VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1"
@@ -32,7 +32,7 @@ VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=a
killall plasma_store || true killall plasma_store || true
bazel run //:redis-cli -- -p 6379 shutdown || true bazel run //:redis-cli -- -p 6379 shutdown || true
sleep 1s sleep 1s
bazel run //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & bazel run //:redis-server -- --loglevel warning "${LOAD_MODULE_ARGS[@]}" --port 6379 &
sleep 1s sleep 1s
# Run tests. Use timeout=10000ms for the Wait tests since tests run slower # Run tests. Use timeout=10000ms for the Wait tests since tests run slower
streaming/src/test/run_streaming_queue_test.sh
+5 -6
View File
@@ -7,8 +7,7 @@
PORTS="2000 2001 2002 2003 2004 2005 2006 2007 2008 2009" PORTS="2000 2001 2002 2003 2004 2005 2006 2007 2008 2009"
RAYLET_PORT=0 RAYLET_PORT=0
for port in $PORTS; do for port in $PORTS; do
nc -z localhost $port if ! nc -z localhost "$port"; then
if [[ $? != 0 ]]; then
RAYLET_PORT=$port RAYLET_PORT=$port
break break
fi fi
@@ -24,13 +23,13 @@ set -e
set -x set -x
# Get the directory in which this script is executing. # Get the directory in which this script is executing.
SCRIPT_DIR="`dirname \"$0\"`" SCRIPT_DIR="$(dirname "$0")"
# Get the directory in which this script is executing. # Get the directory in which this script is executing.
SCRIPT_DIR="`dirname \"$0\"`" SCRIPT_DIR="$(dirname "$0")"
RAY_ROOT="$SCRIPT_DIR/../../.." RAY_ROOT="$SCRIPT_DIR/../../.."
# Makes $RAY_ROOT an absolute path. # Makes $RAY_ROOT an absolute path.
RAY_ROOT="`( cd \"$RAY_ROOT\" && pwd )`" RAY_ROOT="$(cd "$RAY_ROOT" && pwd)"
if [ -z "$RAY_ROOT" ] ; then if [ -z "$RAY_ROOT" ] ; then
exit 1 exit 1
fi fi
@@ -55,5 +54,5 @@ GCS_SERVER_EXEC="./bazel-bin/gcs_server"
# Allow cleanup commands to fail. # Allow cleanup commands to fail.
# Run tests. # Run tests.
./bazel-bin/streaming/streaming_queue_tests $STORE_EXEC $RAYLET_EXEC $RAYLET_PORT $STREAMING_TEST_WORKER_EXEC $GCS_SERVER_EXEC $REDIS_SERVER_EXEC $REDIS_MODULE $REDIS_CLIENT_EXEC ./bazel-bin/streaming/streaming_queue_tests $STORE_EXEC $RAYLET_EXEC "$RAYLET_PORT" $STREAMING_TEST_WORKER_EXEC $GCS_SERVER_EXEC $REDIS_SERVER_EXEC $REDIS_MODULE $REDIS_CLIENT_EXEC
sleep 1s sleep 1s