From b14728d999003dd206450ce17895ad0f65549893 Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Tue, 21 Jul 2020 19:56:41 -0700 Subject: [PATCH] Shellcheck quoting (#9596) * Fix SC2006: Use $(...) notation instead of legacy backticked `...`. * Fix SC2016: Expressions don't expand in single quotes, use double quotes for that. * Fix SC2046: Quote this to prevent word splitting. * Fix SC2053: Quote the right-hand side of == in [[ ]] to prevent glob matching. * Fix SC2068: Double quote array expansions to avoid re-splitting elements. * Fix SC2086: Double quote to prevent globbing and word splitting. * Fix SC2102: Ranges can only match single chars (mentioned due to duplicates). * Fix SC2140: Word is of the form "A"B"C" (B indicated). Did you mean "ABC" or "A\"B\"C"? * Fix SC2145: Argument mixes string and array. Use * or separate argument. * Fix SC2209: warning: Use var=$(command) to assign output (or quote to assign string). Co-authored-by: Mehrdad --- ci/asan_tests/run_asan_tests.sh | 2 +- ci/jenkins_tests/entry_point.sh | 6 +- ci/jenkins_tests/run_doc_tests.sh | 14 ++-- ci/jenkins_tests/run_multi_node_tests.sh | 2 +- ci/jenkins_tests/run_sgd_tests.sh | 40 +++++----- ci/jenkins_tests/run_tune_tests.sh | 79 +++++++++---------- ci/keep_alive | 2 +- ci/suppress_output | 10 +-- ci/travis/bazel-format.sh | 12 +-- ci/travis/build-autoscaler-images.sh | 30 +++---- ci/travis/ci.sh | 9 ++- ci/travis/format.sh | 8 +- ci/travis/install-bazel.sh | 2 +- ci/travis/install-dependencies.sh | 4 +- doc/azure/azure-init.sh | 16 ++-- doc/dev/bin/pip_download_test.sh | 2 +- doc/dev/download_wheels.sh | 16 ++-- doc/examples/lm/ray_train.sh | 6 +- doc/tools/install-prometheus-server.sh | 2 +- java/generate_jni_header_files.sh | 10 +-- java/test.sh | 8 +- python/build-wheel-macos.sh | 10 +-- python/build-wheel-manylinux1.sh | 6 +- .../autoscaler/kubernetes/kubectl-rsync.sh | 4 +- setup_hooks.sh | 4 +- src/ray/test/run_object_manager_tests.sh | 8 +- src/ray/test/run_object_manager_valgrind.sh | 8 +- .../src/test/run_streaming_queue_test.sh | 11 ++- 28 files changed, 166 insertions(+), 165 deletions(-) diff --git a/ci/asan_tests/run_asan_tests.sh b/ci/asan_tests/run_asan_tests.sh index 2333a2bb8..ff67c442a 100755 --- a/ci/asan_tests/run_asan_tests.sh +++ b/ci/asan_tests/run_asan_tests.sh @@ -52,7 +52,7 @@ asan_recompile() { } if [ 0 -lt "$#" ]; then - asan_"$@" + "asan_$1" "${@:2}" else echo "Available commands: setup, run, recompile" fi diff --git a/ci/jenkins_tests/entry_point.sh b/ci/jenkins_tests/entry_point.sh index 4813ff18d..78c09b3c2 100755 --- a/ci/jenkins_tests/entry_point.sh +++ b/ci/jenkins_tests/entry_point.sh @@ -32,13 +32,13 @@ function retry { } if [[ ! -z "$RUN_TUNE_TESTS" ]]; then - retry bash $ROOT_DIR/run_tune_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} + retry bash "$ROOT_DIR"/run_tune_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} fi if [[ ! -z "$RUN_DOC_TESTS" ]]; then - retry bash $ROOT_DIR/run_doc_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} + retry bash "$ROOT_DIR"/run_doc_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} fi if [[ ! -z "$RUN_SGD_TESTS" ]]; then - retry bash $ROOT_DIR/run_sgd_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} + retry bash "$ROOT_DIR"/run_sgd_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} fi \ No newline at end of file diff --git a/ci/jenkins_tests/run_doc_tests.sh b/ci/jenkins_tests/run_doc_tests.sh index e1444b8c8..519826441 100755 --- a/ci/jenkins_tests/run_doc_tests.sh +++ b/ci/jenkins_tests/run_doc_tests.sh @@ -25,7 +25,7 @@ if [ "$DOCKER_SHA" == "" ]; then # Add Ray source git rev-parse HEAD > ./docker/tune_test/git-rev - git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD) + git archive -o ./docker/tune_test/ray.tar "$(git rev-parse HEAD)" if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile) else @@ -33,23 +33,23 @@ if [ "$DOCKER_SHA" == "" ]; then fi fi -echo "Using Docker image" $DOCKER_SHA +echo "Using Docker image" "$DOCKER_SHA" ######################## EXAMPLE TESTS ################################# -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/doc/examples/plot_pong_example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/doc/examples/plot_parameter_server.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/doc/examples/plot_hyperparameter.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/doc/examples/doc_code/torch_example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/doc/examples/doc_code/tf_example.py diff --git a/ci/jenkins_tests/run_multi_node_tests.sh b/ci/jenkins_tests/run_multi_node_tests.sh index 23299a5a1..f04b0e645 100755 --- a/ci/jenkins_tests/run_multi_node_tests.sh +++ b/ci/jenkins_tests/run_multi_node_tests.sh @@ -9,4 +9,4 @@ export RUN_DOC_TESTS=1 export RUN_SGD_TESTS=1 export CI_BUILD_FROM_SOURCE=1 -bash $ROOT_DIR/entry_point.sh +bash "$ROOT_DIR"/entry_point.sh diff --git a/ci/jenkins_tests/run_sgd_tests.sh b/ci/jenkins_tests/run_sgd_tests.sh index 12a35c310..be660cb66 100755 --- a/ci/jenkins_tests/run_sgd_tests.sh +++ b/ci/jenkins_tests/run_sgd_tests.sh @@ -25,7 +25,7 @@ if [ "$DOCKER_SHA" == "" ]; then # Add Ray source git rev-parse HEAD > ./docker/tune_test/git-rev - git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD) + git archive -o ./docker/tune_test/ray.tar "$(git rev-parse HEAD)" if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile) else @@ -33,62 +33,62 @@ if [ "$DOCKER_SHA" == "" ]; then fi fi -echo "Using Docker image" $DOCKER_SHA +echo "Using Docker image" "$DOCKER_SHA" ######################## SGD TESTS ################################# -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python -m pytest /ray/python/ray/util/sgd/tests -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/raysgd_torch_signatures.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/image_models/train.py --no-gpu --mock-data --smoke-test --ray-num-workers=2 --model mobilenetv3_small_075 data -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/train_example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/train_example.py --num-workers=2 -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/tune_example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/tune_example.py --num-workers=2 -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test --num-workers=2 -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_pbt.py --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/dcgan.py --smoke-test --num-workers=2 -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/torch/examples/benchmarks/benchmark.py --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --num-replicas=2 -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --tune -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test --augment-data diff --git a/ci/jenkins_tests/run_tune_tests.sh b/ci/jenkins_tests/run_tune_tests.sh index 1cc12c0cc..7998c0c0e 100755 --- a/ci/jenkins_tests/run_tune_tests.sh +++ b/ci/jenkins_tests/run_tune_tests.sh @@ -25,8 +25,7 @@ if [ "$DOCKER_SHA" == "" ]; then # Add Ray source git rev-parse HEAD > ./docker/tune_test/git-rev - git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD) - + git archive -o ./docker/tune_test/ray.tar "$(git rev-parse HEAD)" if [ "$CI_BUILD_FROM_SOURCE" == "1" ]; then DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test -f docker/tune_test/build_from_source.Dockerfile) else @@ -34,142 +33,142 @@ if [ "$DOCKER_SHA" == "" ]; then fi fi -echo "Using Docker image" $DOCKER_SHA +echo "Using Docker image" "$DOCKER_SHA" ######################## TUNE TESTS ################################# -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ pytest /ray/python/ray/tune/tests/test_actor_reuse.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ pytest /ray/python/ray/tune/tests/test_tune_restore.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/tests/example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ bash -c 'pip install -U tensorflow && python /ray/python/ray/tune/tests/test_logger.py' -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ bash -c 'pip install -U tensorflow==1.15 && python /ray/python/ray/tune/tests/test_logger.py' -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ bash -c 'pip install -U tensorflow==1.14 && python /ray/python/ray/tune/tests/test_logger.py' -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e MPLBACKEND=Agg $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e MPLBACKEND=Agg "$DOCKER_SHA" \ python /ray/python/ray/tune/tests/tutorial.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/pbt_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/hyperband_example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/async_hyperband_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/tf_mnist_example.py --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/lightgbm_example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/xgboost_example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/logging_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/mlflow_example.py -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/bayesopt_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/cifar10_pytorch.py --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/hyperopt_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/doc/source/tune/_tutorials/tune-sklearn.py # if [[ ! -z "$SIGOPT_KEY" ]]; then -# $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e SIGOPT_KEY $DOCKER_SHA \ +# $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e SIGOPT_KEY "$DOCKER_SHA" \ # python /ray/python/ray/tune/examples/sigopt_example.py \ # --smoke-test # fi # Runs only on Python3 -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/nevergrad_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/tune_mnist_keras.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/mnist_pytorch.py --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/mnist_pytorch_lightning.py --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/mnist_pytorch_trainable.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/genetic_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/skopt_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/dragonfly_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/zoopt_example.py \ --smoke-test # Commenting out because flaky -# $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +# $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ # python /ray/python/ray/tune/examples/pbt_memnn_example.py \ # --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/pbt_convnet_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/hyperband_function_example.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/pbt_function.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/pbt_dcgan_mnist/pbt_dcgan_mnist.py \ --smoke-test -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/ci/long_running_distributed_tests/workloads/pytorch_pbt_failure.py \ --smoke-test # uncomment once statsmodels is updated. -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ python /ray/python/ray/tune/examples/bohb_example.py # Moved to bottom because flaky -$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \ +$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 "$DOCKER_SHA" \ pytest /ray/python/ray/tune/tests/test_cluster.py diff --git a/ci/keep_alive b/ci/keep_alive index a8198fd97..4489ea904 100755 --- a/ci/keep_alive +++ b/ci/keep_alive @@ -5,7 +5,7 @@ PID=$$ # Print output to avoid travis killing us watchdog() { - for i in `seq 2 2 150`; do + for i in $(seq 2 2 150); do sleep 120 echo "(running, ${i}m total)" done diff --git a/ci/suppress_output b/ci/suppress_output index 583bd6275..15b0825b9 100755 --- a/ci/suppress_output +++ b/ci/suppress_output @@ -1,17 +1,17 @@ #!/bin/bash # Run a command, suppressing output unless it hangs or crashes. -TMPFILE=`mktemp` +TMPFILE="$(mktemp)" PID=$$ # Print output to avoid travis killing us watchdog() { - for i in `seq 5 5 150`; do + for i in $(seq 5 5 150); do sleep 300 echo "This command has been running for more than $i minutes..." done echo "Command timed out after 2.5h, dumping logs:" - cat $TMPFILE + cat "$TMPFILE" echo "TIMED OUT" kill -SIGKILL $PID } @@ -19,11 +19,11 @@ watchdog() { watchdog & 2>/dev/null WATCHDOG_PID=$! -time "$@" >$TMPFILE 2>&1 +time "$@" >"$TMPFILE" 2>&1 CODE=$? if [ $CODE != 0 ]; then - tail -n 2000 $TMPFILE + tail -n 2000 "$TMPFILE" echo "FAILED $CODE" kill $WATCHDOG_PID exit $CODE diff --git a/ci/travis/bazel-format.sh b/ci/travis/bazel-format.sh index c31b1daf6..1ebcb68b7 100755 --- a/ci/travis/bazel-format.sh +++ b/ci/travis/bazel-format.sh @@ -17,7 +17,7 @@ function usage() echo } -RUN_TYPE=diff +RUN_TYPE="diff" # Parse options while [[ $# > 0 ]]; do @@ -28,7 +28,7 @@ while [[ $# > 0 ]]; do exit 0 ;; -c|--check) - RUN_TYPE=diff + RUN_TYPE="diff" ;; -f|--fix) RUN_TYPE=fix @@ -43,8 +43,8 @@ while [[ $# > 0 ]]; do shift done -pushd $ROOT_DIR/../.. -BAZEL_FILES="bazel/BUILD bazel/BUILD.arrow bazel/ray.bzl BUILD.bazel java/BUILD.bazel - cpp/BUILD.bazel streaming/BUILD.bazel streaming/java/BUILD.bazel WORKSPACE" -buildifier -mode=$RUN_TYPE -diff_command="diff -u" $BAZEL_FILES +pushd "$ROOT_DIR"/../.. +BAZEL_FILES=(bazel/BUILD bazel/BUILD.arrow bazel/ray.bzl BUILD.bazel java/BUILD.bazel \ + cpp/BUILD.bazel streaming/BUILD.bazel streaming/java/BUILD.bazel WORKSPACE) +buildifier -mode=$RUN_TYPE -diff_command="diff -u" "${BAZEL_FILES[@]}" popd diff --git a/ci/travis/build-autoscaler-images.sh b/ci/travis/build-autoscaler-images.sh index 87428aa44..146b14d9e 100755 --- a/ci/travis/build-autoscaler-images.sh +++ b/ci/travis/build-autoscaler-images.sh @@ -6,7 +6,7 @@ set -e set -x SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -ROOT_DIR=$(cd $SCRIPT_DIR/../../; pwd) +ROOT_DIR=$(cd "$SCRIPT_DIR"/../../; pwd) DOCKER_USERNAME="raytravisbot" # We will only build and push when we are building branch build. @@ -14,33 +14,33 @@ if [[ "$TRAVIS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin wheel=$(cd $ROOT_DIR/.whl; ls | grep cp36m-manylinux) - commit_sha=$(echo $TRAVIS_COMMIT | head -c 6) - cp -r $ROOT_DIR/.whl $ROOT_DIR/docker/autoscaler/.whl + commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6) + cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/autoscaler/.whl docker build -q -t rayproject/base-deps docker/base-deps docker build \ --build-arg WHEEL_PATH=".whl/$wheel" \ --build-arg WHEEL_NAME=$wheel \ - -t rayproject/autoscaler:$commit_sha \ - $ROOT_DIR/docker/autoscaler + -t rayproject/autoscaler:"$commit_sha" \ + "$ROOT_DIR"/docker/autoscaler - docker tag rayproject/base-deps rayproject/base-deps:$commit_sha - docker push rayproject/base-deps:$commit_sha - docker push rayproject/autoscaler:$commit_sha + docker tag rayproject/base-deps rayproject/base-deps:"$commit_sha" + docker push rayproject/base-deps:"$commit_sha" + docker push rayproject/autoscaler:"$commit_sha" # We have a branch build, e.g. release/v0.7.0 if [[ "$TRAVIS_BRANCH" != "master" ]]; then # Replace / in branch name to - so it is legal tag name - normalized_branch_name=$(echo $TRAVIS_BRANCH | sed -e "s/\//-/") - docker tag rayproject/autoscaler:$commit_sha rayproject/autoscaler:$normalized_branch_name - docker tag rayproject/base-deps:$commit_sha rayproject/base-deps:$normalized_branch_name - docker push rayproject/autoscaler:$normalized_branch_name - docker push rayproject/base-deps:$normalized_branch_name + normalized_branch_name=$(echo "$TRAVIS_BRANCH" | sed -e "s/\//-/") + docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:"$normalized_branch_name" + docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:"$normalized_branch_name" + docker push rayproject/autoscaler:"$normalized_branch_name" + docker push rayproject/base-deps:"$normalized_branch_name" else - docker tag rayproject/autoscaler:$commit_sha rayproject/autoscaler:latest - docker tag rayproject/base-deps:$commit_sha rayproject/base-deps:latest + docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:latest + docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:latest docker push rayproject/autoscaler:latest docker push rayproject/base-deps:latest fi diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh index 4a21afdf8..d36e9efe5 100755 --- a/ci/travis/ci.sh +++ b/ci/travis/ci.sh @@ -312,7 +312,7 @@ lint_bazel() { # Run buildifier without affecting external environment variables ( mkdir -p -- "${GOPATH}" - export PATH="${GOPATH}/bin":"${GOROOT}/bin":"${PATH}" + export PATH="${GOPATH}/bin:${GOROOT}/bin:${PATH}" # Build buildifier go get github.com/bazelbuild/buildtools/buildifier @@ -329,8 +329,11 @@ lint_web() { . "${HOME}/.nvm/nvm.sh" install_npm_project nvm use --silent node - node_modules/.bin/eslint --max-warnings 0 $(find src -name "*.ts" -or -name "*.tsx") - node_modules/.bin/prettier --check $(find src -name "*.ts" -or -name "*.tsx") + local filenames + # shellcheck disable=SC2207 + filenames=($(find src -name "*.ts" -or -name "*.tsx")) + node_modules/.bin/eslint --max-warnings 0 "${filenames[@]}" + node_modules/.bin/prettier --check "${filenames[@]}" node_modules/.bin/prettier --check public/index.html ) } diff --git a/ci/travis/format.sh b/ci/travis/format.sh index 4102b5c5f..bdb44481d 100755 --- a/ci/travis/format.sh +++ b/ci/travis/format.sh @@ -21,7 +21,7 @@ check_command_exist() { echo "$1 is not a required dependency" exit 1 esac - if ! [ -x "$(command -v $1)" ]; then + if ! [ -x "$(command -v "$1")" ]; then echo "$1 not installed. pip install $1==$VERSION" exit 1 fi @@ -31,7 +31,7 @@ check_command_exist yapf check_command_exist flake8 ver=$(yapf --version) -if ! echo $ver | grep -q 0.23.0; then +if ! echo "$ver" | grep -q 0.23.0; then echo "Wrong YAPF version installed: 0.23.0 is required, not $ver. $YAPF_DOWNLOAD_COMMAND_MSG" exit 1 fi @@ -52,7 +52,7 @@ YAPF_VERSION=$(yapf --version | awk '{print $2}') # params: tool name, tool version, required version tool_version_check() { - if [[ $2 != $3 ]]; then + if [ "$2" != "$3" ]; then echo "WARNING: Ray uses $1 $3, You currently are using $2. This might generate different results." fi } @@ -62,7 +62,7 @@ tool_version_check "yapf" $YAPF_VERSION $YAPF_VERSION_REQUIRED if which clang-format >/dev/null; then CLANG_FORMAT_VERSION=$(clang-format --version | awk '{print $3}') - tool_version_check "clang-format" $CLANG_FORMAT_VERSION "7.0.0" + tool_version_check "clang-format" "$CLANG_FORMAT_VERSION" "7.0.0" else echo "WARNING: clang-format is not installed!" fi diff --git a/ci/travis/install-bazel.sh b/ci/travis/install-bazel.sh index 3cd4534e8..1a8ab6ef8 100755 --- a/ci/travis/install-bazel.sh +++ b/ci/travis/install-bazel.sh @@ -38,7 +38,7 @@ esac fi done if [ ! 0 -eq "${#missing_symlinks[@]}" ]; then - echo "error: expected symlink: ${missing_symlinks[@]}" 1>&2 + echo "error: expected symlink: ${missing_symlinks[*]}" 1>&2 echo "For a correct build, please run 'git config --local core.symlinks true' and re-run git checkout." 1>&2 false fi diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 4c613b1ee..96a78df25 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -159,7 +159,7 @@ install_nvm() { ) printf "%s\n" \ "export NVM_HOME=\"$(cygpath -w -- "${NVM_HOME}")\"" \ - 'nvm() { "${NVM_HOME}/nvm.exe" "$@"; }' \ + "nvm() { \"\${NVM_HOME}/nvm.exe\" \"\$@\"; }" \ > "${NVM_HOME}/nvm.sh" fi else @@ -277,7 +277,7 @@ install_dependencies() { # Additional RLlib dependencies. if [ "${RLLIB_TESTING-}" = 1 ]; then pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 \ - torch=="${TORCH_VERSION-1.4}" torchvision atari_py gym[atari] lz4 smart_open + torch=="${TORCH_VERSION-1.4}" torchvision atari_py "gym[atari]" lz4 smart_open fi # Additional streaming dependencies. diff --git a/doc/azure/azure-init.sh b/doc/azure/azure-init.sh index 4875e789d..7c0628200 100755 --- a/doc/azure/azure-init.sh +++ b/doc/azure/azure-init.sh @@ -7,11 +7,11 @@ RAY_HEAD_IP=$4 TYPE=$5 echo "Installing wheel..." -sudo -u $USERNAME -i /bin/bash -l -c "conda init bash" -sudo -u $USERNAME -i /bin/bash -l -c "conda activate $CONDA_ENV; pip install $WHEEL" +sudo -u "$USERNAME" -i /bin/bash -l -c "conda init bash" +sudo -u "$USERNAME" -i /bin/bash -l -c "conda activate $CONDA_ENV; pip install $WHEEL" echo "Setting up service scripts..." -cat > /home/$USERNAME/ray-head.sh << EOM +cat > /home/"$USERNAME"/ray-head.sh << EOM #!/bin/bash conda activate $CONDA_ENV @@ -22,7 +22,7 @@ ulimit -n 65536 ray start --head --redis-port=6379 --object-manager-port=8076 --num-gpus=\$NUM_GPUS --block --webui-host 0.0.0.0 EOM -cat > /home/$USERNAME/ray-worker.sh << EOM +cat > /home/"$USERNAME"/ray-worker.sh << EOM #!/bin/bash conda activate $CONDA_ENV @@ -39,7 +39,7 @@ do done EOM -cat > /home/$USERNAME/tensorboard.sh << EOM +cat > /home/"$USERNAME"/tensorboard.sh << EOM #!/bin/bash conda activate $CONDA_ENV @@ -48,9 +48,9 @@ mkdir -p /home/$USERNAME/ray_results tensorboard --bind_all --logdir=/home/$USERNAME/ray_results EOM -chmod +x /home/$USERNAME/ray-head.sh -chmod +x /home/$USERNAME/ray-worker.sh -chmod +x /home/$USERNAME/tensorboard.sh +chmod +x /home/"$USERNAME"/ray-head.sh +chmod +x /home/"$USERNAME"/ray-worker.sh +chmod +x /home/"$USERNAME"/tensorboard.sh cat > /lib/systemd/system/ray.service << EOM [Unit] diff --git a/doc/dev/bin/pip_download_test.sh b/doc/dev/bin/pip_download_test.sh index 69e9dcf71..0690b4164 100755 --- a/doc/dev/bin/pip_download_test.sh +++ b/doc/dev/bin/pip_download_test.sh @@ -54,7 +54,7 @@ do echo "=========================================================\n\n\n" conda deactivate - conda remove -y --name ${env_name} --all + conda remove -y --name "${env_name}" --all if [ "$failed" = true ]; then echo "PYTHON ${PYTHON_VERSION} failed sanity check." exit 1 diff --git a/doc/dev/download_wheels.sh b/doc/dev/download_wheels.sh index 5aa7f00a6..f55c6b839 100644 --- a/doc/dev/download_wheels.sh +++ b/doc/dev/download_wheels.sh @@ -11,13 +11,13 @@ if [[ -z "$RAY_VERSION" ]]; then fi # Make sure Linux wheels are downloadable without errors. -wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-manylinux1_x86_64.whl -wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl -wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl -wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl +wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-manylinux1_x86_64.whl" +wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl" +wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl" +wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl" # Make sure macos wheels are downloadable without errors. -wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-macosx_10_13_intel.whl -wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl -wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl +wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp35-cp35m-macosx_10_13_intel.whl" +wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl" +wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl" # Wheel name convention has been changed from Python 3.8. -wget https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-macosx_10_13_x86_64.whl +wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-macosx_10_13_x86_64.whl" diff --git a/doc/examples/lm/ray_train.sh b/doc/examples/lm/ray_train.sh index 0437f31fa..8fe014007 100644 --- a/doc/examples/lm/ray_train.sh +++ b/doc/examples/lm/ray_train.sh @@ -11,9 +11,9 @@ SAVE_INTERVAL_UPDATES=1000 # save a checkpoint every N updates LOG_DIR=$HOME/efs/lm/log/ DATA_DIR=$HOME/efs/lm/data-bin/wikitext-103/ -mkdir -p $LOG_DIR +mkdir -p "$LOG_DIR" -python $HOME/efs/lm/ray_train.py --fp16 $DATA_DIR \ +python "$HOME"/efs/lm/ray_train.py --fp16 "$DATA_DIR" \ --task masked_lm --criterion masked_lm \ --arch roberta_base --sample-break-mode complete --tokens-per-sample $TOKENS_PER_SAMPLE \ --optimizer adam --adam-betas '(0.9, 0.98)' --adam-eps 1e-6 --clip-norm 0.0 \ @@ -23,4 +23,4 @@ python $HOME/efs/lm/ray_train.py --fp16 $DATA_DIR \ --fix-batch-size $FIX_BATCH_SIZE \ --max-update $TOTAL_UPDATES --log-format simple --log-interval 1 \ --save-interval-updates $SAVE_INTERVAL_UPDATES \ - --save-dir $LOG_DIR --ddp-backend=no_c10d + --save-dir "$LOG_DIR" --ddp-backend=no_c10d diff --git a/doc/tools/install-prometheus-server.sh b/doc/tools/install-prometheus-server.sh index 71db6fd2f..9adfbd561 100755 --- a/doc/tools/install-prometheus-server.sh +++ b/doc/tools/install-prometheus-server.sh @@ -5,7 +5,7 @@ set -e TOOLS_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -pushd $TOOLS_DIR +pushd "$TOOLS_DIR" # Download Prometheus server. unamestr="$(uname)" diff --git a/java/generate_jni_header_files.sh b/java/generate_jni_header_files.sh index 618bc4571..ab0513f07 100755 --- a/java/generate_jni_header_files.sh +++ b/java/generate_jni_header_files.sh @@ -10,10 +10,10 @@ cd "$(dirname "$0")" function generate_one() { file=${1//./_}.h - javah -classpath ../bazel-bin/java/all_tests_deploy.jar $1 - clang-format -i $file + javah -classpath ../bazel-bin/java/all_tests_deploy.jar "$1" + clang-format -i "$file" - cat < ../src/ray/core_worker/lib/java/$file + cat < ../src/ray/core_worker/lib/java/"$file" // Copyright 2017 The Ray Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,8 +29,8 @@ function generate_one() // limitations under the License. EOF - cat $file >> ../src/ray/core_worker/lib/java/$file - rm -f $file + cat "$file" >> ../src/ray/core_worker/lib/java/"$file" + rm -f "$file" } generate_one io.ray.runtime.RayNativeRuntime diff --git a/java/test.sh b/java/test.sh index 30c305993..088fef546 100755 --- a/java/test.sh +++ b/java/test.sh @@ -20,7 +20,7 @@ run_testng() { fi } -pushd $ROOT_DIR/.. +pushd "$ROOT_DIR"/.. echo "Linting Java code with checkstyle." # NOTE(hchen): The `test_tag_filters` option causes bazel to ignore caches. # Thus, we add the `build_tests_only` option to avoid re-building everything. @@ -36,15 +36,15 @@ echo "Running tests under cluster mode." # TODO(hchen): Ideally, we should use the following bazel command to run Java tests. However, if there're skipped tests, # TestNG will exit with code 2. And bazel treats it as test failure. # bazel test //java:all_tests --action_env=ENABLE_MULTI_LANGUAGE_TESTS=1 --test_output="errors" || cluster_exit_code=$? -ENABLE_MULTI_LANGUAGE_TESTS=1 run_testng java -cp $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output $ROOT_DIR/testng.xml +ENABLE_MULTI_LANGUAGE_TESTS=1 run_testng java -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml echo "Running tests under single-process mode." # bazel test //java:all_tests --jvmopt="-Dray.run-mode=SINGLE_PROCESS" --test_output="errors" || single_exit_code=$? -run_testng java -Dray.run-mode="SINGLE_PROCESS" -cp $ROOT_DIR/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output $ROOT_DIR/testng.xml +run_testng java -Dray.run-mode="SINGLE_PROCESS" -cp "$ROOT_DIR"/../bazel-bin/java/all_tests_deploy.jar org.testng.TestNG -d /tmp/ray_java_test_output "$ROOT_DIR"/testng.xml popd -pushd $ROOT_DIR +pushd "$ROOT_DIR" echo "Testing maven install." mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests # Ensure mvn test works diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 17ef828ee..eca1f60f9 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -39,7 +39,7 @@ mkdir -p $DOWNLOAD_DIR mkdir -p .whl # Use the latest version of Node.js in order to build the dashboard. -source $HOME/.nvm/nvm.sh +source "$HOME"/.nvm/nvm.sh nvm use node # Build the dashboard so its static assets can be included in the wheel. @@ -61,11 +61,11 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do # Install Python. INST_PATH=python_downloads/$PY_INST - curl $MACPYTHON_URL/$PY_VERSION/$PY_INST > $INST_PATH - sudo installer -pkg $INST_PATH -target / + curl $MACPYTHON_URL/"$PY_VERSION"/"$PY_INST" > "$INST_PATH" + sudo installer -pkg "$INST_PATH" -target / PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM - PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM" + PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM" pushd /tmp # Install latest version of pip to avoid brownouts. @@ -80,7 +80,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $PIP_CMD install -q setuptools_scm==3.1.0 # Fix the numpy version because this will be the oldest numpy version we can # support. - $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.15 + $PIP_CMD install -q numpy=="$NUMPY_VERSION" cython==0.29.15 # Install wheel to avoid the error "invalid command 'bdist_wheel'". $PIP_CMD install -q wheel # Set the commit SHA in __init__.py. diff --git a/python/build-wheel-manylinux1.sh b/python/build-wheel-manylinux1.sh index 3579a04af..1b5bb8c60 100755 --- a/python/build-wheel-manylinux1.sh +++ b/python/build-wheel-manylinux1.sh @@ -32,7 +32,7 @@ export PATH=$PATH:/root/bin # Install and use the latest version of Node.js in order to build the dashboard. set +x curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash -source $HOME/.nvm/nvm.sh +source "$HOME"/.nvm/nvm.sh nvm install node nvm use node @@ -57,7 +57,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do pushd python # Fix the numpy version because this will be the oldest numpy version we can # support. - /opt/python/${PYTHON}/bin/pip install -q numpy==${NUMPY_VERSION} cython==0.29.15 + /opt/python/"${PYTHON}"/bin/pip install -q numpy=="${NUMPY_VERSION}" cython==0.29.15 # Set the commit SHA in __init__.py. if [ -n "$TRAVIS_COMMIT" ]; then sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak @@ -66,7 +66,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do exit 1 fi - PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/${PYTHON}/bin/python setup.py bdist_wheel + PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/"${PYTHON}"/bin/python setup.py bdist_wheel # In the future, run auditwheel here. mv dist/*.whl ../.whl/ popd diff --git a/python/ray/autoscaler/kubernetes/kubectl-rsync.sh b/python/ray/autoscaler/kubernetes/kubectl-rsync.sh index eb756c2d7..583da4ad1 100755 --- a/python/ray/autoscaler/kubernetes/kubectl-rsync.sh +++ b/python/ray/autoscaler/kubernetes/kubectl-rsync.sh @@ -6,7 +6,7 @@ if [ -z "$KRSYNC_STARTED" ]; then export KRSYNC_STARTED=true - exec rsync --blocking-io --rsh "$0" $@ + exec rsync --blocking-io --rsh "$0" "$@" fi # Running as --rsh @@ -22,4 +22,4 @@ if [ "X$pod" = "X-l" ]; then shift fi -exec kubectl $namespace exec -i $pod -- "$@" +exec kubectl "$namespace" exec -i "$pod" -- "$@" diff --git a/setup_hooks.sh b/setup_hooks.sh index 105597f01..9aa0cb0be 100755 --- a/setup_hooks.sh +++ b/setup_hooks.sh @@ -1,3 +1,3 @@ #!/bin/bash -chmod +x $PWD/scripts/pre-push -ln -s $PWD/scripts/pre-push $PWD/.git/hooks/pre-push +chmod +x "$PWD"/scripts/pre-push +ln -s "$PWD"/scripts/pre-push "$PWD"/.git/hooks/pre-push diff --git a/src/ray/test/run_object_manager_tests.sh b/src/ray/test/run_object_manager_tests.sh index b6b0730bd..641c53050 100755 --- a/src/ray/test/run_object_manager_tests.sh +++ b/src/ray/test/run_object_manager_tests.sh @@ -9,10 +9,10 @@ set -x bazel build "//:object_manager_stress_test" "//:object_manager_test" "//:plasma_store_server" # Get the directory in which this script is executing. -SCRIPT_DIR="`dirname \"$0\"`" +SCRIPT_DIR="$(dirname "$0")" RAY_ROOT="$SCRIPT_DIR/../../.." # Makes $RAY_ROOT an absolute path. -RAY_ROOT="`( cd \"$RAY_ROOT\" && pwd )`" +RAY_ROOT="$(cd "$RAY_ROOT" && pwd)" if [ -z "$RAY_ROOT" ] ; then exit 1 fi @@ -23,13 +23,13 @@ if [ ! -d "$RAY_ROOT/python" ]; then fi REDIS_MODULE="./bazel-bin/libray_redis_module.so" -LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" +LOAD_MODULE_ARGS=(--loadmodule "${REDIS_MODULE}") STORE_EXEC="./bazel-bin/plasma_store_server" # Allow cleanup commands to fail. bazel run //:redis-cli -- -p 6379 shutdown || true sleep 1s -bazel run //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & +bazel run //:redis-server -- --loglevel warning "${LOAD_MODULE_ARGS[@]}" --port 6379 & sleep 1s # Run tests. ./bazel-bin/object_manager_stress_test $STORE_EXEC diff --git a/src/ray/test/run_object_manager_valgrind.sh b/src/ray/test/run_object_manager_valgrind.sh index 32c6e8340..7a269eb78 100755 --- a/src/ray/test/run_object_manager_valgrind.sh +++ b/src/ray/test/run_object_manager_valgrind.sh @@ -9,10 +9,10 @@ set -x bazel build "//:object_manager_stress_test" "//:object_manager_test" "//:plasma_store_server" # Get the directory in which this script is executing. -SCRIPT_DIR="`dirname \"$0\"`" +SCRIPT_DIR="$(dirname "$0")" RAY_ROOT="$SCRIPT_DIR/../../.." # Makes $RAY_ROOT an absolute path. -RAY_ROOT="`( cd \"$RAY_ROOT\" && pwd )`" +RAY_ROOT="$(cd "$RAY_ROOT" && pwd)" if [ -z "$RAY_ROOT" ] ; then exit 1 fi @@ -23,7 +23,7 @@ if [ ! -d "$RAY_ROOT/python" ]; then fi REDIS_MODULE="./bazel-bin/libray_redis_module.so" -LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}" +LOAD_MODULE_ARGS=(--loadmodule "${REDIS_MODULE}") STORE_EXEC="./bazel-bin/plasma_store_server" VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=all --leak-check-heuristics=stdstring --error-exitcode=1" @@ -32,7 +32,7 @@ VALGRIND_CMD="valgrind --track-origins=yes --leak-check=full --show-leak-kinds=a killall plasma_store || true bazel run //:redis-cli -- -p 6379 shutdown || true sleep 1s -bazel run //:redis-server -- --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 & +bazel run //:redis-server -- --loglevel warning "${LOAD_MODULE_ARGS[@]}" --port 6379 & sleep 1s # Run tests. Use timeout=10000ms for the Wait tests since tests run slower diff --git a/streaming/src/test/run_streaming_queue_test.sh b/streaming/src/test/run_streaming_queue_test.sh index 8ba7d5bd4..c53e6295f 100755 --- a/streaming/src/test/run_streaming_queue_test.sh +++ b/streaming/src/test/run_streaming_queue_test.sh @@ -7,8 +7,7 @@ PORTS="2000 2001 2002 2003 2004 2005 2006 2007 2008 2009" RAYLET_PORT=0 for port in $PORTS; do - nc -z localhost $port - if [[ $? != 0 ]]; then + if ! nc -z localhost "$port"; then RAYLET_PORT=$port break fi @@ -24,13 +23,13 @@ set -e set -x # Get the directory in which this script is executing. -SCRIPT_DIR="`dirname \"$0\"`" +SCRIPT_DIR="$(dirname "$0")" # Get the directory in which this script is executing. -SCRIPT_DIR="`dirname \"$0\"`" +SCRIPT_DIR="$(dirname "$0")" RAY_ROOT="$SCRIPT_DIR/../../.." # Makes $RAY_ROOT an absolute path. -RAY_ROOT="`( cd \"$RAY_ROOT\" && pwd )`" +RAY_ROOT="$(cd "$RAY_ROOT" && pwd)" if [ -z "$RAY_ROOT" ] ; then exit 1 fi @@ -55,5 +54,5 @@ GCS_SERVER_EXEC="./bazel-bin/gcs_server" # Allow cleanup commands to fail. # Run tests. -./bazel-bin/streaming/streaming_queue_tests $STORE_EXEC $RAYLET_EXEC $RAYLET_PORT $STREAMING_TEST_WORKER_EXEC $GCS_SERVER_EXEC $REDIS_SERVER_EXEC $REDIS_MODULE $REDIS_CLIENT_EXEC +./bazel-bin/streaming/streaming_queue_tests $STORE_EXEC $RAYLET_EXEC "$RAYLET_PORT" $STREAMING_TEST_WORKER_EXEC $GCS_SERVER_EXEC $REDIS_SERVER_EXEC $REDIS_MODULE $REDIS_CLIENT_EXEC sleep 1s