From f7450dbdd7065014517630092eca23c34549fca8 Mon Sep 17 00:00:00 2001 From: Richard Liaw Date: Tue, 26 Feb 2019 14:24:37 -0800 Subject: [PATCH] [tests] Stress tests for Jenkins (#3789) Stress testing for Jenkins. TODO: - [x] Enable a common keypair for autoscaling - [x] Add automatic timeouts? - [x] Switch out key pair one last time before merge --- .../application_cluster_template.yaml | 9 ++++-- .../run_application_stress_tests.sh | 20 +++++++------ ci/stress_tests/run_jenkins_stress_test.sh | 28 +++++++++++++++++++ ci/stress_tests/run_stress_tests.sh | 12 +++++--- ci/stress_tests/stress_testing_config.yaml | 9 ++++-- .../test_many_tasks_and_transfers.py | 5 +++- docker/stress_test/Dockerfile | 13 +++++++++ 7 files changed, 79 insertions(+), 17 deletions(-) create mode 100755 ci/stress_tests/run_jenkins_stress_test.sh create mode 100644 docker/stress_test/Dockerfile diff --git a/ci/stress_tests/application_cluster_template.yaml b/ci/stress_tests/application_cluster_template.yaml index 5549fadd1..5e6b33968 100644 --- a/ci/stress_tests/application_cluster_template.yaml +++ b/ci/stress_tests/application_cluster_template.yaml @@ -1,3 +1,7 @@ +#################################################################### +# All nodes in this cluster will auto-terminate in 1 hour +#################################################################### + # An unique identifier for the head node and workers of this cluster. cluster_name: <<>> @@ -86,9 +90,10 @@ file_mounts: { # List of shell commands to run to set up nodes. setup_commands: - echo 'export PATH="$HOME/anaconda3/envs/tensorflow_<<>>/bin:$PATH"' >> ~/.bashrc - - ray || wget https://s3-us-west-2.amazonaws.com/ray-wheels/latest/<<>>-<<>>-manylinux1_x86_64.whl - - rllib || pip install -U <<>>-<<>>-manylinux1_x86_64.whl[rllib] + - ray || wget https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.7.0.dev0-<<>>-manylinux1_x86_64.whl + - rllib || pip install -U ray-0.7.0.dev0-<<>>-manylinux1_x86_64.whl[rllib] - pip install -U tensorflow-gpu + - echo "sudo halt" | at now + 60 minutes # Consider uncommenting these if you also want to run apt-get commands during setup # - sudo pkill -9 apt-get || true # - sudo pkill -9 dpkg || true diff --git a/ci/stress_tests/run_application_stress_tests.sh b/ci/stress_tests/run_application_stress_tests.sh index 0c4531f06..a8ded40fa 100755 --- a/ci/stress_tests/run_application_stress_tests.sh +++ b/ci/stress_tests/run_application_stress_tests.sh @@ -12,11 +12,16 @@ ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -RAY_VERSION=$(git describe --tags --abbrev=0) RESULT_FILE=$ROOT_DIR/"results-$(date '+%Y-%m-%d_%H-%M-%S').log" -echo "Testing on latest version of Ray: $RAY_VERSION" echo "Logging to" $RESULT_FILE +echo -e $RAY_AWS_SSH_KEY > /root/.ssh/ray-autoscaler_us-west-2.pem && chmod 400 /root/.ssh/ray-autoscaler_us-west-2.pem || true + + +# Show explicitly which commands are currently running. This should only be AFTER +# the private key is placed. +set -x + touch $RESULT_FILE # This function identifies the right string for the Ray wheel. @@ -46,7 +51,6 @@ test_impala(){ cat application_cluster_template.yaml | sed -e " s/<<>>/$TEST_NAME/; - s/<<>>/$RAY_VERSION/; s/<<>>/g3.16xlarge/; s/<<>>/m5.24xlarge/; s/<<>>/5/; @@ -57,10 +61,11 @@ test_impala(){ echo "Try running IMPALA stress test." { RLLIB_DIR=../../python/ray/rllib/ - ray up -y $CLUSTER && + ray --logging-level=DEBUG up -y $CLUSTER && ray rsync_up $CLUSTER $RLLIB_DIR/tuned_examples/ tuned_examples/ && sleep 1 && - ray exec $CLUSTER " + ray --logging-level=DEBUG exec $CLUSTER "rllib || true" && + ray --logging-level=DEBUG exec $CLUSTER " rllib train -f tuned_examples/atari-impala-large.yaml --redis-address='localhost:6379' --queue-trials" && echo "PASS: IMPALA Test for" $PYTHON_VERSION >> $RESULT_FILE } || echo "FAIL: IMPALA Test for" $PYTHON_VERSION >> $RESULT_FILE @@ -89,7 +94,6 @@ test_sgd(){ cat application_cluster_template.yaml | sed -e " s/<<>>/$TEST_NAME/; - s/<<>>/$RAY_VERSION/; s/<<>>/g3.16xlarge/; s/<<>>/g3.16xlarge/; s/<<>>/3/; @@ -100,11 +104,11 @@ test_sgd(){ echo "Try running SGD stress test." { SGD_DIR=$ROOT_DIR/../../python/ray/experimental/sgd/ - ray up -y $CLUSTER && + ray --logging-level=DEBUG up -y $CLUSTER && # TODO: fix submit so that args work ray rsync_up $CLUSTER $SGD_DIR/mnist_example.py mnist_example.py && sleep 1 && - ray exec $CLUSTER " + ray --logging-level=DEBUG exec $CLUSTER " python mnist_example.py --redis-address=localhost:6379 --num-iters=2000 --num-workers=8 --devices-per-worker=2 --gpu" && echo "PASS: SGD Test for" $PYTHON_VERSION >> $RESULT_FILE } || echo "FAIL: SGD Test for" $PYTHON_VERSION >> $RESULT_FILE diff --git a/ci/stress_tests/run_jenkins_stress_test.sh b/ci/stress_tests/run_jenkins_stress_test.sh new file mode 100755 index 000000000..7df661396 --- /dev/null +++ b/ci/stress_tests/run_jenkins_stress_test.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Cause the script to exit if a single command fails. +set -e + +# Show explicitly which commands are currently running. +set -x + +MEMORY_SIZE="20G" +SHM_SIZE="20G" + +docker build -q --no-cache -t ray-project/base-deps docker/base-deps + +# Add Ray source +git rev-parse HEAD > ./docker/stress_test/git-rev +git archive -o ./docker/stress_test/ray.tar $(git rev-parse HEAD) +DOCKER_SHA=$(docker build --no-cache -q -t ray-project/stress_test docker/stress_test) + +echo "Using Docker image" $DOCKER_SHA +docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e RAY_AWS_SSH_KEY \ + $DOCKER_SHA \ + bash /ray/ci/stress_tests/run_stress_tests.sh + +# docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} \ +# -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e RAY_AWS_SSH_KEY \ +# $DOCKER_SHA \ +# bash /ray/ci/stress_tests/run_application_stress_tests.sh diff --git a/ci/stress_tests/run_stress_tests.sh b/ci/stress_tests/run_stress_tests.sh index 394f65deb..1d4d10209 100755 --- a/ci/stress_tests/run_stress_tests.sh +++ b/ci/stress_tests/run_stress_tests.sh @@ -1,11 +1,15 @@ #!/usr/bin/env bash -# Show explicitly which commands are currently running. -set -x - ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) RESULT_FILE=$ROOT_DIR/results-$(date '+%Y-%m-%d_%H-%M-%S').log echo "Logging to" $RESULT_FILE +echo -e $RAY_AWS_SSH_KEY > /root/.ssh/ray-autoscaler_us-west-2.pem && chmod 400 /root/.ssh/ray-autoscaler_us-west-2.pem || true + + +# Show explicitly which commands are currently running. This should only be AFTER +# the private key is placed. +set -x + touch $RESULT_FILE run_test(){ @@ -16,7 +20,7 @@ run_test(){ { ray up -y $CLUSTER --cluster-name "$test_name" && sleep 1 && - ray submit $CLUSTER --cluster-name "$test_name" "$test_name.py" + ray --logging-level=DEBUG submit $CLUSTER --cluster-name "$test_name" "$test_name.py" } || echo "FAIL: $test_name" >> $RESULT_FILE # Tear down cluster. diff --git a/ci/stress_tests/stress_testing_config.yaml b/ci/stress_tests/stress_testing_config.yaml index 65237d0a3..70b61d3c1 100644 --- a/ci/stress_tests/stress_testing_config.yaml +++ b/ci/stress_tests/stress_testing_config.yaml @@ -1,13 +1,17 @@ +#################################################################### +# All nodes in this cluster will auto-terminate in 1 hour +#################################################################### + # An unique identifier for the head node and workers of this cluster. cluster_name: stress-testing # The minimum number of workers nodes to launch in addition to the head # node. This number should be >= 0. -min_workers: 100 +min_workers: 105 # The maximum number of workers nodes to launch in addition to the head # node. This takes precedence over min_workers. -max_workers: 100 +max_workers: 105 # The autoscaler will scale up the cluster to this target fraction of resource # usage. For example, if a cluster of 10 nodes is 100% busy and @@ -97,6 +101,7 @@ setup_commands: - pip install boto3==1.4.8 cython==0.29.0 # - cd ray/python; git checkout master; git pull; pip install -e . --verbose - pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.7.0.dev0-cp36-cp36m-manylinux1_x86_64.whl + - echo "sudo halt" | at now + 60 minutes # Custom commands that will be run on the head node after common setup. head_setup_commands: [] diff --git a/ci/stress_tests/test_many_tasks_and_transfers.py b/ci/stress_tests/test_many_tasks_and_transfers.py index 4abe8af5b..c4c4825b1 100644 --- a/ci/stress_tests/test_many_tasks_and_transfers.py +++ b/ci/stress_tests/test_many_tasks_and_transfers.py @@ -10,11 +10,14 @@ import time import ray +logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) ray.init(redis_address="localhost:6379") -# These numbers need to match the values in the autoscaler config file. +# These numbers need to correspond with the autoscaler config file. +# The number of remote nodes in the autoscaler should upper bound +# these because sometimes nodes fail to update. num_remote_nodes = 100 head_node_cpus = 2 num_remote_cpus = num_remote_nodes * head_node_cpus diff --git a/docker/stress_test/Dockerfile b/docker/stress_test/Dockerfile new file mode 100644 index 000000000..94c1f2f0a --- /dev/null +++ b/docker/stress_test/Dockerfile @@ -0,0 +1,13 @@ +# The stress_test Docker image build a self-contained Ray instance for launching Ray. + +FROM ray-project/base-deps + +# We install ray and boto3 to enable the ray autoscaler as +# a test runner. +RUN pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.7.0.dev0-cp27-cp27mu-manylinux1_x86_64.whl boto3 +RUN mkdir -p /root/.ssh/ + +# We port the source code in so that we run the most up-to-date stress tests. +ADD ray.tar /ray +ADD git-rev /ray/git-rev +WORKDIR /ray