wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-08-12 12:20:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

[tests] Stress tests for Jenkins (#3789)

Browse Source 
Stress testing for Jenkins.

<!--
Thank you for your contribution!

Please review https://github.com/ray-project/ray/blob/master/CONTRIBUTING.rst before opening a pull request.
-->


<!-- Please give a short brief about these changes. -->
TODO:
 - [x] Enable a common keypair for autoscaling 
 - [x] Add automatic timeouts?
 - [x] Switch out key pair one last time before merge
This commit is contained in:
Richard Liaw
2019-02-26 14:24:37 -08:00
committed by GitHub
parent 89ce4c56aa
commit f7450dbdd7
7 changed files with 79 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci/stress_tests/application_cluster_template.yaml
+7 -2
View File
@@ -1,3 +1,7 @@
####################################################################
# All nodes in this cluster will auto-terminate in 1 hour
####################################################################
# An unique identifier for the head node and workers of this cluster.
cluster_name: <<<CLUSTER_NAME>>>
@@ -86,9 +90,10 @@ file_mounts: {
# List of shell commands to run to set up nodes.
setup_commands:
- echo 'export PATH="$HOME/anaconda3/envs/tensorflow_<<<PYTHON_VERSION>>>/bin:$PATH"' >> ~/.bashrc
- ray || wget https://s3-us-west-2.amazonaws.com/ray-wheels/latest/<<<RAY_VERSION>>>-<<<WHEEL_STR>>>-manylinux1_x86_64.whl
- rllib || pip install -U <<<RAY_VERSION>>>-<<<WHEEL_STR>>>-manylinux1_x86_64.whl[rllib]
- ray || wget https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.7.0.dev0-<<<WHEEL_STR>>>-manylinux1_x86_64.whl
- rllib || pip install -U ray-0.7.0.dev0-<<<WHEEL_STR>>>-manylinux1_x86_64.whl[rllib]
- pip install -U tensorflow-gpu
- echo "sudo halt" | at now + 60 minutes
# Consider uncommenting these if you also want to run apt-get commands during setup
# - sudo pkill -9 apt-get || true
# - sudo pkill -9 dpkg || true
ci/stress_tests/run_application_stress_tests.sh
+12 -8
View File
@@ -12,11 +12,16 @@
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
RAY_VERSION=$(git describe --tags --abbrev=0)
RESULT_FILE=$ROOT_DIR/"results-$(date '+%Y-%m-%d_%H-%M-%S').log"
echo "Testing on latest version of Ray: $RAY_VERSION"
echo "Logging to" $RESULT_FILE
echo -e $RAY_AWS_SSH_KEY > /root/.ssh/ray-autoscaler_us-west-2.pem && chmod 400 /root/.ssh/ray-autoscaler_us-west-2.pem || true
# Show explicitly which commands are currently running. This should only be AFTER
# the private key is placed.
set -x
touch $RESULT_FILE
# This function identifies the right string for the Ray wheel.
@@ -46,7 +51,6 @@ test_impala(){
cat application_cluster_template.yaml |
sed -e "
s/<<<CLUSTER_NAME>>>/$TEST_NAME/;
s/<<<RAY_VERSION>>>/$RAY_VERSION/;
s/<<<HEAD_TYPE>>>/g3.16xlarge/;
s/<<<WORKER_TYPE>>>/m5.24xlarge/;
s/<<<MIN_WORKERS>>>/5/;
@@ -57,10 +61,11 @@ test_impala(){
echo "Try running IMPALA stress test."
{
RLLIB_DIR=../../python/ray/rllib/
ray up -y $CLUSTER &&
ray --logging-level=DEBUG up -y $CLUSTER &&
ray rsync_up $CLUSTER $RLLIB_DIR/tuned_examples/ tuned_examples/ &&
sleep 1 &&
ray exec $CLUSTER "
ray --logging-level=DEBUG exec $CLUSTER "rllib || true" &&
ray --logging-level=DEBUG exec $CLUSTER "
rllib train -f tuned_examples/atari-impala-large.yaml --redis-address='localhost:6379' --queue-trials" &&
echo "PASS: IMPALA Test for" $PYTHON_VERSION >> $RESULT_FILE
} || echo "FAIL: IMPALA Test for" $PYTHON_VERSION >> $RESULT_FILE
@@ -89,7 +94,6 @@ test_sgd(){
cat application_cluster_template.yaml |
sed -e "
s/<<<CLUSTER_NAME>>>/$TEST_NAME/;
s/<<<RAY_VERSION>>>/$RAY_VERSION/;
s/<<<HEAD_TYPE>>>/g3.16xlarge/;
s/<<<WORKER_TYPE>>>/g3.16xlarge/;
s/<<<MIN_WORKERS>>>/3/;
@@ -100,11 +104,11 @@ test_sgd(){
echo "Try running SGD stress test."
{
SGD_DIR=$ROOT_DIR/../../python/ray/experimental/sgd/
ray up -y $CLUSTER &&
ray --logging-level=DEBUG up -y $CLUSTER &&
# TODO: fix submit so that args work
ray rsync_up $CLUSTER $SGD_DIR/mnist_example.py mnist_example.py &&
sleep 1 &&
ray exec $CLUSTER "
ray --logging-level=DEBUG exec $CLUSTER "
python mnist_example.py --redis-address=localhost:6379 --num-iters=2000 --num-workers=8 --devices-per-worker=2 --gpu" &&
echo "PASS: SGD Test for" $PYTHON_VERSION >> $RESULT_FILE
} || echo "FAIL: SGD Test for" $PYTHON_VERSION >> $RESULT_FILE
ci/stress_tests/run_jenkins_stress_test.sh
Executable
+28
View File
@@ -0,0 +1,28 @@
#!/usr/bin/env bash
# Cause the script to exit if a single command fails.
set -e
# Show explicitly which commands are currently running.
set -x
MEMORY_SIZE="20G"
SHM_SIZE="20G"
docker build -q --no-cache -t ray-project/base-deps docker/base-deps
# Add Ray source
git rev-parse HEAD > ./docker/stress_test/git-rev
git archive -o ./docker/stress_test/ray.tar $(git rev-parse HEAD)
DOCKER_SHA=$(docker build --no-cache -q -t ray-project/stress_test docker/stress_test)
echo "Using Docker image" $DOCKER_SHA
docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} \
-e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e RAY_AWS_SSH_KEY \
$DOCKER_SHA \
bash /ray/ci/stress_tests/run_stress_tests.sh
# docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} \
# -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e RAY_AWS_SSH_KEY \
# $DOCKER_SHA \
# bash /ray/ci/stress_tests/run_application_stress_tests.sh
ci/stress_tests/run_stress_tests.sh
+8 -4
View File
@@ -1,11 +1,15 @@
#!/usr/bin/env bash
# Show explicitly which commands are currently running.
set -x
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
RESULT_FILE=$ROOT_DIR/results-$(date '+%Y-%m-%d_%H-%M-%S').log
echo "Logging to" $RESULT_FILE
echo -e $RAY_AWS_SSH_KEY > /root/.ssh/ray-autoscaler_us-west-2.pem && chmod 400 /root/.ssh/ray-autoscaler_us-west-2.pem || true
# Show explicitly which commands are currently running. This should only be AFTER
# the private key is placed.
set -x
touch $RESULT_FILE
run_test(){
@@ -16,7 +20,7 @@ run_test(){
{
ray up -y $CLUSTER --cluster-name "$test_name" &&
sleep 1 &&
ray submit $CLUSTER --cluster-name "$test_name" "$test_name.py"
ray --logging-level=DEBUG submit $CLUSTER --cluster-name "$test_name" "$test_name.py"
} || echo "FAIL: $test_name" >> $RESULT_FILE
# Tear down cluster.
ci/stress_tests/stress_testing_config.yaml
+7 -2
View File
@@ -1,13 +1,17 @@
####################################################################
# All nodes in this cluster will auto-terminate in 1 hour
####################################################################
# An unique identifier for the head node and workers of this cluster.
cluster_name: stress-testing
# The minimum number of workers nodes to launch in addition to the head
# node. This number should be >= 0.
min_workers: 100
min_workers: 105
# The maximum number of workers nodes to launch in addition to the head
# node. This takes precedence over min_workers.
max_workers: 100
max_workers: 105
# The autoscaler will scale up the cluster to this target fraction of resource
# usage. For example, if a cluster of 10 nodes is 100% busy and
@@ -97,6 +101,7 @@ setup_commands:
- pip install boto3==1.4.8 cython==0.29.0
# - cd ray/python; git checkout master; git pull; pip install -e . --verbose
- pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.7.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
- echo "sudo halt" | at now + 60 minutes
# Custom commands that will be run on the head node after common setup.
head_setup_commands: []
ci/stress_tests/test_many_tasks_and_transfers.py
+4 -1
View File
@@ -10,11 +10,14 @@ import time
import ray
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
ray.init(redis_address="localhost:6379")
# These numbers need to match the values in the autoscaler config file.
# These numbers need to correspond with the autoscaler config file.
# The number of remote nodes in the autoscaler should upper bound
# these because sometimes nodes fail to update.
num_remote_nodes = 100
head_node_cpus = 2
num_remote_cpus = num_remote_nodes * head_node_cpus
docker/stress_test/Dockerfile
+13
View File
@@ -0,0 +1,13 @@
# The stress_test Docker image build a self-contained Ray instance for launching Ray.
FROM ray-project/base-deps
# We install ray and boto3 to enable the ray autoscaler as
# a test runner.
RUN pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.7.0.dev0-cp27-cp27mu-manylinux1_x86_64.whl boto3
RUN mkdir -p /root/.ssh/
# We port the source code in so that we run the most up-to-date stress tests.
ADD ray.tar /ray
ADD git-rev /ray/git-rev
WORKDIR /ray