From bcc379556b135ee2e472b0e4b388c9e1f8274dc9 Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Fri, 28 Jun 2019 15:42:54 -0700 Subject: [PATCH] Make some fixes to long running stress tests. (#5056) --- ci/long_running_tests/README.rst | 15 +++++-- ci/long_running_tests/config.yaml | 11 ++--- ci/long_running_tests/start_workloads.sh | 52 +++++++++++++++++++----- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/ci/long_running_tests/README.rst b/ci/long_running_tests/README.rst index 736862abe..3395f781a 100644 --- a/ci/long_running_tests/README.rst +++ b/ci/long_running_tests/README.rst @@ -7,10 +7,17 @@ intended to run forever until they fail. Running the Workloads --------------------- -To run the workloads, run ``./start_workloads.sh``. This will start one EC2 -instance per workload and will start the workloads running (one per instance). -Running the ``./start_workloads.sh`` script again will clean up any state from -the previous runs and will start the workloads again. +To run the workloads, run + +.. code-block:: bash + + ./start_workloads.sh + +using the appropriate values of ````, ````, and +````. This will start one EC2 instance per workload and will start +the workloads running (one per instance). Running the ``./start_workloads.sh`` +script again will clean up any state from the previous runs and will start the +workloads again. Check Workload Statuses ----------------------- diff --git a/ci/long_running_tests/config.yaml b/ci/long_running_tests/config.yaml index b9667ae64..e623d4ea6 100644 --- a/ci/long_running_tests/config.yaml +++ b/ci/long_running_tests/config.yaml @@ -38,6 +38,8 @@ worker_nodes: # List of shell commands to run to set up nodes. setup_commands: + - sudo apt-get update + - sudo apt-get install -y build-essential curl unzip # Install Anaconda. - wget https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true @@ -45,15 +47,14 @@ setup_commands: - echo 'termcapinfo xterm* ti@:te@' >> ~/.screenrc # Some Python dependencies. - pip install boto3==1.4.8 cython==0.29.0 - # # Uncomment the following if you wish to install Ray instead. - # - sudo apt-get update - # - sudo apt-get install -y build-essential curl unzip + # Uncomment the following if you wish to install Ray instead. # - git clone https://github.com/ray-project/ray || true # - ray/ci/travis/install-bazel.sh # - cd ray/python; git checkout master; git pull; pip install -e . --verbose # Install nightly Ray wheels. - - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/<>/ray-<>-cp36-cp36m-manylinux1_x86_64.whl - - pip install ray[rllib] ray[debug] tensorflow + - wget https://s3-us-west-2.amazonaws.com/ray-wheels/<<>>/<<>>/ray-<<>>-cp36-cp36m-manylinux1_x86_64.whl + - pip install ray-<<>>-cp36-cp36m-manylinux1_x86_64.whl[rllib,debug] + - pip install tensorflow - pip install -U dask # fix error importing lz4 # Custom commands that will be run on the head node after common setup. diff --git a/ci/long_running_tests/start_workloads.sh b/ci/long_running_tests/start_workloads.sh index 21fdf2bb4..1dda7f6d6 100755 --- a/ci/long_running_tests/start_workloads.sh +++ b/ci/long_running_tests/start_workloads.sh @@ -4,28 +4,60 @@ set -e ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +if [[ -z "$1" ]]; then + echo "ERROR: The first argument must be the Ray branch to test (e.g., 'master')." + exit 1 +else + RAY_BRANCH=$1 +fi + +if [[ -z "$2" ]]; then + echo "ERROR: The second argument must be the Ray version to test (e.g., '0.8.0.dev1')." + exit 1 +else + RAY_VERSION=$2 +fi + +if [[ -z "$3" ]]; then + echo "ERROR: The third argument must be the Ray commit to test (e.g., '62e4b591e3d6443ce25b0f05cc32b43d5e2ebb3d')." + exit 1 +else + RAY_COMMIT=$3 +fi + +echo "Testing ray==$RAY_VERSION at commit $RAY_COMMIT." +echo "The wheels used will live under https://s3-us-west-2.amazonaws.com/ray-wheels/$RAY_BRANCH/$RAY_VERSION/$RAY_COMMIT/" + + pushd "$ROOT_DIR" +# Substitute in the appropriate Ray version and commit in the config file and +# store it in a temporary file. +CLUSTER_CONFIG="config_temporary.yaml" +sed -e "s/<<>>/$RAY_BRANCH/g; + s/<<>>/$RAY_VERSION/g; + s/<<>>/$RAY_COMMIT/;" config.yaml > "$CLUSTER_CONFIG" + # Start one instance per workload. for workload_file in "$ROOT_DIR"/workloads/*; do - file_name=$(basename -- $workload_file) + file_name=$(basename -- "$workload_file") workload_name="${file_name%.*}" - ray up -y config.yaml --cluster-name="$workload_name" & + ray up -y $CLUSTER_CONFIG --cluster-name="$workload_name" & done # Wait for all of the nodes to be up. wait # Start the workloads running. for workload_file in "$ROOT_DIR"/workloads/*; do - file_name=$(basename -- $workload_file) + file_name=$(basename -- "$workload_file") workload_name="${file_name%.*}" ( # Copy the workload to the cluster. - ray rsync_up config.yaml --cluster-name="$workload_name" "$workload_file" "$file_name" + ray rsync_up $CLUSTER_CONFIG --cluster-name="$workload_name" "$workload_file" "$file_name" # Clean up previous runs if relevant. - ray exec config.yaml --cluster-name="$workload_name" "ray stop; rm -r /tmp/ray; tmux kill-server | true" + ray exec $CLUSTER_CONFIG --cluster-name="$workload_name" "ray stop; rm -r /tmp/ray; tmux kill-server | true" # Start the workload. - ray exec config.yaml --cluster-name="$workload_name" "python $file_name" --tmux + ray exec $CLUSTER_CONFIG --cluster-name="$workload_name" "python $file_name" --tmux ) & done # Wait for child processes to finish. @@ -41,9 +73,9 @@ echo "" echo "To kill the instances, use the following commands." echo "" for workload_file in "$ROOT_DIR"/workloads/*; do - file_name=$(basename -- $workload_file) + file_name=$(basename -- "$workload_file") workload_name="${file_name%.*}" - echo " ray down -y $ROOT_DIR/config.yaml --cluster-name=$workload_name" + echo " ray down -y $ROOT_DIR/$CLUSTER_CONFIG --cluster-name=$workload_name" done echo "" @@ -52,9 +84,9 @@ echo "" echo "Use the following commands to attach to the relevant drivers." echo "" for workload_file in "$ROOT_DIR"/workloads/*; do - file_name=$(basename -- $workload_file) + file_name=$(basename -- "$workload_file") workload_name="${file_name%.*}" - echo " ray attach $ROOT_DIR/config.yaml --cluster-name=$workload_name --tmux" + echo " ray attach $ROOT_DIR/$CLUSTER_CONFIG --cluster-name=$workload_name --tmux" done echo ""