From c00742f10370b14cf77e47c407e6c0710b66f4f2 Mon Sep 17 00:00:00 2001 From: SangBin Cho Date: Tue, 28 Jul 2020 10:44:06 -0700 Subject: [PATCH] [Release] Fix release tests (#9733) --- ci/asan_tests/ray-project/cluster.yaml | 2 + ci/asan_tests/run.sh | 33 +++++++++++ .../ray-project/cluster.yaml | 5 +- .../ray-project/project.yaml | 13 +++-- ci/long_running_distributed_tests/run.sh | 45 +++++++++++++++ ci/long_running_tests/run.sh | 57 +++++++++++++++++++ .../workloads/many_tasks_serialized_ids.py | 8 +-- ci/microbenchmark/ray-project/cluster.yaml | 1 + ci/microbenchmark/run.sh | 51 +++++++++++++++++ .../ray-project/cluster.yaml | 3 +- .../ray-project/project.yaml | 3 + .../rllib_regresssion_tests/run.sh | 4 ++ .../ray-project/cluster.yaml | 3 +- .../ray-project/project.yaml | 3 + .../stress_tests/ray-project/cluster.yaml | 2 + ci/regression_test/stress_tests/run.sh | 45 +++++++++++++++ 16 files changed, 264 insertions(+), 14 deletions(-) create mode 100644 ci/asan_tests/run.sh create mode 100644 ci/long_running_distributed_tests/run.sh create mode 100644 ci/long_running_tests/run.sh create mode 100644 ci/microbenchmark/run.sh create mode 100644 ci/regression_test/rllib_regresssion_tests/run.sh create mode 100644 ci/regression_test/stress_tests/run.sh diff --git a/ci/asan_tests/ray-project/cluster.yaml b/ci/asan_tests/ray-project/cluster.yaml index 952a14921..4784fd729 100644 --- a/ci/asan_tests/ray-project/cluster.yaml +++ b/ci/asan_tests/ray-project/cluster.yaml @@ -36,6 +36,8 @@ setup_commands: - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc - pip install -U pip || true + - conda uninstall -y terminado || true + - pip install terminado || true # Clone Ray. - cd $HOME; rm -rf $HOME/ray; git clone https://github.com/ray-project/ray || true # Install Node.js in order to build the dashboard. diff --git a/ci/asan_tests/run.sh b/ci/asan_tests/run.sh new file mode 100644 index 000000000..9dde1e53f --- /dev/null +++ b/ci/asan_tests/run.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +git_sha="" + +usage() { + echo "Run ASAN tests." +} + +for i in "$@" +do +case $i in + --git-sha=*) + git_sha="${i#*=}" + ;; + --help) + usage + exit + ;; + *) + echo "unknown arg, $2" + exit 1 + ;; +esac +done + +echo "git-sha: $git_sha" + +./run_asan_tests.sh setup +if [[ git_sha != "" ]] +then + git_sha=$git_sha ./run_asan_tests.sh recompile +fi +./run_asan_tests.sh run diff --git a/ci/long_running_distributed_tests/ray-project/cluster.yaml b/ci/long_running_distributed_tests/ray-project/cluster.yaml index ea66d51ad..d1224e6b6 100644 --- a/ci/long_running_distributed_tests/ray-project/cluster.yaml +++ b/ci/long_running_distributed_tests/ray-project/cluster.yaml @@ -60,8 +60,11 @@ worker_nodes: setup_commands: # Install ray. + - conda uninstall -y terminado || true - pip install -U pip - - ray || pip install -U {{ray-wheel}} + - pip install terminado + - wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl + - ray || pip install -U ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl || true # Installing this without -U to make sure we don't replace the existing Ray installation - pip install ray[rllib] - pip install -U ipdb diff --git a/ci/long_running_distributed_tests/ray-project/project.yaml b/ci/long_running_distributed_tests/ray-project/project.yaml index e70a4d2ae..09f6e8f53 100644 --- a/ci/long_running_distributed_tests/ray-project/project.yaml +++ b/ci/long_running_distributed_tests/ray-project/project.yaml @@ -5,9 +5,14 @@ name: long-running-distributed-tests cluster: config: ray-project/cluster.yaml params: - - name: ray-wheel - help: "URL to the ray wheel to test (defaults to latest)" - default: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl + - name: ray_version # Ray version string. + default: "0.8.2" + + - name: commit # Ray commit SHA string. + default: "f5a1307a608fe5fdbdb04616b22c91f029af329a" + + - name: ray_branch + default: "releases/0.8.2" commands: - name: test_workload @@ -21,7 +26,7 @@ commands: [ "pytorch_pbt_failure" ] - + # Pathnames for files and directories that should be saved # in a snapshot but that should not be synced with a# session. Pathnames can be relative to the project # directory or absolute. Generally, this should be files diff --git a/ci/long_running_distributed_tests/run.sh b/ci/long_running_distributed_tests/run.sh new file mode 100644 index 000000000..8e52eaedc --- /dev/null +++ b/ci/long_running_distributed_tests/run.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +ray_version="" +commit="" +ray_branch="" +workload="" + +usage() { + echo "Start one microbenchmark trial." +} + +for i in "$@" +do +echo $i +case $i in + --ray-version=*) + ray_version="${i#*=}" + + ;; + --commit=*) + commit="${i#*=}" + ;; + --ray-branch=*) + ray_branch="${i#*=}" + ;; + --workload=*) + workload="${i#*=}" + ;; + --help) + usage + exit + ;; + *) + echo "unknown arg, $i" + exit 1 + ;; +esac +done + +echo "version: $ray_version" +echo "commit: $commit" +echo "branch: $ray_branch" +echo "workload: $workload" + +python "workloads/$workload.py" diff --git a/ci/long_running_tests/run.sh b/ci/long_running_tests/run.sh new file mode 100644 index 000000000..32a81c729 --- /dev/null +++ b/ci/long_running_tests/run.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +ray_version="" +commit="" +ray_branch="" +workload="" + +usage() { + echo "Start one microbenchmark trial." +} + +for i in "$@" +do +echo $i +case $i in + --ray-version=*) + ray_version="${i#*=}" + + ;; + --commit=*) + commit="${i#*=}" + ;; + --ray-branch=*) + ray_branch="${i#*=}" + ;; + --workload=*) + workload="${i#*=}" + ;; + --help) + usage + exit + ;; + *) + echo "unknown arg, $i" + exit 1 + ;; +esac +done + +if [[ $ray_version == "" || $commit == "" || $ray_branch == "" ]] +then + echo "Provide --ray-version, --commit, and --ray-branch" + exit 1 +fi + +echo "version: $ray_version" +echo "commit: $commit" +echo "branch: $ray_branch" +echo "workload: $workload" + +wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl" + +pip install -U pip +source activate tensorflow_p36 && pip install -q -U $wheel Click +source activate tensorflow_p36 && pip install -q ray[all] gym[atari] +source activate tensorflow_p36 && python "workloads/$workload.py" + diff --git a/ci/long_running_tests/workloads/many_tasks_serialized_ids.py b/ci/long_running_tests/workloads/many_tasks_serialized_ids.py index 0fea51623..5cd8445f9 100644 --- a/ci/long_running_tests/workloads/many_tasks_serialized_ids.py +++ b/ci/long_running_tests/workloads/many_tasks_serialized_ids.py @@ -2,7 +2,6 @@ # returning serialized ObjectRefs. import time -import json import random import numpy as np @@ -22,9 +21,6 @@ assert (num_nodes * object_store_memory + num_redis_shards * redis_max_memory < # Simulate a cluster on one machine. -config = json.dumps({ - "distributed_ref_counting_enabled": 1, -}) cluster = Cluster() for i in range(num_nodes): cluster.add_node( @@ -35,9 +31,7 @@ for i in range(num_nodes): resources={str(i): 2}, object_store_memory=object_store_memory, redis_max_memory=redis_max_memory, - dashboard_host="0.0.0.0", - _internal_config=config if i == 0 else None, - ) + dashboard_host="0.0.0.0") ray.init(address=cluster.address) # Run the workload. diff --git a/ci/microbenchmark/ray-project/cluster.yaml b/ci/microbenchmark/ray-project/cluster.yaml index 9df6628b1..0ee31dbc6 100644 --- a/ci/microbenchmark/ray-project/cluster.yaml +++ b/ci/microbenchmark/ray-project/cluster.yaml @@ -45,6 +45,7 @@ head_setup_commands: - bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc - pip install -U pip + - conda uninstall -y terminado # Custom commands that will be run on worker nodes after common setup. worker_setup_commands: [] diff --git a/ci/microbenchmark/run.sh b/ci/microbenchmark/run.sh new file mode 100644 index 000000000..1c66e9ce8 --- /dev/null +++ b/ci/microbenchmark/run.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +ray_version="" +commit="" +ray_branch="" + +usage() { + echo "Start one microbenchmark trial." +} + +for i in "$@" +do +case $i in + --ray-version=*) + ray_version="${i#*=}" + + ;; + --commit=*) + commit="${i#*=}" + ;; + --ray-branch=*) + ray_branch="${i#*=}" + ;; + --help) + usage + exit + ;; + *) + echo "unknown arg, $2" + exit 1 + ;; +esac +done + +if [[ $ray_version == "" || $commit == "" || $ray_branch == "" ]] +then + echo "Provide --ray-version, --commit, and --ray-branch" + exit 1 +fi + +echo "version: $ray_version" +echo "commit: $commit" +echo "branch: $ray_branch" + +rm ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl || true +wget https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl + +pip uninstall -y -q ray +pip install -U ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl + +OMP_NUM_THREADS=64 ray microbenchmark diff --git a/ci/regression_test/rllib_regresssion_tests/ray-project/cluster.yaml b/ci/regression_test/rllib_regresssion_tests/ray-project/cluster.yaml index a8ace8e07..fe7a15092 100644 --- a/ci/regression_test/rllib_regresssion_tests/ray-project/cluster.yaml +++ b/ci/regression_test/rllib_regresssion_tests/ray-project/cluster.yaml @@ -27,7 +27,8 @@ head_node: # List of shell commands to run to set up nodes. setup_commands: - - wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/releases/{{ray_version}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl + - wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl + - conda uninstall -y terminado - source activate tensorflow_p36 && pip install -U ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl - source activate tensorflow_p36 && pip install ray[rllib] ray[debug] - source activate tensorflow_p36 && pip install boto3==1.4.8 cython==0.29.0 diff --git a/ci/regression_test/rllib_regresssion_tests/ray-project/project.yaml b/ci/regression_test/rllib_regresssion_tests/ray-project/project.yaml index 7f2db2cbf..4b894f9ef 100644 --- a/ci/regression_test/rllib_regresssion_tests/ray-project/project.yaml +++ b/ci/regression_test/rllib_regresssion_tests/ray-project/project.yaml @@ -15,6 +15,9 @@ cluster: - name: commit # Ray commit SHA string. default: "f5a1307a608fe5fdbdb04616b22c91f029af329a" + - name: ray_branch + default: "releases/0.8.2" + environment: # dockerfile: The dockerfile to be built and ran the commands with. diff --git a/ci/regression_test/rllib_regresssion_tests/run.sh b/ci/regression_test/rllib_regresssion_tests/run.sh new file mode 100644 index 000000000..4352773a4 --- /dev/null +++ b/ci/regression_test/rllib_regresssion_tests/run.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +source activate tensorflow_p36 && pip install torch torchvision +source activate tensorflow_p36 && rllib train -f compact-regression-test.yaml diff --git a/ci/regression_test/rllib_stress_tests/ray-project/cluster.yaml b/ci/regression_test/rllib_stress_tests/ray-project/cluster.yaml index 73e3ef5b6..56deae83c 100644 --- a/ci/regression_test/rllib_stress_tests/ray-project/cluster.yaml +++ b/ci/regression_test/rllib_stress_tests/ray-project/cluster.yaml @@ -87,7 +87,8 @@ file_mounts: { # List of shell commands to run to set up nodes. setup_commands: - - wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/releases/{{ray_version}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl + - wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl + - conda uninstall -y terminado - source activate tensorflow_p36 && pip install -U ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl - source activate tensorflow_p36 && pip install ray[rllib] ray[debug] - source activate tensorflow_p36 && pip install boto3==1.4.8 cython==0.29.0 diff --git a/ci/regression_test/rllib_stress_tests/ray-project/project.yaml b/ci/regression_test/rllib_stress_tests/ray-project/project.yaml index 48f6685ff..a9ce5488b 100644 --- a/ci/regression_test/rllib_stress_tests/ray-project/project.yaml +++ b/ci/regression_test/rllib_stress_tests/ray-project/project.yaml @@ -15,6 +15,9 @@ cluster: - name: commit # Ray commit SHA string. default: "f5a1307a608fe5fdbdb04616b22c91f029af329a" + - name: ray_branch + default: "releases/0.8.2" + environment: # dockerfile: The dockerfile to be built and ran the commands with. diff --git a/ci/regression_test/stress_tests/ray-project/cluster.yaml b/ci/regression_test/stress_tests/ray-project/cluster.yaml index 67fde4bf9..8d4ef87ed 100644 --- a/ci/regression_test/stress_tests/ray-project/cluster.yaml +++ b/ci/regression_test/stress_tests/ray-project/cluster.yaml @@ -97,6 +97,8 @@ setup_commands: # - git clone https://github.com/ray-project/ray || true # - ray/ci/travis/install-bazel.sh - pip install -U pip + - conda uninstall -y terminado + - pip install terminado - pip install boto3==1.4.8 cython==0.29.0 # - cd ray/python; git checkout master; git pull; pip install -e . --verbose - "pip install https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl" diff --git a/ci/regression_test/stress_tests/run.sh b/ci/regression_test/stress_tests/run.sh new file mode 100644 index 000000000..22f2ef9e4 --- /dev/null +++ b/ci/regression_test/stress_tests/run.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +ray_version="" +commit="" +ray_branch="" +workload="" + +usage() { + echo "Start one microbenchmark trial." +} + +for i in "$@" +do +echo $i +case $i in + --ray-version=*) + ray_version="${i#*=}" + + ;; + --commit=*) + commit="${i#*=}" + ;; + --ray-branch=*) + ray_branch="${i#*=}" + ;; + --workload=*) + workload="${i#*=}" + ;; + --help) + usage + exit + ;; + *) + echo "unknown arg, $i" + exit 1 + ;; +esac +done + +echo "version: $ray_version" +echo "commit: $commit" +echo "branch: $ray_branch" +echo "workload: $workload" + +python "$workload.py"