wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-07-19 11:27:32 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Release] Fix release tests (#9733)

Browse Source
This commit is contained in:
SangBin Cho
2020-07-28 10:44:06 -07:00
committed by GitHub
parent 7e3ba289dc
commit c00742f103
16 changed files with 264 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci/asan_tests/ray-project/cluster.yaml
+2
View File
@@ -36,6 +36,8 @@ setup_commands:
- bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true
- echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
- pip install -U pip || true
- conda uninstall -y terminado || true
- pip install terminado || true
# Clone Ray.
- cd $HOME; rm -rf $HOME/ray; git clone https://github.com/ray-project/ray || true
# Install Node.js in order to build the dashboard.
ci/asan_tests/run.sh
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
git_sha=""
usage() {
echo "Run ASAN tests."
}
for i in "$@"
do
case $i in
--git-sha=*)
git_sha="${i#*=}"
;;
--help)
usage
exit
;;
*)
echo "unknown arg, $2"
exit 1
;;
esac
done
echo "git-sha: $git_sha"
./run_asan_tests.sh setup
if [[ git_sha != "" ]]
then
git_sha=$git_sha ./run_asan_tests.sh recompile
fi
./run_asan_tests.sh run
ci/long_running_distributed_tests/ray-project/cluster.yaml
+4 -1
View File
@@ -60,8 +60,11 @@ worker_nodes:
setup_commands:
# Install ray.
- conda uninstall -y terminado || true
- pip install -U pip
- ray || pip install -U {{ray-wheel}}
- pip install terminado
- wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl
- ray || pip install -U ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl || true
# Installing this without -U to make sure we don't replace the existing Ray installation
- pip install ray[rllib]
- pip install -U ipdb
ci/long_running_distributed_tests/ray-project/project.yaml
+9 -4
View File
@@ -5,9 +5,14 @@ name: long-running-distributed-tests
cluster:
config: ray-project/cluster.yaml
params:
- name: ray-wheel
help: "URL to the ray wheel to test (defaults to latest)"
default: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
- name: ray_version # Ray version string.
default: "0.8.2"
- name: commit # Ray commit SHA string.
default: "f5a1307a608fe5fdbdb04616b22c91f029af329a"
- name: ray_branch
default: "releases/0.8.2"
commands:
- name: test_workload
@@ -21,7 +26,7 @@ commands:
[
"pytorch_pbt_failure"
]
# Pathnames for files and directories that should be saved
# in a snapshot but that should not be synced with a# session. Pathnames can be relative to the project
# directory or absolute. Generally, this should be files
ci/long_running_distributed_tests/run.sh
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
ray_version=""
commit=""
ray_branch=""
workload=""
usage() {
echo "Start one microbenchmark trial."
}
for i in "$@"
do
echo $i
case $i in
--ray-version=*)
ray_version="${i#*=}"
;;
--commit=*)
commit="${i#*=}"
;;
--ray-branch=*)
ray_branch="${i#*=}"
;;
--workload=*)
workload="${i#*=}"
;;
--help)
usage
exit
;;
*)
echo "unknown arg, $i"
exit 1
;;
esac
done
echo "version: $ray_version"
echo "commit: $commit"
echo "branch: $ray_branch"
echo "workload: $workload"
python "workloads/$workload.py"
ci/long_running_tests/run.sh
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env bash
ray_version=""
commit=""
ray_branch=""
workload=""
usage() {
echo "Start one microbenchmark trial."
}
for i in "$@"
do
echo $i
case $i in
--ray-version=*)
ray_version="${i#*=}"
;;
--commit=*)
commit="${i#*=}"
;;
--ray-branch=*)
ray_branch="${i#*=}"
;;
--workload=*)
workload="${i#*=}"
;;
--help)
usage
exit
;;
*)
echo "unknown arg, $i"
exit 1
;;
esac
done
if [[ $ray_version == "" || $commit == "" || $ray_branch == "" ]]
then
echo "Provide --ray-version, --commit, and --ray-branch"
exit 1
fi
echo "version: $ray_version"
echo "commit: $commit"
echo "branch: $ray_branch"
echo "workload: $workload"
wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl"
pip install -U pip
source activate tensorflow_p36 && pip install -q -U $wheel Click
source activate tensorflow_p36 && pip install -q ray[all] gym[atari]
source activate tensorflow_p36 && python "workloads/$workload.py"
ci/long_running_tests/workloads/many_tasks_serialized_ids.py
+1 -7
View File
@@ -2,7 +2,6 @@
# returning serialized ObjectRefs.
import time
import json
import random
import numpy as np
@@ -22,9 +21,6 @@ assert (num_nodes * object_store_memory + num_redis_shards * redis_max_memory <
# Simulate a cluster on one machine.
config = json.dumps({
"distributed_ref_counting_enabled": 1,
})
cluster = Cluster()
for i in range(num_nodes):
cluster.add_node(
@@ -35,9 +31,7 @@ for i in range(num_nodes):
resources={str(i): 2},
object_store_memory=object_store_memory,
redis_max_memory=redis_max_memory,
dashboard_host="0.0.0.0",
_internal_config=config if i == 0 else None,
)
dashboard_host="0.0.0.0")
ray.init(address=cluster.address)
# Run the workload.
ci/microbenchmark/ray-project/cluster.yaml
+1
View File
@@ -45,6 +45,7 @@ head_setup_commands:
- bash Anaconda3-5.0.1-Linux-x86_64.sh -b -p $HOME/anaconda3 || true
- echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
- pip install -U pip
- conda uninstall -y terminado
# Custom commands that will be run on worker nodes after common setup.
worker_setup_commands: []
ci/microbenchmark/run.sh
+51
View File
@@ -0,0 +1,51 @@
#!/usr/bin/env bash
ray_version=""
commit=""
ray_branch=""
usage() {
echo "Start one microbenchmark trial."
}
for i in "$@"
do
case $i in
--ray-version=*)
ray_version="${i#*=}"
;;
--commit=*)
commit="${i#*=}"
;;
--ray-branch=*)
ray_branch="${i#*=}"
;;
--help)
usage
exit
;;
*)
echo "unknown arg, $2"
exit 1
;;
esac
done
if [[ $ray_version == "" || $commit == "" || $ray_branch == "" ]]
then
echo "Provide --ray-version, --commit, and --ray-branch"
exit 1
fi
echo "version: $ray_version"
echo "commit: $commit"
echo "branch: $ray_branch"
rm ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl || true
wget https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl
pip uninstall -y -q ray
pip install -U ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl
OMP_NUM_THREADS=64 ray microbenchmark
ci/regression_test/rllib_regresssion_tests/ray-project/cluster.yaml
+2 -1
View File
@@ -27,7 +27,8 @@ head_node:
# List of shell commands to run to set up nodes.
setup_commands:
- wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/releases/{{ray_version}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl
- wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl
- conda uninstall -y terminado
- source activate tensorflow_p36 && pip install -U ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl
- source activate tensorflow_p36 && pip install ray[rllib] ray[debug]
- source activate tensorflow_p36 && pip install boto3==1.4.8 cython==0.29.0
ci/regression_test/rllib_regresssion_tests/ray-project/project.yaml
+3
View File
@@ -15,6 +15,9 @@ cluster:
- name: commit # Ray commit SHA string.
default: "f5a1307a608fe5fdbdb04616b22c91f029af329a"
- name: ray_branch
default: "releases/0.8.2"
environment:
# dockerfile: The dockerfile to be built and ran the commands with.
ci/regression_test/rllib_regresssion_tests/run.sh
+4
View File
@@ -0,0 +1,4 @@
#!/usr/bin/env bash
source activate tensorflow_p36 && pip install torch torchvision
source activate tensorflow_p36 && rllib train -f compact-regression-test.yaml
ci/regression_test/rllib_stress_tests/ray-project/cluster.yaml
+2 -1
View File
@@ -87,7 +87,8 @@ file_mounts: {
# List of shell commands to run to set up nodes.
setup_commands:
- wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/releases/{{ray_version}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl
- wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl
- conda uninstall -y terminado
- source activate tensorflow_p36 && pip install -U ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl
- source activate tensorflow_p36 && pip install ray[rllib] ray[debug]
- source activate tensorflow_p36 && pip install boto3==1.4.8 cython==0.29.0
ci/regression_test/rllib_stress_tests/ray-project/project.yaml
+3
View File
@@ -15,6 +15,9 @@ cluster:
- name: commit # Ray commit SHA string.
default: "f5a1307a608fe5fdbdb04616b22c91f029af329a"
- name: ray_branch
default: "releases/0.8.2"
environment:
# dockerfile: The dockerfile to be built and ran the commands with.
ci/regression_test/stress_tests/ray-project/cluster.yaml
+2
View File
@@ -97,6 +97,8 @@ setup_commands:
# - git clone https://github.com/ray-project/ray || true
# - ray/ci/travis/install-bazel.sh
- pip install -U pip
- conda uninstall -y terminado
- pip install terminado
- pip install boto3==1.4.8 cython==0.29.0
# - cd ray/python; git checkout master; git pull; pip install -e . --verbose
- "pip install https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl"
ci/regression_test/stress_tests/run.sh
+45
View File
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
ray_version=""
commit=""
ray_branch=""
workload=""
usage() {
echo "Start one microbenchmark trial."
}
for i in "$@"
do
echo $i
case $i in
--ray-version=*)
ray_version="${i#*=}"
;;
--commit=*)
commit="${i#*=}"
;;
--ray-branch=*)
ray_branch="${i#*=}"
;;
--workload=*)
workload="${i#*=}"
;;
--help)
usage
exit
;;
*)
echo "unknown arg, $i"
exit 1
;;
esac
done
echo "version: $ray_version"
echo "commit: $commit"
echo "branch: $ray_branch"
echo "workload: $workload"
python "$workload.py"