From 580820a5308bbff2eb041d14f07f7b2847f58b66 Mon Sep 17 00:00:00 2001 From: Sven Mika Date: Mon, 12 Oct 2020 21:05:09 +0200 Subject: [PATCH] [RLlib] Create ci/rllib_tests and organize a little (#11342) --- .../compact-regression-test.yaml | 0 .../ray-project/cluster.yaml | 1 + .../ray-project/project.yaml | 3 +- .../ray-project/requirements.txt | 0 .../regresssion_tests}/run.sh | 1 - .../stress_tests}/atari_impala_xlarge.yaml | 0 .../stress_tests}/ray-project/cluster.yaml | 0 .../stress_tests}/ray-project/project.yaml | 0 .../ray-project/requirements.txt | 0 .../stress_tests}/run.sh | 0 .../stress_tests}/wait_cluster.py | 0 .../unit_gpu_tests/ray-project/cluster.yaml | 42 +++++++++++++++++++ .../ray-project/requirements.txt | 6 +++ ci/rllib_tests/unit_gpu_tests/run.sh | 5 +++ 14 files changed, 55 insertions(+), 3 deletions(-) rename ci/{regression_test/rllib_regresssion_tests => rllib_tests/regresssion_tests}/compact-regression-test.yaml (100%) rename ci/{regression_test/rllib_regresssion_tests => rllib_tests/regresssion_tests}/ray-project/cluster.yaml (95%) rename ci/{regression_test/rllib_regresssion_tests => rllib_tests/regresssion_tests}/ray-project/project.yaml (95%) rename ci/{regression_test/rllib_regresssion_tests => rllib_tests/regresssion_tests}/ray-project/requirements.txt (100%) rename ci/{regression_test/rllib_regresssion_tests => rllib_tests/regresssion_tests}/run.sh (58%) rename ci/{regression_test/rllib_stress_tests => rllib_tests/stress_tests}/atari_impala_xlarge.yaml (100%) rename ci/{regression_test/rllib_stress_tests => rllib_tests/stress_tests}/ray-project/cluster.yaml (100%) rename ci/{regression_test/rllib_stress_tests => rllib_tests/stress_tests}/ray-project/project.yaml (100%) rename ci/{regression_test/rllib_stress_tests => rllib_tests/stress_tests}/ray-project/requirements.txt (100%) rename ci/{regression_test/rllib_stress_tests => rllib_tests/stress_tests}/run.sh (100%) rename ci/{regression_test/rllib_stress_tests => rllib_tests/stress_tests}/wait_cluster.py (100%) create mode 100644 ci/rllib_tests/unit_gpu_tests/ray-project/cluster.yaml create mode 100644 ci/rllib_tests/unit_gpu_tests/ray-project/requirements.txt create mode 100644 ci/rllib_tests/unit_gpu_tests/run.sh diff --git a/ci/regression_test/rllib_regresssion_tests/compact-regression-test.yaml b/ci/rllib_tests/regresssion_tests/compact-regression-test.yaml similarity index 100% rename from ci/regression_test/rllib_regresssion_tests/compact-regression-test.yaml rename to ci/rllib_tests/regresssion_tests/compact-regression-test.yaml diff --git a/ci/regression_test/rllib_regresssion_tests/ray-project/cluster.yaml b/ci/rllib_tests/regresssion_tests/ray-project/cluster.yaml similarity index 95% rename from ci/regression_test/rllib_regresssion_tests/ray-project/cluster.yaml rename to ci/rllib_tests/regresssion_tests/ray-project/cluster.yaml index 708cb30cd..7a4c643c5 100644 --- a/ci/regression_test/rllib_regresssion_tests/ray-project/cluster.yaml +++ b/ci/rllib_tests/regresssion_tests/ray-project/cluster.yaml @@ -31,6 +31,7 @@ setup_commands: - conda uninstall -y terminado - source activate tensorflow_p36 && pip install -U ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl - source activate tensorflow_p36 && pip install ray[rllib] ray[debug] + - source activate tensorflow_p36 && pip install torch==1.6 torchvision - source activate tensorflow_p36 && pip install boto3==1.4.8 cython==0.29.0 # Command to start ray on the head node. You don't need to change this. diff --git a/ci/regression_test/rllib_regresssion_tests/ray-project/project.yaml b/ci/rllib_tests/regresssion_tests/ray-project/project.yaml similarity index 95% rename from ci/regression_test/rllib_regresssion_tests/ray-project/project.yaml rename to ci/rllib_tests/regresssion_tests/ray-project/project.yaml index 4b894f9ef..b9f1273ed 100644 --- a/ci/regression_test/rllib_regresssion_tests/ray-project/project.yaml +++ b/ci/rllib_tests/regresssion_tests/ray-project/project.yaml @@ -42,7 +42,6 @@ commands: - name: run-regression-tests command: | - source activate tensorflow_p36 && pip install torch torchvision source activate tensorflow_p36 && rllib train -f compact-regression-test.yaml help: "Run rllib regression tests" @@ -55,4 +54,4 @@ output_files: [ # For example, uncomment this to save the logs from the # last ray job. # "/tmp/ray/session_latest", -] \ No newline at end of file +] diff --git a/ci/regression_test/rllib_regresssion_tests/ray-project/requirements.txt b/ci/rllib_tests/regresssion_tests/ray-project/requirements.txt similarity index 100% rename from ci/regression_test/rllib_regresssion_tests/ray-project/requirements.txt rename to ci/rllib_tests/regresssion_tests/ray-project/requirements.txt diff --git a/ci/regression_test/rllib_regresssion_tests/run.sh b/ci/rllib_tests/regresssion_tests/run.sh similarity index 58% rename from ci/regression_test/rllib_regresssion_tests/run.sh rename to ci/rllib_tests/regresssion_tests/run.sh index 8d9dffc8b..f6ecb5454 100644 --- a/ci/regression_test/rllib_regresssion_tests/run.sh +++ b/ci/rllib_tests/regresssion_tests/run.sh @@ -1,4 +1,3 @@ #!/usr/bin/env bash -source activate tensorflow_p36 && pip install torch==1.4 torchvision source activate tensorflow_p36 && rllib train -f compact-regression-test.yaml diff --git a/ci/regression_test/rllib_stress_tests/atari_impala_xlarge.yaml b/ci/rllib_tests/stress_tests/atari_impala_xlarge.yaml similarity index 100% rename from ci/regression_test/rllib_stress_tests/atari_impala_xlarge.yaml rename to ci/rllib_tests/stress_tests/atari_impala_xlarge.yaml diff --git a/ci/regression_test/rllib_stress_tests/ray-project/cluster.yaml b/ci/rllib_tests/stress_tests/ray-project/cluster.yaml similarity index 100% rename from ci/regression_test/rllib_stress_tests/ray-project/cluster.yaml rename to ci/rllib_tests/stress_tests/ray-project/cluster.yaml diff --git a/ci/regression_test/rllib_stress_tests/ray-project/project.yaml b/ci/rllib_tests/stress_tests/ray-project/project.yaml similarity index 100% rename from ci/regression_test/rllib_stress_tests/ray-project/project.yaml rename to ci/rllib_tests/stress_tests/ray-project/project.yaml diff --git a/ci/regression_test/rllib_stress_tests/ray-project/requirements.txt b/ci/rllib_tests/stress_tests/ray-project/requirements.txt similarity index 100% rename from ci/regression_test/rllib_stress_tests/ray-project/requirements.txt rename to ci/rllib_tests/stress_tests/ray-project/requirements.txt diff --git a/ci/regression_test/rllib_stress_tests/run.sh b/ci/rllib_tests/stress_tests/run.sh similarity index 100% rename from ci/regression_test/rllib_stress_tests/run.sh rename to ci/rllib_tests/stress_tests/run.sh diff --git a/ci/regression_test/rllib_stress_tests/wait_cluster.py b/ci/rllib_tests/stress_tests/wait_cluster.py similarity index 100% rename from ci/regression_test/rllib_stress_tests/wait_cluster.py rename to ci/rllib_tests/stress_tests/wait_cluster.py diff --git a/ci/rllib_tests/unit_gpu_tests/ray-project/cluster.yaml b/ci/rllib_tests/unit_gpu_tests/ray-project/cluster.yaml new file mode 100644 index 000000000..d0fa67149 --- /dev/null +++ b/ci/rllib_tests/unit_gpu_tests/ray-project/cluster.yaml @@ -0,0 +1,42 @@ +cluster_name: ray-rllib-regression-tests + +min_workers: 0 +max_workers: 0 + +# Cloud-provider specific configuration. +provider: + type: aws + region: us-west-2 + availability_zone: us-west-2a + cache_stopped_nodes: False + +# How Ray will authenticate with newly launched nodes. +auth: + ssh_user: ubuntu + +head_node: + InstanceType: p2.xlarge # Cheaper 1GPU K80 instance + ImageId: ami-07728e9e2742b0662 # Deep Learning AMI (Ubuntu 16.04) + + # Set primary volume to 25 GiB + BlockDeviceMappings: + - DeviceName: /dev/sda1 + Ebs: + VolumeSize: 100 + + +# List of shell commands to run to set up nodes. +setup_commands: + - wget --quiet https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl + - conda uninstall -y terminado + - source activate tensorflow_p36 && pip install -U ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl + +# Command to start ray on the head node. You don't need to change this. +head_start_ray_commands: + - source activate tensorflow_p36 && ray stop + - ulimit -n 65536; source activate tensorflow_p36 && OMP_NUM_THREADS=1 ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml + +# Command to start ray on worker nodes. You don't need to change this. +worker_start_ray_commands: + - source activate tensorflow_p36 && ray stop + - ulimit -n 65536; source activate tensorflow_p36 && OMP_NUM_THREADS=1 ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 diff --git a/ci/rllib_tests/unit_gpu_tests/ray-project/requirements.txt b/ci/rllib_tests/unit_gpu_tests/ray-project/requirements.txt new file mode 100644 index 000000000..a86eb47e6 --- /dev/null +++ b/ci/rllib_tests/unit_gpu_tests/ray-project/requirements.txt @@ -0,0 +1,6 @@ +ray[rllib] +ray[debug] +torch==1.6+cu101 +torchvision==0.7.0+cu101 +boto3==1.4.8 +cython==0.29.0 diff --git a/ci/rllib_tests/unit_gpu_tests/run.sh b/ci/rllib_tests/unit_gpu_tests/run.sh new file mode 100644 index 000000000..57024310f --- /dev/null +++ b/ci/rllib_tests/unit_gpu_tests/run.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +# Run all test cases, but with a forced num_gpus=1. +# TODO: (sven) chose correct dir and run over all RLlib tests and example scripts! +source activate tensorflow_p36 && export RAY_FORCE_NUM_GPUS=1 && cd ~ && python -m pytest test_attention_net_learning.py