Clean up release tests (#11420)

release/long_running_distributed_tests/README.rst

@@ -0,0 +1,45 @@
+Long Running Distributed Tests
+==============================
+
+This directory contains the long-running multi-node workloads which are intended to run
+forever until they fail. To set up the project you need to run
+
+.. code-block:: bash
+
+    $ pip install anyscale
+    $ anyscale init
+
+Running the Workloads
+---------------------
+Easiest approach is to use the `Anyscale UI <https://www.anyscale.dev/>`_. First run ``anyscale snapshot create`` from the command line to create a project snapshot. Then from the UI, you can launch an individual session and execute the test_workload command for each test. 
+
+You can also start the workloads using the CLI with:
+
+.. code-block:: bash
+
+    $ anyscale start --ray-wheel=<RAY_WHEEL_LINK>
+    $ anyscale run test_workload --workload=<WORKLOAD_NAME>
+
+
+Doing this for each workload will start one EC2 instance per workload and will start the workloads
+running (one per instance). A list of
+available workload options is available in the `ray_projects/project.yaml` file.
+
+
+Debugging
+---------
+The primary method to debug the test while it is running is to view the logs and the dashboard from the UI. After the test has failed, you can still view the stdout logs in the UI and also inspect
+the logs under ``/tmp/ray/session*/logs/`` and
+``/tmp/ray/session*/debug_state.txt``.
+
+Shut Down the Workloads
+-----------------------
+
+The instances running the workloads can all be killed by running
+``anyscale stop <SESSION_NAME>``.
+
+Adding a Workload
+-----------------
+
+To create a new workload, simply add a new Python file under ``workloads/`` and
+add the workload in the run command in `ray-project/project.yaml`.

release/long_running_distributed_tests/cluster.yaml

@@ -0,0 +1,73 @@
+# This file is generated by `ray project create`.
+
+# A unique identifier for the head node and workers of this cluster.
+cluster_name: long-running-distributed-tests
+
+# The minimum number of workers nodes to launch in addition to the head
+# node. This number should be >= 0.
+min_workers: 3
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers defaults to 0.
+max_workers: 3
+
+# The autoscaler will scale up the cluster to this target fraction of resource
+# usage. For example, if a cluster of 10 nodes is 100% busy and
+# target_utilization is 0.8, it would resize the cluster to 13. This fraction
+# can be decreased to increase the aggressiveness of upscaling.
+# This value must be less than 1.0 for scaling to happen.
+target_utilization_fraction: 0.8
+
+# If a node is idle for this many minutes, it will be removed.
+idle_timeout_minutes: 5
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-west-2
+    availability_zone: us-west-2a
+    cache_stopped_nodes: False
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
+
+# By default Ray creates a new private keypair, but you can also use your own.
+# If you do so, make sure to also set "KeyName" in the head and worker node
+# configurations below.
+#    ssh_private_key: /path/to/your/key.pem
+
+# Provider-specific config for the head node, e.g. instance type. By default
+# Ray will auto-configure unspecified fields such as SubnetId and KeyName.
+# For more documentation on available fields, see:
+# http://boto3.readthedocs.io/en/latest/reference/services/ec2.html#EC2.ServiceResource.create_instances
+head_node:
+    InstanceType: g3.8xlarge
+    ImageId: ami-0888a3b5189309429  # DLAMI 7/1/19
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+
+worker_nodes:
+  InstanceType: g3.8xlarge
+  ImageId: ami-0888a3b5189309429  # DLAMI 7/1/19
+  BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 150
+  InstanceMarketOptions:
+    MarketType: spot
+
+setup_commands: []
+
+# Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands:
+    - ray stop
+    - export RAY_BACKEND_LOG_LEVEL=debug
+    - ray start --head --port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands:
+    - ray stop
+    - export RAY_BACKEND_LOG_LEVEL=debug
+    - ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076

release/long_running_distributed_tests/run.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+ray_version="" 
+commit=""
+ray_branch=""
+workload=""
+
+usage() {
+    echo "Start one microbenchmark trial."
+}
+
+for i in "$@"
+do
+echo "$i"
+case "$i" in
+    --ray-version=*)
+    ray_version="${i#*=}"
+
+    ;;
+    --commit=*)
+    commit="${i#*=}"
+    ;;
+    --ray-branch=*)
+    ray_branch="${i#*=}"
+    ;;
+    --workload=*)
+    workload="${i#*=}"
+    ;;
+    --help)
+    usage
+    exit
+    ;;
+    *)
+    echo "unknown arg, $i"
+    exit 1
+    ;;
+esac
+done
+
+echo "version: $ray_version"
+echo "commit: $commit"
+echo "branch: $ray_branch"
+echo "workload: $workload"
+
+wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl"
+
+conda uninstall -y terminado || true
+pip install -U pip
+pip install terminado
+pip install -U "$wheel"
+pip install "ray[rllib]"
+pip install -U ipdb
+# There have been some recent problems with torch 1.5 and torchvision 0.6
+# not recognizing GPUs.
+# So, we force install torch 1.4 and torchvision 0.5. 
+# https://github.com/pytorch/pytorch/issues/37212#issuecomment-623198624.
+pip install torch==1.4.0 torchvision==0.5.0
+echo set-window-option -g mouse on > ~/.tmux.conf
+echo 'termcapinfo xterm* ti@:te@' > ~/.screenrc
+
+python "workloads/$workload.py"

release/long_running_distributed_tests/workloads/pytorch_pbt_failure.py

@@ -0,0 +1,155 @@
+import argparse
+import numpy as np
+import os
+import random
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, Subset
+from torchvision.datasets import CIFAR10
+import torchvision.transforms as transforms
+
+import ray
+from ray import tune
+from ray.autoscaler._private.commands import kill_node
+from ray.tune import CLIReporter
+from ray.tune.ray_trial_executor import RayTrialExecutor
+from ray.tune.schedulers import PopulationBasedTraining
+from ray.tune.utils.util import merge_dicts
+from ray.util.sgd.torch import TorchTrainer
+from ray.util.sgd.torch.resnet import ResNet18
+from ray.util.sgd.utils import BATCH_SIZE
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--smoke-test",
+    action="store_true",
+    default=False,
+    help="Finish quickly for training.")
+args = parser.parse_args()
+
+
+class FailureInjectorExecutor(RayTrialExecutor):
+    """Adds random failure injection to the TrialExecutor."""
+
+    def on_step_begin(self, trial_runner):
+        """Before step(), update available resources and inject failure."""
+        self._update_avail_resources()
+        # With 10% probability inject failure to a worker.
+        if random.random() < 0.1 and not args.smoke_test:
+            # With 10% probability fully terminate the node.
+            should_terminate = random.random() < 0.1
+            kill_node(
+                "/home/ubuntu/ray_bootstrap_config.yaml",
+                yes=True,
+                hard=should_terminate,
+                override_cluster_name=None)
+
+
+def initialization_hook():
+    # Need this for avoiding a connection restart issue on AWS.
+    os.environ["NCCL_SOCKET_IFNAME"] = "^docker0,lo"
+    os.environ["NCCL_LL_THRESHOLD"] = "0"
+
+    # set the below if needed
+    # print("NCCL DEBUG SET")
+    # os.environ["NCCL_DEBUG"] = "INFO"
+
+
+def cifar_creator(config):
+    transform_train = transforms.Compose([
+        transforms.RandomCrop(32, padding=4),
+        transforms.RandomHorizontalFlip(),
+        transforms.ToTensor(),
+        transforms.Normalize((0.4914, 0.4822, 0.4465),
+                             (0.2023, 0.1994, 0.2010)),
+    ])  # meanstd transformation
+
+    transform_test = transforms.Compose([
+        transforms.ToTensor(),
+        transforms.Normalize((0.4914, 0.4822, 0.4465),
+                             (0.2023, 0.1994, 0.2010)),
+    ])
+    train_dataset = CIFAR10(
+        root="~/data", train=True, download=True, transform=transform_train)
+    validation_dataset = CIFAR10(
+        root="~/data", train=False, download=False, transform=transform_test)
+
+    if config.get("test_mode"):
+        train_dataset = Subset(train_dataset, list(range(64)))
+        validation_dataset = Subset(validation_dataset, list(range(64)))
+
+    train_loader = DataLoader(
+        train_dataset, batch_size=config[BATCH_SIZE], num_workers=2)
+    validation_loader = DataLoader(
+        validation_dataset, batch_size=config[BATCH_SIZE], num_workers=2)
+    return train_loader, validation_loader
+
+
+def optimizer_creator(model, config):
+    """Returns optimizer"""
+    return torch.optim.SGD(
+        model.parameters(),
+        lr=config.get("lr", 0.1),
+        momentum=config.get("momentum", 0.9))
+
+
+ray.init(address="auto" if not args.smoke_test else None, _log_to_driver=True)
+num_training_workers = 1 if args.smoke_test else 3
+
+executor = FailureInjectorExecutor(queue_trials=True)
+
+TorchTrainable = TorchTrainer.as_trainable(
+    model_creator=ResNet18,
+    data_creator=cifar_creator,
+    optimizer_creator=optimizer_creator,
+    loss_creator=nn.CrossEntropyLoss,
+    initialization_hook=initialization_hook,
+    num_workers=num_training_workers,
+    config={
+        "test_mode": args.smoke_test,
+        BATCH_SIZE: 128 * num_training_workers,
+    },
+    use_gpu=not args.smoke_test)
+
+
+class NoFaultToleranceTrainable(TorchTrainable):
+    def _train(self):
+        train_stats = self.trainer.train(max_retries=0, profile=True)
+        validation_stats = self.trainer.validate(profile=True)
+        stats = merge_dicts(train_stats, validation_stats)
+        return stats
+
+
+pbt_scheduler = PopulationBasedTraining(
+    time_attr="training_iteration",
+    metric="val_loss",
+    mode="min",
+    perturbation_interval=1,
+    hyperparam_mutations={
+        # distribution for resampling
+        "lr": lambda: np.random.uniform(0.001, 1),
+        # allow perturbations within this set of categorical values
+        "momentum": [0.8, 0.9, 0.99],
+    })
+
+reporter = CLIReporter()
+reporter.add_metric_column("val_loss", "loss")
+reporter.add_metric_column("val_accuracy", "acc")
+
+analysis = tune.run(
+    NoFaultToleranceTrainable,
+    num_samples=4,
+    config={
+        "lr": tune.choice([0.001, 0.01, 0.1]),
+        "momentum": 0.8,
+        "head_location": None,
+        "worker_locations": None
+    },
+    max_failures=-1,  # used for fault tolerance
+    checkpoint_freq=2,  # used for fault tolerance
+    progress_reporter=reporter,
+    scheduler=pbt_scheduler,
+    trial_executor=executor,
+    stop={"training_iteration": 1} if args.smoke_test else None)
+
+print(analysis.get_best_config(metric="val_loss", mode="min"))