From fbac2569827558227e1e2cca71f837f5a3baa3b9 Mon Sep 17 00:00:00 2001
From: Richard Liaw <rliaw@berkeley.edu>
Date: Wed, 11 Mar 2020 01:09:08 -0700
Subject: [PATCH] [sgd] Add benchmarks (#7454)

* Init fp16

* fp16 and schedulers

* scheduler linking and fp16

* to fp16

* loss scaling and documentation

* more documentation

* add tests, refactor config

* moredocs

* more docs

* fix logo, add test mode, add fp16 flag

* fix tests

* fix scheduler

* fix apex

* improve safety

* fix tests

* fix tests

* remove pin memory default

* rm

* fix

* Update doc/examples/doc_code/raysgd_torch_signatures.py

* fix

* migrate changes from other PR

* ok thanks

* pass

* signatures

* lint'

* Update python/ray/experimental/sgd/pytorch/utils.py

* Apply suggestions from code review

Co-Authored-By: Edward Oakes <ed.nmi.oakes@gmail.com>

* should address most comments

* comments

* fix this ci

* first_pass

* add overrides

* override

* fixing up operators

* format

* sgd

* constants

* rm

* revert

* save

* failures

* fixes

* trainer

* run test

* operator

* code

* op

* ok done

* operator

* sgd test fixes

* ok

* trainer

* format

* Apply suggestions from code review

Co-Authored-By: Edward Oakes <ed.nmi.oakes@gmail.com>

* Update doc/source/raysgd/raysgd_pytorch.rst

* docstring

* dcgan

* doc

* commits

* nit

* testing

* revert

* Start renaming pytorch to torch

* Rename PyTorchTrainer to TorchTrainer

* Rename PyTorch runners to Torch runners

* Finish renaming API

* Rename to torch in tests

* Finish renaming docs + tests

* Run format + fix DeprecationWarning

* fix

* move tests up

* benchmarks

* rename

* remove some args

* better metrics output

* fix up the benchmark

* benchmark-yaml

* horovod-benchmark

* benchmarks

* Remove benchmark code for cleanups

* benchmark-code

* nits

* benchmark yamls

* benchmark yaml

* ok

* ok

* ok

* benchmark

* nit

* finish_bench

* makedatacreator

* relax

* metrics

* autosetsampler

* profile

* movements

* OK

* smoothen

* fix

* nitdocs

* loss

* envflag

* comments

* nit

* format

* visible

* images

* move_images

* fix

* rernder

* rrender

* rest

* multgpu

* fix

* nit

* finish

* extrra

* setup

* revert

Co-authored-by: Edward Oakes <ed.nmi.oakes@gmail.com>
Co-authored-by: Maksim Smolin <maximsmol@gmail.com>
---
 doc/source/raysgd/raysgd_pytorch.rst          |  47 +++++
 .../sgd/torch/distributed_torch_runner.py     |   4 +-
 .../sgd/torch/examples/benchmarks/README.rst  | 162 ++++++++++++++++++
 .../torch/examples/benchmarks/benchmark.py    | 126 ++++++++++++++
 .../torch/examples/benchmarks/dp_benchmark.py | 106 ++++++++++++
 .../benchmarks/horovod-benchmark.yaml         |  85 +++++++++
 .../benchmarks/horovod_benchmark_apex.py      | 144 ++++++++++++++++
 .../benchmarks/raysgd_multigpu_benchmark.png  | Bin 0 -> 15349 bytes
 .../benchmarks/raysgd_multinode_benchmark.png | Bin 0 -> 15172 bytes
 .../sgd/torch/examples/sgd-development.yaml   |  94 ++++++++++
 python/ray/util/sgd/torch/torch_runner.py     |   2 +-
 python/ray/util/sgd/torch/torch_trainer.py    |   2 -
 12 files changed, 768 insertions(+), 4 deletions(-)
 create mode 100644 python/ray/util/sgd/torch/examples/benchmarks/README.rst
 create mode 100644 python/ray/util/sgd/torch/examples/benchmarks/benchmark.py
 create mode 100644 python/ray/util/sgd/torch/examples/benchmarks/dp_benchmark.py
 create mode 100644 python/ray/util/sgd/torch/examples/benchmarks/horovod-benchmark.yaml
 create mode 100644 python/ray/util/sgd/torch/examples/benchmarks/horovod_benchmark_apex.py
 create mode 100644 python/ray/util/sgd/torch/examples/benchmarks/raysgd_multigpu_benchmark.png
 create mode 100644 python/ray/util/sgd/torch/examples/benchmarks/raysgd_multinode_benchmark.png
 create mode 100644 python/ray/util/sgd/torch/examples/sgd-development.yaml

diff --git a/doc/source/raysgd/raysgd_pytorch.rst b/doc/source/raysgd/raysgd_pytorch.rst
index d6865a8fb..a7cc457aa 100644
--- a/doc/source/raysgd/raysgd_pytorch.rst
+++ b/doc/source/raysgd/raysgd_pytorch.rst
@@ -525,6 +525,53 @@ You can see the `DCGAN script <https://github.com/ray-project/ray/blob/master/py
 
     stats = trainer.train()
 
+Benchmarks
+----------
+
+RaySGD TorchTrainer provides comparable or better performance than other existing solutions for parallel or distributed training.
+
+**Multi-GPU (Single Node) benchmarks**:
+
+
+.. code-block:: bash
+
+    # Images per second for ResNet50
+    # Batch size per worker = 128
+    # GPU Type = V100
+    # Run on AWS us-east-1c, p3dn.24xlarge instance.
+
+
+    Number   DataParallel  Ray (PyTorch)  DataParallel  Ray (PyTorch)
+    of GPUs                               + Apex        + Apex
+    =======  ============  =============  ============  ==============
+    1        2769.7        5143           2962.7        6172
+    2        5492.2        9463           5886.1        10052.8
+    4        10733.4       18807          11705.9       20319.5
+    8        21872.5       36911.8        23317.9       38642
+
+**Multi-node benchmarks**:
+
+.. code-block:: bash
+
+    # Images per second for ResNet50
+    # Batch size per worker = 128
+    # GPU Type = V100
+    # Run on AWS us-east-1c, p3dn.24xlarge instances.
+
+    Number   Horovod  Ray (PyTorch)  Horovod  Ray (PyTorch)
+    of GPUs                          + Apex   + Apex
+    =======  =======  =============  =======  ==============
+    1 * 8    2769.7   5143           2962.7   6172
+    2 * 8    5492.2   9463           5886.1   10052.8
+    4 * 8    10733.4  18807          11705.9  20319.5
+    8 * 8    21872.5  36911.8        23317.9  38642
+
+
+
+You can see more details in the `benchmarking README <https://github.com/ray-project/ray/blob/master/python/ray/util/sgd/torch/examples/benchmarks/README.rst>`_.
+
+DISCLAIMER: RaySGD does not provide any custom communication primitives. If you see any performance issues, you may need to file them on the PyTorch github repository.
+
 
 Feature Requests
 ----------------
diff --git a/python/ray/util/sgd/torch/distributed_torch_runner.py b/python/ray/util/sgd/torch/distributed_torch_runner.py
index 58b01c48f..5c588ed80 100644
--- a/python/ray/util/sgd/torch/distributed_torch_runner.py
+++ b/python/ray/util/sgd/torch/distributed_torch_runner.py
@@ -18,13 +18,15 @@ class DistributedTorchRunner(TorchRunner):
 
     Args:
         args: Arguments for TorchRunner.
-        backend (string): backend used by distributed PyTorch.
+        backend (string): Backend used by distributed PyTorch.
         kwargs: Keyword arguments for TorchRunner.
 
     """
 
     def __init__(self, *args, backend="gloo", **kwargs):
         super(DistributedTorchRunner, self).__init__(*args, **kwargs)
+        if backend not in ("gloo", "nccl"):
+            raise ValueError("Backend must be one of 'gloo' or 'nccl'.")
         self.backend = backend
 
     def setup(self, url, world_rank, world_size):
diff --git a/python/ray/util/sgd/torch/examples/benchmarks/README.rst b/python/ray/util/sgd/torch/examples/benchmarks/README.rst
new file mode 100644
index 000000000..42aa31f96
--- /dev/null
+++ b/python/ray/util/sgd/torch/examples/benchmarks/README.rst
@@ -0,0 +1,162 @@
+Running benchmarks
+==================
+
+RaySGD provides comparable or better performance than other existing solutions for parallel or distributed training.
+
+You can run ``ray/python/ray/util/sgd/torch/examples/benchmarks/benchmark.py`` for benchmarking the RaySGD TorchTrainer implementation. To benchmark training on a multi-node multi-gpu cluster, you can use the `Ray Autoscaler <https://ray.readthedocs.io/en/latest/autoscaling.html#aws>`_.
+
+DISCLAIMER: RaySGD does not provide any custom communication primitives. If you see any performance issues, you may need to file them on the PyTorch github repository.
+
+Single Node Results
+-------------------
+
+Here are benchmarking results comparing the following:
+
+* torch.nn.DataParallel
+* torch.nn.Parallel with ``apex.amp`` enabled (``O1``)
+* Ray (wrapping Pytorch DistributedDataParallel)
+* Ray (wrapping Pytorch DistributedDataParallel) with ``apex.amp`` enabled (``O1``)
+
+on synthetic ImageNet data (via ``benchmark.py`` and ``dp_benchmark.py``) as of 03/04/2020.
+
+Framework versions used:
+
+* PyTorch Version: torch-1.4.0-cp36-cp36m
+* Torchvision Version: torchvision-0.5.0-cp36-cp36m
+* Apex Version: commit hash 5633f6d
+
+.. code-block::
+
+    # Images per second for ResNet50
+    # Batch size per worker = 128
+    # GPU Type = V100
+    # Run on AWS us-east-1c, p3dn.24xlarge instance.
+
+
+    Number   DataParallel  Ray (PyTorch)  DataParallel  Ray (PyTorch)
+    of GPUs                               + Apex        + Apex
+    =======  ============  =============  ============  ==============
+    1        2769.7        5143           2962.7        6172
+    2        5492.2        9463           5886.1        10052.8
+    4        10733.4       18807          11705.9       20319.5
+    8        21872.5       36911.8        23317.9       38642
+
+
+.. image:: raysgd_multigpu_benchmark.png
+    :scale: 30%
+    :align: center
+
+
+Multi Node Results
+------------------
+
+Here are benchmarking results comparing the following:
+
+* Horovod
+* Horovod with ``apex.amp`` enabled  (``O1``)
+* Pytorch DistributedDataParallel
+* Pytorch DistributedDataParallel with ``apex.amp`` enabled (``O1``)
+
+on synthetic ImageNet data (via ``benchmark.py`` and ``horovod_benchmark_apex.py``) as of 03/04/2020.
+
+
+Framework versions used:
+
+* PyTorch Version: torch-1.4.0-cp36-cp36m
+* Torchvision Version: torchvision-0.5.0-cp36-cp36m
+* Apex Version: commit hash 5633f6d
+* Horovod Version: horovod-0.19.0
+
+.. code-block:: bash
+
+    # Images per second for ResNet50
+    # Batch size per worker = 128
+    # GPU Type = V100
+    # Run on AWS us-east-1c, p3dn.24xlarge instances.
+
+    Number   Horovod  Ray (PyTorch)  Horovod  Ray (PyTorch)
+    of GPUs                          + Apex   + Apex
+    =======  =======  =============  =======  ==============
+    1 * 8    2769.7   5143           2962.7   6172
+    2 * 8    5492.2   9463           5886.1   10052.8
+    4 * 8    10733.4  18807          11705.9  20319.5
+    8 * 8    21872.5  36911.8        23317.9  38642
+
+
+.. image:: raysgd_multinode_benchmark.png
+    :scale: 30%
+    :align: center
+
+
+Simple Instructions
+-------------------
+
+Note that these instructions are not maintained and may require a bit of wrangling to get working.
+
+First, ``git clone https://github.com/ray-project/ray && cd ray/python/ray/util/sgd/torch/examples/``.
+
+You can use ``sgd-development.yaml`` to setup your cluster configuration and ``ray up sgd-development.yaml`` to launch the cluster.
+
+You can specify the number of nodes you want to use with the following configuration:
+
+.. code-block::
+
+    # The maximum number of workers nodes to launch in addition to the head
+    # node. This takes precedence over min_workers. min_workers default to 0.
+    min_workers: <NUMBER_OF_NODES>  # Change this to a custom quantity
+    initial_workers:  <NUMBER_OF_NODES>  # same as above
+    max_workers:  <NUMBER_OF_NODES>  # same as above
+
+You may want to install FP16 support for PyTorch with the following configuration in the YAML file:
+
+.. code-block:: yaml
+
+    setup_commands:
+        - ray || pip install -U ray[rllib]
+        - pip install -U ipdb torch torchvision
+        # Install apex, but continue if this command fails.
+        # For faster installation purposes, we do not install the apex cpp bindings
+        # The cpp bindings can improve your benchmarked performance.
+        - git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --no-cache-dir  ./ || true
+
+You should then run ``ray monitor sgd-development.yaml`` to monitor the progress of the cluster setup. When the cluster is done setting up, you should see something like the following:
+
+.. code-block:: bash
+
+    2020-03-05 01:24:53,613 INFO log_timer.py:17 -- AWSNodeProvider: Set tag ray-node-status=up-to-date on ['i-07ba946522fcb1d3d'] [LogTimer=134ms]
+    2020-03-05 01:24:53,734 INFO log_timer.py:17 -- AWSNodeProvider: Set tag ray-runtime-config=c12bae3df69d4d6a207e90948dc4bf763319d7ed on ['i-07ba946522fcb1d3d'] [LogTimer=121ms]
+    2020-03-05 01:24:58,475 INFO autoscaler.py:733 -- StandardAutoscaler: 7/7 target nodes (0 pending)
+    2020-03-05 01:24:58,476 INFO autoscaler.py:734 -- LoadMetrics: MostDelayedHeartbeats={'172.31.38.189': 0.21588897705078125, '172.31.38.95': 0.21587467193603516, '172.31.42.196': 0.21586227416992188, '172.31.34.227': 0.2158496379852295, '172.31.42.101': 0.2158372402191162}, NodeIdleSeconds=Min=6 Mean=27 Max=40, NumNodesConnected=8, NumNodesUsed=0.0, ResourceUsage=0.0/512.0 CPU, 0.0/64.0 GPU, 0.0 GiB/4098.67 GiB memory, 0.0/1.0 node:172.31.34.227, 0.0/1.0 node:172.31.36.8, 0.0/1.0 node:172.31.36.82, 0.0/1.0 node:172.31.38.189, 0.0/1.0 node:172.31.38.95, 0.0/1.0 node:172.31.42.101, 0.0/1.0 node:172.31.42.196, 0.0/1.0 node:172.31.45.185, 0.0 GiB/5.45 GiB object_store_memory, TimeSinceLastHeartbeat=Min=0 Mean=0 Max=0
+
+You can then launch a synthetic benchmark run with the following command:
+
+.. code-block:: bash
+
+    $ ray submit sgd-development.yaml benchmarks/benchmark.py --args="--batch-size 128"
+
+    # Or with apex fp16
+    $ ray submit sgd-development.yaml benchmarks/benchmark.py --args="--batch-size 128 --use-fp16"
+
+You should see something like:
+
+.. code-block:: bash
+
+    Model: resnet50
+    Batch size: 128
+    Number of GPUs: 16
+    Iter #0: 354.2 img/sec per GPU
+    Iter #1: 354.0 img/sec per GPU
+    Iter #2: 353.0 img/sec per GPU
+    Iter #3: 353.3 img/sec per GPU
+    Iter #4: 352.8 img/sec per GPU
+    Iter #5: 348.5 img/sec per GPU
+    Iter #6: 352.5 img/sec per GPU
+    Iter #7: 352.5 img/sec per GPU
+    Iter #8: 352.1 img/sec per GPU
+    Iter #9: 352.2 img/sec per GPU
+    Img/sec per GPU: 352.5 +-3.0
+    Total img/sec on 16 GPU(s): 5640.2 +-47.2
+
+
+You can run ``ray up benchmarks/horovod-benchmark.yaml`` to launch an AWS cluster that sets up Horovod on each machine.
+See ``https://github.com/horovod/horovod`` for launching Horovod training. ``horovod_benchmark_apex.py`` can be used with ``horovodrun`` to obtain benchmarking results.
diff --git a/python/ray/util/sgd/torch/examples/benchmarks/benchmark.py b/python/ray/util/sgd/torch/examples/benchmarks/benchmark.py
new file mode 100644
index 000000000..6155d9e21
--- /dev/null
+++ b/python/ray/util/sgd/torch/examples/benchmarks/benchmark.py
@@ -0,0 +1,126 @@
+from __future__ import print_function
+
+import argparse
+import torch.nn.functional as F
+import torch.optim as optim
+import torch.utils.data.distributed
+from torchvision import models
+import timeit
+import numpy as np
+
+import ray
+from ray.util.sgd import TorchTrainer
+from ray.util.sgd.torch import TrainingOperator
+
+# Benchmark settings
+parser = argparse.ArgumentParser(
+    description="PyTorch Synthetic Benchmark",
+    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument(
+    "--fp16", action="store_true", default=False, help="use fp16 training")
+
+parser.add_argument(
+    "--model", type=str, default="resnet50", help="model to benchmark")
+parser.add_argument(
+    "--batch-size", type=int, default=32, help="input batch size")
+
+parser.add_argument(
+    "--num-warmup-batches",
+    type=int,
+    default=10,
+    help="number of warm-up batches that don't count towards benchmark")
+parser.add_argument(
+    "--num-batches-per-iter",
+    type=int,
+    default=10,
+    help="number of batches per benchmark iteration")
+parser.add_argument(
+    "--num-iters", type=int, default=10, help="number of benchmark iterations")
+
+parser.add_argument(
+    "--no-cuda",
+    action="store_true",
+    default=False,
+    help="Disables CUDA training")
+parser.add_argument(
+    "--local",
+    action="store_true",
+    default=False,
+    help="Disables cluster training")
+
+args = parser.parse_args()
+args.cuda = not args.no_cuda and torch.cuda.is_available()
+device = "GPU" if args.cuda else "CPU"
+
+
+def init_hook():
+    import torch.backends.cudnn as cudnn
+    cudnn.benchmark = True
+
+
+class Training(TrainingOperator):
+    def setup(self, config):
+        data = torch.randn(args.batch_size, 3, 224, 224)
+        target = torch.LongTensor(args.batch_size).random_() % 1000
+        if args.cuda:
+            data, target = data.cuda(), target.cuda()
+
+        self.data, self.target = data, target
+
+    def train_epoch(self, *pargs, **kwargs):
+        # print(self.model)
+        def benchmark():
+            self.optimizer.zero_grad()
+            output = self.model(self.data)
+            loss = F.cross_entropy(output, self.target)
+            loss.backward()
+            self.optimizer.step()
+
+        # print("Running warmup...")
+        if self.global_step == 0:
+            timeit.timeit(benchmark, number=args.num_warmup_batches)
+            self.global_step += 1
+        # print("Running benchmark...")
+        time = timeit.timeit(benchmark, number=args.num_batches_per_iter)
+        img_sec = args.batch_size * args.num_batches_per_iter / time
+        return {"img_sec": img_sec}
+
+
+if __name__ == "__main__":
+    ray.init(address=None if args.local else "auto")
+    num_workers = 2 if args.local else int(ray.cluster_resources().get(device))
+    from ray.util.sgd.torch.examples.train_example import LinearDataset
+
+    print("Model: %s" % args.model)
+    print("Batch size: %d" % args.batch_size)
+    print("Number of %ss: %d" % (device, num_workers))
+
+    trainer = TorchTrainer(
+        model_creator=lambda cfg: getattr(models, args.model)(),
+        optimizer_creator=lambda model, cfg: optim.SGD(
+            model.parameters(), lr=0.01 * cfg.get("lr_scaler")),
+        data_creator=lambda cfg: LinearDataset(4, 2),
+        initialization_hook=init_hook,
+        config=dict(
+            lr_scaler=num_workers),
+        training_operator_cls=Training,
+        num_workers=num_workers,
+        use_gpu=args.cuda,
+        use_fp16=args.fp16,
+    )
+
+    img_secs = []
+    for x in range(args.num_iters):
+        result = trainer.train()
+        # print(result)
+        img_sec = result["img_sec"]
+        print("Iter #%d: %.1f img/sec per %s" % (x, img_sec, device))
+        img_secs.append(img_sec)
+
+    # Results
+    img_sec_mean = np.mean(img_secs)
+    img_sec_conf = 1.96 * np.std(img_secs)
+    print("Img/sec per %s: %.1f +-%.1f" % (device, img_sec_mean, img_sec_conf))
+    print("Total img/sec on %d %s(s): %.1f +-%.1f" %
+          (num_workers, device, num_workers * img_sec_mean,
+           num_workers * img_sec_conf))
diff --git a/python/ray/util/sgd/torch/examples/benchmarks/dp_benchmark.py b/python/ray/util/sgd/torch/examples/benchmarks/dp_benchmark.py
new file mode 100644
index 000000000..80fd80a5b
--- /dev/null
+++ b/python/ray/util/sgd/torch/examples/benchmarks/dp_benchmark.py
@@ -0,0 +1,106 @@
+from __future__ import print_function
+
+import argparse
+import timeit
+import torch.backends.cudnn as cudnn
+import torch.nn.functional as F
+import torch.optim as optim
+import torch.utils.data.distributed
+from torch.nn import DataParallel
+from torchvision import models
+import numpy as np
+import os
+# Apex
+from apex import amp
+
+# Benchmark settings
+parser = argparse.ArgumentParser(
+    description="PyTorch DP Synthetic Benchmark",
+    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument(
+    "--fp16-allreduce",
+    action="store_true",
+    default=False,
+    help="use fp16 compression during allreduce")
+
+parser.add_argument(
+    "--model", type=str, default="resnet50", help="model to benchmark")
+parser.add_argument(
+    "--batch-size", type=int, default=32, help="input batch size")
+parser.add_argument("--num-gpus", type=int, default=1, help="number of gpus")
+
+parser.add_argument(
+    "--num-warmup-batches",
+    type=int,
+    default=10,
+    help="number of warm-up batches that don\"t count towards benchmark")
+parser.add_argument(
+    "--num-batches-per-iter",
+    type=int,
+    default=10,
+    help="number of batches per benchmark iteration")
+parser.add_argument(
+    "--num-iters", type=int, default=10, help="number of benchmark iterations")
+parser.add_argument(
+    "--amp-fp16",
+    action="store_true",
+    default=False,
+    help="Enables FP16 training with Apex.")
+
+args = parser.parse_args()
+os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(
+    str(i) for i in range(args.num_gpus))
+
+cudnn.benchmark = True
+
+# Set up standard model.
+model = getattr(models, args.model)().cuda()
+model = DataParallel(model)
+
+optimizer = optim.SGD(model.parameters(), lr=0.01)
+
+# Apex
+if args.amp_fp16:
+    model, optimizer = amp.initialize(model, optimizer, opt_level="O1")
+
+# Set up fixed fake data
+data = torch.randn(args.batch_size, 3, 224, 224)
+target = torch.LongTensor(args.batch_size).random_() % 1000
+data, target = data.cuda(), target.cuda()
+
+
+def benchmark_step():
+    optimizer.zero_grad()
+    output = model(data)
+    loss = F.cross_entropy(output, target)
+    loss.backward()
+    optimizer.step()
+
+
+print("Model: %s" % args.model)
+print("Batch size: %d" % args.batch_size)
+device = "GPU"
+print("Number of %ss: %d" % (device, args.num_gpus))
+
+# Warm-up
+print("Running warmup...")
+timeit.timeit(benchmark_step, number=args.num_warmup_batches)
+
+# Benchmark
+print("Running benchmark...")
+img_secs = []
+for x in range(args.num_iters):
+    time = timeit.timeit(benchmark_step, number=args.num_batches_per_iter)
+    img_sec = args.batch_size * args.num_batches_per_iter / time
+    print("Iter #%d: %.1f img/sec per %s" % (x, img_sec, device))
+    img_secs.append(img_sec)
+
+# Results
+img_sec_mean = np.mean(img_secs)
+img_sec_conf = 1.96 * np.std(img_secs)
+print("Img/sec per %s: %.1f +-%.1f" % (device, img_sec_mean, img_sec_conf))
+print("Total img/sec on %d %s(s): %.1f +-%.1f" % (
+    args.num_gpus,
+    device,
+    img_sec_mean,  # we do NOT scale this by number workers
+    args.num_gpus * img_sec_conf))
diff --git a/python/ray/util/sgd/torch/examples/benchmarks/horovod-benchmark.yaml b/python/ray/util/sgd/torch/examples/benchmarks/horovod-benchmark.yaml
new file mode 100644
index 000000000..b72651500
--- /dev/null
+++ b/python/ray/util/sgd/torch/examples/benchmarks/horovod-benchmark.yaml
@@ -0,0 +1,85 @@
+# An unique identifier for the head node and workers of this cluster.
+cluster_name: horovod-pytorch
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers default to 0.
+min_workers: 1
+initial_workers: 1
+max_workers: 1
+
+target_utilization_fraction: 0.9
+
+# If a node is idle for this many minutes, it will be removed.
+idle_timeout_minutes: 50
+# docker:
+#     image: tensorflow/tensorflow:1.5.0-py3
+#     container_name: ray_docker
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-east-1
+    availability_zone: us-east-1c
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
+
+
+head_node:
+    InstanceType: p3dn.24xlarge
+    ImageId: ami-0698bcaf8bd9ef56d
+    InstanceMarketOptions:
+        MarketType: spot
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 250
+           # SpotOptions:
+           #     MaxPrice: "9.0"
+
+
+worker_nodes:
+    InstanceType: p3dn.24xlarge
+    ImageId: ami-0698bcaf8bd9ef56d
+    InstanceMarketOptions:
+        MarketType: spot
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 250
+        # SpotOptions:
+        #     MaxPrice: "9.0"
+    #     # Run workers on spot by default. Comment this out to use on-demand.
+    #     InstanceMarketOptions:
+    #         MarketType: spot
+
+setup_commands:
+    - pip install torch torchvision ipdb
+    - pip install ray[rllib]  # enable autoscaling
+    - git clone https://github.com/horovod/horovod || true
+    - git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --no-cache-dir  ./ || true
+    - tmux new -d -s my-session "HOROVOD_GPU_ALLREDUCE=NCCL HOROVOD_GPU_BROADCAST=NCCL pip install horovod"
+
+
+file_mounts: {}
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands:
+    - cat ~/ray_bootstrap_key.pem > ~/.ssh/id_rsa
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands:
+    - pip install horovod
+
+# # Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands:
+    - ray stop
+    - ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --object-store-memory=1000000000
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands:
+    - ray stop
+    - ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076
+    # - nvidia-docker run -it --network=host -d --rm -p 4321:22 horovod:latest bash -c "pip install Pillow==6.1; sleep infinity"
+
diff --git a/python/ray/util/sgd/torch/examples/benchmarks/horovod_benchmark_apex.py b/python/ray/util/sgd/torch/examples/benchmarks/horovod_benchmark_apex.py
new file mode 100644
index 000000000..251511da2
--- /dev/null
+++ b/python/ray/util/sgd/torch/examples/benchmarks/horovod_benchmark_apex.py
@@ -0,0 +1,144 @@
+from __future__ import print_function
+
+import argparse
+import torch.backends.cudnn as cudnn
+import torch.nn.functional as F
+import torch.optim as optim
+import torch.utils.data.distributed
+from torchvision import models
+import horovod.torch as hvd
+import timeit
+import numpy as np
+# Apex
+from apex import amp
+
+# Benchmark settings
+parser = argparse.ArgumentParser(
+    description="PyTorch Synthetic Benchmark",
+    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument(
+    "--fp16-allreduce",
+    action="store_true",
+    default=False,
+    help="use fp16 compression during allreduce")
+
+parser.add_argument(
+    "--model", type=str, default="resnet50", help="model to benchmark")
+parser.add_argument(
+    "--batch-size", type=int, default=32, help="input batch size")
+
+parser.add_argument(
+    "--num-warmup-batches",
+    type=int,
+    default=10,
+    help="number of warm-up batches that don\"t count towards benchmark")
+parser.add_argument(
+    "--num-batches-per-iter",
+    type=int,
+    default=10,
+    help="number of batches per benchmark iteration")
+parser.add_argument(
+    "--num-iters", type=int, default=10, help="number of benchmark iterations")
+
+parser.add_argument(
+    "--no-cuda",
+    action="store_true",
+    default=False,
+    help="disables CUDA training")
+parser.add_argument(
+    "--amp-fp16",
+    action="store_true",
+    default=False,
+    help="Enables FP16 training with Apex.")
+
+args = parser.parse_args()
+args.cuda = not args.no_cuda and torch.cuda.is_available()
+
+hvd.init()
+
+if args.cuda:
+    # Horovod: pin GPU to local rank.
+    torch.cuda.set_device(hvd.local_rank())
+
+cudnn.benchmark = True
+
+# Set up standard model.
+model = getattr(models, args.model)()
+
+if args.cuda:
+    # Move model to GPU.
+    model.cuda()
+
+optimizer = optim.SGD(model.parameters(), lr=0.01)
+
+# Horovod: (optional) compression algorithm.
+compression = (hvd.Compression.fp16
+               if args.fp16_allreduce else hvd.Compression.none)
+
+# Horovod: wrap optimizer with DistributedOptimizer.
+optimizer = hvd.DistributedOptimizer(
+    optimizer,
+    named_parameters=model.named_parameters(),
+    compression=compression)
+
+# Horovod: broadcast parameters & optimizer state.
+hvd.broadcast_parameters(model.state_dict(), root_rank=0)
+hvd.broadcast_optimizer_state(optimizer, root_rank=0)
+
+# Apex
+if args.amp_fp16:
+    model, optimizer = amp.initialize(model, optimizer, opt_level="O1")
+
+# Set up fixed fake data
+data = torch.randn(args.batch_size, 3, 224, 224)
+target = torch.LongTensor(args.batch_size).random_() % 1000
+if args.cuda:
+    data, target = data.cuda(), target.cuda()
+
+
+def benchmark_step():
+    optimizer.zero_grad()
+    output = model(data)
+    loss = F.cross_entropy(output, target)
+    # Apex
+    if args.amp_fp16:
+        with amp.scale_loss(loss, optimizer) as scaled_loss:
+            scaled_loss.backward()
+            optimizer.synchronize()
+        with optimizer.skip_synchronize():
+            optimizer.step()
+    else:
+        loss.backward()
+        optimizer.step()
+
+
+def log(s, nl=True):
+    if hvd.rank() != 0:
+        return
+    print(s, end="\n" if nl else "")
+
+
+log("Model: %s" % args.model)
+log("Batch size: %d" % args.batch_size)
+device = "GPU" if args.cuda else "CPU"
+log("Number of %ss: %d" % (device, hvd.size()))
+
+# Warm-up
+log("Running warmup...")
+timeit.timeit(benchmark_step, number=args.num_warmup_batches)
+
+# Benchmark
+log("Running benchmark...")
+img_secs = []
+for x in range(args.num_iters):
+    time = timeit.timeit(benchmark_step, number=args.num_batches_per_iter)
+    img_sec = args.batch_size * args.num_batches_per_iter / time
+    log("Iter #%d: %.1f img/sec per %s" % (x, img_sec, device))
+    img_secs.append(img_sec)
+
+# Results
+img_sec_mean = np.mean(img_secs)
+img_sec_conf = 1.96 * np.std(img_secs)
+log("Img/sec per %s: %.1f +-%.1f" % (device, img_sec_mean, img_sec_conf))
+log("Total img/sec on %d %s(s): %.1f +-%.1f" %
+    (hvd.size(), device, hvd.size() * img_sec_mean, hvd.size() * img_sec_conf))
diff --git a/python/ray/util/sgd/torch/examples/benchmarks/raysgd_multigpu_benchmark.png b/python/ray/util/sgd/torch/examples/benchmarks/raysgd_multigpu_benchmark.png
new file mode 100644
index 0000000000000000000000000000000000000000..325891d95e146b65dee8719afcd8704aa4749446
GIT binary patch
literal 15349
zcmb_@c|4Tu+qWSUMH?+brPW%AvL{KAeH$~DNo1$&J1Hr$v=|{|9m`-a_GM5~WM}Lq
zF?L3FGnV(f)P3K-=lMO)`+T1F^Ugn9GuL&l$9Wvz@Ao*aml|qH`&f>%(9zNDyL;!B
z79AZuoQ@9Ww3`Wh^ONj`r=ydqxO+=p$GvZMkoDc+uDJTgmo6VTAkXml<?hQ>H+G-<
zD4hTG{=Vbl0pG0W%|b6=+d@lbl1>HG=D)sh@Z^n~aGe{kSdQ~79DSrQDsiAzVd&;)
z=*g4c=Y5kmt`WE#5@KR}-C9~jz2(L`pG;=v%;g|;SM`hSb#<?R0hsBY<%EI1FjhJ`
ze}6_gx*OH>|I5phDA2$^&;C!B!<2{L-`^)}kVOXPo24k9<5seZhl1}R;qdpw@zA4}
zYb&>xCOcktugvyaNV(2j5FgCSt(r=ca-PsoUhJ+I@>w<Fx?=WXb4s6(kBRqei1(V4
z@EQrdW;;#H)+C9#%?*5|u&@o5yA^M5lC}$nKLzzKkQ+lP$2V6=ZXMs_Wu(U3BwT0S
z$3FZ~{p2xW5p<U9>x;kN`cd9MXxll;DUBk7Y`-|I(@mH!Gb9>&kU3Uk!%j%N=^`(U
zzE$t*P?xjYq%IfiIdrZEli(Zk%dN@TBhuC5p^7`<h@IT~Z?a3a3$$9xr^G0W1Rvt8
zK=X<#jGkz)56-}}_l3mZc!HmOi}d0>A_H-(DR%I7iDg^TD*+0->{@k`fbk6v+HCRL
zp}Rprt3zJl{<`uEdKLL<dRUjzq+Qtgk58l6<u)$OwtW4vz~8iR24&8JtDSBHOO@fg
z2R?3GJw+lAXl~s(Mw%ET!Pn=^GjmbY#lw@7djhFqHpE-vqn96i>G=}MHM{m}WX@@2
zruTuUMf0m?xw=_;ii3As666TC(sj<J?oUl#<&p8cReMXELGLPyR<z!m7e=QrTEvW9
z8@AZS(!;ob#>wx|qV}=WwUO}IaKDWQDS@Vxp-L;C+u~?_EA)AoK%qZ}or(L;&wKid
z>|!hDYgz4MUh^sCA3BYS^Ma`eN&Qkg6vlO}Iq(w6`n3v7g)7g&NZ`T+ut3lEaSb%<
zVc6gH-enHuk?B)Srl?eBy2AvF4G!UwI-a~r^H^-U<|K?VPnO;Tk7maHsM4Sf#mjj8
zY1?Lbs%y^B1K;s1!Dl6Ud2X;j5N8P!ATm)Hp1ZEKrEyES+P-{fM#|-j>`+&U5_ajR
zT54C>gNeIh9&FRpe!!kz%8K{K=2Z(RROQA(BeB@Qq`%UuSU_~PB!DB18f$fIi`q?a
zRiadSF6YZ_46_%R)&-j`?h&zwjI7N>Y<4OqI_h^+4VBvumSHRPgM)(yBtNchv}|vL
z(+KEjba!i_LJOjr1zkvtgr5RN$dJ>n(fXoTxDnW@U3<#i9J!AT3jKYbUmlwMQS<a-
zWENW8*f?Hhdt=eQ*BY@bu{Wnh7Bwj<rk0S3J}1->E=#>&(|C6}%{QbfdF4O`*6Sr&
zKY!XmvfrM6v4!Z0zE`a^{R3~+Vc1J`xwUEhiRN!ck0y$#9?GuhN=kVgQ|u7^LDV9w
zf80{0&6V~uMCA71!;XD$>b^d`P7&b?(d4KZ@!j-#%}j!Eri)zWYgZGg1`{rEaxHcK
zWP!thpW8E?Ifkx=lIERGp{$s(9yD`Im0`Z?ETa3{H9eEvg`;(>cl*9sD;>>#>x4_T
zK1b0lVOg16e0tL#q1^HiEX4BGCT-BI`(oIoCWQ_wk9uU37<;5HK3}6Q$^7-bQfng(
z=0(dzP0Zr_$`AFCz(#q@T7KS#(d{kD_JFf*{I?vhUub>)f0^+G)H*{5c;bf^T<t7s
zeSfMbdmGyov1XH{w+RWn%T9$kXAb3?&a!e0sKrZLA&YGD?tgk7na=vg$oHH5(~~H&
zbF~&B)e^I(P_*;48&0d~c6G2@%*jaq<$lr@Q8g<$+E%a3nnQz(NmdT+Zp-r3&e<gx
z+U?No*6FsKeN>V$^J8*#MzU$4u;p>xTF>yD^jAmTEEV5u&o^%zpt+=Wr)wq$b1y%n
z8vFI#9?{9OHkg!=^jwk85`>OBv8}>&et20$sgIgnwS10PHq)pa`6}QAKX;%iNknl`
zJ<ZT(eRFG-;1-!YmK@v}BuskJ6ZI6qxEy!Y2<!OuDr$35-OsUOy?G88LmpU~#XA^j
z?P$NEqJmqnMuHLzQHY*&X=)5gM6s&oV|M#3#*#{AFZPtBw<c@kwvM%^rJvPd7L#(D
zb6(VmVzmolk^8n3tz7iByh;T}Fy?O*>9yOx3;SUoyzjB4`M&sb(k@f)`X#IsTDttn
z$eT+>pPZvz@u5b9YZY(Rt^!Z*cQ?bJLXe)=5Gn9B9upSelYaN`n_6-rce1hmx;b?y
zSt-0Mu2`+|S@$s`EfY?dKxF>PbovmNtXN`)_b>XVzFBG09hmT}k^p>_^|1*VmK(ns
zbo<lzFT~y4<8Vl_ot@|Cy5sIGrZ(|dTcr^*2DC3BO*`<zR9TcYFV^y(*}KD3hHuC#
z^jFYp;5o0`iLf+e>=KOQVJn`q?ZbboOxvuojIWBkmnh?&DPi!E9!)<8{A5UBXIgpA
znLZ}@>V{r_4(Zd&z%CxNk;+me^%>BF{L2Vdy>l=XA9}o#hD}IL(q04(^(6Ko5o>Un
z5XG<Ww}eE%1cua9gcSJY0&j5dqc%uKM;xwtAx}>2e%iD&S=G%<ANPi>gUvYdt^qUF
z-8}nb3QE*{{>MSgxeeC?HbyJiwV9}-UqXj{eO~G2P^AoA^FH%4>kef`biP4n`;(>0
zn2RxH-Icds0?{5R+VL6R<&gab75=H{WV4$|2m$Rqa9IszWPtZcwiUNUp^U9e4Mxep
zG&7MmFI;gMKBUoaZ~3wv+Sq)x4gNO&h0W}ZHXOfZ;r?~LEFF3@`_X77K{-|T`NesR
z>>CO8T(;UgdcCBEGhh8MmaAP0o&3Y<`y%zK7cerlQi7!MQ+*Nls#*MU0>X{;QB6yg
zQ>=$5L$@-I9%97an!c4*bLyUtK>)|wssy!IX5@*a?8dE?Q`V|4R(&>_E<K3MQrkZ3
zRcBn0wVT7Z;26$1=Dhn2bVJz@n|^h3p2>F>JARE*1huZ0dh3(PHnsa5J=#RwB@^CZ
zE!Ei+^+{jThSnvSJkMQqH(mIo@_AGsgArbBB14d1RxH0C=O~G?Qs}%WU&xZN4(Bq!
zG1%e#Q?=PVo37mt3je4qh00i$+mJ}*#yPu$4K4QB<evR-E}9aV?VlQ#9zzpTlhP`T
z*jR7zbB)v0-7|Kxy4~$%*O3dooY;IexTNxm{5Q9<<+!*JMkW&CNETxwFPWBAX&RZ8
z^Zi~SV-}tPjea_Y(T=Ke(Bhvq#*-tbq_u=vbjB01X9bO`7lzlDyG_)muB9Brd9Sr?
zHB-U?OgQ3#Flx*6$)Gq*JFp!WGCaqH>s%P6?Pij@*7e6dt1$~|TF#G8eu|>rW`$RM
z%9pfYQe>4)eH^SIMaLa&<!Lwb%Qa`KA@d$E+T4&Zwx{}QyD|z-Y072WG7Byz&lh8N
z^Y*T9sn)1<S@5B(JcY_FeQ=$_%eKyr>d*OEbeZ_on-nmv2N?zZJUp?o@T$|=fv?kV
zcGrXvi{}?~>(W1t2u$R&z$N37t-N{P>vUIIHVSlkq#)umn*SK&@(s+@u5GdC4oB$R
zDYjO0QIjKbT`qguL|87xZj7HPqj@H6rOWnQ#IAet{WY^@W<f%L&8NL4fd_?9s&SHH
z`FRug_lW$f?3MabOYX2nKlIpv8C3Mt1NcUUD(^(+hn%=bStXLLPuhlgD^;z``Fe+P
zH?vBUzLN2l@=>w-H1njmU`n+iC#;coGvg<b!-gt56`R)am@0f6BjC{du==1MLZ8yM
z(4MbdvT#Ij)U<kGbwm>$>^giC>(Uz)X>YZ^B3OgtPKgP!LR+iKFNOXMzl`qpOw@)I
zC+R>ro*qqjTVBNNw?B<&Pw>T*%_$;{Gn)Cj7Vf!h_4YL%G*X;!K>jKxZ8pbU(dnHM
zb&)h;4)2Sy?!_F(Z8)?_X4wgN=icyfBd|?fTd&5T_5B?91M&Ch(Q6DypD1UvNq+yy
z-EawBoO8;o{sM~m8yuz*sfedVPjn{DojN3fVx;j}TEsQ`-~zIFx2|l%p9(9nJ<qzw
zZ=@?&6)i<%uy{N9DWb_gHCS0{(!KG96s7<eyZQZ!{NN%dnSNMf0{Vq+^k+m=VSIXi
zur6nf-9_VSEtL!k-3gw6emDSyNfwQaG?}!&W)`RY?6=oR@=whVEfi^!JTefTavbNK
zptMf~E%?&o<59a-nf!xy&!MdPe}(gRNWPs~P;uqj1rr!&(8C3u@?LC)hc~nx2zMRm
zd!ZAZ{<cCq=@v{ysZO%)qc-pHjnLV4p9q|*PTyo^0Fh~m@&e0uf%~#j`Cf^)%KB2q
z%X?b8gtnT=eVY95A1*Ad-IFY}iJn|a@hz?jkLg;b725h_y`(OEX!1Xssj<vsvV!pQ
zRx!KcMi!a&KPzcXWRpc?nPFWFoDm=#pF^DIc<0Pv?!FnOpX0l!M-pqQCr#b#e}u~y
z0&6yl+!S3NOqLNAh;$JlE-3%wk<j1c26T<Bo3x{?x!bl>X!ng?9)D+D-R_L2qJ~v*
z2;LFWvzBP@YK~De$IchH|N5CC8FBs)?p8wLB>uxvoYo)r3wu#R3cG>%<fvL9rObfh
zI)j2;pLSNh<D=s2oWp17Cl*OjcNXXU&Xp#6!CGQr^NwCNLJ>x9`*fEeU1_}>8(-hY
z4u(=^hu9xZyIdF#tq#I^nD)Tw9N$pQGbqogp6Qers!uxm3+m(1KBqe8x(~n$tE}~3
zq}%E0f8uicaeyE0u*4mv2QJ@bj?{-omDlUOtEcRx+a-8XA-9T}>Gk#TBX@G7{!E<P
zP#&LBm>F60b~uk+2S#1w7Zojpv>pBW`fiI7yY<#Ya`<UKhdzA9XSW1N7i+VPmxs>V
zj(v|_S!fg~Czaav=08hFByA9p4y6{H{3op9nb~>n(TalzP8BE9S{iO2mGW3HBaXQg
zrO#|FL=k#o9jYdY2eRL;udf%`bmy3MZ2k)O6V%({yZs6$2cS`r=kk>4^jeIVjk5O_
zhzzAh88=Yfx)*q*T(LgXNlcxw+@l}<w0(U%ORvh;7st!3DnA9W_rE$OdU^!d5>N0}
zmuH3*9X+Rp8oeHFp{?U*NN}+2#@a7U1<f*;&G+}0h4-5u#OXX{`qYr57-9=S0gS$R
zD$8t7-h-Tc?bPLN!j{;ag(bBq_Sz9vnP@=m+=N5?b%!!4@KvK^a_>;3*Pud4IFHN}
zeLgV?^;N5yI{%5C&rjfmZGTY{WA^fNca%r97G2Es$4LT>sOt7rE%?S_!uCvUGV!A7
z5ko)Bz3888w%%Eon|*-&@{JT4TW{{#4Zk6=7d}i()W)cl5_D@SX>;Y`)}0r;HkZ5Y
zY62KtsY{bzmoi6nPNf&#QR<UTS@SOhFkU}#ZC#NS{!3OtNV_W1bg05(PK>b{P{um!
zMD6gM?Bi6nyqVsDdLR4iOX+VQE<WJUVUbn4suP`Ms>2%M+XxVb(T&jMxUQRN$~uc$
z*DqYS@Mw|Mmss+6WZ=qsal07gxxCMau^LWnq<}u!lDwGk_Fn02)+d0{J9T6ljyK04
zZK>{y9oH*OYT%SOQ46)s7R`xPie?XchhyhL2nVjBnB*&U#mInBFq@TXWF5`Ij0F-v
zdFwbb#SFA!FX3@~`&T`$(w1?_qx&zPo)FT`#sJad0tVKoFI(lP^GG8wU0E+tG4UqH
z%L^LmZ&tJSSxad2w7xE5hsTfrB{dA>5<u0_URh?#Q|B`MJFlb$>qX|uk&|$VE&H_2
zVn~OvqX7?F;*Dl#>o*cxbm47o?HA>?;|`;jHkS=z#uF8S3@>Rk8sI6|>W^soxFQ{a
z(Ctu`6I%sKzLhzbk}jpw9a60@=b;=PX6E(&fsT@ui#F#GLS>x2oj{3FnwR<#o4^id
zk!9&?Q*)bi*<{NWX|wdkZJ06&25LuIlmTRW0dZ*=CK?0yZOMEx<p>+RZQ>)}(TOfo
z%%jXJos+i$sVG)-#r78M^3#jp06ShsWjJb(GxQGg9(J7@m<+=|<%s1!sH>Ja;BqFX
zL6$fv#EbLJdh0N}hY5Qt;Q*Te%eBnUXknu%{y`twN@;`fu%gbM%m9}5)ebN3k{3mS
zOUT+MH-!$xBaU{q@3*aELFAbRvfgc@NBbrZO+U53&mXj{(U>;Gau&NcO6vjhZ5{Vp
z_;&uT$f6}llRRcesv2z4mX~hZ@UCXr`hu&it$n8!`5d76a&)n5M-<XvwMO|IYTXxs
z@OwIqsm*0~WHB6eY|Zbcz#!A!4R4byWkx>Lc~U1CqmR=(`;n7@M4znp2F1LNs@6K2
z6wG%5chKiHo)deCky$nO<VWHY5I9F31Tt?M)N}`6CcW>>!)`R*Nl2{{qZ+Xst<gBV
z(M$~;rW~d$avWUVvivEuXbDpxA<en5t#1v++NvHr@>uIW9CN6tMWozvai#P0k*&EY
zCyw|8_Q%6&bBF}zNBj30u?XeSqigRQFbf817S2r4ec5IcQi=>$W6j*?3`iy3@T=Z$
zTiRKX9Ow*Gv{6e|G$O#6{DwdM$0S#1ILF=qXBIU~xiiaMjs*UfkE#!<CeQER3;*VQ
zJIS9+Gs-yFZY9N7H@_O<60CtKSTc?qV+{&Ig+w_>EMk5+2U`F^LHI{{-7VNTz3O0M
zCg&Cwg?n)X2d46<x;^9WGP5?_-W40=BEwIl%SSbyCSGjY*uei1KHlo4Ke4Ci4P(Z#
zzzJMZ-08jT`I?QU?}goW=kgg#Vs^jX14p<bOgdWxOL#!Ib9l)q{`867v_;wxV-N#A
zzG6UQh)0d;$ZkI+3sX>UX6(3h9#3rgRt@{7^0^1@*VU0lon1UugG;)Uyvig6XEs^n
z(7beMlJ;4WJf`qcO*4=oP%iac)q{NB<S`8KHOf-cFBuLP4nP`ASj$G~6S%|GI?cmu
ziy3GeJUOvCSv%5NQ{^fFXdCHWx1U@>oL_aON9R<85FPGr&UPy6=n26Q!u;yopPx-8
z9ke@NLz-dGyVOeU4k&h`gUn|SJoDt?gPoYW92i$B6^TjdR89%L8E2=PNMj>X35c*I
zW8G>NNo%8*oVZ&;hdsmD;V)$st{O*8Wu>Z~QuYDh^PaA>b;fQz<0(pN);FEJERl#>
zjS3>OP{@z2a9NG?I(d@P6Ge^``jGUFs#^3|HYM4EM!V!uLz&sun+_byp2w&)Q7TuB
z8vtRv!p<bfE6Fie`vw>qJzDMVRC!k6y|@(@gh2HASuWg?1iwmTf^RrF+gMveH+ovs
zXFot!7Y}C{cABO-YE<j?UOYNQ%HIv=e>+b-mO@Ec*4B@-MhKY%exJ8HcKGweQ${F&
zF`&~IbQn`uv~=!IO^GnFFrH(KLLG(Yo4w4sqx*XDs^+|}Ht0T`U3s3r79f@CL`sUr
z>EjsX9@3qw?hkIaWyr|M=^76U<kvcjeb}z7xsizEw%nqsSr0BIBq|$oB&msJBz0R{
z++}%ejq~}un_gW8DV3DHnr^Pxk=7BwVH3^FfR+{Lvl2FWd>#Q!l|D^YpMV_ZFZn*w
zB+~1DmgT18VCJP%536IDq&L*^vrZiM@tmg_&6|`KweP%gNj0_FC7A7%rzSz2&+fn?
z5*`e`(82psk-oqV>Bs?uV^(-=?H>=*En)*=iRz>)dm+riW!<}n^q6NC*6<@|v8{{|
z44|0{+FP3E^XzXcR|Ao9UzLg3eA2!hVH<?{e-*|5)8+ppncp;6o1S}6v@ycBC#cCQ
zJgV1C(uazVlW=-oyB<4Y)7{sWs&a_!ytut!H^FJN+-9a!PDt(gy|qP8qQb4hmB1sm
zlexZ;k9yX|Vg&{F<20zl)=RYVw>)q57$_<#u7o;x^Ihr8(r>XZpB0al_3_jqO)j>`
z6$4b*3y_o+^@F6#R8P}2lC4PPl9pan8-OG;3ES%lDjRaWt7gY@Rw~v^+ZC>yng(iP
z9Xj^8O>QNT$NdMa_+E=%Nv3BD?;EHqY)~(fy-+abs+B2-FyAL1Et{q)-(~;y3~M-`
z*PvkF?;l9eY}Db0sdSu2=2?xOsSo3JitOvsH?gA-wtcm-nLPdSIbyd@%)0C48ruN`
zQw*S${)=E$A#K10RTFGe4`5FgIgsq-sBPgh-_9pU3rr^>(zN!#W8#-_GsI6Gi(>|{
z;&uZL)TIt}Cr6VsbOul+(nP}hl|3%7f`z#aYn%oG_+~$06FEu&lAb{br;uUe%|^74
z-EeK-+_Ds<LevqT&V2_vY3kFg?z`~z&a^ys)px#zdF3hZMs;72-E<gJIIkR!Q=dBs
zh8l&zhHir{D*$`-KVe{QFt2_SW#Zf0S7;N3^WK^!Y&A0NJAOFY{>!Ulbq!+r44l*V
zovCv#ohRGtsLRt9>@rIckWLp~QDs`kLK|lHvy78+D-y`9XrC*a7Va-|v8knq0RUD$
z5+-%J!-@wW)C)&oSyziAgS9wMq7q**5Z|0tnTfH<S*JD6JIH@DD0nz$sUgJbLtV=A
znk&WJYLu^-tW9v5>b!^qs*_D^H|=MoK1b{jZODrPp~w+6t}IhXzNTR}HoQR+B?749
zjXYj6d5%YLo%MI5OlQrOxei9E%gaCM*Yr}C{`D;Mm}uI-E<xg_PX1+p+e_9Rdh_cT
zpCRo_&$f%GMpDU9CiYL6*<A?)H_FgT?lv`d4<QP%ept|=C7#>r_4-Ufi=!+ma-~R1
zvw?n>;Je4weYpEi`(c((+4uzo@kfl^Pv!FkyI7x-HY%Ysa_Af0-(CyfK3YgoS*y(K
zIWBH{beR@}3DwPm#D5;|`I@gZyi+!^I429$s8G$aDlpksnAN|24#C3@(pwd6kr>xG
zYmaYuIb{PTL83x|RYzJRocZd#c<Iv5c)(1G)#4-->A!$1({YKqOyx}}(z=?N0kwtb
z7QR|())XU_)1`FkepjaMF$aOvV>o3eWX20~7(Hjhb(EDwF88b43eS}ptI*?Ojy`*w
zJ4XcqP<5ORW$=WNI<H)QaTk=@!#+3%l2qlBJa^Q^*}xb9Sb1h#k($je_4A*RJX^)n
zok8eiwLtSsLC8R?v(JIlj>Ah28j)RhO0p8T=B+j*-nvcWc!XMjkaX!lfG%ghvf@R^
zqQAgws{LWaL<Q#6ZGB^px~lC>3dpDQ50Id|4I-~DR#>x+lTz8&%jg#HD-A4KMnV7;
zeoK3ZirQQ$Cb%d8;=-#|BJC{@@kNd{Q?LTmtjH|?I?wmssZ~znYTXv&0Gz+>cAe?D
zPoffw4S9KVr>wI>u(?2zMIQ(aS*<)HBGSS*$Yozr!x&}A4mZ!s)_Io?u(O`4{sFb<
zD|Ot1>&*!S+91VHj!~=t@VIZ>%VG{g(pT7xrgGxyBTgxpszs%nhO8bN4i*~1=kbd#
zm$tI0`y5%4UCy=yv@h#@BNAw6#I`Jo{+o2~TXEGc&#>D2$)`|xJZbh5t%)nV+rN0v
z8tXGVxP<AY=8$r<HL4t1)cvR(&qykM)6Qt>uF7R15~34#Fw<a+7B<L>>d+o4O8_ZE
z(DM8xi{=le9hMm9E?xanwq(G6T=c8fl5dK6O$UFd3prB%VP_phYdMe=RwG80AXz3@
zT+pSN64RrfV`w0Z#BI3gZ6}f&qs|NR`b{Zd1oIQGLc2+wPE<3wJ<XcJ`I_QG@ygKx
zgA_V%bp4~iXh1RBi{;8PH;TO+t+(ZL$7GdO^w7#j!Q9i*T)<=TL;I@?W9`*G4inIx
zI-14z*d<suL?eeu@SQ9$3?nqFG9QZkvq*~vI!5rxfp6U4UN^^<&Vf=VBb$gY?}Fu{
z(d{K~rJA$-b)g^X>;nB6>GdR74N6x8ZyshhkPexjkq+5UW^FmW3(D|90TJY%{|aXR
z_sjoF<jZo#8kV-(mrb54{}I571t+e^dwt<+NvYE~0^~-sl_U8fo0WR}^VXLU&?7&{
zr_5*Q=f*?%T{dn>GvGgpD88%Z_5*FnO4%S%iQli?B>=pG>ujGvu5sm+<IHq)8p8}k
zke$tTnI>x1lgVV@{F+Yt<su3jrij_GZP$TP!NQB9uFOuW4HsH<e2Hte>CI2I8>;9H
z<+^Ga;WE>6`ARmToL@K7OeyTdU9S^Bw+Lj1L#bgAzxBpB=b@UX%(E}g$K!y)TK*XF
zsm*axM`EiS$qhxox8ds*3&0UUmN{Q!*!d_(aOnhr9OT4cdGVlYzujv>IJeZp@i@15
zRwM&68y+~CNXwV)S0}?5=;+etVbdV^6`oU#xwS>KlIF#wsl{0fA@DC=b#y&<nd*GJ
zt?u?`NM#MVK$BQ$R}tw2Ilx)MHjOa5((e%dEE|XveZZ_w>od4c6NQo=gT>^KMSc5O
z7c%IB6Rw!Q=~niLwle$rF(V@*F<p}{3sIO^=s+483^>*C=Vln4|4g5*2Fe5Afsjc4
zRI$-wI~&C;{X>wDQ}HYGcq(^Jz^M=@`9B|c?^7i2Zcv{9Oan5Llc)pGU9g>Bcf<er
zvO#W8-k3ea3<Z7CFDMkz>*Z{#bqzxXc7v5u2sGjEuL#u={96-{dUX0fxBFjxiFG>E
za(btS9|q5)Ln5FNTGk{n0z=-9g9iEkp@-w_UJs!u{J7sDJrfOjxWR}=4XW96c)iiB
z9A~%MCpRb6Bs-vYi<J{PK<?LMWW{G6aqKnty%cLV(}o+^|5=uQTPx@%=)r#md>WKX
z{1*#g#{6d_Ujt7eVIe^>JKG}!$3xW>|Fi+E@4xQD?+$gRVAIdJ?C*dh0o^MAjDgYK
z9Z>7pu~>(qi`r>D!5jjDa&jd5%4NN2aO5(!alHi={b!>LwPp(cn^OyIT*DcZN1=6B
zzPjvsN{>ys8>L7^q!O>=*9qE~oyy8Bn(E3r;-yNmsC~9a4-U7S?JF!WuJkm&%C<9*
ztnUnZ;427x)fvZ%*`T{X9M|#F0R;-Pua$T%a^e;i7O>!4zb5EsBRJN1ppygbD*vMY
z4lY{z88TbL%!`K#J$$Jh2?#J4NN7u4iDq%WHsqdc*Mm!hS0Dd#+!u<~7Imq}`@9m4
zXJ&!<L#qx-B+LNjSpl#F5hZMdbs$w0`EF9RXo7(WK>RD;G6wlyFcP4BJpkxTSlox)
z5NQY1GK7cRZn=W^>-2vz;Lhj3UCp&+#2T7B%>2}-TFtK_<F)c?DIADR039MQYH`Z=
zTL2sjLVdReD6;BBGYb};8T1&BBYu=;C<mY~cweggqVC!r_Hx(R{0waj0wC1W9XY_8
zPIYFCY$eQrEd=|Ey}}MJBX#+0jr;WlFtXZydvmt#c{`_ljaEX&zq^D#)9kF-1PI=O
z92Cfao)|I&h$TN$HwyvW!=TX7E8M`Nf^)2!2Ai&z+sfO=E%~kxxF5(|kj~}1mY=kT
zDmEsu?;LvFHGhsdH(j(3IQ)<q@S9aHER{fkvfNkm_ql7R4&#=>nm3|XB6Gd#gb5pt
zs}3D$s&+tZ!><2c`N>up9FNaZ+gDWym*u#vDgbifi?E#r+Y`sxc&<PC*keOmr4+M@
zR78mJ{vHcAL=4=9JTN^rq7T30rgI7LfB*FMyn2y}aOr1B=ZPQj2h#mq7TbVr>+~>s
zuFc<H#ob|oKF<pz%mh$Y#=R`?(yH$|K~W+8XSu{}AF<z?1Go{i-m<yxw;vHu^;((9
z3*zxi@3FD#FLKd80^Aj}9{Wh!TdOL&uom$iUn+qmLmP5cCqt_a)N%w@rV*A~&3^l<
z)L{V4^ENg%Ommw)NOS-*td3#dS?0(vK)w1Dj~JOjYz7O;Ok7t7-H^Z#?SP^Ey+VBz
z9tB`0Iz3nC>;O_Lp6xF#ATsRu2lFWinDvVS_7+^o^Lt=MU@r@RXxq{(x9y#^R2Pt~
zjR_M%7QOm=iB<=k!l%7bgQRKx%eAZ`a1b*8S@ZgzYe6zJ|5SoUkLZqv*NZtWW^MNN
zoGL#j&bj}&gCDJ*!}ZC6;LV+tj92_xuv<k8)%5=Ir@^#DqjFba)S%z?=A8IG2g0U!
z5HOV;BQIMTkO#jjh4YA;eccaOjew2J@8-_|<$GkSi_h^e?snJ-D~x)=!zSjP1Fo!;
zAnP-e-ze;~4a_EV(0j4TwNa?Jr^Ro}iRk@%$(=j1k=^2UeWPlJ^l}V)A=U^s?D0R_
z1TcGnLJ(W2(kMiKD<KC2yawv8F6tTBrBT4)6al`5+}_$8v?)u%f&2en{Z=PtKvHcy
zJUm`^GwnT!st@P2`13HZguh%ju!yXmyRZQJ^+$_Zz{`JIa6b9G-1<u~+n!dJhkmn#
z-L|W9gV@R6h96DcSZuNT^|Nke+GM*AQDi%5f2FU|%Pm&Mt0+#|BfnNygJ<P?HT3xu
z@6qEFUH1(DqP~>7Z)X4#C<I=~$r1#KuNg+=wtmL(vmU?d@Xs06*ImofUvOfB2LeDK
z)GBb;Si^tPy<~P^Cl>x-jeag5w-aXkS#0&?pO6c!BV7YOkczT7vW-=DB5v)@b_ntF
z=but|-t+Y}-z<PvGcI&LPF$+eOnxO`9tcIN5PyzVj5=>Q*qwpxe!SI4&Y21_8aXpA
zw=LntO?RDH<MJWUW$H&z-&ADPajrk_L6Gi0`;!+UQuQ93n$dK|X;(z-(A)N$2)_Zh
z7@v~t7s1hxZx{NrZ@u%wpRz%;m@BNUy_uz%%;a^6?yDvgVMmk=PKErw`7FEiynmtn
zV4rn&j@>40)h)Nsrn`A?n42ylHvPSJ0qw`dMw5b;+XRAC_Hbn(&5qR80d`Jr0e~^{
z2lZiv6%`d<Z9wTA4p6}YW(GQXy}<PM=S4*BNf|OKK(md!B6Hm0)AN0=TN#0G(tQ^!
z(ea4P4EX`8|K_-1!3KiE0k%(cra9Z9$NZ1KdG~<Pk8@aNb>M+fnR6Z}YPn5M1ilRg
zIOfRy13TCygi{cKYftRIeB%0}9~542gu6%(BL-8494+K&oUdAUeg&veM-W9GAMY<F
zY}&1${y>J^@X7a=A3Qg5Zd05i?qz|mlKcp|xyFM-zO?^vQ@hbX5F59(jOWs$meT{^
zM0$z*)jj|au-|Lj@Y+AMe&ksYq(|773^T1(k0&LaUNQT-uZ)TA+=N2Qw%UhHG3H#?
z?C}y#WAH4!?3oNqydx|NBmri=?VM%^&S@oD?1wyTk4=IvHN)s}9mm|M)+xFKyB1j*
zwmdq+p7C+NU2!RDyeUqqd#XFvK{r$9af^cVZ;k*3eW;a&t0<olE)ou!SdR^rw&NiP
zfMn!TJ<6<ql}+b3B~Ag$T>*86$i=ay!Rv!F_<#6=v%dJsRF?q2$v3K#OwU0LN~0hh
z!MZd3J}8~U$t->sT`5S|{Jtq;`g!-YtuI^-f?d2AqNq&TI4EeGd(u%O_4N<MnBED;
zCsCR^_733)u#rS?G^Phi3&1WrK{(=T39v`>l!f|->uo?A`?)1&*?Mvs<oV1#s4Y$o
z8Cp*&1C+lXm;Qba*xQ1GxFpT(tad=*#k-H#fF$Rf1MflrzkZIb*Xo?Am*G`U_-K2o
z%CVzIk2cIkWCt58Ht5xTXlxWf!c@S&)k!Bh00#jvjhuVDlg|mLB@@p<eXQiPPj0Qa
z!;o#ap;Krmx0LuoAE+w?(LwDjA_)X)^vZR*kBlK`b_C9{27s&7h>mRJwIf<F;&xM1
z(Mi(dkuQ;b7CFe`=A6nE1Q6K}dEEfcOxYhB&IZ_|@R9iyHV_7NzC%X~fEt%e`YPe}
zO&^<Q7l6l#M2x!*mgrm)jg#~9effAwRJY)v>XEsPE`Juk=vUy#adL=4&|?fXotc#r
zEbXOn3IGQo*B{j|z*np@yoW$h?W`IpFbKrJ@?QYF&N-*bnp~6U5;|JpTn~eA3uGRV
zR@GDe#kpQr!=$Mb$#cO~AdOtc<xl?y5yJ#jL90&N@schLAY(T+htJWr+X<Y|<H{Xi
z9c+R4S@dYQ*7P^bge<OozR_f>U51b=|CH%<|JAj>(D`n-RqLk@E=WM-{&a8t&5b}a
zT1^M2>922I5-wtR7B)gmHj;XNL!}+8u+bs6`4(s(1D;D0_W?!}6#omg190Iv_5;*A
zwq!77#>PzDf4vuc$Sz+&GS$O8%e0wHhU$zwfJ@N~o7}hcWfW9-Y=J`NK_+&d@53i|
zA}*j;c$B#cIEh$5RaXjgeVSza*TLBc+569QKLO>?^fwIOnSRX&38#bqaxD+aSO4Xj
zo^BMYUGCbuRZ)Q#82uA$c3dz-SSox*>MaZ}3Yh&q3xI`i^yCE4H`;XZ&vJYC_7~~l
zJ(nd4Dj+L=#|LA*!C3g4MQQa(E%2_!J><2g(X287*)*>Z)&F{W=S*EtqUZE?oK35*
ze+YdksqZ>b&K{q?Aw~lsgM=(6t#Hez*4+a(52_dVueeTRL|0T?`a3pA9K`WULvC1f
z#(m{j3oiM@MWuMfJJs|M|K!DcYlPuZZ2^qHv((+IyC<O)21dP0&~-xR`#U}|xNXNQ
z$#+0AfJ;KQ{nrnW>qs+A1I+;n44vGIO2AbDGW#EXfQ*_$bmw`<2ma{+Jn^qTKu>4<
ze%?iG<bKOuKr(|G7a~v%uN_D4x9k!eefC$B9D;3>P_EJ*Az#OK|Gl&~elM+9;7w>w
z(6}KUO72$;4{vfm|1sk12$xs`^bg48{29z{_(#)+N)hP4Hq&jCd0}zZS@Qt-&dwNT
z^!LC|qVo9WS3-Fcel;7O3UN%?ne0l|E|6VlSPnz2)@Y*%4^6<<trF*oeU#J-6firt
zfMoz3NzpqjEY!Yo8)Gq;Y5hGr$1|uW{&ivg+>&5Hw89`00DUN6z#4*?f&vz3;b&U^
zTphsOs5T91W{Y!7*}1`=aX+pnQ5!tAvz!quU{`lG^-p6hr$O_af?@9^{)_?&q7g4`
zhd#bh!r~Mxg4BNZ2~>*4xqbW|pIM}wT_^suwi}L8s{XqxXrSCMDS9xfzxswI^v@;a
zXY%+^f12DGPjkS(Gvr;c5N_C?;Q?bhjY^%KXiL`1(i4ikbU%QDlM?|59#%is7)*TE
z&PaAv1)z<biVA5P0K`Tqpn}3+u*_u-d}j9rXqlh^L9oN<Dbv1u;H++M5hrVu{g8LT
z+XpwRO_gr!%yMT)00&Hff>@WifzlouP%}|f>9u+g*K<JQ_ZmXBPOm4)#CF0mPSS<H
zEdXd?dFdU5wPKEPAl5-H2Ykr3?!Q7FDOd9|fH+aeDTKJSWnAi*+LfUl#0b0|=tIjS
z>u<Js00>Z_zrT+L^&7a`p~npXkjn-rV2%hF)qf)<I=;`z@!nb=A5jyvYCmH$R8b0n
z7kkUWvO=JReGTnsLb`-smiquG9{~6=T+;41E`r~-@2cn8x!mJ^KwQ(JJLb94YcVKv
z2hzLBXN&r6N4|u<dmth1wPIU7`bMRPr!fOUCScjHAe^{@3~C=x_MpXV76X{RxSrQ<
zz%Jz%JrKx}2c8y~_-(rM7g&S?WukHLYayh(fo#()*Ij)-$WY=8#3lnko}m(?ncd25
zpzLR62X`nJpVRcP4pNmstCzK}1NFnzymke?1qfP0WE03l45Jr~+y*%oS8hP|&(Q;3
zIHA~^B>FvI6r8;z403N!Tz&btFT#bPdp@kXd~>CLQ1n#qfp^@qDJ_7|NjZ(}CBxZV
zX8XE$JVsx?hlMzPw*${O3h56$+jD3J6hNf{S9tx1zypt&yn0h6EL5AsEnia0#zNT*
zRkAycH@is1FtT#u_%0|MI<JNYFj6Co+*%vRIt!2vejk_k=e_WE59~YgtH7%bvmhzx
zjA5dbj`Ps!M119eD}0m_d<5rwegp7s*@aagMusf2wt9W?VXRt873sZ3SXXs>cXnoX
z!xw;H3{3>%WIvv0JR&1d2mnM8$Z_@Oo2z{HSCkm50s;{d2q}YCs(uCw3{1-IxDLHN
zQ@9KzHpLELuBK7}kF<+0%(gZa4wn}p`sGIQxyBg57<<j3$R?-s%a<^26$!Kg2%mUB
z5=yu0S)q|@H|ju&5)edk)Ki3Slny`VQpW3O0)yX=GgNY)ogo3W1@E>iDJ(kVh=}m7
zQUJvh>8{}I2PMlv@*{a=9E_ORD1E<e_Pj#LLdppzD0O3$pr_8=0lk!@z0*>X*E%_B
z7O=On@+-GDSpe;~?ZMr*XTIz_(VF7A3rg&j2l|IKyQIF$@zv&i22zunbjLIdCfe<|
zCC=ZJK(qfs;4ah;j3LwTC%Qn9$C)@{2(deO$4`<+$H&X}t%DaW!Ul55gKi|3kvF{u
z(!-zRgtfDgvrINKAzBG?il1s*S^+ZRE+i{7#yVnu8w4a1K!`n4FWVYWe?0Ia(Conh
z+|l*_h`0Ywm;c)o!4+bW9TEw6rVlvaE7=Y{0_0V>t+kPru3X<PkU@U()dm!}6_Yl`
zuANQe1{xhu>C<{_fX^!cZ$VY{*Z_Mi1c_2)B)={e5Odd`pI@eY2~>@_`pqXQM7Zqh
zK8U?5hH#cX2I#mDWEB*jFE&fH0MXF`%0P~+Qb-izZfE%&Kau|vazQU<0mRG)oa4Lf
z#=HWphTS(}ftID4^^n(l{t2t)5MZa4;BfO_wgXONHuT!iqis3gjS^N1mdiE%E=v=d
z;9j)q`%5(}tgP8U@*rmE=RO;$1>TOF<^$EqwZzMSszPBdpk6`DOLl$q?VW0^h=V@c
z|Kc}x2}0SE3~d`#9AiB@ga9wkzPqT61sGHKUPhe_kR9fh8$}4x3$lfr9?WbfaIC^5
z)r_3#r|u2ku2G%?33D4#RVD4((SH=ut}?x1`@v@;0T8o|GWN_+jTO)Se4Q}`s+rpj
zADS&D%^~{U?1581fC{o!;i@&$@`VP0+0ya&2X*woE}C6fP6CC~b(_+j8p5h|AE;tD
zNKOOq;|<@ePY#!b?CI4jz<x4qxLL|UmePnd60<J1@Rjd1Vbcpr8k>SLvoye}UlUwd
zG~~wt5qn(E*pwhgV#I=k<mZjqVSkr}(YtO`%~Ot`pztmuHt#`w-)hyWjFkKQM>~)k
z+#2THssZ8eQ!ohnbRQbyJ;shfhiz~hiMPDpU+f@qIaDnxBloQvNZ-&J^s9cV71Xi9
zLD^_=qj356IA0z2^}g<^u>ckLb=2}?$N7WQ3h-J<`Nu;JhOHn|GDDrt4d3utn;$mK
zcBX!R#ETX2nN(S6a`5d4ybcOvK>c_ZBqe}j6po*Qsy?m<o%IUFfkM%<REf#XA0V|d
z!X(Hjz=#DUh5cW`c^$fqz3e(E2iV{&&LElwZ_FYy4$u4yWsLzZO?6=nM5o4GfLsi(
zW=S}`m+5XO`*snGfvE?aYwmSq+<KuP$I=$rWUCImrB~nZd(z&_ZgIrmu9`{{f^QuH
z*V$$$1P3bgOD^0fNJl;L!TPR!;{7s{aF`8l&7oxzgW1vo38TB$h3{-ubdsh)MeSWF
z)Jw?<+R0-ZRg#0pKt-!{`D9wG?bnD?SUAYdR+lO#esw$Q2kLLJAd|K8Hels+R0=Ms
zd3x$;7f_nMu7Q_fV0XByT~q?Y<+iS8_@{2^So^eF$@&3hha6#P_+HJ&cXQgrO<yQ8
zTt`#$H3%Nx8FN7$<2eo-kLScGC=wgK*TLz?U0t4yx#kehMI!k5l~3oC8)ik0VURap
zftMyD0S<&+*AJ8`$#S`XE*|m}aHM2N&6V)Adq!SBeRHc?<aXQ6fY-uQSXfvF?0Bv^
z7+Ix1di%Wu2g((WL>yk&@WDZ^d|nfd_D`)sPQB0lnC9j4$3GrAmW$4Z<j^LQQ%l1^
zysrA3^q@OPgaiKXKh!8ga0Yz$&#(Xc%N-xP)ABz)Hf3%b=GW@RBuyu;44S9AtEhG>
J=O!xP{{Zm$I<^1+

literal 0
HcmV?d00001

diff --git a/python/ray/util/sgd/torch/examples/benchmarks/raysgd_multinode_benchmark.png b/python/ray/util/sgd/torch/examples/benchmarks/raysgd_multinode_benchmark.png
new file mode 100644
index 0000000000000000000000000000000000000000..f2578522530269015d86855026d27dee1fd64a40
GIT binary patch
literal 15172
zcmb7rc|4Tu_rE1eB$cfQZIZpGEM-WPgtCUQQz^Ub8CfE2LK_NW&6aKKyO|U!+4nt_
zZN|=+?7wrXp6B^~zTemH`}_Sf+%xxbUFSO2Iq&m6@B5LKh6)Q4Hxm^V6^rV{^Ey;i
zG-xU+YF9>j_~zS)X8{!zidpr%;+5OIQv=&?v+GrDIKL`w+>sc-^!@&EGv%{dGbM*;
zcN`Kw8Gzv!eVyTdeEQhJN4;%yyvhW+^ZT!B^<26BOgZ8J_3nq~Ep}a3xpJ2C8BV}b
z-T%PaM%9=TIcwTY5Ub&NQE;efA%>x~!_?F?HKWluRYT9<IE>(f9-Tfl{H125qVo5r
zqoO)Xp!xqiJYWGg_~+XH_3)wn=$F@>g?0u8HyWb!zPc)`j3*IXDU&^HZL2K2vh9@{
z<1I;d>+<sQ?E8uw6q~#^*C_)FOEW{}#7;v;$82M1v1=7~*5>N&x*MbS9k)1ZqM+&A
z>28xPNVQEg{>rvY-}M#cJA=8`>L!D@%pQ6ZR;J5mCSTu-&8-RMV#6__sTk2`dzjEU
zRV!~-Wey)cJh@P{zSMV?G5987WxifwU?_n)LsI=bfk{PCZN_!>t726ea>)t$%zY<s
zu-}flwf1vP{GOLQ=QdFqPAVEse<hkuVn;@n=ggp`Cfml(?_o>*u1%%8IK?f3cnn5Z
z56x7qI(CnR?mKyOXd(xT@+=$U7-@eon$s3p?lF}|8T4)vBeiKNI5<@;r(_uyiw`ZZ
z?muPmLQu2ljL!<1GCeRb7AtA<e5$WlV#pPq;=q4r_OATq+Ja-1ZptO+99AyePF~sT
zZK;~Yk!Bd7<K`j8!BPh}IB^%A?6ve+T`J^7d6ZyCElCeTcx1$n`z(w$IPNpQGpAfE
zC-1%FM!H_}m_u=4yJ?rQVO)(@*7X}V%4Y7GEllP&m2Hx|+wgvVJd)PCc#eT@JV#k%
z8Rpjph2Bct-j!`i;@jL<ol$onw4TcLTl>zxJW6TT)udyIZ)$AxcqiE3eUxsu)K9*k
zZFCvUV<G!bpD-uPD=C&N3*QoYDR9VQBLC-*!L`yK897yXjss=N`|Ao}@o3KJ6p->y
z<#d!Sg^?B>2ZUqPx1+&BkFinPy}!=_HWp?~FRmMkuRUy``bPJjG7B1$%}utdk0^~j
zV*2{jou9WTC75Sx3FpID_~g2a90zPwLiam|Y#2LLj_~=dP8HKOkpo_i=HQZ>l1B72
z)vXCpr<@bLFOk&b)mH8>?b1!va1RUPlb1b6t=?wtJtajkE8C^0Ouuvgct9xTXyKW5
z=Jc`K^`|{1yZ?-p+?kUigtl8JN)gHP0f<*TU8GhI$>+YXX;WQMA?Ha{RZJ*g$ZYl+
zUySAJ@>bK(2qh^`bYw*Gp1E^symTc-b|7U;WY@91X$%+Nomh1Bs-L724dM#gN0<1$
zE^N%W%5%`Gu0h>Wd^fe-$mekFrZ!Y%x|S~bqOiRW=UCxe<}9|Xg=TX3on;pqPFSeV
z8iePp3Twy5B@?)v*0cJAUM~4_$j+M0B-C}F1k6|m?rmqQZmc6Yw3aphOupEpw^vtd
zfu?h5sxPUmvN}S{c+Vc(TB_8!^qUvmPkBz8blHsreY956vD{i2A+`0aw$=ppMUE=i
zPp~v6sq1;1MzBY((7ts(Vb*j>n<fGy^ldk;V(cEL6Q5ZUANTooEz;b<bN5T|<b7m{
zVBDQO9;R=SlS?OviY@;3>*gfE{n(?6G@Z2}JV{JwbH8<WC7Or;`x1-fw)kfe0#W{Y
z-7BJvc2dn|ets(xsg?@SweNT1z|dMZXd;w?HMSEa6Etil+9PYH?n$R@_)FP$AE%(l
zg^tUoO45nubRE%QjmX~Bz6hJ_XmemPU*#=%^oH&|?zJ4ed!eOV4L5GFR0?O-CdsKT
z?;7W~QOX=bd@agCUC)f$_{(t7f*OL$7kbfsQnpv(DYeY|>F7kAaC!8etLv*no8m3H
z_kyH|QD)71MMH{P7u=H*Qb?Wj!?C20-SadNW*K9)YesT2h8%W;`RasP-=NU#3lRef
zuQtzb6O}i2-|e^~f`0TQJu?JPD{%|25S&tA_IT@o=>VF}0-H7(mNNQ>8G`By&2}AZ
z+FFG9LG^{%Y=8ADg!u>0{nkn5iUzxdZ`})(Xbtx>+pFCaMqia<RQP$I?41I=dWEq@
z#au1V^veXL$bc}7ATLUDD@py3g;(VjrY9>;-r_60DP?WWWrE=;6IE9PPgp$iv`R2b
z(v31wU@{oY;`z0{y433O(q`u-@i7WI1f8?nJ(~ouM(sXPOibvFIdxg7eIy@TqC-0e
zl~k2rzmN)9e_0{YQI1HqT@Kcuvo&0&RyP>)N*$>l*tD5wO%~`3Om51GH)y_n+^3U`
z7w2Y`;HiH9Rt-N5^)8(M8&>qjARYM_J16e^_h`v9qiaR%7LpguLToLm)l+e9T<$E~
z26NL(*fjqUI>E}+h|863vX_%Hda5fUV>9Bm&reHcqmsRgvl%O2ln<KAJ4eNNVzf!x
z1`@T2rvm%Qh6R}`PnhZ@@N0SH`DL9sUB;>2-q?@+0!;0g5QEK?onBl<X)5eUPTa*%
zHg*0D^A4(AI4UYSEs6Nr)bt0Ml>x+#c8=GBQvz<(+rR^<;u$OArnVcnai8oror5XB
zxy>b+bj+M3)XUtH*>m12rpg2EgMK5z$l3qOw*PO3$J&ta-qX_~dJ+cW-#WEU8S+%P
zPLhDclHCv`Rlk!@Zf_)I?`fAeK9Coe`GyK?Ua;^;ixQW|>u%$hN23WtJTi`7iYMDr
zE;C<JxbqX&5G&sY*<tw1Se{iqlMk!UXxN#RkL%*t>Sv>+PD53%6uiezaIt?_G@E0*
z2GR3TUxfyH#{hnF-3jg#vl=>%dL0#IyKmn<NYFEtlmMdNcyq$i@B{u{NQHq_-|adx
zKT-I2r0sDKxCE!`v&jlKzU=KQb-mgA-J3G)9;ev%K21Bxeq(KUx-^!SVQ2T3*C%P)
z$jfAT_VtXJ<XBpm>5=W|joPuFbKfIgzqVmJdWn(e@l_Udl1iAz-LzBG>MW@WnWAPD
z-GC?9=9X<U+v7(@N6Tt8LOQ$5{Y<}lk1r4TP)xPPg)|6ErlZdp(VsNP$usI|YLBM|
zEBn@)T_6ZEe_W~hc`kycv$xQW)=%W2eNSEuU6vDmz32mN1WofB)fXnFE3^?2>Xvg0
zO$wgF+CxI}O`GdjS@*Xn+#E)pqcuXpobOfkOmbBmc+ACWe~w<%%gD&F4Sib5C&9Sx
z+2Qw7fvjh6LCjpeElDc0O$LWk=@l!LWm}h&9&_QUR)`{9@wXS4xo}JvR`MmUO7=LP
zrPtn<sBd+=)!WTe)_m)Y8H?DtBLEd#W_Y+U?H@yUQJp^0mBrk5!nj?fo)9ftZoWQY
z^_V&BS%jhdb=&h&5<!8Bi*7Oso8BS(3WHgJ6K$#G?t4_<xKGc#Gb(klsxmt><CB=)
zb?e}*fim~$(zjg&1qJbAT#^cx+ejQw$h}?*qd`9WK|6i;g4VT+!i@Z$(L~&=eX{TF
zSojq}tEXa|y1Z@zwOw5(ulS;uyiT6i!j0lsxu`04QYT?3m{aVt=hGzjw=+TrV334x
z7S?r2_*9zWmBSg$iCPKw&$AzE|L9%GLOiY2-TbVK3nz&ule)_3WpoI_<MP#HdYhgO
zs`Q&`ELq&<GcSXy?+<oqhQyqCfq%9Tjq-GCp7*+|G?Rf#v}zSnaC*%iQLu)QXmH)!
zSQp=$HYVDCLgaYbO|4Y14)yOo7wC4*R4!uVvyUepPorYfP$X5v>)tc>T~0PO$81`6
zdX@Mm$L_7f7oS<HM>nrk_$Pl&iszkni{$GXYo-Y7zf9>ef8(Eg>+Q-DDGQ<fLCh&;
zgdH~lmx`}hFDudNpOPJ)Wx$ZTg<{_BbeGjm7#s=|VCEEkxc3q@WTwZo5!?(?u|(y$
zfNIFB1J8=^mb&K>YgI$aLqBwMtH@T4AgSpSX?TtyvwHBbWahE{wB5#KZs~Sz`FVLJ
z*K%eL*>3ANyKJ+($Sq-V&zNgPG~uq2JR2YRcv|Yaq2Q6?t2Inq&#UP~eJ)gXGRF>A
z2jdyeRk}XI^fMWa!K%^t4{ko-_kD=LJ}|z}5k0m#gZEsxvP;6=*NialSF7c{G%3`Y
z-rDS++>}kuqAU`#-qcwrv*9Y7jaD{R@NrCdv!}XWnX(wNynXlp<R4KaFv)xz7~*+D
zFWTBt5fH*nP2PK3{y0%!j`b;pK~(CH?I|#e<;?DE29cKmfw4IU;s@_u1RQlZh+(F3
zEo@n{&FkPhW-22Uj6_IQ%a{&f_Q9de6Qq;(#dQ^Da-ODjJ<x5I1w>oW|JKegkgq|O
zwfM;fd00NB*`#}|!kx=;kdOQ^NW8q(B&e~8**O6Mx=-u7y9#bX1?QR2Ob-NjaqJg5
z@>6N|&bV63zx8!9(juqwiIV9t3_CVr)f-SI7a0J}3r)S#(R`Y1qC-2P;FC1*EZ}aO
z#=8l(NR5N(!@7rebK}l8Y2Th@$h00z!ugMcl<ORR)8@hSIH8_vf;~aw=0u@v0~a^n
zn!K4=(`!#|&D7gRHDf1fLNS-vEe_FT-m0`sm>FT^q@&9W==8ECw5F?uFz8Y(hS@zX
zGCLR9!mqWSkd9mSY7!eyq!uez3d2W!(X2F2pcd&DQc}p`A&cH`rDIJP+N~*?%9ykr
zZREL9WgYW+(>rbE;TiQ3+yraEiZ3Unni-bl<t@T|n#=_QXD^g9)=200qopDi+f$1e
zr>X>Mb(LABgSKfE{u~Y7Ki@yDequ)Ax0v$#V)t<Eu4+Bo#LH+dHMhsYl3KS@+duHU
z%4|woPWSx4ZzMT*Oo#Azl-*LBR=>oZj~mBMxYCqBI~r`1Gu-)rCQL)`tx6Vd864$&
zr8LvMVfxa#Jvg@!(-LQOZl_eU#OK6!tr%?|;(?<t#f4iuzHNR1cVjS``D=5E7joXy
zBM`jrDFxI@rVqDknER1lzQjCg*RsZCal1MB7jERl9<J19!2#Gd0N2)L4Kn*q2m9lk
zq-i8EBlYq@wQJ}K-Pg9a+jWPXDkKNh4PK;McA*ahB?Msj3&ht96fm$!b{g@~aY{X9
z5=~TGWipN`j!;=xntuOdP9mRDjFFIZ3Q}OP-B@hX4Bn4}y|Upy5~LASuGNyca0^hz
zk-|x$SX;agw)61~-xhk&$}pl1IkhRtJ)@5OiryGTQ{v%yHXQq7Jqcp2j{LE&Z>QOJ
zkeyVwiFT-tQ`$7^-^`?D3wO8{`f{8&=Tskmpmwh)lFwMsiM6^cPq{B@2c~D>*^jZm
zVmDc{wD|N~Ap4-+xs<l^8>^Rtn9+%U9oxuYWcFsI+bQ-!>5+95x8=-*#0{R4k2R*e
z7IeRJV3hTbnX-)enEAywXYG{q6MKQ7R8X`I=g@1XzHV<ORkG6*?5yCs-)yD-KD`OB
zR#)0h<))Bi29Z`-*rk;QC2b~V?@harjmCbQT(#2AWU%3G*1VPoBRue?EsIGvDzY=7
zR?MctSkhqkebPZ4t<F1=3PB$(8VDbS!mXTZ;$;$7m3x%l(AY(pLmWN|mBzkMe^aTx
zE8k~RqO1G$Mw(z$Ky+1{S<`Vy15YK}^Vu+UUb}E=wB(W+QWpoff&ZS7^>)=a(a-ik
zRW2AHBi`!QLJueZ8zKEqhyNSJ5H8{E8rD0X#gHUi>Q#rfr2F|7s?Q}tJml^E`cFcx
zahpIect8>V#(3JP$jUH|760(<<j2P0;MgB|U0M0bO$z*apFVxGK+!RxbN5kd#Cb|B
zRIZF9S$Q*Ihlbxjn1b@Y5bAh(`He=@3>q@C+Q683#qHce>y|^}3%w_8Uru3Gr_$6$
zAMkU^;-LaBb)S&nGQX2s=rlBV;^fKiWqfA7%U0^@>aGi9ym<(>)MS)hwlmq_>9x90
z!~7c!ChK+YA9ILgs=he7yGtF?jp@1Fteaj#j2U;jqP4?^Hhm=f&QcHE70E8CmUSD~
zdQ0(2YCO|7n5}j2?Y!kcnSE-q1?qCd3@0_SVj_!yS%UK;(cW7l;ntr08`wq4ke`!h
zzw-lma=Y#v>D}@uPx<O}`M_o+Z}DJXUv8FJWiN4~z4=R=0-lg`wKDsNNB<xt9anqA
zsO7#$4vys>rRQc`k63a4w<?i@xJt=K(k5O(Q;Q#*Qa{ZP9I{9N4hAOC!@v2nj_}N^
zw#jr3Nx19FO@2jCSe6}*IQL+u@||fwu}+6CJYoFeI@z7u349IB*o^!PeuIRc%&QmA
z2kzsD9$}BqsD73a-3FiBZJd;2f46PNTdT{_(uK;woTYV}8x(w>KQ%2)Q}k)ql=Y^~
zHO(qY%L0Vr{T_8w%OvwnBJp^uoKq#alh9H?tU5+sovY*T<KLKND4(rn9@y=quOHFp
zZ0<SWz66n=9Q)Se8|F*WNW}^<Lrg+K!iHA=gahr0T+l>+X+Fua6!7+B+~(R%t;2VK
zMF0{FI~l+%j_Y$QZ;*Bx-2W>3Y;%Q|Yld!WsQfH}w#akFjPet~e<{(9OmdnFdu83M
zb+(BXwT&0I<t>7xhVL<C!ek69F=YF}3NJ^7mP&8W@`XkeEl-iz=0s{-@AN=9^PTDT
zbloh#R(C1Q7dBf|_&I(?!BJ_51H_G@Yjrz|*;FI^lbzXmy{zk+^KWWWx?7Xgvn=1=
zpMH2+My3lxCR)J^Bcmf?W2ZgDV>j1_HVXlYj-e8>Ov{U=hVTlLp~k3MK&WNBxIvLt
zxpkFV<@6yj<Dzb5F4OC-qhE3-C+)qzJrXo&3Ds>{?ZR)2oaq8MJj2bZK;AX}QF<S(
zex3tukW>HzE1%oY?#dhhre%Yt_&Y|t#;`YY@Ec{u5o595jT@`@&A!WRt0edI^z>;|
z+6*}<><ssKcuQUb7A4ahIQKnFA!5b9#>1#!szB7Im=S&KnLkJN?T~tey9vd~$;!5B
zW2G4#x&bh<UJ@BEJ$?Fgc0%*Bm(mB;&3UdV=6HP%kqJndV9Z9ymR#O2g^k-5Dx~6@
zO?;G|O5O~QLk6bq;aTSR!KtTiN4POB*E<Z%djuj)_PBkR@WaELhO#S{de6%Ez42P{
z>{m`V=N;Lca`I#LCMs5n#E#TQj-fabD6{+x5k>ytlYNB6iMIE?_l0y*et1s<r9vZr
zbHywzn^P3MD@~moF;RZl_92u)6ID5fRYD7(Qj$5&?-bSbJ@-ZfcNZz?f;)ix<Q4XV
zeKw1IHiqktyK%Rwy>T0JVSe2rITd>(O;J=1F}*k`#x~t%*O7M&{BEhJqnD$8XhgCv
zar4$xEVioOtpg%1nq3|(Th<zSA(eB@^lPu>(s;{zZb|E}W#jHgJl4rw3@MbUzAmhA
zGloSzj~$!REELep>p7mzfWNh{I8zd=gI`}EGw&aHFkP~avYpydczDr!u7=B0wU#gQ
zdLKO#$?9jl`Ns2{+(iuYl}Oo+AE`4Lwd@n+?AZx=PC+X2EE|KbTKfZG;TC@{+^Jz|
zQiIIU4q2DhXF;Jcz`_xw?KeYH<Hs;6O0s1w%kEflp-CxOjNfE#9Zg%ia?GV`I5QyI
ziqD~Lef3{m9q=3~jCQK(aUZUq?8y)8j0?4pss#2^+O|FI&deOS%e)+GtL@Q0JUTJt
z%$4jr<hQwg_Si9Es4?wBsceB?E=ea9&{yo_DSs1c*zIyf4t>(ANR*HnRd!Dw<OstI
zw7-bA;ih~uZ`2YJ5}IE1S{{ik|EkkV%h_ks@~>SAJHOFAf#h-xKKk(<+xC=MaxI$V
z7#o{@=)LITTUC5TF{h`*IVpN+c`U9k+qArYdF472x>u3-upSk4<yv~~{*{;&d)X6?
z?P=Q0cO-FoD(*yS*5qlI8Ft+Q)bz!L_T+E9%MRVRgT$3}D5LvV?T)`4bp1-W>xgrZ
zL^-k?(3d}YjUcSvM~GJpoJ@GY_nA^cR0(`k=`1U2^*to06!urYtK?Bta?aCK8*V|j
zk#K{unbM8UvDUrZAtG(*$DE$J*s!AQQ|Ed#UQCrM6RNkfqEB3_-O&}2eR#L&z&GBI
z2aerz@?8rb4~LZ`QLN3AlI}QIbGOAi{<vQLSSzx_?n_YFywqisqiIi)Vj*w|t_E$l
zRT>A=Y25Qw5(jroU4NYSeW#43dmo7rJ+J^FMDxt@2YG`y#iow%OF>g5g)!XM_Jny)
z#-wF1)_<0c*Hs;^ZA@brzR%-$%RQZXaK+a;z9^yw?<jf8$nt!Bx~{Z{LAG0%C#h2`
zt?N7sM$XY&OUGb$9+uM&b-4)1ZrfSVBZd{U`py$+1}fRyA1AP~Oe(4T6(UjJq?*pR
z2|Z}V@QZE}nKV-s5F4Zpo+`Lr)OP(RY_nZQ20dLQn!fo6QoiYQ;Wenot+Q}(wauqk
z+>`5EV;?0Zd9d_~WqG=&OgL`)?jz_~)b_kbV>^0V=9~9UyK6PQHpQy1*EsEtkQiAn
zW`ttz-=O9Hjn?^h*90M({xnh4q+Q1v3!xz(Kb~7qAemuYoUQlv$|oQwZa!e;*BF;X
zx-`NkU0hry6NAO`g!SHTlh?1rxJLttJ=K$6XH;lw1-$;H?q4VB^7*Yfd(XaO2&B;m
z>I30LXDM&>GBTn4vOOBwP>g7++C=5x$+sEVc@S0^=rXBk>Cq2%^3wQ`u9yWeaZG-F
z7r6cPuQLN^8m%HtZ;0q;;vjQDePdRMF@!`I5Nn=$Rk;uHmr_blzR#)$RIx%qN(oxW
zUh1IRHP~>~0J>Bxc3iP05Qp%1_sQ<R%$;-{9UYZYpnZhFeZD8_PAu_fGdgkNgne&;
zO=pSocY>=ywuz0DZTqcF%@4mO<ErG1lN5goQ<U~y`$^#N0}s1ijogAp|1~T6d2FmB
z3%4ZhbGV3cw6sIN1LvO8EL2<U-LK9T0v-L2ym%JtDAW+u)5jsqUizlU-sW|N0UNM(
zRBK0}(V<&1{{ZRiQ2A>*SBD!gqA$JzeyXJWgU&H_rl9AS1o)^$_MJ)XwL#8_X9(sK
zMHb``(gxdww3lEVl@SoTJ@3|lNBKzUC;jDYnP3wm3f9rC_&S1hsi+7uMDc4^0dzlA
z{&i*Cw|gQv|6DHYRMK(`F`ruAhr<@mN!tJYSu~u~!AQprKDQpX)s3P$V6l7a0m!WY
zj{g}LoXq^F4%;sTGpNS)m%c##9y)Sg<Ql|%w{C}AhJ1;9B2M|wn2;GF^W2U5Oywx4
z220ECDET?$0?b<;z4)K^`1R=Ue1ECyA9rPFC*9Y}HtCV|n6w7RV!8y-H^0oCWR$EH
zo@rX{;Q=t|o(Aof<<v0hk>Z6&CI)8B5eSN0QVxC)Lz$jR8Ts)8O(u3YeEaao(I`Vd
zEAwVk?3N3|(DjbO!hbX>cAVOXd;a_?(4RXl85i1ycVrr#v~2q|$<HwerYY<)vjYU`
z?#H_KI04Y}u2<Q*UL3w}X9F2nk6!2An!5h$6CRUjctjNGvQfE5mUpH>j#+2AE~GEL
zOmvIK%pmru&G7iq8?qs)h#}NeU*6s(+8~IyMrS0rhCVyee1n*ry$%h*PUxY@jLR1k
z3(3gH9E8yzZj*UJHL#iEwj`C$AAc_h7cnTyU=i5;XS~}?8&Zvr!XE2|{-+Zds&EJ}
zuVYlL@<hQf@S?N^Yx2uKUi@u?8x;U+JsMAsb*HDs+7=IDG8_ggP9GPOmmfgT!1S|@
zl*>F_@54XZ{O7_#$olWAjeMC=aSGCLxw`1qW){$frjBIX?QNOY(dM?P>h6THwfTBx
zF>rc5Q4H?!S;~DKFJgKsQqSaqxq6|B5tWkah?2CqI#BL8X3V<fXB@%668AxWYS&_S
zC@*UA$;rmAe|};f(DrFbR8|aYczv?Y7HH&RKG@xy)FPbkAFvj;X`mXWD=B8)&;y!^
z^`3TCusms5y{(G;YeKoLtP;>L?UVJKwnsXjK;%X^$^G)?Up}CwN@xuaL`>6>VbDZu
zPiMRG%lA2G2WE)5b$s~<Q~O*%JwVOtL&$}#4HI&qw8c8t#(%ZSXZ6&*-*f$z3?N+E
zy7@rW@ULgjxpe3m7zV}ZzjYM-ychN!Y^BUojJ%$DgLS{RPEClf_*k$+6%sq=92mCz
zT{zUCugDOx>68I}B^(QM`ScKAzgDOyzWhD)GKS8>VRBbXOG~F`Vq*4FD3@ghKK%Xb
z%D_A5;?T=SV6hI+SvxU=`SoygdzNt*gn5odgk7I{t}Z_O$I8I~S-YXaaGdHb%#W4#
zF7tC=Xn_TbsF#rSbV&u5X7Y3y)0lS7^<Br1AY#iyV|xkO<K_6@`@O~*aHj__>~!*-
zc~<J4?gxJv`0A~N2AMHtv_!>hfcJhXs(X=<&n$a{go;R0#la>^DJi`zUoO$;8}3Ah
z3+v^5i4*Bb!b6(>(n(8ocAFvF(GTp|YX@OJ->21wv5Aq#)Rw+Lw*nldZtIkq?FMEC
zEy9QWrEFS{phjNmPKKhUagA71AK1<YGG#7+{#RKHznn%N28cx>1Nh2mo)gy}`hc9k
z{`9XEt8xGF@fl%i-~9Z1x6AL*hlht-c@Fs$W8GR+^@qp8jgv007#G<amiAlx4#|l^
zbBySI>w+Ca){EKY(=OL?77sU>5iq2z;!G&I_Qz6DeU9N>ofmb#91&uIOZqs{mXjwY
zGIk7!k{lWe!Qa=r^SN+~G73vOsE!4vhz_y+PC3*WT1fW#^g@t<PIRDXxK&-kN@N?=
z%hSK+yc@?P<vM!FGx|Q1#HIcnMGnUBKZp`=95jftiPGq2E|v$DT|Ng}zyY9R5fJ(J
zs?28wD+kVEu=k<HEJYCJaWOHm24ZIxY5Mq3wkZk)4RfVb>fd)zg4(@yJtOX;Hi28(
z{uQr0M)KfHGG9KDp0(UT4MA#64u+ik@p9YQRb&@`c*N$=6f2)?ko7wn|4KpITQUqJ
zP^8TK{!ORS5zg|_D7(orQg%MD(Iw~4m|VDUzg+pl-1j;$0Re$=VvSygz9F=-1~%FL
zV)>nA_9$6i`(7I|Eolk*1P6%wMRr}{p6-^a&6;Er=NgXbO<MXLi1J}u%EXn}+Mx7Q
z5hQSKB`%SJ8uH{TeRx0o>Z2Gy3#D~#!dbu_Y~Qqi#dN-wyZF5|aj-xGD@!M@lHdgP
zLIBQBg?$qf5v`eqmsnhF>mwqT+o)x?((8)h>e5sea1L9s732y;zP3{WEcBX3);BD{
zOC7$v`s;4p3T4I#q!wSg`d&syM<e+`JQs3VGGHJVgF*ak^;N0H?@^MfkDK;|`6n+!
zVz)U1t`wyyN27nDJ+61+4I_GGKrv{4h~E4nsjCU5_7X=iDR#t~-kGbB_E}ksuS>lg
zeFIb+JJ%{*M${mZr`b^XZ|a(1&W^uJWXL#Aqp#W){#cgdwBN?sowolGDgAPk*Rx+v
zHkIhQCDg3W)k3$(({uSB&?NWh75#4{;hYbO0#CbbTLA?eEM|?I<5##sCY8UNOu7(~
z2lK^wQX}yhfg!@#Kw$L%(`6}9^)pe*p|^eDdVz7VqckW8P%b|hy?LlE(lB>96dT7z
z@7I0bUpC`rTrThet~_KR&<fGje*VX&C8$5T0c6&iF;hJ!bePt%XE^JBL~sX!oPIW+
zMMON?GIcZ!P7;saKp>f@03S=iC|el4?7mI|IhqLg*I#*^qw=;VF9P@w(?GhN?+z@4
zvYV3QNF#ge>sjbcBHsP7l5-2=AiGC&@!H3mSoet4O@!RaK2kAVMw*0T>_!gVFSiQz
z=Ln9Hvg-wr0t9UbeSo~im6$^CfO-wol;o1pZ`TFh@Dz}14$za52KYnaxtlmNnPF5Y
z>@D`&uG=!v1waNG6<Ax2#khu60^>CO*ZP6H>DirspRxDH$VkttzpjbG7MVrkeb*<O
z6V6Q*Ze<O`9FNZ_gye%@=Il7l-&1%3x~WMJJQcmrko_dQR^ejTo@QAeAtn-u65axB
z5X5A-M=m5m0PBY7iIafZ0$jC?rV}U@D}YTuaJzn63=;a)gucHf2g)g%_?9LS1HZUC
z^Pm1TfU7O>inP&kUM^Y$wab?a)g>zbi#2aF#@M9BxI8YGg3<%bm8BLgEDZ(czs=NY
z+RwnkeI2lYCtkS(8i`Bjzio&_a281f+o{n+`0@!BnB00t=Ydq{{C66^e7QdJ_p0cz
zBi($2AB0};HKgkdkq;qKCn3doP-j58ovi%FZc2E?SI?g9UhIafF}1eP<f%ZDMLoNf
zFt)rnz_m%f3*m4UUu~S(3SoW=(L*mY!W*^_^j2c*|Aa-Aaw&WY+iwH0tB`^3Ho@%n
zFkLwaq6UT#c-fXyMik!3O1%Y)wsvP)XQtsb;QI0)>=?$K@tN#%!n0gjC1&JwdsF|y
z@rNvG0&C5G$6~-d^n-eVE1zPJ{pQl5%oSdqYw)*uoOR1R9K7d031<0iZg@bsZv^Gt
zswWWJ|6U^JR#ibnHV2D9UtBLUbm#q7kH0+(ZF+k+=<X0-CIDTB-hxj{(*t&ZLH~=t
z!>%lgt9_=z=k+1G<QtP+Z$Y}0(_qZMwoN9mx}zHn_4WDh0vU`U+aUR1#6_iM3qJk!
zwwF91wKYJ%*w1~dDY9-!jHmfkJ7I9+?;a}xl*rOaR@2Wht0XvZSI|&hjH!6Cqh3Sx
zQM%<|bd;^i$&)tmNcG(cI-P5cfETW7GEfPK+Ew3!pvq(@*^KO8Ar>?!6lMlAN2sXO
zwp?dL<UO@|r2$eyD$K8aVAI?de!h)G!yGe_9ADSk+L{Tnv*`jWpztSwsNIgG|9hW%
z`u%**pQ)X6p%ZFa(9Wr7slJ3|HH;KD&8*f8pI}7;_4udkE`OX0PSBZuv+3St+CRZi
zjd6X{!o$Pkk#9WA+X$&g@f-fhaT7M(ILogyzcU`2Gz-<$+-v5s$=RPCr(13;X4uz%
zKCMQ3nUJ{g8ADj<8``)51S$sZ^5n@5$_zr7vg7D??6u&Pbz@x1()A*a$lji{fI9lD
zC}0WQ%v!^P!SqoO0K@0}k?JP{otudD8hz~ngq|@77ia`pphXLo&IF;GVpB^Vs3A~L
zl=yF5v>UhQD)7p?)m0fXH`Z5;p~=5MmY%3fWBW^N|A2r8$amP8k{=2<bRXw&9SQ$y
zs=ZFi=IR8$?<>hTso!DPXyK6xZPxOKq`N5_Y54P8g>A{Q77fudIn6q))ZjqBs{5fq
zj?hO@^ap&Q=+?^Q^@fbQXwgEew65*<X($lPlcfZM5Df`_3*HGzhBLQn;v7ewLdN@+
zDmT@S_(MBCb;cEDmS1(huk4;|lhVBMR;^;gi$5geD~wAV%LnHBf4uS_X02)z7aE_~
zjz%1X7zTB-uYu=2-AvwcCCFUACr(9pfJGqzXXP+t4diS3DEJqOW26X{Y;NZ#s}N|;
z60VMEY%LEX`e7=<3`D=uJ~op4;o4aCGJM2}l0<*oAxiem#%JU8bnH<J>Mf!Gb2d$Z
z7zTvFHc^l1zlX9-l-rv4^#A@O*!78bQ+B2+K&EBhiWRh%!B9}*BY5Y}pp#85V9Jsu
z?Ks;s4i;Le*PB#Eg<t&V!VJl88q<Q=Pe5dfM%@>Z8T^R(p)uXRJqE^E@&ZN8Jo|k-
z!+QS22~m`QKniwcu>A@u>h-6Vlkx`^Ewuf%L~tHEeF3Di+tp=1t!c#mu^g~3-KQtJ
zBX6eL<z4Mf_gm2`vKrZMg&2xg2S!Q5`ME5JRMaQdbUD-}wfR@x&o`OczvZ#WDxf_7
zHiA~I>44RxT#p@0hLin0rEIvgbHp%YP0md2@7_8rQbg8p7p}WB-!*k-5B)E*AV!_+
zPd^8?Oxj>Xt3KfgaY<5hbh)^${lGr3onMBVZv5T1@z+T=iBDn1<L?ex-1Ub^U{Tb;
zaGVCV-CEq=r&EI*FW2?vm<F%T18}rp{~$Jhe&!ov8h`)()3Cy~KJ-7n+lRNRYs9W3
z?Sz2<3?a~<7(8|&*Z+P*_i3DKJUoo})B#!ZgO1OKVGV*_m0#5LYd`3Iaot(alZNxu
z%D3_0l_Z8=S7C63#=BTrvdZ6&hPm#9xx!uGwg?=0{xTxs22`YlkYo*i$1~(4#A$i+
zSdc~dAS@_Y_tqfCkxfve02o4q-#{po|8|;yKn-C3y+6I7JE5O{<1!5=iUC$2>9?h?
zN?8!7NPtDYI$`-TXWJsjfgt?Pn<V}rOzMR7Gp)-#XO4m~|2=Qc^;JqS6#t{7cb^*?
zSXS4H{Z^{Yy+*<T(ILc_rPu11g&INjUCO^9FEB~;BJ@=LE}{N<JGkxivW%p76nyV|
z6|RQD_c;eG&<o5p&;El3%;G^xd?2%6v%m*JZ~Zwu@-X|!or&bgLmEvVKjI*fU5Gj3
zG$@y0kQ0l=5Yv|;@oP08-Qk)mMfBJgLb!W`Qe(cUfyfC-WR6jeU*E1i@fzX6{|o|S
z)tiBCRYo3#>XYZgBpF=BlRJ$A`MgJ8-?WF^zdQ2W!b)4*#&wl2zRSHB(m;7}xx+^g
ziP=MJEC%SM<)>;p-Cybf+(tgj(bv{pC<r)8r(MUJ4<c1p-&c^9beVZK@L|RemM`_%
znI1+YSMd{GUS87-xe%#I(k0N06#JGXdY_elI(8KrM%_p=LPJHy74qxj_1X8Vcd>z^
zhvD3O5KJ4wm>rfTd$ecvcV?T~0XlF^h%;9Rdh`4V0^Zz-?am2m#cbO0K=rH(@SDEA
z10bZ7{89RqT%zd=I5qQ#767|}EWt&mH~#Kl3Dgu<#hbBSDLw#8kQRIV1jq&5R7_(Z
zu?ZTzIC{By`bp!d?>nN;_+YARdQ05dx|*?pW3@a^ZamHfs0U!_J^;KlGl-_J*CrKv
zgUjUBUGhYm;H+GOio&1|)a!3@)TgX9WhdU;A)i|h_l07=ME>X3z}>W*?r2%%@<AjP
zd_E}kd44=8?7|Lo?=46Cx<^;gI>_1YV|@p`#-LC0-K2QQ)N{Z}Q*Q2j)mTHE!qC)y
zB~NkR1QbI93X0>YJDlPsdpjBxQMB&SGXv#a>si<jEHeFHBQB%z@|(`|!D_PI&YEt|
z!J6eogErq3>d|)d=Y|E=&rIDr^ojtLKZG4Giv43Iv?8?#H}nAA6aunXC<x{>?MQHw
zSyLA;Ju;rwsGxtgewVE8S8%uF^=VH$vzwwyzH75~jE556dVkC5v6yS;?(fKjqGQB3
z+heNNaHz`XMcGD?eb2Yrs}t;0<Vd;2zLZE)kJnk~+-hluUiQB%v>zlJNRqG+WA6HM
zwAr~|Ne*#>%<d@ql->*)@*~Tkd2Wj-alA~aP@yTJ9yCuog!x`KZmW!y(L{<3&^EYj
zZi3nREwu3lW?nTci|sov1)cz98ckc7#}rX54|%mB6NYnl`VLXTYZ01`n5D^y4f7a(
zW!~7mE!mz(n5NOcl+*Dnxb~KpWZJyK#&?Bz2LiWN4*ijSJ)xHA{pSOv@ISu=->Dx0
zaVW*l-87?IJ_fkCKIqIkKziLFmy#>d>vbdvT)n4;%Y47bC3>>K%ETc$Kc_$gr7rn@
zmYos(S7j&aZL4-zXqDTMH{uhwRGdL!fE(IaDxS98hiz*5@}&SUYB{zRnrMl=r~85F
znO57+J4Yglm|jkuwy(rnf4NJ4jaXMY0*`k$7#V-4zR(r!0q-3!f0;q}i_@;7M@+Fc
z0}~H*RPZ&5>)|_VCf+6P)O?bd`DeCnj%Egt;|#fg_-rA>KK|L?-hQRn*^YK;diWd5
z(<e8^;++{}ueNH8LMOejBgcH`LKq+KVvP56so?CZ(3NR;UxebzJ-$q!Wi3&R8C<og
zICo;OPMMOny=yfA@%kLZzNeH8x3=I#4Dp*IIAlVTZn1|~)dOZS192C!H;+9}jm3QC
zols6ZEH}VYi`ql`=NF_$0j-1oNnEoP82|b8f799k8TcP3Bl728cZO>(dO+kaV&V|)
zE}yC5S^W9*;JL}AJ|~I!wbr(_DWDU>`sS^!Uq?fQs(!uaFtUF-fQ39yN(SO&2?QLy
zy~R#(PQ5=qy~qMA?(tbDwWKvsnc2AFwsW)W+-C{jIpz$#jEKsO6_O*y!=uRCq-mjT
z5lY!`78-c|!<z)1W4+~`PU-N%PvugS{9^MtkEy~gBX|eKfz+9`-E^sy9fxp5t1y_U
z?%ZpNVXXXT`jIS&yk&z__H@k%4hFJv61XeIOg1{97E+{`@eQQx>&d?fjU$jUJ>P+b
zkQKYWKI0d*S0_aSYYT*xD+?a;60kpK`*@sd<LM>9@}1E2;AF?Ory(VX2ZT677=@UK
zNWFY)WaLeF7bwSj4bhZ<`sIgl5x74ZQJDZ&Q-JhRL-`fVepnfN%`nINc2kvH|Fj3h
z1+NjYiGI3{fL(t{Zk9=DD^jL)oop|(>pDv)Lv$_+>{qg(gy;jBHX%+s2jSunrqTm~
zA(Vz7s8uma#bkHUCJzdSb@uGL8Dk0W5;YZPvp+4EOrWlJ1x-aS$ZQgs&@JsO;57cn
zXd<fXLEobS9TvT+r+L5;qt+MO9H9EnzU%wrnM$TjTZ$th1vfocVeyWpD=_iwFx}qu
ztl9Xd@q8%=Sqts`sg#g?Cu<!`hUuY4ALr0t($aWl)!rBYM{^$UX_pUT?<>3(CP8Zy
zV$;LSjOGy0FOV6${T^B&GaBU`Y1;37js}Gft&PfvFCIHK;e$x3D5u?4G~{iAd@jtm
zGo43yPVX>U3g}F~0=*})#xfG<=`x$+_c>y3Q>&AJ2W=4hULC{?FP!`{u}lb{<i>mV
z*|7n(P?mm`B0gH~$nhPiR))$puy%X8K`rxf<W;Rg+YaI5VgRW$WG9YvjcZ0rH;^$C
zK5Lbhb7o7`@B&Po9MBrYAh}hTEyhGj7O5(#zN=@JUfGXd8<8-L7al)Jbl+HB<tWfN
zOqU5%Xdj|O`DrYFXJbwu<oMQAf1!;UhaeqfQ`8c-qvz6iV5|B2>kV&u><g1$dKCh%
zC>?tv2gE#5dB%55NOl7CHjJ^4^<9}Dd}oq1^VhbRlUXv;o(ZX#|19E2RfKuAv6!kr
zr;%0TikK8mceXHsSDhvw3C2P(*(G!&J<YQo-X5`C0)YkB&wXTBzXS6k*4PV7XyT`~
z)%!`h`5XQmn8(fhKP2Vd20=BFkD94CIqT_6qKEx<z+zeHG4~C+rv~OeQoD5AY`+Zk
zC7N-Qi=e0bDWnMzDbTd(=<3o^-vEAEF(>$Z>W}QI6<f;MZHaOX)0J~CHAa?cBm8DD
ziqz`Yp2^8#(_#~#FUPqxl9GjABY_s8OJL7UXl%F=_JkZ>2#8V1(L;wC3?TS!TPD-H
z@+Z3wUntQ%Z+3ARqRqfwKB-Arv6ML_ZbtE;I}0Jv(l;CT_vK!DfV2>HSwN>{lIE^b
zWbMKZc>fS<+jhV<C>{}Yk)K%aYd<Gx-;FDg00OvgKnUfuEcsOq)K-+`F@?>Efcp2W
z3ihC-Q%Q)+jrZ4UmJMOYRWzjrZ?6R~HYjoOWE(vz-(-YkbG)K9T_&jPnnDihzS70D
zojM*ewSwf<#oJq3>MVh2yAZeCn*-nQk7uFA&x1HomNDg6Hj#2xGx!OQOk)s}RI9Aj
zDE{#+P~vw2@kWs)-xD=0lWgWmdiRz4Ku&yu$RfMip4>_9nKxRAR#z@Q7O=4Cac8>x
zZmh{~!?ih{d~s`#-N+!%TWV0Nk{Zk5rKr5coEHH!5w(s8jB><|n;9Ejz#5T1Bsu;<
z$_VTFFW~$ihyOFIhAaM!vi~oKTiIjl<g+%5n^dde0e`(e^Yt)XL8Yp!aXwqgBH;f3
Dmmvdn

literal 0
HcmV?d00001

diff --git a/python/ray/util/sgd/torch/examples/sgd-development.yaml b/python/ray/util/sgd/torch/examples/sgd-development.yaml
new file mode 100644
index 000000000..e6697a272
--- /dev/null
+++ b/python/ray/util/sgd/torch/examples/sgd-development.yaml
@@ -0,0 +1,94 @@
+# An unique identifier for the head node and workers of this cluster.
+cluster_name: sgd-pytorch
+
+# The maximum number of workers nodes to launch in addition to the head
+# node. This takes precedence over min_workers. min_workers default to 0.
+min_workers: 0
+initial_workers: 0
+max_workers: 0
+
+target_utilization_fraction: 0.9
+
+# If a node is idle for this many minutes, it will be removed.
+idle_timeout_minutes: 10
+# docker:
+#     image: tensorflow/tensorflow:1.5.0-py3
+#     container_name: ray_docker
+
+# Cloud-provider specific configuration.
+provider:
+    type: aws
+    region: us-east-1
+    availability_zone: us-east-1c
+
+# How Ray will authenticate with newly launched nodes.
+auth:
+    ssh_user: ubuntu
+    # ssh_private_key: ...
+
+head_node:
+    InstanceType: p3dn.24xlarge
+    ImageId: ami-0698bcaf8bd9ef56d
+    # KeyName: ...
+    InstanceMarketOptions:
+        MarketType: spot
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 300
+           # SpotOptions:
+           #     MaxPrice: "9.0"
+
+
+worker_nodes:
+    InstanceType: p3.16xlarge
+    ImageId: ami-0698bcaf8bd9ef56d
+    # KeyName: ...
+    InstanceMarketOptions:
+        MarketType: spot
+    BlockDeviceMappings:
+        - DeviceName: /dev/sda1
+          Ebs:
+              VolumeSize: 300
+        # SpotOptions:
+        #     MaxPrice: "9.0"
+    #     # Run workers on spot by default. Comment this out to use on-demand.
+    #     InstanceMarketOptions:
+    #         MarketType: spot
+
+setup_commands:
+    # This replaces the standard anaconda Ray installation
+    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+    # Uncomment this and the filemount to update the Ray installation with your local Ray code
+    # - rm -rf ./anaconda3/lib/python3.6/site-packages/ray/util/sgd/
+    # - cp -rf ~/sgd ./anaconda3/lib/python3.6/site-packages/ray/util/
+
+    # Installing this without -U to make sure we don't replace the existing Ray installation
+    - pip install ray[rllib]
+    - pip install -U ipdb torch torchvision
+    # Install Apex
+    - rm -rf apex || true
+    - git clone https://github.com/NVIDIA/apex && cd apex && pip install -v --no-cache-dir  ./ || true
+
+
+file_mounts: {
+    # This should point to ray/python/ray/util/sgd.
+    # ~/sgd: ../../../sgd,
+}
+
+# Custom commands that will be run on the head node after common setup.
+head_setup_commands: []
+
+# Custom commands that will be run on worker nodes after common setup.
+worker_setup_commands: []
+
+# # Command to start ray on the head node. You don't need to change this.
+head_start_ray_commands:
+    - ray stop
+    - ray start --head --redis-port=6379 --object-manager-port=8076 --autoscaling-config=~/ray_bootstrap_config.yaml --object-store-memory=1000000000
+
+# Command to start ray on worker nodes. You don't need to change this.
+worker_start_ray_commands:
+    - ray stop
+    - ray start --address=$RAY_HEAD_IP:6379 --object-manager-port=8076 --object-store-memory=1000000000
+
diff --git a/python/ray/util/sgd/torch/torch_runner.py b/python/ray/util/sgd/torch/torch_runner.py
index bf812dfb7..022801e7d 100644
--- a/python/ray/util/sgd/torch/torch_runner.py
+++ b/python/ray/util/sgd/torch/torch_runner.py
@@ -114,7 +114,7 @@ class TorchRunner:
         else:
             self.criterion = self.loss_creator(self.config)
 
-        if torch.cuda.is_available() and hasattr("cuda", self.criterion):
+        if torch.cuda.is_available() and hasattr(self.criterion, "cuda"):
             self.criterion = self.criterion.cuda()
 
     def _create_schedulers_if_available(self):
diff --git a/python/ray/util/sgd/torch/torch_trainer.py b/python/ray/util/sgd/torch/torch_trainer.py
index 4a638b1f7..269cab493 100644
--- a/python/ray/util/sgd/torch/torch_trainer.py
+++ b/python/ray/util/sgd/torch/torch_trainer.py
@@ -525,7 +525,6 @@ class TorchTrainer:
                 return
             else:
                 delay = 2**i
-                logger.info("Resources: {}".format(resources))
                 logger.warning(
                     "No new workers found. Retrying in %d sec." % delay)
                 time.sleep(delay)
@@ -562,7 +561,6 @@ class TorchTrainable(Trainable):
         validation_stats = self._trainer.validate()
 
         train_stats.update(validation_stats)
-        # output {"mean_loss": test_loss, "mean_accuracy": accuracy}
         return train_stats
 
     def _save(self, checkpoint_dir):