From d0faf97893930d3c71c0818a5ae2a87bb2f7f454 Mon Sep 17 00:00:00 2001
From: William Falcon <waf2107@columbia.edu>
Date: Fri, 24 Apr 2020 16:51:26 -0400
Subject: [PATCH] fixed dataset stuff + docs (#1599)

* Fixed dataset docs and disabled auto-sampler for iterable dataset
---
 docs/source/multi_gpu.rst                 | 45 +++++++++++++++++++++++
 pytorch_lightning/trainer/data_loading.py | 15 +++++++-
 2 files changed, 59 insertions(+), 1 deletion(-)

diff --git a/docs/source/multi_gpu.rst b/docs/source/multi_gpu.rst
index ab6669e6..07a9f5a7 100644
--- a/docs/source/multi_gpu.rst
+++ b/docs/source/multi_gpu.rst
@@ -69,6 +69,10 @@ when needed.
         dataset = MNIST(...)
         return DataLoader(dataset)
 
+.. note:: If you don't want this behavior, disable it with `Trainer(replace_sampler_ddp=False)`
+
+.. note:: For iterable datasets, we don't do this automatically.
+
 Distributed modes
 -----------------
 Lightning allows multiple ways of training
@@ -279,3 +283,44 @@ Implement Your Own Distributed (DDP) training
 If you need your own way to init PyTorch DDP you can override :meth:`pytorch_lightning.core.LightningModule.`.
 
 If you also need to use your own DDP implementation, override:  :meth:`pytorch_lightning.core.LightningModule.configure_ddp`.
+
+
+Batch size
+----------
+When using distributed training make sure to modify your learning rate according to your effective
+batch size.
+
+Let's say you have a batch size of 7 in your dataloader.
+
+.. code-block::
+
+    class LitModel(LightningModule):
+
+        def train_dataloader(self):
+            return Dataset(..., batch_size=7)
+
+In (DDP, Horovod) your effective batch size will be 7 * gpus * num_nodes.
+
+.. code-block::
+
+    # effective batch size = 7 * 8
+    Trainer(gpus=8, distributed_backend='ddp|horovod')
+
+    # effective batch size = 7 * 8 * 10
+    Trainer(gpus=8, num_nodes=10, distributed_backend='ddp|horovod')
+
+
+In DDP2, your effective batch size will be 7 * num_nodes.
+The reason is that the full batch is visible to all GPUs on the node when using DDP2.
+
+.. code-block::
+
+    # effective batch size = 7
+    Trainer(gpus=8, distributed_backend='ddp2')
+
+    # effective batch size = 7 * 10
+    Trainer(gpus=8, num_nodes=10, distributed_backend='ddp2')
+
+
+.. note:: Huge batch sizes are actually really bad for convergence. Check out:
+        `Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour <https://arxiv.org/abs/1706.02677>`_
diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py
index 2dde0137..71ef0934 100644
--- a/pytorch_lightning/trainer/data_loading.py
+++ b/pytorch_lightning/trainer/data_loading.py
@@ -10,6 +10,12 @@ from pytorch_lightning.core import LightningModule
 from pytorch_lightning.utilities import rank_zero_warn
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
 
+try:
+    from torch.utils.data import IterableDataset
+    ITERABLE_DATASET_EXISTS = True
+except ImportError:
+    ITERABLE_DATASET_EXISTS = False
+
 try:
     from apex import amp
 except ImportError:
@@ -95,7 +101,14 @@ class TrainerDataLoadingMixin(ABC):
     def auto_add_sampler(self, dataloader: DataLoader, train: bool) -> DataLoader:
 
         # don't do anything if it's not a dataloader
-        if not isinstance(dataloader, DataLoader):
+        # don't manipulate iterable datasets
+        is_dataloader = isinstance(dataloader, DataLoader)
+
+        is_iterable_ds = False
+        if ITERABLE_DATASET_EXISTS and hasattr(dataloader, 'dataset'):
+            is_iterable_ds = isinstance(dataloader.dataset, IterableDataset)
+
+        if not is_dataloader or is_iterable_ds:
             return dataloader
         need_dist_sampler = (self.use_ddp or self.use_ddp2 or self.use_horovod or self.use_tpu)
         if self.replace_sampler_ddp and need_dist_sampler: