release v0.3.4.1

docs/Trainer/Distributed training.md

@@ -3,6 +3,26 @@ Lightning makes multi-gpu training and 16 bit training trivial.
 *Note:*   
 None of the flags below require changing anything about your lightningModel definition. 
 
+---
+#### Choosing a backend  
+Lightning supports two backends. DataParallel and DistributedDataParallel. Both can be used for single-node multi-GPU training.
+For multi-node training you must use DistributedDataParallel.   
+
+You can toggle between each mode by setting this flag.
+``` {.python}
+# DEFAULT uses DataParallel
+trainer = Trainer(distributed_backend='dp')
+
+# change to distributed data parallel
+trainer = Trainer(distributed_backend='ddp')
+```
+
+If you request multiple nodes, the back-end will auto-switch to ddp.
+We recommend you use DistributedDataparallel even for single-node multi-GPU training. It is MUCH faster than DP but *may*
+have configuration issues depending on your cluster.
+
+For a deeper understanding of what lightning is doing, feel free to read [this guide](https://medium.com/@_willfalcon/9-tips-for-training-lightning-fast-neural-networks-in-pytorch-8e63a502f565).   
+
 ---
 #### 16-bit mixed precision
 16 bit precision can cut your memory footprint by half. If using volta architecture GPUs it can give a dramatic training speed-up as well.    
@@ -37,11 +57,17 @@ Make sure you're on a GPU machine. You can set as many GPUs as you want.
 In this setting, the model will run on all 8 GPUs at once using DataParallel under the hood.
 ```python
 # set these flags
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3,4,5,6,7"
+# lightning sets these flags for you automatically
+# no need to set yourself
+# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3,4,5,6,7"
 
 
-trainer = Trainer(gpus=[0,1,2,3,4,5,6,7])
+# to use DataParallel (default)
+trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='dp')
+
+# RECOMMENDED use DistributedDataParallel
+trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='ddp')
 ```
 
 ---
@@ -60,6 +86,20 @@ cluster = SlurmCluster(
     log_path='/some/path/to/save',
 )
 
+# OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT
+# which interface your nodes use for communication
+cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo')
+
+# see output of the NCCL connection process
+# NCCL is how the nodes talk to each other
+cluster.add_command('export NCCL_DEBUG=INFO')
+
+# setting a master port here is a good idea.
+cluster.add_command(f'export MASTER_PORT={PORT}')
+
+# good to load the latest NCCL version
+cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0'])
+
 # configure cluster
 cluster.per_experiment_nb_nodes = 12 
 cluster.per_experiment_nb_gpus = 8
@@ -67,6 +107,20 @@ cluster.per_experiment_nb_gpus = 8
 cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu')
 ```
 
+Finally, make sure to add a distributed sampler to your dataset. The distributed sampler copies a 
+portion of your dataset onto each GPU. (World_size = gpus_per_node * nb_nodes).   
+
+```python
+# ie: this:
+dataset = myDataset()
+dataloader = Dataloader(dataset)
+
+# becomes:
+dataset = myDataset()
+dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
+dataloader = Dataloader(dataset, sampler=dist_sampler)
+```
+
 ---
 #### Self-balancing architecture
 Here lightning distributes parts of your module across available GPUs to optimize for speed and memory.   

pytorch_lightning/models/trainer.py

@@ -178,9 +178,9 @@ class Trainer(TrainerIO):
             self.use_ddp = distributed_backend == 'ddp'
 
             # use ddp automatically if nb_gpu_nodes > 1
-            if nb_gpu_nodes > 1:
+            if nb_gpu_nodes > 1 and self.use_dp:
                 self.use_ddp = True
-                self.use_ddp = False
+                self.use_dp = False
                 w = 'DataParallel does not support nb_gpu_nodes > 1. ' \
                     'Switching to DistributedDataParallel for you. ' \
                     'To silence this warning set distributed_backend=ddp'
@@ -393,9 +393,10 @@ class Trainer(TrainerIO):
 
             # whenever we have the correct number of tasks, we let slurm manage processes
             # otherwise we launch the required number of processes
+            nb_requested_gpus = len(self.data_parallel_device_ids) * self.nb_gpu_nodes
+            nb_slurm_tasks = 0
             try:
                 nb_slurm_tasks = int(os.environ['SLURM_NTASKS'])
-                nb_requested_gpus = len(self.data_parallel_device_ids) * self.nb_gpu_nodes
                 is_slurm_managing_tasks = nb_slurm_tasks == nb_requested_gpus
             except Exception as e:
                 # likely not on slurm, so set the slurm managed flag to false
@@ -442,7 +443,6 @@ class Trainer(TrainerIO):
         self.optimizers = model.configure_optimizers()
 
         model.cuda(self.data_parallel_device_ids[0])
-        model = LightningDataParallel(model, device_ids=self.data_parallel_device_ids)
 
         # run through amp wrapper
         if self.use_amp:
@@ -452,6 +452,8 @@ class Trainer(TrainerIO):
             )
             self.optimizers = optimizers
 
+        model = LightningDataParallel(model, device_ids=self.data_parallel_device_ids)
+
         self.__run_pretrain_routine(model)
 
     def ddp_train(self, gpu_nb, model):
@@ -505,7 +507,7 @@ class Trainer(TrainerIO):
             )
             self.optimizers = optimizers
 
-        model = LightningDistributedDataParallel(model, device_ids=[gpu_nb])
+        model = LightningDistributedDataParallel(model, device_ids=[gpu_nb], find_unused_parameters=True)
 
         # continue training routine
         self.__run_pretrain_routine(model)
@@ -774,6 +776,11 @@ class Trainer(TrainerIO):
         else:
             loss.backward()
 
+        # insert after step hook
+        if self.__is_function_implemented('on_after_backward'):
+            model_ref = self.__get_model()
+            response = model_ref.on_after_backward()
+
         if self.print_nan_grads:
             model = self.__get_model()
             for param in model.parameters():
@@ -794,6 +801,11 @@ class Trainer(TrainerIO):
             for optimizer in self.optimizers:
                 optimizer.step()
 
+                # insert after step hook
+                if self.__is_function_implemented('on_before_zero_grad'):
+                    model_ref = self.__get_model()
+                    response = model_ref.on_before_zero_grad(optimizer)
+
                 # clear gradients
                 optimizer.zero_grad()
 

pytorch_lightning/root_module/hooks.py

@@ -22,3 +22,24 @@ class ModelHooks(torch.nn.Module):
     def on_tng_metrics(self, metrics):
         pass
 
+    def on_before_zero_grad(self, optimizer):
+        """
+        Called after optimizer.step() and before optimizer.zero_grad()
+
+        for optimizer in optimizers:
+            optimizer.step()
+            model.on_before_zero_grad(optimizer) # < ---- called here
+            optimizer.zero_grad
+
+        :param optimizer:
+        :return:
+        """
+        pass
+
+    def on_after_backward(self):
+        """
+        Called after loss.backward() and before optimizers do anything
+        :return:
+        """
+        pass
+

setup.py

@@ -7,7 +7,7 @@ from setuptools import setup, find_packages
 # http://blog.ionelmc.ro/2014/05/25/python-packaging/
 setup(
     name="pytorch-lightning",
-    version='0.3.1',
+    version='0.3.4.1',
     description="The Keras for ML researchers using PyTorch",
     author="William Falcon",
     author_email="waf2107@columbia.edu",