release v0.5.2.1

Update trainer.py
Update README.md
2026-09-12 12:40:20 +08:00 · 2019-10-10 18:19:16 -04:00 · 2019-10-10 18:17:26 -04:00 · 2019-10-10 16:31:12 -04:00 · 2019-10-10 15:17:51 -04:00 · 2019-10-10 15:17:19 -04:00
61 changed files with 2360 additions and 2094 deletions
@@ -0,0 +1,5 @@
+# use this to run tests
+rm -rf tests/save_dir*
+rm -rf tests/mlruns_*
+coverage run --source pytorch_lightning -m py.test pytorch_lightning tests examples -v --doctest-modules   
+coverage report -m
@@ -17,6 +17,8 @@
 [![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=latest)](https://pytorch-lightning.readthedocs.io/en/latest)
 [![Gitter](https://badges.gitter.im/PyTorch-Lightning/community.svg)](https://gitter.im/PyTorch-Lightning/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/williamFalcon/pytorch-lightning/blob/master/LICENSE)
+[![Next Release](https://img.shields.io/badge/Next%20Release-Nov%206-<COLOR>.svg)](https://shields.io/)
+
 <!-- 
 removed until codecov badge isn't empy. likely a config error showing nothing on master.
 [![codecov](https://codecov.io/gh/Borda/pytorch-lightning/branch/master/graph/badge.svg)](https://codecov.io/gh/Borda/pytorch-lightning)
@@ -50,9 +52,9 @@ Lightning sets up all the boilerplate state-of-the-art training for you so you c
 - [What lightning automates](https://github.com/williamFalcon/pytorch-lightning#what-does-lightning-control-for-me)    
 - [Tensorboard integration](https://github.com/williamFalcon/pytorch-lightning#tensorboard)    
 - [Lightning features](https://github.com/williamFalcon/pytorch-lightning#lightning-automates-all-of-the-following-each-is-also-configurable)    
- [Demos](https://github.com/williamFalcon/pytorch-lightning#demo)    
+- [Examples](https://github.com/williamFalcon/pytorch-lightning#examples)    
 - [Tutorials](https://github.com/williamFalcon/pytorch-lightning#tutorials)
- [Contributing](https://github.com/williamFalcon/pytorch-lightning/blob/master/CONTRIBUTING.md)    
+- [Contributing](https://github.com/williamFalcon/pytorch-lightning/blob/master/.github/CONTRIBUTING.md)
 - [Bleeding edge install](https://github.com/williamFalcon/pytorch-lightning#bleeding-edge)   
 - [Lightning Design Principles](https://github.com/williamFalcon/pytorch-lightning#lightning-design-principles)   
 - [Asking for help](https://github.com/williamFalcon/pytorch-lightning#asking-for-help)
@@ -66,7 +68,9 @@ Think about Lightning as refactoring your research code instead of using a new f
 The LightningModule defines a *system* such as seq-2-seq, GAN, etc... It can ALSO define a simple classifier such as the example below.     

 To use lightning do 2 things:  
-1. [Define a LightningModule](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/)         
+1. [Define a LightningModule](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/)     
+
+**WARNING:** This syntax is for version 0.5.0+ where abbreviations were removed.    
 ```python
 import os
 import torch
@@ -91,7 +95,9 @@ class CoolSystem(pl.LightningModule):
        # REQUIRED
        x, y = batch
        y_hat = self.forward(x)
-        return {'loss': F.cross_entropy(y_hat, y)}
+        loss = F.cross_entropy(y_hat, y)
+        tensorboard_logs = {'train_loss': loss}
+        return {'loss': loss, 'log': tensorboard_logs}

    def validation_step(self, batch, batch_nb):
        # OPTIONAL
@@ -102,15 +108,17 @@ class CoolSystem(pl.LightningModule):
    def validation_end(self, outputs):
        # OPTIONAL
        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
-        return {'avg_val_loss': avg_loss}
+        tensorboard_logs = {'val_loss': avg_loss}
+        return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}

    def configure_optimizers(self):
        # REQUIRED
        # can return multiple optimizers and learning_rate schedulers
+        # (LBFGS it is automatically supported, no need for closure function)
        return torch.optim.Adam(self.parameters(), lr=0.02)

    @pl.data_loader
-    def tng_dataloader(self):
+    def train_dataloader(self):
        # REQUIRED
        return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)

@@ -135,27 +143,27 @@ trainer = Trainer()
 trainer.fit(model)   
 ```     

-Or with tensorboard logger and some options turned on such as multi-gpu, etc...
+Trainer sets up a tensorboard logger, early stopping and checkpointing by default (you can modify all of them or
+use something other than tensorboard).   
+
+Here are more advanced examples
 ```python   
-from test_tube import Experiment    
-
-# PyTorch summarywriter with a few bells and whistles    
-exp = Experiment(save_dir=os.getcwd())
-
 # train on cpu using only 10% of the data (for demo purposes)
-# pass in experiment for automatic tensorboard logging.    
-trainer = Trainer(experiment=exp, max_nb_epochs=1, train_percent_check=0.1)
+trainer = Trainer(max_nb_epochs=1, train_percent_check=0.1)

-# train on 4 gpus
-# trainer = Trainer(experiment=exp, max_nb_epochs=1, gpus=[0, 1, 2, 3])
+# train on 4 gpus (lightning chooses GPUs for you)
+# trainer = Trainer(max_nb_epochs=1, gpus=4, distributed_backend='ddp')  
+
+# train on 4 gpus (you choose GPUs)
+# trainer = Trainer(max_nb_epochs=1, gpus=[0, 1, 3, 7], distributed_backend='ddp')   

 # train on 32 gpus across 4 nodes (make sure to submit appropriate SLURM job)
-# trainer = Trainer(experiment=exp, max_nb_epochs=1, gpus=[0, 1, 2, 3, 4, 5, 6, 7], nb_gpu_nodes=4)
+# trainer = Trainer(max_nb_epochs=1, gpus=8, nb_gpu_nodes=4, distributed_backend='ddp')

 # train (1 epoch only here for demo)
 trainer.fit(model)

-# view tensorflow logs 
+# view tensorboard logs 
 print('View tensorboard logs by running\ntensorboard --logdir %s' % os.getcwd())
 print('and going to http://localhost:6006 on your browser')
 ```    
@@ -170,20 +178,20 @@ trainer.test()
 Everything in gray!    
 You define the blue parts using the LightningModule interface:  

-![Ouverview](./docs/source/_static/overview_flat.jpg)
+![Overview](./docs/source/_static/overview_flat.jpg)

 ```python
 # what to do in the training loop
-def training_step(self, data_batch, batch_nb):
+def training_step(self, batch, batch_nb):

 # what to do in the validation loop
-def validation_step(self, data_batch, batch_nb):
+def validation_step(self, batch, batch_nb):

 # how to aggregate validation_step outputs
 def validation_end(self, outputs):

 # and your dataloaders
-def tng_dataloader():
+def train_dataloader():
 def val_dataloader():
 def test_dataloader():
 ```
@@ -192,8 +200,8 @@ def test_dataloader():

 ```python
 # define what happens for training here
-def training_step(self, data_batch, batch_nb):
-    x, y = data_batch
+def training_step(self, batch, batch_nb):
+    x, y = batch
    
    # define your own forward and loss calculation
    hidden_states = self.encoder(x)
@@ -219,8 +227,8 @@ def training_step(self, data_batch, batch_nb):

 ```python
 # define what happens for validation here
-def validation_step(self, data_batch, batch_nb):    
-    x, y = data_batch
+def validation_step(self, batch, batch_nb):    
+    x, y = batch
    
    # or as basic as a CNN classification
    out = self.forward(x)
@@ -245,12 +253,13 @@ def validation_end(self, outputs):

    val_loss_mean /= len(outputs)
    val_acc_mean /= len(outputs)
-    tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-    return tqdm_dic
+    logs = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
+    result = {'log': logs}
+    return result
 ```
   
 ## Tensorboard    
-Lightning is fully integrated with tensorboard.   
+Lightning is fully integrated with tensorboard, MLFlow and supports any logging module.   

 ![tensorboard-support](./docs/source/_static/tf_loss.png)

@@ -258,26 +267,11 @@ Lightning also adds a text column with all the hyperparameters for this experime

 ![tensorboard-support](./docs/source/_static/tf_tags.png)

-Simply note the path you set for the [Experiment](https://williamfalcon.github.io/test-tube/experiment_tracking/experiment/) from [test_tube](https://github.com/williamFalcon/test-tube)
-```python
-from test_tube import Experiment
-from pytorch_lightning import Trainer
-
-exp = Experiment(save_dir='/some/path')
-trainer = Trainer(experiment=exp)
-...
-```   
-
-And run tensorboard from that dir   
-```bash
-tensorboard --logdir /some/path     
-```    
-
 ## Lightning automates all of the following ([each is also configurable](https://williamfalcon.github.io/pytorch-lightning/Trainer/)):

-
 #### Checkpointing    

+- [Checkpoint callback](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
 - [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
 - [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics) 
 - [Restoring training session](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#restoring-training-session)
@@ -321,6 +315,7 @@ tensorboard --logdir /some/path

 - [Accumulate gradients](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients)
 - [Force training for min or max epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs)
+- [Early stopping callback](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#early-stopping)
 - [Force disable early stop](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop)
 - [Gradient Clipping](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping)
 - [Hooks](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/)
@@ -341,27 +336,11 @@ tensorboard --logdir /some/path
 #### Testing loop  
 - [Run test set](https://williamfalcon.github.io/pytorch-lightning/Trainer/Testing%20loop/)  

-## Demo
-```bash
-# install lightning
-pip install pytorch_lightning
-
-# clone lightning for the demo
-git clone https://github.com/williamFalcon/pytorch-lightning.git
-cd pytorch-lightning
-cd examples/new_project_templates/
-
-# all of the following demos use the SAME model to show no modification needs to be made to your code
-
-# train on cpu 
-python single_cpu_template.py
-
-# train on multiple-gpus 
-python single_gpu_node_template.py --gpus "0,1"
-
-# train on 32 gpus on a cluster (run on a SLURM managed cluster)
-python multi_node_cluster_template.py --nb_gpu_nodes 4 --gpus '0,1,2,3,4,5,6,7'
-```
+## Examples   
+- [GAN](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/domain_templates/gan.py)    
+- [MNIST](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/basic_examples)      
+- [Other projects using Lightning](https://github.com/williamFalcon/pytorch-lightning/network/dependents?package_id=UGFja2FnZS0zNzE3NDU4OTM%3D)    
+- [Multi-node](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/multi_node_examples)   

 ## Tutorials   
 - [Basic Lightning use](https://towardsdatascience.com/supercharge-your-ai-research-with-pytorch-lightning-337948a99eec)    
@@ -10,7 +10,7 @@ Otherwise, to Define a Lightning Module, implement the following methods:
 **Required**:  

 - [training_step](RequiredTrainerInterface.md#training_step)      
- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)    
+- [train_dataloader](RequiredTrainerInterface.md#train_dataloader)    
 - [configure_optimizers](RequiredTrainerInterface.md#configure_optimizers)    

 **Optional**:   
@@ -23,7 +23,6 @@ Otherwise, to Define a Lightning Module, implement the following methods:
 - [test_dataloader](RequiredTrainerInterface.md#test_dataloader)    
 - [on_save_checkpoint](RequiredTrainerInterface.md#on_save_checkpoint)    
 - [on_load_checkpoint](RequiredTrainerInterface.md#on_load_checkpoint)    
- [update_tng_log_metrics](RequiredTrainerInterface.md#update_tng_log_metrics)    
 - [add_model_specific_args](RequiredTrainerInterface.md#add_model_specific_args)    

 ---
@@ -78,10 +77,10 @@ class CoolModel(pl.LightningModule):

    def configure_optimizers(self):
        # REQUIRED
-        return [torch.optim.Adam(self.parameters(), lr=0.02)]
+        return torch.optim.Adam(self.parameters(), lr=0.02)

    @pl.data_loader
-    def tng_dataloader(self):
+    def train_dataloader(self):
        return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)

    @pl.data_loader
@@ -111,7 +110,7 @@ The LightningModule interface is on the right. Each method corresponds to a part
 ### training_step

 ``` {.python}
-def training_step(self, data_batch, batch_nb)
+def training_step(self, batch, batch_nb)
 ```

 In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model.
@@ -120,7 +119,7 @@ In this step you'd normally do the forward pass and calculate the loss for a bat

 | Param  | description  |
 |---|---|
-|  data_batch | The output of your dataloader. A tensor, tuple or list  |
+|  batch | The output of your dataloader. A tensor, tuple or list  |
 |  batch_nb | Integer displaying which batch this is  |

 **Return**   
@@ -130,14 +129,15 @@ Dictionary or OrderedDict
 | key  | value  | is required |
 |---|---|---|
 |  loss | tensor scalar  | Y |
-|  prog | Dict for progress bar display. Must have only tensors  | N |
+|  progress_bar | Dict for progress bar display. Must have only tensors  | N |
+|  log | Dict of metrics to add to logger. Must have only tensors (no images, etc)  | N |


 **Example**

 ``` {.python}
-def training_step(self, data_batch, batch_nb):
-    x, y, z = data_batch
+def training_step(self, batch, batch_nb):
+    x, y, z = batch
    
    # implement your own
    out = self.forward(x)
@@ -145,7 +145,8 @@ def training_step(self, data_batch, batch_nb):
    
    output = {
        'loss': loss, # required
-        'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional
+        'progress_bar': {'training_loss': loss}, # optional (MUST ALL BE TENSORS)
+        'log': {'training_loss': loss} # optional (MUST ALL BE TENSORS)
    }
    
    # return a dict
@@ -155,21 +156,25 @@ def training_step(self, data_batch, batch_nb):
 If you define multiple optimizers, this step will also be called with an additional ```optimizer_idx``` param.    
 ``` {.python}
 # Multiple optimizers (ie: GANs)     
-def training_step(self, data_batch, batch_nb, optimizer_idx):
+def training_step(self, batch, batch_nb, optimizer_idx):
    if optimizer_idx == 0:
        # do training_step with encoder
    if optimizer_idx == 1:
        # do training_step with decoder    
 ```    

+You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to
+break out of the current training epoch early.
+
 --- 
-### tng_dataloader 
+### train_dataloader 

 ``` {.python}
@pl.data_loader
-def tng_dataloader(self)
+def train_dataloader(self)
 ```
-Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.
+Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.   
+If you want to change the data during every epoch DON'T use the data_loader decorator.   

 ##### Return
 PyTorch DataLoader
@@ -178,7 +183,7 @@ PyTorch DataLoader

 ``` {.python}
@pl.data_loader
-def tng_dataloader(self):
+def train_dataloader(self):
    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
    dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True)
    loader = torch.utils.data.DataLoader(
@@ -200,8 +205,7 @@ Set up as many optimizers and (optionally) learning rate schedulers as you need.
 Lightning will call .backward() and .step() on each one in every epoch.  If you use 16 bit precision it will also handle that.

 **Note:** If you use multiple optimizers, training_step will have an additional ```optimizer_idx``` parameter.    
-
-
+**Note 2:** If you use LBFGS lightning handles the closure function automatically for you.

 ##### Return    
 Return any of these 3 options:   
@@ -240,10 +244,10 @@ the [optimizer_step](https://williamfalcon.github.io/pytorch-lightning/Trainer/h

 ``` {.python}
 # if you have one val dataloader:
-def validation_step(self, data_batch, batch_nb)   
+def validation_step(self, batch, batch_nb)   

 # if you have multiple val dataloaders:  
-def validation_step(self, data_batch, batch_nb, dataloader_idx)
+def validation_step(self, batch, batch_nb, dataloader_idxdx)
 ```
 **OPTIONAL**    
 If you don't need to validate you don't need to implement this method. In this step you'd normally generate examples or calculate anything of interest such as accuracy. 
@@ -256,22 +260,22 @@ The dict you return here will be available in the `validation_end` method.

 | Param  | description  |
 |---|---|
-|  data_batch | The output of your dataloader. A tensor, tuple or list  |
+|  batch | The output of your dataloader. A tensor, tuple or list  |
 |  batch_nb | Integer displaying which batch this is  |
-|  dataloader_i | Integer displaying which dataloader this is (only if multiple val datasets used)  |
+|  dataloader_idx | Integer displaying which dataloader this is (only if multiple val datasets used)  |

 **Return**   

 | Return  | description  | optional |
 |---|---|---|   
-|  dict | Dict or OrderedDict with metrics to display in progress bar. All keys must be tensors. | Y |
+|  dict | Dict or OrderedDict - passed to the validation_end step | N |

 **Example**

 ``` {.python}
 # CASE 1: A single validation dataset
-def validation_step(self, data_batch, batch_nb):
-    x, y = data_batch
+def validation_step(self, batch, batch_nb):
+    x, y = batch
    
    # implement your own
    out = self.forward(x)
@@ -281,7 +285,7 @@ def validation_step(self, data_batch, batch_nb):
    # or generated text... or whatever
    sample_imgs = x[:6]
    grid = torchvision.utils.make_grid(sample_imgs)
-    self.experiment.add_image('example_images', grid, 0) 
+    self.logger.experiment.add_image('example_images', grid, 0) 
    
    # calculate acc
    labels_hat = torch.argmax(out, dim=1)
@@ -302,7 +306,7 @@ If you pass in multiple validation datasets, validation_step will have an additi

 ```python
 # CASE 2: multiple validation datasets
-def validation_step(self, data_batch, batch_nb, dataset_idx):
+def validation_step(self, batch, batch_nb, dataset_idx):
    # dataset_idx tells you which dataset this is.   
 ```   

@@ -328,9 +332,12 @@ The outputs here are strictly for the progress bar. If you don't need to display

 **Return**   

-| Return  | description  | optional |
-|---|---|---|   
-|  dict | Dict of OrderedDict with metrics to display in progress bar | Y |
+Dictionary or OrderedDict   
+
+| key  | value  | is required |
+|---|---|---|
+|  progress_bar | Dict for progress bar display. Must have only tensors  | N |
+|  log | Dict of metrics to add to logger. Must have only tensors (no images, etc)  | N |

 **Example**

@@ -351,8 +358,14 @@ def validation_end(self, outputs):

    val_loss_mean /= len(outputs)
    val_acc_mean /= len(outputs)
-    tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-    return tqdm_dic
+    tqdm_dict = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
+       
+    # show val_loss and val_acc in progress bar but only log val_loss
+    results = {
+        'progress_bar': tqdm_dict,
+        'log': {'val_loss': val_loss_mean.item()}
+    }
+    return results
 ```

 With multiple dataloaders, `outputs` will be a list of lists. The outer list contains
@@ -377,18 +390,24 @@ def validation_end(self, outputs):

    val_loss_mean /= i
    val_acc_mean /= i
-    tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-    return tqdm_dic
+    tqdm_dict = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
+    
+    # show val_loss and val_acc in progress bar but only log val_loss
+    results = {
+        'progress_bar': tqdm_dict,
+        'log': {'val_loss': val_loss_mean.item()}
+    }
+    return results
 ```

 ### test_step

 ``` {.python}
 # if you have one test dataloader:
-def test_step(self, data_batch, batch_nb)   
+def test_step(self, batch, batch_nb)   

 # if you have multiple test dataloaders:  
-def test_step(self, data_batch, batch_nb, dataloader_idx)
+def test_step(self, batch, batch_nb, dataloader_idxdx)
 ```
 **OPTIONAL**    
 If you don't need to test you don't need to implement this method. In this step you'd normally generate examples or calculate anything of interest such as accuracy.   
@@ -403,9 +422,9 @@ This function is used when you execute `trainer.test()`.

 | Param  | description  |
 |---|---|
-|  data_batch | The output of your dataloader. A tensor, tuple or list  |
+|  batch | The output of your dataloader. A tensor, tuple or list  |
 |  batch_nb | Integer displaying which batch this is  |
-|  dataloader_i | Integer displaying which dataloader this is (only if multiple test datasets used)  |
+|  dataloader_idx | Integer displaying which dataloader this is (only if multiple test datasets used)  |

 **Return**   

@@ -417,8 +436,8 @@ This function is used when you execute `trainer.test()`.

 ``` {.python}
 # CASE 1: A single test dataset
-def test_step(self, data_batch, batch_nb):
-    x, y = data_batch
+def test_step(self, batch, batch_nb):
+    x, y = batch
    
    # implement your own
    out = self.forward(x)
@@ -443,7 +462,7 @@ If you pass in multiple test datasets, test_step will have an additional argumen

 ```python
 # CASE 2: multiple test datasets
-def test_step(self, data_batch, batch_nb, dataset_idx):
+def test_step(self, batch, batch_nb, dataset_idx):
    # dataset_idx tells you which dataset this is.   
 ```   

@@ -490,8 +509,14 @@ def test_end(self, outputs):

    test_loss_mean /= len(outputs)
    test_acc_mean /= len(outputs)
-    tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
-    return tqdm_dic
+    tqdm_dict = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
+    
+    # show test_loss and test_acc in progress bar but only log test_loss
+    results = {
+        'progress_bar': tqdm_dict,
+        'log': {'test_loss': val_loss_mean.item()}
+    }
+    return results
 ```

 With multiple dataloaders, `outputs` will be a list of lists. The outer list contains
@@ -516,8 +541,14 @@ def test_end(self, outputs):

    test_loss_mean /= i 
    test_acc_mean /= i
-    tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
-    return tqdm_dic
+    tqdm_dict = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
+    
+    # show test_loss and test_acc in progress bar but only log test_loss
+    results = {
+        'progress_bar': tqdm_dict,
+        'log': {'test_loss': val_loss_mean.item()}
+    }
+    return results
 ```

 --- 
@@ -573,6 +604,7 @@ def val_dataloader(self)
 If you don't need a validation dataset and a validation_step, you don't need to implement this method.    

 Called by lightning during validation loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.   
+If you want to change the data during every epoch DON'T use the data_loader decorator.   

 ##### Return
 PyTorch DataLoader or list of PyTorch Dataloaders.    
@@ -612,6 +644,7 @@ def test_dataloader(self)
 If you don't need a test dataset and a test_step, you don't need to implement this method.    

 Called by lightning during test loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.
+If you want to change the data during every epoch DON'T use the data_loader decorator.   

 ##### Return
 PyTorch DataLoader
@@ -632,26 +665,6 @@ def test_dataloader(self):
    return loader
 ```

--- 
-### update_tng_log_metrics 
-
-``` {.python}
-def update_tng_log_metrics(self, logs)
-```
-Called by lightning right before it logs metrics for this batch.
-This is a chance to amend or add to the metrics about to be logged.
-
-##### Return
-Dict 
-
-**Example**
-
-``` {.python}
-def update_tng_log_metrics(self, logs):
-    # modify or add to logs
-    return logs
-```
-
 --- 
 ### add_model_specific_args 

@@ -674,7 +687,7 @@ def add_model_specific_args(parent_parser, root_dir):
    parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])

    # param overwrites
-    # parser.set_defaults(gradient_clip=5.0)
+    # parser.set_defaults(gradient_clip_val=5.0)

    # network params
    parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
@@ -9,12 +9,22 @@ The current epoch
 Current dtype    

 ---
-#### experiment    
-An instance of test-tube Experiment which you can use to log anything for tensorboard (subclass of [PyTorch SummaryWriter](https://pytorch.org/docs/stable/tensorboard.html)).   
+#### logger    
+A reference to the logger you passed into trainer. 
+Passing a logger is optional. If you don't pass one in, Lightning will create one for you automatically.
+This logger saves logs to '''/os.getcwd()/lightning_logs'''
+```python
+Trainer(logger=your_logger)
+```    
+
+Call it from anywhere in your LightningModule to add metrics, images, etc... whatever your logger supports.  
+
+Here is an example using the TestTubeLogger (which is a wrapper on [PyTorch SummaryWriter](https://pytorch.org/docs/stable/tensorboard.html) with versioned folder structure).     
 ```{.python}
-self.experiment.add_embedding(...)   
-self.experiment.log({'val_loss': 0.9})   
-self.experiment.add_scalars(...)   
+# if logger is a tensorboard logger or TestTubeLogger
+self.logger.experiment.add_embedding(...)   
+self.logger.experiment.log({'val_loss': 0.9})   
+self.logger.experiment.add_scalars(...)   
 ```

 --- 
@@ -22,7 +32,7 @@ self.experiment.add_scalars(...)
 Total training batches seen across all epochs   

 --- 
-#### gradient_clip   
+#### gradient_clip_val   
 The current gradient clip value    

 ---
@@ -2,17 +2,25 @@ Lightning can automate saving and loading checkpoints.

 ---
 ### Model saving
-To enable checkpointing, define the checkpoint callback and give it to the trainer.
+Checkpointing is enabled by default to the current working directory.
+To change the checkpoint path pass in :
+```python
+Trainer(default_save_path='/your/path/to/save/checkpoints')
+```
+
+To modify the behavior of checkpointing pass in your own callback.

 ``` {.python}
 from pytorch_lightning.callbacks import ModelCheckpoint

+# DEFAULTS used by the Trainer
 checkpoint_callback = ModelCheckpoint(
-    filepath='/path/to/store/weights.ckpt',
+    filepath=os.getcwd(),
    save_best_only=True,
    verbose=True,
    monitor='val_loss',
-    mode='min'
+    mode='min',
+    prefix=''
 )

 trainer = Trainer(checkpoint_callback=checkpoint_callback)
@@ -8,6 +8,17 @@ None of the flags below require changing anything about your lightningModel defi
 Lightning supports two backends. DataParallel and DistributedDataParallel. Both can be used for single-node multi-GPU training.
 For multi-node training you must use DistributedDataParallel.   

+##### DataParallel (dp)   
+Splits a batch across multiple GPUs on the same node. Cannot be used for multi-node training.   
+
+##### DistributedDataParallel (ddp)   
+Trains a copy of the model on each GPU and only syncs gradients. If used with DistributedSampler, each GPU trains
+on a subset of the full dataset.  
+
+##### DistributedDataParallel-2 (ddp2)   
+Works like DDP, except each node trains a single copy of the model using ALL GPUs on that node. 
+Very useful when dealing with negative samples, etc...
+
 You can toggle between each mode by setting this flag.
 ``` {.python}
 # DEFAULT (when using single GPU or no GPUs)
@@ -18,6 +29,9 @@ trainer = Trainer(distributed_backend='dp')

 # change to distributed data parallel (gpus > 1)
 trainer = Trainer(distributed_backend='ddp')
+
+# change to distributed data parallel (gpus > 1)
+trainer = Trainer(distributed_backend='ddp2')
 ```

 If you request multiple nodes, the back-end will auto-switch to ddp.
@@ -37,7 +51,7 @@ Below are the possible configurations we support.
 |---|---|---|---|---|---|
 | Y  |   |   |   |  | ```Trainer(gpus=1)``` |
 | Y  |   |   |   | Y | ```Trainer(gpus=1, use_amp=True)``` |
-|   | Y | Y |   |   | ```Trainer(gpus=k)``` |
+|   | Y | Y |   |   | ```Trainer(gpus=k, distributed_backend='dp')``` |
 |   | Y |  | Y  |  | ```Trainer(gpus=k, distributed_backend='ddp')``` |
 |   | Y |  | Y  | Y | ```Trainer(gpus=k, distributed_backend='ddp', use_amp=True)``` |

@@ -74,6 +88,21 @@ First, install apex (if install fails, look [here](https://github.com/NVIDIA/ape
 ```bash
 $ git clone https://github.com/NVIDIA/apex
 $ cd apex
+
+# ------------------------
+# OPTIONAL: on your cluster you might need to load cuda 10 or 9
+# depending on how you installed PyTorch
+
+# see available modules
+module avail
+
+# load correct cuda before install
+module load cuda-10.0
+# ------------------------
+
+# make sure you've loaded a cuda version > 4.0 and < 7.0
+module load gcc-6.1.0
+
 $ pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./
 ```

@@ -105,41 +134,54 @@ trainer = Trainer(gpus=8, distributed_backend='ddp')

 ---
 #### Multi-node
-Multi-node training is easily done by specifying these flags.
+Multi-node training is easily done by specifying these flags. 
 ```python
 # train on 12*8 GPUs
-trainer = Trainer(gpus=8, nb_gpu_nodes=12)
+trainer = Trainer(gpus=8, nb_gpu_nodes=12, distributed_backend='ddp')
 ```

-In addition, make sure to set up your SLURM job correctly via the [SlurmClusterObject](https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/). In particular, specify the number of tasks per node correctly.
+You must configure your job submission script correctly for the trainer to work. Here is an example
+script for the above trainer configuration.   

-```python
-cluster = SlurmCluster(
-    hyperparam_optimizer=test_tube.HyperOptArgumentParser(),
-    log_path='/some/path/to/save',
-)
+```sh
+#!/bin/bash -l

-# OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT
-# which interface your nodes use for communication
-cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo')
+# SLURM SUBMIT SCRIPT
+#SBATCH --nodes=12
+#SBATCH --gres=gpu:8
+#SBATCH --ntasks-per-node=8
+#SBATCH --mem=0
+#SBATCH --time=0-02:00:00

-# see output of the NCCL connection process
-# NCCL is how the nodes talk to each other
-cluster.add_command('export NCCL_DEBUG=INFO')
+# activate conda env
+conda activate my_env

-# setting a master port here is a good idea.
-cluster.add_command('export MASTER_PORT=%r' % PORT)
+# -------------------------
+# OPTIONAL
+# -------------------------
+# debugging flags (optional)
+# export NCCL_DEBUG=INFO
+# export PYTHONFAULTHANDLER=1

-# good to load the latest NCCL version
-cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0'])
+# PyTorch comes with prebuilt NCCL support... but if you have issues with it
+# you might need to load the latest version from your  modules
+# module load NCCL/2.4.7-1-cuda.10.0

-# configure cluster
-cluster.per_experiment_nb_nodes = 12 
-cluster.per_experiment_nb_gpus = 8
+# on your cluster you might need these:
+# set the network interface
+# export NCCL_SOCKET_IFNAME=^docker0,lo
+# -------------------------

-cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu')
+# random port between 12k and 20k
+export MASTER_PORT=$((12000 + RANDOM % 20000))
+
+# run script from above
+python my_main_file.py
 ```

+**NOTE:** When running in DDP mode, any errors in your code will show up as an NCCL issue.
+Set the ```NCCL_DEBUG=INFO``` flag to see the ACTUAL error.
+
 Finally, make sure to add a distributed sampler to your dataset. The distributed sampler copies a 
 portion of your dataset onto each GPU. (World_size = gpus_per_node * nb_nodes).   

@@ -154,6 +196,58 @@ dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
 dataloader = Dataloader(dataset, sampler=dist_sampler)
 ```

+#### Auto-slurm-job-submission
+Instead of manually building SLURM scripts, you can use the [SlurmCluster object](https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/) to
+do this for you. The SlurmCluster can also run a grid search if you pass in a [HyperOptArgumentParser](https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/).
+
+Here is an example where you run a grid search of 9 combinations of hyperparams.
+[The full examples are here](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/new_project_templates/multi_node_examples).
+```python
+# grid search 3 values of learning rate and 3 values of number of layers for your net
+# this generates 9 experiments (lr=1e-3, layers=16), (lr=1e-3, layers=32), (lr=1e-3, layers=64), ... (lr=1e-1, layers=64)
+parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
+parser.opt_list('--learning_rate', default=0.001, type=float, options=[1e-3, 1e-2, 1e-1], tunable=True)
+parser.opt_list('--layers', default=1, type=float, options=[16, 32, 64], tunable=True)
+hyperparams = parser.parse_args()
+
+# Slurm cluster submits 9 jobs, each with a set of hyperparams
+cluster = SlurmCluster(
+    hyperparam_optimizer=hyperparams,
+    log_path='/some/path/to/save',
+)
+
+# OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT
+# which interface your nodes use for communication
+cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo')
+
+# see output of the NCCL connection process
+# NCCL is how the nodes talk to each other
+cluster.add_command('export NCCL_DEBUG=INFO')
+
+# setting a master port here is a good idea.
+cluster.add_command('export MASTER_PORT=%r' % PORT)
+
+# ************** DON'T FORGET THIS ***************
+# MUST load the latest NCCL version
+cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0'])
+
+# configure cluster
+cluster.per_experiment_nb_nodes = 12 
+cluster.per_experiment_nb_gpus = 8
+
+cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu')  
+
+# submit a script with 9 combinations of hyper params
+# (lr=1e-3, layers=16), (lr=1e-3, layers=32), (lr=1e-3, layers=64), ... (lr=1e-1, layers=64)
+cluster.optimize_parallel_cluster_gpu(
+    main,
+    nb_trials=9, # how many permutations of the grid search to run
+    job_name='name_for_squeue'
+)
+```
+
+The other option is that you generate scripts on your own via a bash command or use another library...
+
 ---
 #### Self-balancing architecture
 Here lightning distributes parts of your module across available GPUs to optimize for speed and memory.   
@@ -1,7 +1,113 @@
-Lighting offers a few options for logging information about model, gpu usage, etc (via test-tube). It also offers printing options for training monitoring.
+Lighting offers options for logging information about model, gpu usage, etc, via several different logging frameworks. It also offers printing options for training monitoring.
+
+---   
+### default_save_path   
+Lightning sets a default TestTubeLogger and CheckpointCallback for you which log to
+```os.getcwd()``` by default. To modify the logging path you can set:
+```python
+Trainer(default_save_path='/your/path/to/save/checkpoints')
+```
+ 
+If you need more custom behavior (different paths for both, different metrics, etc...)
+from the logger and the checkpointCallback, pass in your own instances as explained below.


 ---
+### Setting up logging
+
+The trainer inits a default logger for you (TestTubeLogger). All logs will
+go to the current working directory under a folder named ```os.getcwd()/lightning_logs``. 
+
+If you want to modify the default logging behavior even more, pass in a logger
+(which should inherit from `LightningBaseLogger`).   
+
+```{.python}
+my_logger = MyLightningLogger(...)
+trainer = Trainer(logger=my_logger)
+```
+
+The path in this logger will overwrite default_save_path.
+
+Lightning supports several common experiment tracking frameworks out of the box
+
+---
+#### Test tube
+
+Log using [test tube](https://williamfalcon.github.io/test-tube/).
+
+```{.python}
+from pytorch_lightning.logging import TestTubeLogger
+tt_logger = TestTubeLogger(
+    save_dir=".",
+    name="default",
+    debug=False,
+    create_git_tag=False
+)
+trainer = Trainer(logger=tt_logger)
+```
+
+---
+#### MLFlow
+
+Log using [mlflow](https://mlflow.org)
+
+```{.python}
+from pytorch_lightning.logging import MLFlowLogger
+mlf_logger = MLFlowLogger(
+    experiment_name="default",
+    tracking_uri="file:/."
+)
+trainer = Trainer(logger=mlf_logger)
+```
+
+---
+#### Custom logger
+
+You can implement your own logger by writing a class that inherits from
+`LightningLoggerBase`. Use the `rank_zero_only` decorator to make sure that
+only the first process in DDP training logs data.
+
+```{.python}
+from pytorch_lightning.logging import LightningLoggerBase, rank_zero_only
+
+class MyLogger(LightningLoggerBase):
+
+    @rank_zero_only
+    def log_hyperparams(self, params):
+        # params is an argparse.Namespace
+        # your code to record hyperparameters goes here
+        pass
+    
+    @rank_zero_only
+    def log_metrics(self, metrics, step_num):
+        # metrics is a dictionary of metric names and values
+        # your code to record metrics goes here
+        pass
+    
+    def save(self):
+        # Optional. Any code necessary to save logger data goes here
+        pass
+    
+    @rank_zero_only
+    def finalize(self, status):
+        # Optional. Any code that needs to be run after training
+        # finishes goes here
+```
+
+If you write a logger than may be useful to others, please send
+a pull request to add it to Lighting!
+
+---
+#### Using loggers
+You can call the logger anywhere from your LightningModule by doing:
+```python
+self.logger
+
+# add an image if using TestTubeLogger
+self.logger.experiment.add_image(...)
+```
+
+
 #### Display metrics in progress bar 
 ``` {.python}
 # DEFAULT
@@ -13,15 +119,21 @@ trainer = Trainer(show_progress_bar=True)
 Every k batches lightning will make an entry in the metrics log
 ``` {.python}
 # DEFAULT (ie: save a .csv log file every 10 batches)
-trainer = Trainer(add_log_row_interval=10)
+trainer = Trainer(row_log_interval=10)
 ```   

 ---
-#### Log metric row every k batches 
+#### Log GPU memory
 Logs GPU memory when metrics are logged.   
 ``` {.python}
 # DEFAULT
-trainer = Trainer(log_gpu_memory=False)
+trainer = Trainer(log_gpu_memory=None)
+
+# log only the min/max utilization
+trainer = Trainer(log_gpu_memory='min_max')
+
+# log all the GPU memory (if on DDP, logs only that node)
+trainer = Trainer(log_gpu_memory='all')
 ```

 ---
@@ -38,61 +150,21 @@ trainer = Trainer(process_position=1)

 ---
 #### Save a snapshot of all hyperparameters 
-Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use.
-Give lightning a test-tube Experiment object to automate this for you.
+Automatically log hyperparameters stored in the `hparams` attribute as an `argparse.Namespace` 
 ``` {.python}
-from test_tube import Experiment

-exp = Experiment(...)
-Trainer(experiment=exp)
-```
+class MyModel(pl.Lightning):
+    def __init__(self, hparams):
+        self.hparams = hparams

---
-#### Snapshot code for a training run
-Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag.
-Give lightning a test-tube Experiment object to automate this for you.
-``` {.python}
-from test_tube import Experiment
+    ...

-exp = Experiment(create_git_tag=True)
-Trainer(experiment=exp)
-```
+args = parser.parse_args()
+model = MyModel(args)

---
-### Tensorboard support   
-In the LightningModule you can access the experiment logger by doing:
-```python
-self.experiment
-
-# add image
-# Look at PyTorch SummaryWriter docs for what you can do.   
-self.experiment.add_image(...)
-```
-
-The experiment object is a strict subclass of PyTorch SummaryWriter. However, this class
-also snapshots every detail about the experiment (data folder paths, code, hyperparams),
-and allows you to visualize it using tensorboard.
-``` {.python}
-from test_tube import Experiment, HyperOptArgumentParser
-
-# exp hyperparams
-args = HyperOptArgumentParser()
-hparams = args.parse_args()
-
-# this is a summaryWriter with nicer logging structure
-exp = Experiment(save_dir='/some/path', create_git_tag=True)
-
-# track experiment details (must be ArgumentParser or HyperOptArgumentParser).
-# each option in the parser is tracked
-exp.argparse(hparams)
-exp.tag({'description': 'running demo'})
-
-# trainer uses the exp object to log exp data
-trainer = Trainer(experiment=exp)
+logger = TestTubeLogger(...)
+t = Trainer(logger=logger)
 trainer.fit(model)
-
-# view logs at:
-# tensorboard --logdir /some/path   
 ```

 ---
@@ -25,6 +25,9 @@ parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4,
 hparams = parser.parse_args()    
 ```    
    
+**NOTE** You must set ```Tunable=True``` for that argument to be considered in the permutation set. Otherwise
+test-tube will use the default value. This flag is useful when you don't want to search over an argument and
+want to use the default instead.   
     
 (2). Define the cluster options in the [SlurmCluster object](https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/) (over 5 nodes and 8 gpus)    

@@ -83,6 +86,11 @@ cluster.optimize_parallel_cluster_gpu(
    job_display_name='my_exp')
 ```

+**NOTE** nb_trials specifies how many of the possible permutations to use. If using ```grid_search``` it will use
+the depth first ordering. If using ```random_search``` it will use the first k shuffled options. FYI, random search
+has been shown to be just as good as any Bayesian optimization method when using a reasonable number of samples (60),
+[see this paper for more information](http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf).
+
 ---
 #### Walltime auto-resubmit
 Lightning automatically resubmits jobs when they reach the walltime. Make sure to set the SIGUSR1 signal in 
@@ -19,6 +19,25 @@ It can be useful to force training for a minimum number of epochs or limit to a
 trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)
 ```

+---
+#### Early stopping
+The trainer already sets up default early stopping for you. 
+To modify this behavior, pass in your own EarlyStopping callback.
+``` {.python}
+from pytorch_lightning.callbacks import EarlyStopping
+
+# DEFAULTS used by Trainer
+early_stop_callback = EarlyStopping(
+    monitor='val_loss',
+    min_delta=0.00,
+    patience=3,
+    verbose=False,
+    mode='min'
+)
+
+trainer = Trainer(early_stop_callback=early_stop_callback)
+```
+
 ---
 #### Force disable early stop 
 Use this to turn off early stopping and run training to the [max_epoch](#force-training-for-min-or-max-epochs)
@@ -34,10 +53,10 @@ Specifically, this will [clip the gradient norm computed over all model paramete

 ``` {.python}
 # DEFAULT (ie: don't clip)
-trainer = Trainer(gradient_clip=0)
+trainer = Trainer(gradient_clip_val=0)

 # clip gradients with norm above 0.5
-trainer = Trainer(gradient_clip=0.5)
+trainer = Trainer(gradient_clip_val=0.5)
 ```

 ---
@@ -38,6 +38,14 @@ trainer = Trainer(overfit_pct=0.01)
 #### Print the parameter count by layer
 By default lightning prints a list of parameters *and submodules* when it starts training.

+``` {.python}
+# DEFAULT print a full list of all submodules and their parameters.
+trainer = Trainer(weights_summary='full')
+
+# only print the top-level modules (i.e. the children of LightningModule).
+trainer = Trainer(weights_summary='top')
+```
+
 ---
 #### Print which gradients are nan 
 This option prints a list of tensors with nan gradients.
@@ -58,16 +58,6 @@ def on_post_performance_check(self):
 ```

 ---
-#### on_tng_metrics
-Called in the training loop, right before metrics are logged.
-Although you can log at any time by using self.experiment, you can use
-this callback to modify what will be logged.
-```python
-def on_tng_metrics(self, metrics):
-    # do something before validation end
-```
-
---   
 #### optimizer_step 
 Calls .step() and .zero_grad for each optimizer.  
 You can override this method to adjust how you do the optimizer step for each optimizer
@@ -75,12 +65,12 @@ You can override this method to adjust how you do the optimizer step for each op
 Called once per optimizer
 ```python
 # DEFAULT
-def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
+def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None):
    optimizer.step()   
    optimizer.zero_grad()   
    
 # Alternating schedule for optimizer steps (ie: GANs)    
-def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
+def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None):
    # update generator opt every 2 steps
    if optimizer_i == 0:
        if batch_nb % 2 == 0 :
@@ -101,7 +91,7 @@ This step allows you to do a lot of non-standard training tricks such as learnin

 ```python
 # learning rate warm-up
-def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
+def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None):
    # warm up lr
    if self.trainer.global_step < 500:
        lr_scale = min(1., float(self.trainer.global_step + 1) / 500.)
@@ -137,5 +127,5 @@ def on_after_backward(self):
        for k, v in params.items():
            grads = v
            name = k
-            self.experiment.add_histogram(tag=name, values=grads, global_step=self.trainer.global_step)
+            self.logger.experiment.add_histogram(tag=name, values=grads, global_step=self.trainer.global_step)
 ```
@@ -19,6 +19,7 @@ But of course the fun is in all the advanced things it can do:

 **Checkpointing**    

+- [Checkpoint callback](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)    
 - [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
 - [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics) 
 - [Restoring training session](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#restoring-training-session)
@@ -35,7 +36,7 @@ But of course the fun is in all the advanced things it can do:
 - [Log GPU usage](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage)
 - [Make model overfit on subset of data](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data)
 - [Print the parameter count by layer](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer)
- [Pring which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
+- [Print which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
 - [Print input and output size of every module in system](https://williamfalcon.github.io/pytorch-lightning/LightningModule/properties/#example_input_array)


@@ -60,8 +61,9 @@ But of course the fun is in all the advanced things it can do:

 **Training loop**    

- [Accumulate gradients](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients)
+- [Accumulate gradients](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients)    
 - [Force training for min or max epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs)
+- [Early stopping callback](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#early-stopping)    
 - [Force disable early stop](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop)
 - [Gradient Clipping](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping)
 - [Hooks](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/)
@@ -82,4 +84,4 @@ But of course the fun is in all the advanced things it can do:

 **Testing loop**    

- [Run test set](https://williamfalcon.github.io/pytorch-lightning/Trainer/Testing%20loop/)  
+- [Run test set](https://williamfalcon.github.io/pytorch-lightning/Trainer/Testing%20loop/)  
@@ -1,5 +1,5 @@
 ### Template model definition
-In 99% of cases you want to just copy [this template](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py) to start a new lightningModule and change the core of what your model is actually trying to do.
+In 99% of cases you want to just copy [one of the examples](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples) to start a new lightningModule and change the core of what your model is actually trying to do.

 ```bash
 # get a copy of the module template
@@ -47,57 +47,17 @@ def main(hparams, cluster, results_dict):
    :param hparams:
    :return:
    """
-    # init experiment
-    log_dir = os.path.dirname(os.path.realpath(__file__))
-    exp = Experiment(
-        name='test_tube_exp',
-        debug=True,
-        save_dir=log_dir,
-        version=0,
-        autosave=False,
-        description='test demo'
-    )
-    
-    # set the hparams for the experiment
-    exp.argparse(hparams)
-    exp.save()
-
    # build model
    model = MyLightningModule(hparams)

-    # callbacks
-    early_stop = EarlyStopping(
-        monitor=hparams.early_stop_metric,
-        patience=hparams.early_stop_patience,
-        verbose=True,
-        mode=hparams.early_stop_mode
-    )
-
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_function=None,
-        save_best_only=True,
-        verbose=True,
-        monitor=hparams.model_save_monitor_value,
-        mode=hparams.model_save_monitor_mode
-    )
-
    # configure trainer
-    trainer = Trainer(
-        experiment=exp,
-        cluster=cluster,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-    )
+    trainer = Trainer()

    # train model
    trainer.fit(model)
 ```


-
-
 The __main__ function will start training on your **main** function. If you use the HyperParameterOptimizer
 in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple
 argument parser you get the default arguments in the argument parser.
@@ -47,7 +47,7 @@ if use_bert:
 else:
    model = CoolerNotBERT()
    
-trainer = Trainer(gpus=[0, 1, 2, 3], use_amp=True)
+trainer = Trainer(gpus=4, use_amp=True)
 trainer.fit(model)
 ```

@@ -60,9 +60,8 @@ Notice a few things about this flow:
 ###### Templates 
 1. [MNIST LightningModule](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#minimal-example) 
 2. [Trainer](https://williamfalcon.github.io/pytorch-lightning/Trainer/)
-    - [Basic CPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/single_cpu_template.py) 
-    - [Multi-GPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/single_gpu_node_template.py)
-    - [GPU cluster Trainer Template](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/multi_node_cluster_template.py)
+    - [Basic CPU, GPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/basic_examples) 
+    - [GPU cluster Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/multi_node_examples)

 ###### Docs shortcuts
 - [LightningModule](LightningModule/RequiredTrainerInterface/)  
@@ -77,6 +76,7 @@ Notice a few things about this flow:

 ###### Checkpointing    

+- [Checkpoint callback](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)    
 - [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
 - [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics) 
 - [Restoring training session](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#restoring-training-session)
@@ -120,6 +120,7 @@ Notice a few things about this flow:

 - [Accumulate gradients](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients)
 - [Force training for min or max epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs)
+- [Early stopping callback](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#early-stopping)    
 - [Force disable early stop](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop)
 - [Gradient Clipping](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping)
 - [Hooks](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/)
@@ -0,0 +1,11 @@
+# Examples   
+This folder has 3 sections:   
+
+### Domain templates   
+These are templates to show common approaches such as GANs and RL.   
+
+### Basic examples   
+These show the most common use of Lightning for either CPU or GPU training.   
+
+### Multi-node examples   
+These show how to run jobs on a GPU cluster using lightning.
@@ -1,4 +1,4 @@
-from .new_project_templates.lightning_module_template import LightningTemplateModel
+from .basic_examples.lightning_module_template import LightningTemplateModel

 __all__ = [
    'LightningTemplateModel'
@@ -0,0 +1,39 @@
+# Basic Examples   
+Use these examples to test how lightning works.   
+
+#### Test on CPU  
+```bash
+python cpu_template.py
+```
+
+---   
+#### Train on a single GPU
+```bash
+python gpu_template.py --gpus 1
+```   
+
+---    
+#### DataParallel (dp)   
+Train on multiple GPUs using DataParallel.
+
+```bash
+python gpu_template.py --gpus 2 --distributed_backend dp
+```   
+
+---
+#### DistributedDataParallel (ddp)    
+
+Train on multiple GPUs using DistributedDataParallel   
+```bash
+python gpu_template.py --gpus 2 --distributed_backend ddp
+```
+
+---
+#### DistributedDataParallel+DP (ddp2)    
+
+Train on multiple GPUs using DistributedDataParallel + dataparallel.
+On a single node, uses all GPUs for 1 model. Then shares gradient information
+across nodes.   
+```bash
+python gpu_template.py --gpus 2 --distributed_backend ddp2
+```
@@ -0,0 +1,53 @@
+"""
+Runs a model on a single node across N-gpus.
+"""
+import os
+import numpy as np
+import torch
+
+from argparse import ArgumentParser
+from pytorch_lightning import Trainer
+from examples.basic_examples.lightning_module_template import LightningTemplateModel
+
+SEED = 2334
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+
+def main(hparams):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    """
+    # ------------------------
+    # 1 INIT LIGHTNING MODEL
+    # ------------------------
+    model = LightningTemplateModel(hparams)
+
+    # ------------------------
+    # 2 INIT TRAINER
+    # ------------------------
+    trainer = Trainer()
+
+    # ------------------------
+    # 3 START TRAINING
+    # ------------------------
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+    # ------------------------
+    # TRAINING ARGUMENTS
+    # ------------------------
+    # these are project-wide arguments
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    parent_parser = ArgumentParser(add_help=False)
+
+    # each LightningModule defines arguments relevant to it
+    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
+    hyperparams = parser.parse_args()
+
+    # ---------------------
+    # RUN TRAINING
+    # ---------------------
+    main(hyperparams)
@@ -0,0 +1,78 @@
+"""
+Runs a model on a single node across N-gpus.
+"""
+import os
+import numpy as np
+import torch
+
+from argparse import ArgumentParser
+from pytorch_lightning import Trainer
+from examples.basic_examples.lightning_module_template import LightningTemplateModel
+
+SEED = 2334
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+
+def main(hparams):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    """
+    # ------------------------
+    # 1 INIT LIGHTNING MODEL
+    # ------------------------
+    model = LightningTemplateModel(hparams)
+
+    # ------------------------
+    # 2 INIT TRAINER
+    # ------------------------
+    trainer = Trainer(
+        gpus=hparams.gpus,
+        distributed_backend=hparams.distributed_backend,
+        use_amp=hparams.use_16bit
+    )
+
+    # ------------------------
+    # 3 START TRAINING
+    # ------------------------
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+    # ------------------------
+    # TRAINING ARGUMENTS
+    # ------------------------
+    # these are project-wide arguments
+
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    parent_parser = ArgumentParser(add_help=False)
+
+    # gpu args
+    parent_parser.add_argument(
+        '--gpus',
+        type=int,
+        default=2,
+        help='how many gpus'
+    )
+    parent_parser.add_argument(
+        '--distributed_backend',
+        type=str,
+        default='dp',
+        help='supports three options dp, ddp, ddp2'
+    )
+    parent_parser.add_argument(
+        '--use_16bit',
+        dest='use_16bit',
+        action='store_true',
+        help='if true uses 16 bit precision'
+    )
+
+    # each LightningModule defines arguments relevant to it
+    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
+    hyperparams = parser.parse_args()
+
+    # ---------------------
+    # RUN TRAINING
+    # ---------------------
+    main(hyperparams)
@@ -8,7 +8,7 @@ from torchvision.datasets import MNIST
 import torchvision.transforms as transforms
 import torch
 import torch.nn.functional as F
-from test_tube import HyperOptArgumentParser
+from argparse import ArgumentParser
 from torch import optim
 from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
@@ -79,14 +79,14 @@ class LightningTemplateModel(LightningModule):
        nll = F.nll_loss(logits, labels)
        return nll

-    def training_step(self, data_batch, batch_i):
+    def training_step(self, batch, batch_idx):
        """
        Lightning calls this inside the training loop
-        :param data_batch:
+        :param batch:
        :return:
        """
        # forward pass
-        x, y = data_batch
+        x, y = batch
        x = x.view(x.size(0), -1)

        y_hat = self.forward(x)
@@ -95,23 +95,26 @@ class LightningTemplateModel(LightningModule):
        loss_val = self.loss(y, y_hat)

        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
-        if self.trainer.use_dp:
+        if self.trainer.use_dp or self.trainer.use_ddp2:
            loss_val = loss_val.unsqueeze(0)

+        tqdm_dict = {'train_loss': loss_val}
        output = OrderedDict({
-            'loss': loss_val
+            'loss': loss_val,
+            'progress_bar': tqdm_dict,
+            'log': tqdm_dict
        })

        # can also return just a scalar instead of a dict (return loss_val)
        return output

-    def validation_step(self, data_batch, batch_i):
+    def validation_step(self, batch, batch_idx):
        """
        Lightning calls this inside the validation loop
-        :param data_batch:
+        :param batch:
        :return:
        """
-        x, y = data_batch
+        x, y = batch
        x = x.view(x.size(0), -1)
        y_hat = self.forward(x)

@@ -126,7 +129,7 @@ class LightningTemplateModel(LightningModule):
            val_acc = val_acc.cuda(loss_val.device.index)

        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
-        if self.trainer.use_dp:
+        if self.trainer.use_dp or self.trainer.use_ddp2:
            loss_val = loss_val.unsqueeze(0)
            val_acc = val_acc.unsqueeze(0)

@@ -160,15 +163,16 @@ class LightningTemplateModel(LightningModule):

            # reduce manually when using dp
            val_acc = output['val_acc']
-            if self.trainer.use_dp:
+            if self.trainer.use_dp or self.trainer.use_ddp2:
                val_acc = torch.mean(val_acc)

            val_acc_mean += val_acc

        val_loss_mean /= len(outputs)
        val_acc_mean /= len(outputs)
-        tqdm_dic = {'val_loss': val_loss_mean, 'val_acc': val_acc_mean}
-        return tqdm_dic
+        tqdm_dict = {'val_loss': val_loss_mean, 'val_acc': val_acc_mean}
+        result = {'progress_bar': tqdm_dict, 'log': tqdm_dict}
+        return result

    # ---------------------
    # TRAINING SETUP
@@ -189,27 +193,27 @@ class LightningTemplateModel(LightningModule):
        dataset = MNIST(root=self.hparams.data_root, train=train,
                        transform=transform, download=True)

-        # when using multi-node (ddp) we need to add the datasampler
+        # when using multi-node (ddp) we need to add the  datasampler
        train_sampler = None
        batch_size = self.hparams.batch_size

        if self.use_ddp:
-            train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
-            batch_size = batch_size // self.trainer.world_size  # scale batch size
+            train_sampler = DistributedSampler(dataset)

        should_shuffle = train_sampler is None
        loader = DataLoader(
            dataset=dataset,
            batch_size=batch_size,
            shuffle=should_shuffle,
-            sampler=train_sampler
+            sampler=train_sampler,
+            num_workers=0
        )

        return loader

    @pl.data_loader
-    def tng_dataloader(self):
-        print('tng data loader called')
+    def train_dataloader(self):
+        print('training data loader called')
        return self.__dataloader(train=True)

    @pl.data_loader
@@ -230,31 +234,23 @@ class LightningTemplateModel(LightningModule):
        :param root_dir:
        :return:
        """
-        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+        parser = ArgumentParser(parents=[parent_parser])

        # param overwrites
-        # parser.set_defaults(gradient_clip=5.0)
+        # parser.set_defaults(gradient_clip_val=5.0)

        # network params
        parser.add_argument('--in_features', default=28 * 28, type=int)
        parser.add_argument('--out_features', default=10, type=int)
        # use 500 for CPU, 50000 for GPU to see speed difference
        parser.add_argument('--hidden_dim', default=50000, type=int)
-        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=True)
-        parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
-                        options=[0.0001, 0.0005, 0.001],
-                        tunable=True)
+        parser.add_argument('--drop_prob', default=0.2, type=float)
+        parser.add_argument('--learning_rate', default=0.001, type=float)

        # data
        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)

        # training params (opt)
-        parser.opt_list('--optimizer_name', default='adam', type=str,
-                        options=['adam'], tunable=False)
-
-        # if using 2 nodes with 4 gpus each the batch size here
-        #  (256) will be 256 / (2*8) = 16 per gpu
-        parser.opt_list('--batch_size', default=256 * 8, type=int,
-                        options=[32, 64, 128, 256], tunable=False,
-                        help='batch size will be divided over all gpus being used across all nodes')
+        parser.add_argument('--optimizer_name', default='adam', type=str)
+        parser.add_argument('--batch_size', default=64, type=int)
        return parser
@@ -109,7 +109,7 @@ class GAN(pl.LightningModule):
            # log sampled images
            sample_imgs = self.generated_imgs[:6]
            grid = torchvision.utils.make_grid(sample_imgs)
-            self.experiment.add_image('generated_images', grid, 0)
+            self.logger.experiment.add_image('generated_images', grid, 0)

            # ground truth result (ie: all fake)
            valid = torch.ones(imgs.size(0), 1)
@@ -146,7 +146,7 @@ class GAN(pl.LightningModule):
        return [opt_g, opt_d], []

    @pl.data_loader
-    def tng_dataloader(self):
+    def train_dataloader(self):
        transform = transforms.Compose([transforms.ToTensor(),
                                        transforms.Normalize([0.5], [0.5])])
        dataset = MNIST(os.getcwd(), train=True, download=True, transform=transform)
@@ -0,0 +1,21 @@
+# Multi-node example   
+
+This demo launches a job using 2 GPUs on 2 different nodes (4 GPUs total).
+To run this demo do the following:
+
+1. Log into the jumphost node of your SLURM-managed cluster.  
+2. Create a conda environment with Lightning and a GPU PyTorch version.   
+3. Choose a script to submit    
+
+#### DDP  
+Submit this job to run with distributedDataParallel (2 nodes, 2 gpus each)
+```bash
+sbatch ddp_job_submit.sh YourEnv
+```
+
+#### DDP2  
+Submit this job to run with a different implementation of distributedDataParallel.
+In this version, each node acts like DataParallel but syncs across nodes like DDP.
+```bash
+sbatch ddp2_job_submit.sh YourEnv
+```
@@ -1,19 +1,19 @@
 #!/bin/bash -l

 # SLURM SUBMIT SCRIPT
-#SBATCH --nodes=4
-#SBATCH --gres=gpu:4
-#SBATCH --ntasks-per-node=4
+#SBATCH --nodes=2
+#SBATCH --gres=gpu:2
+#SBATCH --ntasks-per-node=1
 #SBATCH --mem=0
 #SBATCH --time=0-02:00:00

 # activate conda env
-conda activate my_env
+source activate $1

 # -------------------------
 # debugging flags (optional)
-# export NCCL_DEBUG=INFO
-# export PYTHONFAULTHANDLER=1
+ export NCCL_DEBUG=INFO
+ export PYTHONFAULTHANDLER=1

 # on your cluster you might need these:
 # set the network interface
@@ -23,8 +23,5 @@ conda activate my_env
 # module load NCCL/2.4.7-1-cuda.10.0
 # -------------------------

-# random port between 12k and 20k
-export MASTER_PORT=$((12000 + RANDOM % 20000))
-
 # run script from above
-python minimal_multi_node_demo.py
+srun python3 multi_node_ddp2_demo.py
@@ -0,0 +1,27 @@
+#!/bin/bash -l
+
+# SLURM SUBMIT SCRIPT
+#SBATCH --nodes=2
+#SBATCH --gres=gpu:2
+#SBATCH --ntasks-per-node=2
+#SBATCH --mem=0
+#SBATCH --time=0-02:00:00
+
+# activate conda env
+source activate $1
+
+# -------------------------
+# debugging flags (optional)
+ export NCCL_DEBUG=INFO
+ export PYTHONFAULTHANDLER=1
+
+# on your cluster you might need these:
+# set the network interface
+# export NCCL_SOCKET_IFNAME=^docker0,lo
+
+# might need the latest cuda
+# module load NCCL/2.4.7-1-cuda.10.0
+# -------------------------
+
+# run script from above
+srun python3 multi_node_ddp_demo.py
@@ -5,9 +5,9 @@ import os
 import numpy as np
 import torch

-from test_tube import HyperOptArgumentParser, Experiment
+from argparse import ArgumentParser
 from pytorch_lightning import Trainer
-from examples.new_project_templates.lightning_module_template import LightningTemplateModel
+from examples.basic_examples.lightning_module_template import LightningTemplateModel

 SEED = 2334
 torch.manual_seed(SEED)
@@ -25,42 +25,27 @@ def main(hparams):
    # ------------------------
    model = LightningTemplateModel(hparams)

-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-    # init experiment
-    exp = Experiment(
-        name='test_exp',
-        save_dir=hyperparams.log_dir,
-        autosave=False,
-        description='test demo'
-    )
-
    # ------------------------
    # 2 INIT TRAINER
    # ------------------------
    trainer = Trainer(
-        experiment=exp,
-        gpus=8,
-        nb_gpu_nodes=2
+        gpus=2,
+        nb_gpu_nodes=2,
+        distributed_backend='ddp2'
    )

    # ------------------------
-    # 5 START TRAINING
+    # 3 START TRAINING
    # ------------------------
    trainer.fit(model)


 if __name__ == '__main__':
-    # use current dir for logging
+
    root_dir = os.path.dirname(os.path.realpath(__file__))
-    log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
+    parent_parser = ArgumentParser(add_help=False)

-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-    parent_parser.add_argument('--log_dir', type=str, default=log_dir,
-                               help='where to save logs')
-
-    # allow model to overwrite or extend args
+    # each LightningModule defines arguments relevant to it
    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
    hyperparams = parser.parse_args()

@@ -0,0 +1,55 @@
+"""
+Multi-node example (GPU)
+"""
+import os
+import numpy as np
+import torch
+
+from argparse import ArgumentParser
+from pytorch_lightning import Trainer
+from examples.basic_examples.lightning_module_template import LightningTemplateModel
+
+SEED = 2334
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+
+def main(hparams):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    :return:
+    """
+    # ------------------------
+    # 1 INIT LIGHTNING MODEL
+    # ------------------------
+    model = LightningTemplateModel(hparams)
+
+    # ------------------------
+    # 2 INIT TRAINER
+    # ------------------------
+    trainer = Trainer(
+        gpus=2,
+        nb_gpu_nodes=2,
+        distributed_backend='ddp'
+    )
+
+    # ------------------------
+    # 3 START TRAINING
+    # ------------------------
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    parent_parser = ArgumentParser(add_help=False)
+
+    # each LightningModule defines arguments relevant to it
+    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
+    hyperparams = parser.parse_args()
+
+    # ---------------------
+    # RUN TRAINING
+    # ---------------------
+    main(hyperparams)
@@ -1,107 +0,0 @@
-# Multi-node examples
-Use these templates for multi-node training. 
-The main complexity around cluster training is how you submit the SLURM jobs.  
-
-## Test-tube   
-Lightning uses test-tube to submit SLURM jobs and to run hyperparameter searches on a cluster.  
-
-To run a hyperparameter search, we normally add the values to search to the Hyperparameter optimizer 
-```python
-from test_tube import HyperOptArgumentParser
-
-parser = HyperOptArgumentParser(strategy='grid_search')
-parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=True)
-parser.opt_list('--learning_rate', default=0.001, type=float,
-                        options=[0.0001, 0.0005, 0.001],
-                        tunable=True)
-                        
-# give your model a chance to add its own parameters
-parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-
-# parse args
-hyperparams = parser.parse_args()
-```
-
-The above sets up a grid search on learning rate and drop probability. You can now add this object to the 
-cluster object to perform the grid search:   
-```python
-cluster = SlurmCluster(
-    hyperparam_optimizer=hyperparams,
-    log_path='/path/to/log/slurm/files',
-)
-
-# ... configure cluster options
-
-# run grid search on cluster
-nb_trials = 6   # (2 drop probs * 3 lrs)
-cluster.optimize_parallel_cluster_gpu(
-    YourMainFunction,
-    nb_trials=nb_trials,
-    job_name=hyperparams.experiment_name
-)
-```
-
-Running the above will launch 6 jobs, each with a different drop prob and learning rate combination.   
-The ```tunable``` parameter must be set to True to add that argument to the space of options, otherwise
-Test-Tube will use the ```default=value```.    
-
-
-## SLURM Flags   
-However you decide to submit your jobs, debugging requires a few flags. Without these flags, you'll
-see a nccl error instead of the actual error which caused the bug.   
-
-```sh
-export NCCL_DEBUG=INFO
-export PYTHONFAULTHANDLER=1
-```
-
-On some clusters you might need to set the network interface with this flag.   
-```sh
-export NCCL_SOCKET_IFNAME=^docker0,lo
-```   
-
-You might also need to load the latest version of NCCL  
-```sh
-module load NCCL/2.4.7-1-cuda.10.0
-```
-
-Finally, you must set the master port (usually a random number between 12k and 20k).   
-```sh
-# random port between 12k and 20k
-export MASTER_PORT=$((12000 + RANDOM % 20000))$   
-```
-
-## Simplest example.   
-1. Modify this script with your CoolModel file.   
-2. Update and submit [this bash script](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/multi_node_examples/minimal_multi_node_demo_script.sh)   
-```bash
-squeue minimal_multi_node_demo_script.sh
-```
-
-## Grid search on a cluster   
-
-#### Option 1: Run on cluster using your own SLURM script    
-The trainer and model will work on a cluster if you configure your SLURM script correctly.   
-
-1. Update [this demo slurm script](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/multi_node_examples/demo_script.sh).  
-2. Submit the script   
-```bash
-$ squeue demo_script.sh
-```
-
-Most people have some way they automatically generate their own scripts.  
-To run a grid search this way, you'd need a way to automatically generate scripts using all the combinations of 
-hyperparameters to search over.   
-
-#### Option 2: Use test-tube for SLURM script
-With test tube we can automatically generate slurm scripts for different hyperparameter options.   
-
-To run this demo:    
-```bash
-source activate YourCondaEnv
-
-python multi_node_cluster_auto_slurm.py --email your@email.com --gpu_partition your_partition --conda_env YourCondaEnv
-```
-
-That will submit 6 jobs. Each job will have a specific combination of hyperparams. Each job will also run on 2 nodes
-where each node has 8 gpus.   
@@ -1,66 +0,0 @@
-#!/bin/bash
-#
-# Auto-generated by test-tube (https://github.com/williamFalcon/test-tube)
-#################
-
-# set a job name
-#SBATCH --job-name=lightning_test
-#################
-
-# a file for job output, you can check job progress
-#SBATCH --output=/slurm_output_%j.out
-#################
-
-# a file for errors
-#SBATCH --error=/slurm_output_%j.err
-#################
-
-# time needed for job
-#SBATCH --time=01:00:00
-#################
-
-# gpus per node
-#SBATCH --gres=gpu:8
-#################
-
-# cpus per job
-#SBATCH --cpus-per-task=10
-#################
-
-# number of requested nodes
-#SBATCH --nodes=2
-#################
-
-# memory per node (0 means all)
-#SBATCH --mem=0
-#################
-
-# slurm will send a signal this far out before it kills the job
-#SBATCH --signal=USR1@300
-#################
-
-# comment
-#SBATCH --comment=lightning_demo
-#################
-
-# 1 task per gpu
-#SBATCH --ntasks-per-node=8
-#################
-
-source activate YourEnv
-
-# debugging flags (optional)
-export NCCL_DEBUG=INFO
-export PYTHONFAULTHANDLER=1
-
-# on your cluster you might need these:
-# set the network interface
-export NCCL_SOCKET_IFNAME=^docker0,lo
-
-# might need the latest cuda
-module load NCCL/2.4.7-1-cuda.10.0
-
-# random port between 12k and 20k
-export MASTER_PORT=$((12000 + RANDOM % 20000))$
-
-srun python multi_node_own_slurm_script.py
@@ -1,24 +0,0 @@
-from pytorch_lightning import Trainer
-from test_tube import Experiment
-import os
-
-
-def main():
-    # use the cool model from the main README.md
-    model = CoolModel()  # noqa: F821
-    exp = Experiment(save_dir=os.getcwd())
-
-    # train on 4 GPUs across 4 nodes
-    trainer = Trainer(
-        experiment=exp,
-        distributed_backend='ddp',
-        max_nb_epochs=10,
-        gpus=4,
-        nb_gpu_nodes=4
-    )
-
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-    main()
@@ -1,191 +0,0 @@
-"""
-Multi-node example (GPU)
-"""
-import os
-import numpy as np
-from time import sleep
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
-from pytorch_lightning import Trainer
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-from examples.new_project_templates.lightning_module_template import LightningTemplateModel
-
-PORT = np.random.randint(12000, 20000, 1)[0]
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-
-def main_local(hparams):
-    main(hparams, None, None)
-
-
-def main(hparams, cluster):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    print('loading model...')
-    model = LightningTemplateModel(hparams)
-    print('model built')
-
-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-    # when using grid search, it's possible for all models to start at once
-    # and use the same test tube experiment version
-    relative_node_id = int(os.environ['SLURM_NODEID'])
-    sleep(relative_node_id + 1)
-
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        version=hparams.hpc_exp_number,  # match the slurm job version number
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-        gpus=hparams.per_experiment_nb_gpus,
-        nb_gpu_nodes=hyperparams.nb_gpu_nodes
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-def optimize_on_cluster(hyperparams):
-    # enable cluster training
-    # log all scripts to the test tube folder
-    cluster = SlurmCluster(
-        hyperparam_optimizer=hyperparams,
-        log_path=hyperparams.slurm_log_path,
-    )
-
-    # email for cluster coms
-    cluster.notify_job_status(email=hyperparams.email, on_done=True, on_fail=True)
-
-    # configure cluster
-    cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus
-    cluster.per_experiment_nb_nodes = hyperparams.nb_gpu_nodes
-    cluster.job_time = '2:00:00'
-    cluster.gpu_type = 'volta'
-    cluster.memory_mb_per_node = 0
-
-    # any modules for code to run in env
-    cluster.add_command(f'source activate {hyperparams.conda_env}')
-
-    # set DDP master port
-    cluster.add_command(f'export MASTER_PORT={PORT}')
-
-    # OPTIONAL for debugging
-    # without these flags errors in your code will 
-    # appear to be nccl errors
-    cluster.add_command('export NCCL_DEBUG=INFO')
-    cluster.add_command('export PYTHONFAULTHANDLER=1')
-
-    # depending on your cluster config, you probably want
-    # to limit the wired connection device
-    # cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo')
-
-    # depending on your cluster, you might need to load
-    # the latest NCCL version
-    # cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0'])
-
-    # run only on 32GB voltas
-    cluster.add_slurm_cmd(cmd='constraint', value='volta32gb',
-                          comment='use 32gb gpus')
-    cluster.add_slurm_cmd(cmd='partition', value=hyperparams.gpu_partition,
-                          comment='use 32gb gpus')
-
-    # run hopt
-    # creates and submits jobs to slurm
-    cluster.optimize_parallel_cluster_gpu(
-        main,
-        nb_trials=hyperparams.num_hyperparam_trials,
-        job_name=hyperparams.experiment_name
-    )
-
-
-if __name__ == '__main__':
-
-    # use default args
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-    slurm_out_dir = os.path.join(demo_log_dir, 'slurm_scripts')
-
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # cluster args not defined inside the model
-
-    parent_parser.add_argument('--per_experiment_nb_gpus', type=int,
-                               default=8, help='how many gpus to use in a node')
-    parent_parser.add_argument('--nb_gpu_nodes', type=int, default=2,
-                               help='how many nodes to use in a cluster')
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir,
-                               help='where to save logs')
-    parent_parser.add_argument('--slurm_log_path', type=str, default=slurm_out_dir,
-                               help='where to save slurm meta')
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir,
-                               help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a',
-                               help='test tube exp name')
-    parent_parser.add_argument('--num_hyperparam_trials', type=int, default=6,
-                               help='how many grid search trials to run')
-
-    parent_parser.add_argument('--email', type=str, default='add@email.com',
-                               help='email for jobs')
-    parent_parser.add_argument('--conda_env', type=str, default='base',
-                               help='email for jobs')
-    parent_parser.add_argument('--gpu_partition', type=str, help='consult your cluster manual')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print('RUNNING ON SLURM CLUSTER')
-    optimize_on_cluster(hyperparams)
@@ -1,106 +0,0 @@
-"""
-Runs a model on a single node on CPU only..
-"""
-import os
-import numpy as np
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment
-from pytorch_lightning import Trainer
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-from examples.new_project_templates.lightning_module_template import LightningTemplateModel
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    model = LightningTemplateModel(hparams)
-
-    # ------------------------
-    # 2 INIT EXP
-    # ------------------------
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # dirs
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-
-    # although we user hyperOptParser, we are using it only as argparse right now
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # gpu args
-    parent_parser.add_argument('--test_tube_save_path', type=str,
-                               default=test_tube_dir, help='where to save logs')
-    parent_parser.add_argument('--model_save_path', type=str,
-                               default=checkpoint_dir, help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str,
-                               default='pt_lightning_exp_a', help='test tube exp name')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print('RUNNING ON CPU')
-    main(hyperparams)
@@ -1,114 +0,0 @@
-"""
-16-bit single node, CPU example
-"""
-import os
-import numpy as np
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment
-from pytorch_lightning import Trainer
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-from examples.new_project_templates.lightning_module_template import LightningTemplateModel
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    print('loading model...')
-    model = LightningTemplateModel(hparams)
-    print('model built')
-
-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-        gpus=hparams.gpus,
-        use_amp=True
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # dirs
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-
-    # although we user hyperOptParser, we are using it only as argparse right now
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # gpu args
-    parent_parser.add_argument('--gpus', type=str, default='-1',
-                               help='how many gpus to use in the node.'
-                                    'value -1 uses all the gpus on the node')
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir,
-                               help='where to save logs')
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir,
-                               help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a',
-                               help='test tube exp name')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
-    main(hyperparams)
@@ -1,114 +0,0 @@
-"""
-Runs a model on a single node across N-gpus.
-"""
-import os
-import numpy as np
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment
-from pytorch_lightning import Trainer
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-from examples.new_project_templates.lightning_module_template import LightningTemplateModel
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    print('loading model...')
-    model = LightningTemplateModel(hparams)
-    print('model built')
-
-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-        gpus=hparams.gpus,
-        distributed_backend='ddp'
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # dirs
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-
-    # although we user hyperOptParser, we are using it only as argparse right now
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # gpu args
-    parent_parser.add_argument('--gpus', type=str, default='-1',
-                               help='how many gpus to use in the node.'
-                                    ' value -1 uses all the gpus on the node')
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir,
-                               help='where to save logs')
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir,
-                               help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a',
-                               help='test tube exp name')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
-    main(hyperparams)
@@ -1,113 +0,0 @@
-"""
-Runs a model on a single node across N-gpus using dataParallel
-"""
-import os
-import numpy as np
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment
-from pytorch_lightning import Trainer
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-from examples.new_project_templates.lightning_module_template import LightningTemplateModel
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    print('loading model...')
-    model = LightningTemplateModel(hparams)
-    print('model built')
-
-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-        gpus=hparams.gpus,
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # dirs
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-
-    # although we user hyperOptParser, we are using it only as argparse right now
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # gpu args
-    parent_parser.add_argument('--gpus', type=str, default='-1',
-                               help='how many gpus to use in the node.'
-                                    ' value -1 uses all the gpus on the node')
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir,
-                               help='where to save logs')
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir,
-                               help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a',
-                               help='test tube exp name')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
-    main(hyperparams)
@@ -1,74 +0,0 @@
-import os
-import sys
-
-from test_tube import HyperOptArgumentParser, Experiment
-from pytorch_lightning import Trainer
-from pytorch_lightning.utilities.arg_parse import add_default_args
-from pytorch_lightning.callbacks.pt_callbacks import EarlyStopping, ModelCheckpoint
-
-from examples.new_project_templates.lightning_module_template import LightningTemplateModel
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # init experiment
-    exp = Experiment(
-        name=hparams.tt_name,
-        debug=hparams.debug,
-        save_dir=hparams.tt_save_path,
-        version=hparams.hpc_exp_number,
-        autosave=False,
-        description=hparams.tt_description
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # build model
-    model = LightningTemplateModel(hparams)
-
-    # callbacks
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        mode='min',
-        verbose=True,
-    )
-
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_acc',
-        mode='min'
-    )
-
-    # configure trainer
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-    )
-
-    # train model
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # use default args given by lightning
-    root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
-    parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False)
-    add_default_args(parent_parser, root_dir)
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser)
-    hyperparams = parser.parse_args()
-
-    # train model
-    main(hyperparams)
@@ -125,7 +125,8 @@ class EarlyStopping(Callback):
            print('Early stopping conditioned on metric `%s` '
                  'which is not available. Available metrics are: %s' %
                  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning)
-            exit(-1)
+            stop_training = True
+            return stop_training

        if self.monitor_op(current - self.min_delta, self.best):
            self.best = current
@@ -0,0 +1,10 @@
+from .base import LightningLoggerBase, rank_zero_only
+
+try:
+    from .test_tube_logger import TestTubeLogger
+except ModuleNotFoundError:
+    pass
+try:
+    from .mlflow_logger import MLFlowLogger
+except ModuleNotFoundError:
+    pass
@@ -0,0 +1,71 @@
+from functools import wraps
+
+
+def rank_zero_only(fn):
+    """Decorate a logger method to run it only on the process with rank 0
+
+    :param fn: Function to decorate
+    """
+
+    @wraps(fn)
+    def wrapped_fn(self, *args, **kwargs):
+        if self.rank == 0:
+            fn(self, *args, **kwargs)
+
+    return wrapped_fn
+
+
+class LightningLoggerBase(object):
+    """Base class for experiment loggers"""
+
+    def __init__(self):
+        self._rank = 0
+
+    def log_metrics(self, metrics, step_num):
+        """Record metrics
+
+        :param metric: Dictionary with metric names as keys and measured
+        quanties as values
+        :param step_num: Step number at which the metrics should be recorded
+        """
+        raise NotImplementedError()
+
+    def log_hyperparams(self, params):
+        """Record hyperparameters
+
+        :param params: argparse.Namespace containing the hyperparameters
+        """
+        raise NotImplementedError()
+
+    def save(self):
+        """Save log data"""
+        pass
+
+    def finalize(self, status):
+        """Do any processing that is necessary to finalize an experiment
+
+        :param status: Status that the experiment finished with (e.g. success, failed, aborted)
+        """
+        pass
+
+    def close(self):
+        """Do any cleanup that is necessary to close an experiment"""
+        pass
+
+    @property
+    def rank(self):
+        """
+        Process rank. In general, metrics should only be logged by the process
+        with rank 0
+        """
+        return self._rank
+
+    @rank.setter
+    def rank(self, value):
+        """Set the process rank"""
+        self._rank = value
+
+    @property
+    def version(self):
+        """Return the experiment version"""
+        return None
@@ -0,0 +1,59 @@
+from time import time
+from logging import getLogger
+
+import mlflow
+
+from .base import LightningLoggerBase, rank_zero_only
+
+logger = getLogger(__name__)
+
+
+class MLFlowLogger(LightningLoggerBase):
+    def __init__(self, experiment_name, tracking_uri=None, tags=None):
+        super().__init__()
+        self.client = mlflow.tracking.MlflowClient(tracking_uri)
+        self.experiment_name = experiment_name
+        self._run_id = None
+        self.tags = tags
+
+    @property
+    def run_id(self):
+        if self._run_id is not None:
+            return self._run_id
+
+        experiment = self.client.get_experiment_by_name(self.experiment_name)
+        if experiment is None:
+            logger.warning(
+                f"Experiment with name f{self.experiment_name} not found. Creating it."
+            )
+            self.client.create_experiment(self.experiment_name)
+            experiment = self.client.get_experiment_by_name(self.experiment_name)
+
+        run = self.client.create_run(experiment.experiment_id, tags=self.tags)
+        self._run_id = run.info.run_id
+        return self._run_id
+
+    @rank_zero_only
+    def log_hyperparams(self, params):
+        for k, v in vars(params).items():
+            self.client.log_param(self.run_id, k, v)
+
+    @rank_zero_only
+    def log_metrics(self, metrics, step_num=None):
+        timestamp_ms = int(time() * 1000)
+        for k, v in metrics.items():
+            if isinstance(v, str):
+                logger.warning(
+                    f"Discarding metric with string value {k}={v}"
+                )
+                continue
+            self.client.log_metric(self.run_id, k, v, timestamp_ms, step_num)
+
+    def save(self):
+        pass
+
+    @rank_zero_only
+    def finalize(self, status="FINISHED"):
+        if status == 'success':
+            status = 'FINISHED'
+        self.client.set_terminated(self.run_id, status)
@@ -0,0 +1,102 @@
+import os.path
+from copy import copy
+
+from .base import LightningLoggerBase, rank_zero_only
+
+from test_tube import Experiment
+
+
+class TestTubeLogger(LightningLoggerBase):
+    __test__ = False
+
+    def __init__(
+        self, save_dir, name="default", description=None, debug=False,
+            version=None, create_git_tag=False
+    ):
+        super().__init__()
+        self.save_dir = save_dir
+        self.name = name
+        self.description = description
+        self.debug = debug
+        self._version = version
+        self.create_git_tag = create_git_tag
+        self._experiment = None
+
+    @property
+    def experiment(self):
+        if self._experiment is not None:
+            return self._experiment
+
+        self._experiment = Experiment(
+            save_dir=self.save_dir,
+            name=self.name,
+            debug=self.debug,
+            version=self.version,
+            description=self.description,
+            create_git_tag=self.create_git_tag,
+            rank=self.rank,
+        )
+        return self._experiment
+
+    @rank_zero_only
+    def log_hyperparams(self, params):
+        # TODO: HACK figure out where this is being set to true
+        self.experiment.debug = self.debug
+        self.experiment.argparse(params)
+
+    @rank_zero_only
+    def log_metrics(self, metrics, step_num=None):
+        # TODO: HACK figure out where this is being set to true
+        self.experiment.debug = self.debug
+        self.experiment.log(metrics, global_step=step_num)
+
+    @rank_zero_only
+    def save(self):
+        # TODO: HACK figure out where this is being set to true
+        self.experiment.debug = self.debug
+        self.experiment.save()
+
+    @rank_zero_only
+    def finalize(self, status):
+        # TODO: HACK figure out where this is being set to true
+        self.experiment.debug = self.debug
+        self.save()
+        self.close()
+
+    @rank_zero_only
+    def close(self):
+        # TODO: HACK figure out where this is being set to true
+        self.experiment.debug = self.debug
+        exp = self.experiment
+        exp.close()
+
+    @property
+    def rank(self):
+        return self._rank
+
+    @rank.setter
+    def rank(self, value):
+        self._rank = value
+        if self._experiment is not None:
+            self.experiment.rank = value
+
+    @property
+    def version(self):
+        if self._experiment is None:
+            return self._version
+        else:
+            return self.experiment.version
+
+    # Test tube experiments are not pickleable, so we need to override a few
+    # methods to get DDP working. See
+    # https://docs.python.org/3/library/pickle.html#handling-stateful-objects
+    # for more info.
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        state["_experiment"] = self.experiment.get_meta_copy()
+        return state
+
+    def __setstate__(self, state):
+        self._experiment = state["_experiment"].get_non_ddp_exp()
+        del state["_experiment"]
+        self.__dict__.update(state)
@@ -87,7 +87,6 @@ class LightningDistributedDataParallel(DistributedDataParallel):
                # --------------
                # normal
                # output = self.module(*inputs[0], **kwargs[0])
-
                # lightning
                if self.module.training:
                    output = self.module.training_step(*inputs[0], **kwargs[0])
@@ -99,6 +98,7 @@ class LightningDistributedDataParallel(DistributedDataParallel):
                outputs = self.parallel_apply(self._module_copies[:len(inputs)], inputs, kwargs)
                output = self.gather(outputs, self.output_device)
        else:
+            # normal
            output = self.module(*inputs, **kwargs)

        if torch.is_grad_enabled():
@@ -171,6 +171,14 @@ def parallel_apply(modules, inputs, kwargs_tup=None, devices=None):  # pragma: n
            with lock:
                results[i] = e

+    # TODO: fix hack (maybe not a hack)
+    # make sure each module knows what training state it's in...
+    # fixes weird bug where copies are out of sync
+    root_m = modules[0]
+    for m in modules[1:]:
+        m.training = root_m.training
+        m.testing = root_m.testing
+
    if len(modules) > 1:
        threads = [threading.Thread(target=_worker,
                                    args=(i, module, input, kwargs, device))
@@ -10,13 +10,18 @@ def data_loader(fn):

    attr_name = '_lazy_' + fn.__name__

-    @property
-    def _data_loader(self):
+    def _get_data_loader(self):
        try:
            value = getattr(self, attr_name)
        except AttributeError:
            try:
                value = fn(self)  # Lazy evaluation, done only once.
+                if (
+                    value is not None and
+                    not isinstance(value, list) and
+                    fn.__name__ in['test_dataloader', 'val_dataloader']
+                ):
+                    value = [value]
            except AttributeError as e:
                # Guard against AttributeError suppression. (Issue #142)
                traceback.print_exc()
@@ -25,4 +30,4 @@ def data_loader(fn):
            setattr(self, attr_name, value)  # Memoize evaluation.
        return value

-    return _data_loader
+    return _get_data_loader
@@ -10,7 +10,7 @@ class ModelHooks(torch.nn.Module):
        """
        pass

-    def on_batch_start(self, data_batch):
+    def on_batch_start(self, batch):
        pass

    def on_batch_end(self):
@@ -28,9 +28,6 @@ class ModelHooks(torch.nn.Module):
    def on_post_performance_check(self):
        pass

-    def on_tng_metrics(self, metrics):
-        pass
-
    def on_before_zero_grad(self, optimizer):
        """
        Called after optimizer.step() and before optimizer.zero_grad()
@@ -12,11 +12,12 @@ import pandas as pd

 class ModelSummary(object):

-    def __init__(self, model):
+    def __init__(self, model, mode='full'):
        '''
        Generates summaries of model layers and dimensions.
        '''
        self.model = model
+        self.mode = mode
        self.in_sizes = []
        self.out_sizes = []

@@ -28,9 +29,20 @@ class ModelSummary(object):
    def __repr__(self):
        return self.summary.__str__()

+    def named_modules(self):
+        if self.mode == 'full':
+            mods = self.model.named_modules()
+            mods = list(mods)[1:]  # do not include root module (LightningModule)
+        elif self.mode == 'top':
+            # the children are the top-level modules
+            mods = self.model.named_children()
+        else:
+            mods = []
+        return list(mods)
+
    def get_variable_sizes(self):
        '''Run sample input through each layer to get output sizes'''
-        mods = list(self.model.modules())
+        mods = self.named_modules()
        in_sizes = []
        out_sizes = []
        input_ = self.model.example_input_array
@@ -43,8 +55,7 @@ class ModelSummary(object):

        with torch.no_grad():

-            for i in range(1, len(mods)):
-                m = mods[i]
+            for _, m in mods:
                if type(input_) is list or type(input_) is tuple:  # pragma: no cover
                    out = m(*input_)
                else:
@@ -72,16 +83,17 @@ class ModelSummary(object):

        self.in_sizes = in_sizes
        self.out_sizes = out_sizes
+        assert len(in_sizes) == len(out_sizes)
        return

    def get_layer_names(self):
        '''Collect Layer Names'''
-        mods = list(self.model.named_modules())
+        mods = self.named_modules()
        names = []
        layers = []
-        for m in mods[1:]:
-            names += [m[0]]
-            layers += [str(m[1].__class__)]
+        for name, m in mods:
+            names += [name]
+            layers += [str(m.__class__)]

        layer_types = [x.split('.')[-1][:-2] for x in layers]

@@ -91,11 +103,9 @@ class ModelSummary(object):

    def get_parameter_sizes(self):
        '''Get sizes of all parameters in `model`'''
-        mods = list(self.model.modules())
+        mods = self.named_modules()
        sizes = []
-
-        for i in range(1, len(mods)):
-            m = mods[i]
+        for _, m in mods:
            p = list(m.parameters())
            modsz = []
            for j in range(len(p)):
@@ -133,6 +143,7 @@ class ModelSummary(object):
        df['Name'] = self.layer_names
        df['Type'] = self.layer_types
        df['Params'] = self.param_nums
+        df['Params'] = df['Params'].map(get_human_readable_count)

        if self.model.example_input_array is not None:

@@ -178,6 +189,33 @@ def count_mem_items():  # pragma: no cover
    return nb_params, nb_tensors


+def get_memory_profile(mode):
+    """
+    'all' means return memory for all gpus
+    'min_max' means return memory for max and min
+    :param mode:
+    :return:
+    """
+    memory_map = get_gpu_memory_map()
+
+    if mode == 'min_max':
+        min_mem = 1000000
+        min_k = None
+        max_mem = 0
+        max_k = None
+        for k, v in memory_map:
+            if v > max_mem:
+                max_mem = v
+                max_k = k
+            if v < min_mem:
+                min_mem = v
+                min_k = k
+
+        memory_map = {min_k: min_mem, max_k: max_mem}
+
+    return memory_map
+
+
 def get_gpu_memory_map():
    """Get the current gpu usage.

@@ -196,6 +234,31 @@ def get_gpu_memory_map():
    gpu_memory = [int(x) for x in result.strip().split('\n')]
    gpu_memory_map = {}
    for k, v in zip(range(len(gpu_memory)), gpu_memory):
-        k = 'gpu_%i' % k
+        k = f'gpu_{k}'
        gpu_memory_map[k] = v
    return gpu_memory_map
+
+
+def get_human_readable_count(number):
+    """
+    Abbreviates an integer number with K, M, B, T for thousands, millions,
+    billions and trillions, respectively.
+    Examples:
+        123     -> 123
+        1234    -> 1 K       (one thousand)
+        2e6     -> 2 M       (two million)
+        3e9     -> 3 B       (three billion)
+        4e12    -> 4 T       (four trillion)
+        5e15    -> 5,000 T
+    :param number: a positive integer number
+    :returns a string formatted according to the pattern described above.
+    """
+    assert number >= 0
+    labels = [' ', 'K', 'M', 'B', 'T']
+    num_digits = int(np.floor(np.log10(number)) + 1 if number > 0 else 1)
+    num_groups = int(np.ceil(num_digits / 3))
+    num_groups = min(num_groups, len(labels))  # don't abbreviate beyond trillions
+    shift = -3 * (num_groups - 1)
+    number = number * (10 ** shift)
+    index = num_groups - 1
+    return f'{int(number):,d} {labels[index]}'
@@ -19,13 +19,14 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
        self.global_step = 0
        self.loaded_optimizer_states_dict = {}
        self.trainer = None
-        self.experiment = None
+        self.logger = None
        self.example_input_array = None

        # track if gpu was requested for checkpointing
        self.on_gpu = False
        self.use_dp = False
        self.use_ddp = False
+        self.use_ddp2 = False
        self.use_amp = False

    def forward(self, *args, **kwargs):
@@ -91,22 +92,26 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
        """
        raise NotImplementedError

-    def optimizer_step(self, epoch_nb, batch_nb, optimizer, optimizer_i):
+    def optimizer_step(self, epoch_nb, batch_nb, optimizer, optimizer_i, second_order_closure=None):
        """
        Do something instead of the standard optimizer behavior
        :param epoch_nb:
        :param batch_nb:
        :param optimizer:
        :param optimizer_i:
+        :param second_order_closure: closure for second order methods
        :return:
        """
-        optimizer.step()
+        if isinstance(optimizer, torch.optim.LBFGS):
+            optimizer.step(second_order_closure)
+        else:
+            optimizer.step()

        # clear gradients
        optimizer.zero_grad()

    @data_loader
-    def tng_dataloader(self):
+    def train_dataloader(self):
        """
        Implement a PyTorch DataLoader
        :return:
@@ -130,17 +135,16 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
        return None

    @classmethod
-    def load_from_metrics(cls, weights_path, tags_csv, on_gpu):
+    def load_from_metrics(cls, weights_path, tags_csv):
        """
        Primary way of loading model from csv weights path
        :param weights_path:
        :param tags_csv:
-        :param on_gpu:
        :param map_location: dic for mapping storage {'cuda:1':'cuda:0'}
        :return:
        """
        hparams = load_hparams_from_tags_csv(tags_csv)
-        hparams.__setattr__('on_gpu', on_gpu)
+        hparams.__setattr__('on_gpu', False)

        # load on CPU only to avoid OOM issues
        # then its up to user to put back on GPUs
@@ -155,8 +159,8 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):

        return model

-    def summarize(self):
-        model_summary = ModelSummary(self)
+    def summarize(self, mode):
+        model_summary = ModelSummary(self, mode=mode)
        print(model_summary)

    def freeze(self):
@@ -23,5 +23,5 @@ class LightningTestModel(LightningValidationMixin, LightningTestMixin, Lightning
    Most common test case. Validation and test dataloaders
    """

-    def on_tng_metrics(self, logs):
+    def on_training_metrics(self, logs):
        logs['some_tensor_to_test'] = torch.rand(1)
@@ -81,14 +81,14 @@ class LightningTestModelBase(LightningModule):
        nll = F.nll_loss(logits, labels)
        return nll

-    def training_step(self, data_batch, batch_i):
+    def training_step(self, batch, batch_idx):
        """
        Lightning calls this inside the training loop
-        :param data_batch:
+        :param batch:
        :return:
        """
        # forward pass
-        x, y = data_batch
+        x, y = batch
        x = x.view(x.size(0), -1)

        y_hat = self.forward(x)
@@ -104,8 +104,10 @@ class LightningTestModelBase(LightningModule):
        if self.trainer.batch_nb % 1 == 0:
            output = OrderedDict({
                'loss': loss_val,
-                'prog': {'some_val': loss_val * loss_val}
+                'progress_bar': {'some_val': loss_val * loss_val},
+                'log': {'train_some_val': loss_val * loss_val},
            })
+
            return output
        if self.trainer.batch_nb % 2 == 0:
            return loss_val
@@ -115,11 +117,14 @@ class LightningTestModelBase(LightningModule):
    # ---------------------
    def configure_optimizers(self):
        """
-        return whatever optimizers we want here
+        return whatever optimizers we want here.
        :return: list of optimizers
        """
        # try no scheduler for this model (testing purposes)
-        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        if self.hparams.optimizer_name == 'lbfgs':
+            optimizer = optim.LBFGS(self.parameters(), lr=self.hparams.learning_rate)
+        else:
+            optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)

        # test returning only 1 list instead of 2
        return optimizer
@@ -153,7 +158,7 @@ class LightningTestModelBase(LightningModule):
        return loader

    @data_loader
-    def tng_dataloader(self):
+    def train_dataloader(self):
        return self._dataloader(train=True)

    @staticmethod
@@ -167,7 +172,7 @@ class LightningTestModelBase(LightningModule):
        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])

        # param overwrites
-        # parser.set_defaults(gradient_clip=5.0)
+        # parser.set_defaults(gradient_clip_val=5.0)

        # network params
        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
@@ -25,13 +25,13 @@ class LightningValidationStepMixin:
    def val_dataloader(self):
        return self._dataloader(train=False)

-    def validation_step(self, data_batch, batch_i):
+    def validation_step(self, batch, batch_idx):
        """
        Lightning calls this inside the validation loop
-        :param data_batch:
+        :param batch:
        :return:
        """
-        x, y = data_batch
+        x, y = batch
        x = x.view(x.size(0), -1)
        y_hat = self.forward(x)

@@ -51,16 +51,16 @@ class LightningValidationStepMixin:
            val_acc = val_acc.unsqueeze(0)

        # alternate possible outputs to test
-        if batch_i % 1 == 0:
+        if batch_idx % 1 == 0:
            output = OrderedDict({
                'val_loss': loss_val,
                'val_acc': val_acc,
            })
            return output
-        if batch_i % 2 == 0:
+        if batch_idx % 2 == 0:
            return val_acc

-        if batch_i % 3 == 0:
+        if batch_idx % 3 == 0:
            output = OrderedDict({
                'val_loss': loss_val,
                'val_acc': val_acc,
@@ -104,8 +104,9 @@ class LightningValidationMixin(LightningValidationStepMixin):
        val_loss_mean /= len(outputs)
        val_acc_mean /= len(outputs)

-        tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-        return tqdm_dic
+        tqdm_dict = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
+        results = {'progress_bar': tqdm_dict, 'log': tqdm_dict}
+        return results


 class LightningValidationStepMultipleDataloadersMixin:
@@ -118,13 +119,13 @@ class LightningValidationStepMultipleDataloadersMixin:
    def val_dataloader(self):
        return [self._dataloader(train=False), self._dataloader(train=False)]

-    def validation_step(self, data_batch, batch_i, dataloader_i):
+    def validation_step(self, batch, batch_idx, dataloader_idx):
        """
        Lightning calls this inside the validation loop
-        :param data_batch:
+        :param batch:
        :return:
        """
-        x, y = data_batch
+        x, y = batch
        x = x.view(x.size(0), -1)
        y_hat = self.forward(x)

@@ -144,26 +145,26 @@ class LightningValidationStepMultipleDataloadersMixin:
            val_acc = val_acc.unsqueeze(0)

        # alternate possible outputs to test
-        if batch_i % 1 == 0:
+        if batch_idx % 1 == 0:
            output = OrderedDict({
                'val_loss': loss_val,
                'val_acc': val_acc,
            })
            return output
-        if batch_i % 2 == 0:
+        if batch_idx % 2 == 0:
            return val_acc

-        if batch_i % 3 == 0:
+        if batch_idx % 3 == 0:
            output = OrderedDict({
                'val_loss': loss_val,
                'val_acc': val_acc,
                'test_dic': {'val_loss_a': loss_val}
            })
            return output
-        if batch_i % 5 == 0:
+        if batch_idx % 5 == 0:
            output = OrderedDict({
-                f'val_loss_{dataloader_i}': loss_val,
-                f'val_acc_{dataloader_i}': val_acc,
+                f'val_loss_{dataloader_idx}': loss_val,
+                f'val_acc_{dataloader_idx}': val_acc,
            })
            return output

@@ -206,8 +207,9 @@ class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipl
        val_loss_mean /= i
        val_acc_mean /= i

-        tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-        return tqdm_dic
+        tqdm_dict = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
+        result = {'progress_bar': tqdm_dict}
+        return result


 class LightningTestStepMixin:
@@ -216,13 +218,13 @@ class LightningTestStepMixin:
    def test_dataloader(self):
        return self._dataloader(train=False)

-    def test_step(self, data_batch, batch_i):
+    def test_step(self, batch, batch_idx):
        """
        Lightning calls this inside the validation loop
-        :param data_batch:
+        :param batch:
        :return:
        """
-        x, y = data_batch
+        x, y = batch
        x = x.view(x.size(0), -1)
        y_hat = self.forward(x)

@@ -242,16 +244,16 @@ class LightningTestStepMixin:
            test_acc = test_acc.unsqueeze(0)

        # alternate possible outputs to test
-        if batch_i % 1 == 0:
+        if batch_idx % 1 == 0:
            output = OrderedDict({
                'test_loss': loss_test,
                'test_acc': test_acc,
            })
            return output
-        if batch_i % 2 == 0:
+        if batch_idx % 2 == 0:
            return test_acc

-        if batch_i % 3 == 0:
+        if batch_idx % 3 == 0:
            output = OrderedDict({
                'test_loss': loss_test,
                'test_acc': test_acc,
@@ -290,8 +292,9 @@ class LightningTestMixin(LightningTestStepMixin):
        test_loss_mean /= len(outputs)
        test_acc_mean /= len(outputs)

-        tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
-        return tqdm_dic
+        tqdm_dict = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
+        result = {'progress_bar': tqdm_dict}
+        return result


 class LightningTestStepMultipleDataloadersMixin:
@@ -300,13 +303,13 @@ class LightningTestStepMultipleDataloadersMixin:
    def test_dataloader(self):
        return [self._dataloader(train=False), self._dataloader(train=False)]

-    def test_step(self, data_batch, batch_i, dataloader_i):
+    def test_step(self, batch, batch_idx, dataloader_idx):
        """
        Lightning calls this inside the validation loop
-        :param data_batch:
+        :param batch:
        :return:
        """
-        x, y = data_batch
+        x, y = batch
        x = x.view(x.size(0), -1)
        y_hat = self.forward(x)

@@ -326,26 +329,26 @@ class LightningTestStepMultipleDataloadersMixin:
            test_acc = test_acc.unsqueeze(0)

        # alternate possible outputs to test
-        if batch_i % 1 == 0:
+        if batch_idx % 1 == 0:
            output = OrderedDict({
                'test_loss': loss_test,
                'test_acc': test_acc,
            })
            return output
-        if batch_i % 2 == 0:
+        if batch_idx % 2 == 0:
            return test_acc

-        if batch_i % 3 == 0:
+        if batch_idx % 3 == 0:
            output = OrderedDict({
                'test_loss': loss_test,
                'test_acc': test_acc,
                'test_dic': {'test_loss_a': loss_test}
            })
            return output
-        if batch_i % 5 == 0:
+        if batch_idx % 5 == 0:
            output = OrderedDict({
-                f'test_loss_{dataloader_i}': loss_test,
-                f'test_acc_{dataloader_i}': test_acc,
+                f'test_loss_{dataloader_idx}': loss_test,
+                f'test_acc_{dataloader_idx}': test_acc,
            })
            return output

@@ -383,5 +386,6 @@ class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloaders
        test_loss_mean /= i
        test_acc_mean /= i

-        tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
-        return tqdm_dic
+        tqdm_dict = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
+        result = {'progress_bar': tqdm_dict}
+        return result
@@ -0,0 +1,14 @@
+import warnings
+
+
+def ignore_scalar_return_in_dp():
+    # Users get confused by this warning so we silence it
+    m_1 = """
+    Was asked to gather along dimension 0, but all
+    input tensors were scalars; will instead unsqueeze
+    and return a vector.
+    """
+    warnings.filterwarnings('ignore', message=m_1)
+
+
+ignore_scalar_return_in_dp()
@@ -5,7 +5,7 @@ import pdb
 from subprocess import call

 import torch
-
+import torch.distributed as dist
 from pytorch_lightning.pt_overrides.override_data_parallel import (
    LightningDistributedDataParallel, LightningDataParallel)

@@ -35,6 +35,11 @@ class TrainerIO(object):
        # if script called from hpc resubmit, load weights
        self.restore_hpc_weights_if_needed(model)

+        # wait for all models to restore weights
+        if self.use_ddp or self.use_ddp2:
+            # wait for all processes to catch up
+            dist.barrier()
+
    def restore_state_if_checkpoint_exists(self, model):
        # do nothing if there's not dir or callback
        no_ckpt_callback = self.checkpoint_callback is None
@@ -88,7 +93,7 @@ class TrainerIO(object):
        if self.proc_rank == 0:
            # save weights
            print('handling SIGUSR1')
-            self.hpc_save(self.weights_save_path, self.experiment)
+            self.hpc_save(self.weights_save_path, self.logger)

            # find job id
            job_id = os.environ['SLURM_JOB_ID']
@@ -105,7 +110,7 @@ class TrainerIO(object):
                print('requeue failed...')

            # close experiment to avoid issues
-            self.experiment.close()
+            self.logger.close()

    def term_handler(self, signum, frame):
        # save
@@ -233,12 +238,12 @@ class TrainerIO(object):
    # ----------------------------------
    # PRIVATE OPS
    # ----------------------------------
-    def hpc_save(self, folderpath, experiment):
+    def hpc_save(self, folderpath, logger):
        # make sure the checkpoint folder exists
        os.makedirs(folderpath, exist_ok=True)

-        # save exp to make sure we get all the metrics
-        experiment.save()
+        # save logger to make sure we get all the metrics
+        logger.save()

        ckpt_number = self.max_ckpt_in_folder(folderpath) + 1

@@ -8,7 +8,7 @@ import os

 def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None):

-    # tng, test, val check intervals
+    # training, test, val check intervals
    parser.add_argument('--eval_test_set', dest='eval_test_set', action='store_true',
                        help='true = run test set also')
    parser.add_argument('--check_val_every_n_epoch', default=1, type=int,
@@ -19,7 +19,7 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None
    parser.add_argument('--max_nb_epochs', default=200, type=int, help='cap epochs')
    parser.add_argument('--min_nb_epochs', default=2, type=int, help='min epochs')
    parser.add_argument('--train_percent_check', default=1.0, type=float,
-                        help='how much of tng set to check')
+                        help='how much of training set to check')
    parser.add_argument('--val_percent_check', default=1.0, type=float,
                        help='how much of val set to check')
    parser.add_argument('--test_percent_check', default=1.0, type=float,
@@ -29,7 +29,7 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None
                        help='how much within 1 epoch to check val')
    parser.add_argument('--log_save_interval', default=100, type=int,
                        help='how many batches between log saves')
-    parser.add_argument('--add_log_row_interval', default=100, type=int,
+    parser.add_argument('--row_log_interval', default=100, type=int,
                        help='add log every k batches')

    # early stopping
@@ -40,7 +40,7 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None
                        help='number of epochs until stop')

    # gradient handling
-    parser.add_argument('--gradient_clip', default=-1, type=int)
+    parser.add_argument('--gradient_clip_val', default=-1, type=int)
    parser.add_argument('--track_grad_norm', default=-1, type=int,
                        help='if > 0, will track this grad norm')

@@ -78,9 +78,9 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None
    # FAST training
    # use these settings to make sure network has no bugs without running a full dataset
    parser.add_argument('--fast_dev_run', dest='fast_dev_run', default=False, action='store_true',
-                        help='runs validation after 1 tng step')
+                        help='runs validation after 1 training step')
    parser.add_argument('--enable_tqdm', dest='enable_tqdm', default=False, action='store_true',
-                        help='false removes the prog bar')
+                        help='false removes the progress bar')
    parser.add_argument('--overfit', default=-1, type=float,
                        help='% of dataset to use with this option. float, or -1 for none')

@@ -93,7 +93,7 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None
    parser.add_argument('--debug', dest='debug', action='store_true',
                        help='enables/disables test tube')
    parser.add_argument('--local', dest='local', action='store_true',
-                        help='enables local tng')
+                        help='enables local training')

    # optimizer
    parser.add_argument('--lr_scheduler_milestones', default=None, type=str)
@@ -1,7 +1,7 @@
 scikit-learn==0.20.2
-tqdm==4.32.1
+tqdm==4.35.0
 twine==1.13.0
 numpy==1.16.4
-torch>=1.1.0
+torch>=1.2.0
 torchvision>=0.3.0
-pandas
+pandas
@@ -14,7 +14,7 @@ from setuptools import setup, find_packages
 # engineer specific practices
 setup(
    name='pytorch-lightning',
-    version='0.4.9',
+    version='0.5.2.1',
    description='The Keras for ML researchers using PyTorch',
    author='William Falcon',
    author_email='waf2107@columbia.edu',
@@ -29,7 +29,7 @@ setup(
    keywords=['deep learning', 'pytorch', 'AI'],
    python_requires='>=3.6',
    install_requires=[
-        'torch==1.2.0',
+        'torch>=1.2.0',
        'tqdm>=4.35.0',
        'test-tube>=0.6.9',
        'pandas>=0.20.3',
@@ -51,7 +51,6 @@ setup(
        # Specify the Python versions you support here. In particular, ensure
        # that you indicate whether you support Python 2, Python 3 or both.
        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
    ],
@@ -14,6 +14,9 @@ from torch.utils.data import DataLoader
 from torchvision.datasets import MNIST
 import numpy as np
 import pdb
+# from test_models import assert_ok_test_acc, load_model, \
+#     clear_save_dir, get_test_tube_logger, get_hparams, init_save_dir, \
+#     init_checkpoint_callback, reset_seed, set_random_master_port


 class CoolModel(pl.LightningModule):
@@ -32,7 +35,7 @@ class CoolModel(pl.LightningModule):
    def training_step(self, batch, batch_nb):
        x, y = batch
        y_hat = self.forward(x)
-        return {'tng_loss': self.my_loss(y_hat, y)}
+        return {'training_loss': self.my_loss(y_hat, y)}

    def validation_step(self, batch, batch_nb):
        x, y = batch
@@ -47,7 +50,7 @@ class CoolModel(pl.LightningModule):
        return [torch.optim.Adam(self.parameters(), lr=0.02)]

    @pl.data_loader
-    def tng_dataloader(self):
+    def train_dataloader(self):
        return DataLoader(MNIST('path/to/save', train=True), batch_size=32)

    @pl.data_loader
@@ -58,240 +61,53 @@ class CoolModel(pl.LightningModule):
    def test_dataloader(self):
        return DataLoader(MNIST('path/to/save', train=False), batch_size=32)

-
-def get_model():
-    # set up model with these hyperparams
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    hparams = Namespace(**{'drop_prob': 0.2,
-                           'batch_size': 32,
-                           'in_features': 28 * 28,
-                           'learning_rate': 0.001 * 8,
-                           'optimizer_name': 'adam',
-                           'data_root': os.path.join(root_dir, 'mnist'),
-                           'out_features': 10,
-                           'hidden_dim': 1000})
-    model = LightningTemplateModel(hparams)
-
-    return model, hparams
-
-
-def get_exp(debug=True, version=None):
-    # set up exp object without actually saving logs
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    save_dir = os.path.join(root_dir, 'save_dir')
-    exp = Experiment(debug=debug, save_dir=save_dir, name='tests_tt_dir', version=version)
-    return exp
-
-
-def init_save_dir():
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    save_dir = os.path.join(root_dir, 'save_dir')
-
-    if os.path.exists(save_dir):
-        shutil.rmtree(save_dir)
-
-    os.makedirs(save_dir, exist_ok=True)
-
-    return save_dir
-
-
-def clear_save_dir():
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    save_dir = os.path.join(root_dir, 'save_dir')
-    if os.path.exists(save_dir):
-        shutil.rmtree(save_dir)
-
-
-def load_model(exp, save_dir, on_gpu, map_location=None, module_class=LightningTemplateModel):
-
-    # load trained model
-    tags_path = exp.get_data_path(exp.name, exp.version)
-    tags_path = os.path.join(tags_path, 'meta_tags.csv')
-
-    checkpoints = [x for x in os.listdir(save_dir) if '.ckpt' in x]
-    weights_dir = os.path.join(save_dir, checkpoints[0])
-
-    trained_model = module_class.load_from_metrics(weights_path=weights_dir,
-                                                   tags_csv=tags_path,
-                                                   on_gpu=on_gpu,
-                                                   map_location=map_location)
-
-    assert trained_model is not None, 'loading model failed'
-
-    return trained_model
-
-
-def run_prediction(dataloader, trained_model):
-    # run prediction on 1 batch
-    for batch in dataloader:
-        break
-
-    x, y = batch
-    x = x.view(x.size(0), -1)
-
-    y_hat = trained_model(x)
-
-    # acc
-    labels_hat = torch.argmax(y_hat, dim=1)
-    val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
-    val_acc = torch.tensor(val_acc)
-    val_acc = val_acc.item()
-    assert val_acc > 0.70, 'this model is expected to get > 0.7 in test set (it got %f)' % val_acc
-
-
-# ------------------------------------------------------------------------
-def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True):
-    save_dir = init_save_dir()
-
-    # exp file to get meta
-    exp = get_exp(False)
-    exp.argparse(hparams)
-    exp.save()
-
-    # exp file to get weights
-    checkpoint = ModelCheckpoint(save_dir)
-
-    # add these to the trainer options
-    trainer_options['checkpoint_callback'] = checkpoint
-    trainer_options['experiment'] = exp
-
-    # fit model
-    trainer = Trainer(**trainer_options)
-    result = trainer.fit(model)
-
-    # correct result and ok accuracy
-    assert result == 1, 'amp + ddp model failed to complete'
-
-    # test model loading
-    pretrained_model = load_model(exp, save_dir, on_gpu)
-
-    # test new model accuracy
-    run_prediction(model.test_dataloader, pretrained_model)
-
-    if trainer.use_ddp:
-        # on hpc this would work fine... but need to hack it for the purpose of the test
-        trainer.model = pretrained_model
-        trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers()
-
-    # test HPC loading / saving
-    trainer.hpc_save(save_dir, exp)
-    trainer.hpc_load(save_dir, on_gpu=on_gpu)
-
-    clear_save_dir()
-
-
-def assert_ok_val_acc(trainer):
-    # this model should get 0.80+ acc
-    acc = trainer.tng_tqdm_dic['val_acc']
-    assert acc > 0.50, f'model failed to get expected 0.50 validation accuracy. Got: {acc}'
-
-
-def assert_ok_test_acc(trainer):
-    # this model should get 0.80+ acc
-    acc = trainer.tng_tqdm_dic['test_acc']
-    assert acc > 0.50, f'model failed to get expected 0.50 validation accuracy. Got: {acc}'
-
-
-def get_hparams(continue_training=False, hpc_exp_number=0):
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-
-    args = {
-        'drop_prob': 0.2,
-        'batch_size': 32,
-        'in_features': 28 * 28,
-        'learning_rate': 0.001 * 8,
-        'optimizer_name': 'adam',
-        'data_root': os.path.join(root_dir, 'mnist'),
-        'out_features': 10,
-        'hidden_dim': 1000}
-
-    if continue_training:
-        args['test_tube_do_checkpoint_load'] = True
-        args['hpc_exp_number'] = hpc_exp_number
-
-    hparams = Namespace(**args)
-    return hparams
-
-
-def main():
-    """
-    Make sure DDP + AMP continue training correctly
-    :return:
-    """
-    hparams = get_hparams()
-    model = LightningTestModel(hparams)
-
-    trainer_options = dict(
-        show_progress_bar=True,
-        max_nb_epochs=4,
-        gpus=2,
-        distributed_backend='dp',
-    )
-
-    save_dir = init_save_dir()
-
-    # exp file to get meta
-    exp = get_exp(False)
-    exp.argparse(hparams)
-    exp.save()
-
-    # exp file to get weights
-    checkpoint = ModelCheckpoint(save_dir)
-
-    # add these to the trainer options
-    trainer_options['experiment'] = exp
-    trainer_options['checkpoint_callback'] = checkpoint
-
-    # fit model
-    trainer = Trainer(**trainer_options)
-    trainer.is_slurm_managing_tasks = True
-    result = trainer.fit(model)
-
-    # track epoch before saving
-    real_global_epoch = trainer.current_epoch
-
-    # correct result and ok accuracy
-    assert result == 1, 'amp + dp model failed to complete'
-
-    # ---------------------------
-    # HPC LOAD/SAVE
-    # ---------------------------
-    # save
-    trainer.hpc_save(save_dir, exp)
-
-    # init new trainer
-    new_exp = get_exp(False, version=exp.version)
-    trainer_options['experiment'] = new_exp
-    trainer_options['checkpoint_callback'] = ModelCheckpoint(save_dir)
-    trainer_options['train_percent_check'] = 0.2
-    trainer_options['val_percent_check'] = 0.2
-    trainer_options['max_nb_epochs'] = 1
-    new_trainer = Trainer(**trainer_options)
-
-    # set the epoch start hook so we can predict before the model does the full training
-    def assert_good_acc():
-        assert trainer.current_epoch == real_global_epoch and trainer.current_epoch > 0
-
-        # if model and state loaded correctly, predictions will be good even though we
-        # haven't trained with the new loaded model
-        dp_model = new_trainer.model
-        dp_model.eval()
-
-        _ = [run_prediction(dataloader, dp_model, dp=True) for dataloader in trainer.val_dataloader]
-
-    # new model
-    model = LightningTestModel(hparams)
-    model.on_sanity_check_start = assert_good_acc
-
-    # fit new model which should load hpc weights
-    new_trainer.fit(model)
-
-    # test freeze on gpu
-    model.freeze()
-    model.unfreeze()
-
-    clear_save_dir()
-
-
-if __name__ == '__main__':
-    main()
+#
+# def main():
+#     reset_seed()
+#     set_random_master_port()
+#
+#     hparams = get_hparams()
+#     model = LightningTestModel(hparams)
+#
+#     save_dir = init_save_dir()
+#
+#     # exp file to get meta
+#     logger = get_test_tube_logger(False)
+#
+#     print(logger.debug)
+#
+#     # exp file to get weights
+#     checkpoint = init_checkpoint_callback(logger)
+#
+#     trainer_options = dict(
+#         show_progress_bar=False,
+#         max_nb_epochs=1,
+#         train_percent_check=0.4,
+#         val_percent_check=0.2,
+#         checkpoint_callback=checkpoint,
+#         logger=logger,
+#         gpus=[0, 1],
+#         distributed_backend='ddp'
+#     )
+#
+#     # fit model
+#     trainer = Trainer(**trainer_options)
+#     result = trainer.fit(model)
+#
+#     exp = logger.experiment
+#     print(os.listdir(exp.get_data_path(exp.name, exp.version)))
+#
+#     # correct result and ok accuracy
+#     assert result == 1, 'training failed to complete'
+#     pretrained_model = load_model(logger.experiment, save_dir,
+#                                   module_class=LightningTestModel)
+#
+#     # run test set
+#     new_trainer = Trainer(**trainer_options)
+#     new_trainer.test(pretrained_model)
+#
+#     # test we have good test accuracy
+#     clear_save_dir()
+#
+# if __name__ == '__main__':
+#     main()
@@ -5,4 +5,5 @@ pytest>=3.0.5
 pytest-cov
 flake8
 check-manifest
-test_tube
+test_tube
+mlflow
@@ -0,0 +1,181 @@
+import os.path
+import pickle
+import shutil
+
+import numpy as np
+import torch
+from pytorch_lightning import Trainer
+
+from pytorch_lightning.testing import LightningTestModel
+from pytorch_lightning.logging import LightningLoggerBase, rank_zero_only
+from .test_models import get_hparams, get_test_tube_logger, init_save_dir, clear_save_dir
+
+RANDOM_SEEDS = list(np.random.randint(0, 10000, 1000))
+
+
+def test_testtube_logger():
+    """
+    verify that basic functionality of test tube logger works
+    """
+    reset_seed()
+
+    hparams = get_hparams()
+    model = LightningTestModel(hparams)
+
+    save_dir = init_save_dir()
+
+    logger = get_test_tube_logger(False)
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        train_percent_check=0.01,
+        logger=logger
+    )
+
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    assert result == 1, "Training failed"
+
+    clear_save_dir()
+
+
+def test_testtube_pickle():
+    """
+    Verify that pickling a trainer containing a test tube logger works
+    """
+    reset_seed()
+
+    hparams = get_hparams()
+    model = LightningTestModel(hparams)
+
+    save_dir = init_save_dir()
+
+    logger = get_test_tube_logger(False)
+    logger.log_hyperparams(hparams)
+    logger.save()
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        train_percent_check=0.01,
+        logger=logger
+    )
+
+    trainer = Trainer(**trainer_options)
+    pkl_bytes = pickle.dumps(trainer)
+    trainer2 = pickle.loads(pkl_bytes)
+    trainer2.logger.log_metrics({"acc": 1.0})
+
+
+def test_mlflow_logger():
+    """
+    verify that basic functionality of mlflow logger works
+    """
+    reset_seed()
+
+    try:
+        from pytorch_lightning.logging import MLFlowLogger
+    except ModuleNotFoundError:
+        return
+
+    hparams = get_hparams()
+    model = LightningTestModel(hparams)
+
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    mlflow_dir = os.path.join(root_dir, "mlruns")
+
+    logger = MLFlowLogger("test", f"file://{mlflow_dir}")
+    logger.log_hyperparams(hparams)
+    logger.save()
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        train_percent_check=0.01,
+        logger=logger
+    )
+
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    assert result == 1, "Training failed"
+
+    n = np.random.randint(0, 10000000, 1)[0]
+    shutil.move(mlflow_dir, mlflow_dir + f'_{n}')
+
+
+def test_mlflow_pickle():
+    """
+    verify that pickling trainer with mlflow logger works
+    """
+    reset_seed()
+
+    try:
+        from pytorch_lightning.logging import MLFlowLogger
+    except ModuleNotFoundError:
+        return
+
+    hparams = get_hparams()
+    model = LightningTestModel(hparams)
+
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    mlflow_dir = os.path.join(root_dir, "mlruns")
+
+    logger = MLFlowLogger("test", f"file://{mlflow_dir}")
+    logger.log_hyperparams(hparams)
+    logger.save()
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        logger=logger
+    )
+
+    trainer = Trainer(**trainer_options)
+    pkl_bytes = pickle.dumps(trainer)
+    trainer2 = pickle.loads(pkl_bytes)
+    trainer2.logger.log_metrics({"acc": 1.0})
+
+
+def test_custom_logger():
+
+    class CustomLogger(LightningLoggerBase):
+        def __init__(self):
+            super().__init__()
+            self.hparams_logged = None
+            self.metrics_logged = None
+            self.finalized = False
+
+        @rank_zero_only
+        def log_hyperparams(self, params):
+            self.hparams_logged = params
+
+        @rank_zero_only
+        def log_metrics(self, metrics, step_num):
+            self.metrics_logged = metrics
+
+        @rank_zero_only
+        def finalize(self, status):
+            self.finalized_status = status
+
+    hparams = get_hparams()
+    model = LightningTestModel(hparams)
+
+    logger = CustomLogger()
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        train_percent_check=0.01,
+        logger=logger
+    )
+
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+    assert result == 1, "Training failed"
+    assert logger.hparams_logged == hparams
+    assert logger.metrics_logged != {}
+    assert logger.finalized_status == "success"
+
+
+def reset_seed():
+    SEED = RANDOM_SEEDS.pop()
+    torch.manual_seed(SEED)
+    np.random.seed(SEED)