release v0.4.4

* split __train up for clarity

* split __train up for clarity

* added lr scheduler after epoch completes

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at waf2107@columbia.edu. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

README.md

@@ -81,36 +81,40 @@ class CoolModel(pl.LightningModule):
     def forward(self, x):
         return torch.relu(self.l1(x.view(x.size(0), -1)))
 
-    def my_loss(self, y_hat, y):
-        return F.cross_entropy(y_hat, y)
-
     def training_step(self, batch, batch_nb):
+        # REQUIRED
         x, y = batch
         y_hat = self.forward(x)
-        return {'loss': self.my_loss(y_hat, y)}
+        return {'loss': F.cross_entropy(y_hat, y)}
 
     def validation_step(self, batch, batch_nb):
+        # OPTIONAL
         x, y = batch
         y_hat = self.forward(x)
         return {'val_loss': self.my_loss(y_hat, y)}
 
     def validation_end(self, outputs):
+        # OPTIONAL
         avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
         return {'avg_val_loss': avg_loss}
 
     def configure_optimizers(self):
+        # REQUIRED
         return [torch.optim.Adam(self.parameters(), lr=0.02)]
 
     @pl.data_loader
     def tng_dataloader(self):
+        # REQUIRED
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 
     @pl.data_loader
     def val_dataloader(self):
+        # OPTIONAL
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 
     @pl.data_loader
     def test_dataloader(self):
+        # OPTIONAL
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 ```
 

docs/LightningModule/RequiredTrainerInterface.md

@@ -10,16 +10,14 @@ Otherwise, to Define a Lightning Module, implement the following methods:
 **Required**:  
 
 - [training_step](RequiredTrainerInterface.md#training_step)   
-- [validation_step](RequiredTrainerInterface.md#validation_step)
-- [validation_end](RequiredTrainerInterface.md#validation_end)
-    
+- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)
 - [configure_optimizers](RequiredTrainerInterface.md#configure_optimizers)
 
-- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)
-- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)
-- [test_dataloader](RequiredTrainerInterface.md#test_dataloader)
-
 **Optional**:   
+- [validation_step](RequiredTrainerInterface.md#validation_step)
+- [validation_end](RequiredTrainerInterface.md#validation_end)
+- [val_dataloader](RequiredTrainerInterface.md#val_dataloader)
+- [test_dataloader](RequiredTrainerInterface.md#test_dataloader)
 
 - [on_save_checkpoint](RequiredTrainerInterface.md#on_save_checkpoint)
 - [on_load_checkpoint](RequiredTrainerInterface.md#on_load_checkpoint)
@@ -48,24 +46,25 @@ class CoolModel(pl.LightningModule):
     def forward(self, x):
         return torch.relu(self.l1(x.view(x.size(0), -1)))
 
-    def my_loss(self, y_hat, y):
-        return F.cross_entropy(y_hat, y)
-
     def training_step(self, batch, batch_nb):
+        # REQUIRED
         x, y = batch
         y_hat = self.forward(x)
-        return {'loss': self.my_loss(y_hat, y)}
+        return {'loss': F.cross_entropy(y_hat, y)(y_hat, y)}
 
     def validation_step(self, batch, batch_nb):
+        # OPTIONAL
         x, y = batch
         y_hat = self.forward(x)
-        return {'val_loss': self.my_loss(y_hat, y)}
+        return {'val_loss': F.cross_entropy(y_hat, y)(y_hat, y)}
 
     def validation_end(self, outputs):
+        # OPTIONAL
         avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
         return {'avg_val_loss': avg_loss}
 
     def configure_optimizers(self):
+        # REQUIRED
         return [torch.optim.Adam(self.parameters(), lr=0.02)]
 
     @pl.data_loader
@@ -74,10 +73,13 @@ class CoolModel(pl.LightningModule):
 
     @pl.data_loader
     def val_dataloader(self):
+        # OPTIONAL
+        # can also return a list of val dataloaders
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 
     @pl.data_loader
     def test_dataloader(self):
+        # OPTIONAL
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 ```
 ---   
@@ -90,7 +92,7 @@ The LightningModule interface is on the right. Each method corresponds to a part
   </a>
 </p>   
 
----
+## Required Methods    
 
 ### training_step
 
@@ -136,15 +138,75 @@ def training_step(self, data_batch, batch_nb):
     return output
 ```
 
----
+--- 
+### tng_dataloader 
+
+``` {.python}
+@pl.data_loader
+def tng_dataloader(self)
+```
+Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.
+
+##### Return
+PyTorch DataLoader
+
+**Example**
+
+``` {.python}
+@pl.data_loader
+def tng_dataloader(self):
+    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+    dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True)
+    loader = torch.utils.data.DataLoader(
+        dataset=dataset,
+        batch_size=self.hparams.batch_size,
+        shuffle=True
+    )
+    return loader
+```
+
+--- 
+### configure_optimizers 
+
+``` {.python}
+def configure_optimizers(self)
+```
+
+Set up as many optimizers and (optionally) learning rate schedulers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. 
+Lightning will call .backward() and .step() on each one in every epoch.  If you use 16 bit precision it will also handle that.
+
+
+##### Return
+List or Tuple - List of optimizers with an optional second list of learning-rate schedulers
+
+**Example**
+
+``` {.python}
+# most cases
+def configure_optimizers(self):
+    opt = Adam(self.parameters(), lr=0.01)
+    return [opt]
+    
+# gan example, with scheduler for discriminator
+def configure_optimizers(self):
+    generator_opt = Adam(self.model_gen.parameters(), lr=0.01)
+    disriminator_opt = Adam(self.model_disc.parameters(), lr=0.02)
+    discriminator_sched = CosineAnnealing(discriminator_opt, T_max=10)
+    return [generator_opt, disriminator_opt], [discriminator_sched]
+```
+
+## Optional Methods    
 
 ### validation_step
 
 ``` {.python}
-def validation_step(self, data_batch, batch_nb)
+def validation_step(self, data_batch, batch_nb, dataloader_i)
 ```
+**OPTIONAL**    
+If you don't need to validate you don't need to implement this method.    
+
+In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes, calculate accuracy, or save example outputs (using self.experiment or whatever you want). Really, anything you want.       
 
-In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model.
 This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation.
 
 **Params**   
@@ -153,6 +215,7 @@ This is most likely the same as your training_step. But unlike training step, th
 |---|---|
 |  data_batch | The output of your dataloader. A tensor, tuple or list  |
 |  batch_nb | Integer displaying which batch this is  |
+|  dataloader_i | Integer displaying which dataloader this is  |
 
 **Return**   
 
@@ -190,9 +253,12 @@ def validation_step(self, data_batch, batch_nb):
 
 ``` {.python}
 def validation_end(self, outputs)
-```
+```   
+If you didn't define a validation_step, this won't be called.       
 
-Called at the end of the validation loop with the output of each validation_step.
+Called at the end of the validation loop with the output of each validation_step.  Called once per validation dataset.   
+
+The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything.    
 
 **Params**    
 
@@ -227,36 +293,6 @@ def validation_end(self, outputs):
     return tqdm_dic
 ```
 
---- 
-### configure_optimizers 
-
-``` {.python}
-def configure_optimizers(self)
-```
-
-Set up as many optimizers and (optionally) learning rate schedulers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. 
-Lightning will call .backward() and .step() on each one in every epoch.  If you use 16 bit precision it will also handle that.
-
-
-##### Return
-List or Tuple - List of optimizers with an optional second list of learning-rate schedulers
-
-**Example**
-
-``` {.python}
-# most cases
-def configure_optimizers(self):
-    opt = Adam(self.parameters(), lr=0.01)
-    return [opt]
-    
-# gan example, with scheduler for discriminator
-def configure_optimizers(self):
-    generator_opt = Adam(self.model_gen.parameters(), lr=0.01)
-    disriminator_opt = Adam(self.model_disc.parameters(), lr=0.02)
-    discriminator_sched = CosineAnnealing(discriminator_opt, T_max=10)
-    return [generator_opt, disriminator_opt], [discriminator_sched]
-```
-
 --- 
 ### on_save_checkpoint 
 
@@ -299,33 +335,6 @@ def on_load_checkpoint(self, checkpoint):
     self.something_cool_i_want_to_save = checkpoint['something_cool_i_want_to_save']
 ```
 
---- 
-### tng_dataloader 
-
-``` {.python}
-@pl.data_loader
-def tng_dataloader(self)
-```
-Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.
-
-##### Return
-PyTorch DataLoader
-
-**Example**
-
-``` {.python}
-@pl.data_loader
-def tng_dataloader(self):
-    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
-    dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True)
-    loader = torch.utils.data.DataLoader(
-        dataset=dataset,
-        batch_size=self.hparams.batch_size,
-        shuffle=True
-    )
-    return loader
-```
-
 --- 
 ### val_dataloader 
 
@@ -333,10 +342,13 @@ def tng_dataloader(self):
 @pl.data_loader
 def tng_dataloader(self)
 ```
-Called by lightning during validation loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.
+**OPTIONAL**    
+If you don't need a validation dataset and a validation_step, you don't need to implement this method.    
+
+Called by lightning during validation loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.   
 
 ##### Return
-PyTorch DataLoader
+PyTorch DataLoader or list of PyTorch Dataloaders.    
 
 **Example**
 
@@ -352,6 +364,11 @@ def val_dataloader(self):
     )
     
     return loader
+
+# can also return multiple dataloaders   
+@pl.data_loader
+def val_dataloader(self):
+    return [loader_a, loader_b, ..., loader_n]   
 ```
 
 --- 
@@ -361,6 +378,9 @@ def val_dataloader(self):
 @pl.data_loader
 def test_dataloader(self)
 ```
+**OPTIONAL**    
+If you don't need a test dataset and a test_step, you don't need to implement this method.    
+
 Called by lightning during test loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.
 
 ##### Return

docs/examples/Examples.md

@@ -3,7 +3,7 @@ In 99% of cases you want to just copy [this template](https://github.com/william
 
 ```bash
 # get a copy of the module template
-wget https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py
+wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/examples/new_project_templates/lightning_module_template.py
 ```
 
 ---    

examples/new_project_templates/lightning_module_template.py

@@ -105,7 +105,7 @@ class LightningTemplateModel(LightningModule):
         # can also return just a scalar instead of a dict (return loss_val)
         return output
 
-    def validation_step(self, data_batch, batch_i):
+    def validation_step(self, data_batch, batch_i, dataloader_i):
         """
         Lightning calls this inside the validation loop
         :param data_batch:
@@ -218,7 +218,7 @@ class LightningTemplateModel(LightningModule):
     @pl.data_loader
     def val_dataloader(self):
         print('val data loader called')
-        return self.__dataloader(train=False)
+        return [self.__dataloader(train=False) for i in range(2)]
 
     @pl.data_loader
     def test_dataloader(self):

pytorch_lightning/models/trainer.py

@@ -13,6 +13,7 @@ from torch.utils.data.distributed import DistributedSampler
 import torch.multiprocessing as mp
 import torch.distributed as dist
 
+from pytorch_lightning.root_module.root_module import LightningModule
 from pytorch_lightning.root_module.memory import get_gpu_memory_map
 from pytorch_lightning.root_module.model_saving import TrainerIO
 from pytorch_lightning.pt_overrides.override_data_parallel import (
@@ -312,6 +313,14 @@ class Trainer(TrainerIO):
         f_op = getattr(model, f_name, None)
         return callable(f_op)
 
+    def __is_overriden(self, f_name):
+        model = self.__get_model()
+        super_object = super(model.__class__, model)
+
+        # when code pointers are different, it was overriden
+        is_overriden = getattr(model, f_name).__code__ is not getattr(super_object, f_name).__code__
+        return is_overriden
+
     @property
     def __tng_tqdm_dic(self):
         tqdm_dic = {
@@ -345,13 +354,17 @@ class Trainer(TrainerIO):
         self.nb_tng_batches = int(self.nb_tng_batches * self.train_percent_check)
 
         # determine number of validation batches
-        self.nb_val_batches = len(self.val_dataloader)
+        # val datasets could be none, 1 or 2+
+        self.nb_val_batches = 0
+        if self.val_dataloader is not None:
+            self.nb_val_batches = sum(len(dataloader) for dataloader in self.val_dataloader)
+
         self.nb_val_batches = int(self.nb_val_batches * self.val_percent_check)
         self.nb_val_batches = max(1, self.nb_val_batches)
         self.nb_val_batches = self.nb_val_batches
 
         # determine number of test batches
-        self.nb_test_batches = len(self.test_dataloader)
+        self.nb_test_batches = len(self.test_dataloader) if self.test_dataloader is not None else 0
         self.nb_test_batches = int(self.nb_test_batches * self.test_percent_check)
 
         # determine when to check validation
@@ -364,7 +377,7 @@ class Trainer(TrainerIO):
 
             self.tqdm_metrics[k] = v
 
-    def validate(self, model, dataloader, max_batches):
+    def validate(self, model, dataloader, max_batches, dataloader_i):
         """
         Run validation code
         :param model: PT model
@@ -372,6 +385,7 @@ class Trainer(TrainerIO):
         :param max_batches: Scalar
         :return:
         """
+
         # enable eval mode
         model.zero_grad()
         model.eval()
@@ -396,9 +410,9 @@ class Trainer(TrainerIO):
             # RUN VALIDATION STEP
             # -----------------
             if self.use_ddp:
-                output = model(data_batch, batch_i)
+                output = model(data_batch, batch_i, dataloader_i)
             elif self.use_dp:
-                output = model(data_batch, batch_i)
+                output = model(data_batch, batch_i, dataloader_i)
             elif self.single_gpu:
                 # put inputs on gpu manually
                 gpu_id = self.data_parallel_device_ids[0]
@@ -407,10 +421,10 @@ class Trainer(TrainerIO):
                         data_batch[i] = x.cuda(gpu_id)
 
                 # do non dp, ddp step
-                output = model.validation_step(data_batch, batch_i)
+                output = model.validation_step(data_batch, batch_i, dataloader_i)
 
             else:
-                output = model.validation_step(data_batch, batch_i)
+                output = model.validation_step(data_batch, batch_i, dataloader_i)
 
             outputs.append(output)
 
@@ -418,11 +432,13 @@ class Trainer(TrainerIO):
             if self.progress_bar and self.prog_bar is not None:
                 self.prog_bar.update(1)
 
-        # give model a chance to do something with the outputs
-        if self.data_parallel:
-            val_results = model.module.validation_end(outputs)
-        else:
-            val_results = model.validation_end(outputs)
+        # give model a chance to do something with the outputs (and method defined)
+        val_results = {}
+        if self.__is_overriden('validation_end'):
+            if self.data_parallel:
+                val_results = model.module.validation_end(outputs)
+            else:
+                val_results = model.validation_end(outputs)
 
         # enable train mode again
         model.train()
@@ -439,13 +455,20 @@ class Trainer(TrainerIO):
         :return:
         """
         self.tng_dataloader = model.tng_dataloader
+
         self.test_dataloader = model.test_dataloader
         self.val_dataloader = model.val_dataloader
 
+        # handle returning an actual dataloader instead of a list of loaders
+        have_val_loaders = self.val_dataloader is not None
+        if have_val_loaders and not isinstance(self.val_dataloader, list):
+            self.val_dataloader = [self.val_dataloader]
+
         if self.use_ddp and not isinstance(self.tng_dataloader.sampler, DistributedSampler):
             msg = """
-when using multiple gpus and multiple nodes you must pass
- a DistributedSampler to DataLoader(sampler).
+You're using multiple gpus and multiple nodes without using a DistributedSampler
+to assign a subset of your data to each process. To silence this warning, pass a
+DistributedSampler to your DataLoader.
 
 ie: this:
 dataset = myDataset()
@@ -455,8 +478,32 @@ becomes:
 dataset = myDataset()
 dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
 dataloader = Dataloader(dataset, sampler=dist_sampler)
+
+If you want each process to load the full dataset, ignore this warning.
 """
-            raise MisconfigurationException(msg)
+            warnings.warn(msg)
+
+        if self.use_ddp and\
+                not all(isinstance(dataloader, DistributedSampler)
+                        for dataloader in self.val_dataloader):
+            msg = """
+You're val_dataloader(s) are not all DistributedSamplers.
+You're using multiple gpus and multiple nodes without using a DistributedSampler
+to assign a subset of your data to each process. To silence this warning, pass a
+DistributedSampler to your DataLoader.
+
+ie: this:
+dataset = myDataset()
+dataloader = Dataloader(dataset)
+
+becomes:
+dataset = myDataset()
+dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
+dataloader = Dataloader(dataset, sampler=dist_sampler)
+
+If you want each process to load the full dataset, ignore this warning.
+"""
+            warnings.warn(msg)
 
     # -----------------------------
     # MODEL TRAINING
@@ -702,32 +749,25 @@ We recommend you switch to ddp if you want to use amp
         if self.cluster is not None:  # pragma: no cover
             self.enable_auto_hpc_walltime_manager()
 
-        # run tiny validation to make sure program won't crash during val
+        # run tiny validation (if validation defined) to make sure program won't crash during val
         ref_model.on_sanity_check_start()
-        _ = self.validate(model, self.val_dataloader, max_batches=self.nb_sanity_val_steps)
+        if self.val_dataloader is not None:
+            for ds_i, dataloader in enumerate(self.val_dataloader):
+                self.validate(model, dataloader, self.nb_sanity_val_steps, ds_i)
 
         # ---------------------------
         # CORE TRAINING LOOP
         # ---------------------------
-
         self.__train()
 
     def __train(self):
         # run all epochs
         for epoch_nb in range(self.current_epoch, self.max_nb_epochs):
-            # update the lr scheduler
-            if self.lr_schedulers is not None:
-                for lr_scheduler in self.lr_schedulers:
-                    lr_scheduler.step()
-
+            # get model
             model = self.__get_model()
+
+            # update training progress in trainer and model
             model.current_epoch = epoch_nb
-
-            # hook
-            if self.__is_function_implemented('on_epoch_start'):
-                model = self.__get_model()
-                model.on_epoch_start()
-
             self.current_epoch = epoch_nb
             self.total_batches = self.nb_tng_batches + self.nb_val_batches
             self.batch_loss_value = 0  # accumulated grads
@@ -737,92 +777,103 @@ We recommend you switch to ddp if you want to use amp
                 self.prog_bar = tqdm.tqdm(range(self.total_batches),
                                           position=self.process_position)
 
-            for batch_nb, data_batch in enumerate(self.tng_dataloader):
-                self.batch_nb = batch_nb
-                self.global_step += 1
+            # -----------------
+            # RUN TNG EPOCH
+            # -----------------
+            self.run_tng_epoch()
 
-                model = self.__get_model()
-                model.global_step = self.global_step
-
-                # stop when the flag is changed or we've gone past the amount
-                #  requested in the batches
-                self.total_batch_nb += 1
-                met_batch_limit = batch_nb > self.nb_tng_batches
-                if met_batch_limit:
-                    break
-
-                # ---------------
-                # RUN TRAIN STEP
-                # ---------------
-                batch_result = self.__run_tng_batch(data_batch, batch_nb)
-                early_stop_epoch = batch_result == -1
-
-                # ---------------
-                # RUN VAL STEP
-                # ---------------
-                is_val_check_batch = (batch_nb + 1) % self.val_check_batch == 0
-                if self.fast_dev_run or is_val_check_batch or early_stop_epoch:
-                    self.__run_validation()
-
-                # when batch should be saved
-                if (batch_nb + 1) % self.log_save_interval == 0 or early_stop_epoch:
-                    if self.proc_rank == 0 and self.experiment is not None:
-                        self.experiment.save()
-
-                # when metrics should be logged
-                if batch_nb % self.add_log_row_interval == 0 or early_stop_epoch:
-                    # count items in memory
-                    # nb_params, nb_tensors = count_mem_items()
-
-                    model = self.__get_model()
-                    metrics = self.__tng_tqdm_dic
-
-                    # add gpu memory
-                    if self.on_gpu:
-                        mem_map = get_gpu_memory_map()
-                        metrics.update(mem_map)
-
-                    # add norms
-                    if self.track_grad_norm > 0:
-                        model = self.__get_model()
-                        grad_norm_dic = model.grad_norm(self.track_grad_norm)
-                        metrics.update(grad_norm_dic)
-
-                    if self.__is_function_implemented('on_tng_metrics'):
-                        model.on_tng_metrics(metrics)
-
-                    # log metrics
-                    scalar_metrics = self.__metrics_to_scalars(
-                        metrics, blacklist=self.__log_vals_blacklist())
-                    if self.proc_rank == 0 and self.experiment is not None:
-                        self.experiment.log(scalar_metrics, global_step=self.global_step)
-                        self.experiment.save()
-
-                # hook
-                if self.__is_function_implemented('on_batch_end'):
-                    model = self.__get_model()
-                    model.on_batch_end()
-
-                # end epoch early
-                if early_stop_epoch:
-                    break
-
-            # hook
-            if self.__is_function_implemented('on_epoch_end'):
-                model = self.__get_model()
-                model.on_epoch_end()
+            # update LR schedulers
+            if self.lr_schedulers is not None:
+                for lr_scheduler in self.lr_schedulers:
+                    lr_scheduler.step()
 
             # early stopping
             met_min_epochs = epoch_nb > self.min_nb_epochs
             if self.enable_early_stop and met_min_epochs:
                 should_stop = self.early_stop_callback.on_epoch_end(epoch=epoch_nb,
                                                                     logs=self.__tng_tqdm_dic)
-
                 # stop training
                 stop = should_stop and met_min_epochs
                 if stop:
                     return
 
+    def run_tng_epoch(self):
+        # before epoch hook
+        if self.__is_function_implemented('on_epoch_start'):
+            model = self.__get_model()
+            model.on_epoch_start()
+
+        # run epoch
+        for batch_nb, data_batch in enumerate(self.tng_dataloader):
+            self.batch_nb = batch_nb
+            self.global_step += 1
+
+            model = self.__get_model()
+            model.global_step = self.global_step
+
+            # stop when the flag is changed or we've gone past the amount
+            #  requested in the batches
+            self.total_batch_nb += 1
+            met_batch_limit = batch_nb > self.nb_tng_batches
+            if met_batch_limit:
+                break
+
+            # ---------------
+            # RUN TRAIN STEP
+            # ---------------
+            batch_result = self.__run_tng_batch(data_batch, batch_nb)
+            early_stop_epoch = batch_result == -1
+
+            # ---------------
+            # RUN VAL STEP
+            # ---------------
+            is_val_check_batch = (batch_nb + 1) % self.val_check_batch == 0
+            if self.fast_dev_run or is_val_check_batch or early_stop_epoch:
+                self.__run_validation()
+
+            # when batch should be saved
+            if (batch_nb + 1) % self.log_save_interval == 0 or early_stop_epoch:
+                if self.proc_rank == 0 and self.experiment is not None:
+                    self.experiment.save()
+
+            # when metrics should be logged
+            if batch_nb % self.add_log_row_interval == 0 or early_stop_epoch:
+                # count items in memory
+                # nb_params, nb_tensors = count_mem_items()
+
+                model = self.__get_model()
+                metrics = self.__tng_tqdm_dic
+
+                # add gpu memory
+                if self.on_gpu:
+                    mem_map = get_gpu_memory_map()
+                    metrics.update(mem_map)
+
+                # add norms
+                if self.track_grad_norm > 0:
+                    model = self.__get_model()
+                    grad_norm_dic = model.grad_norm(self.track_grad_norm)
+                    metrics.update(grad_norm_dic)
+
+                if self.__is_function_implemented('on_tng_metrics'):
+                    model.on_tng_metrics(metrics)
+
+                # log metrics
+                scalar_metrics = self.__metrics_to_scalars(
+                    metrics, blacklist=self.__log_vals_blacklist())
+                if self.proc_rank == 0 and self.experiment is not None:
+                    self.experiment.log(scalar_metrics, global_step=self.global_step)
+                    self.experiment.save()
+
+            # end epoch early
+            if early_stop_epoch:
+                break
+
+        # epoch end hook
+        if self.__is_function_implemented('on_epoch_end'):
+            model = self.__get_model()
+            model.on_epoch_end()
+
     def __metrics_to_scalars(self, metrics, blacklist=set()):
         new_metrics = {}
         for k, v in metrics.items():
@@ -969,30 +1020,30 @@ We recommend you switch to ddp if you want to use amp
         elif not can_check_epoch:
             return
 
-        # hook
-        if self.__is_function_implemented('on_pre_performance_check'):
-            model = self.__get_model()
-            model.on_pre_performance_check()
+        # validate only if model has validation_step defined
+        if self.__is_overriden('validation_step'):
 
-        # use full val set on end of epoch
-        # use a small portion otherwise
-        max_batches = None if not self.fast_dev_run else 1
-        validation_results = self.validate(
-            self.model,
-            self.val_dataloader,
-            max_batches
-        )
-        self.__add_tqdm_metrics(validation_results)
+            # hook
+            if self.__is_function_implemented('on_pre_performance_check'):
+                model = self.__get_model()
+                model.on_pre_performance_check()
 
-        # hook
-        if self.__is_function_implemented('on_post_performance_check'):
-            model = self.__get_model()
-            model.on_post_performance_check()
+            # use full val set on end of epoch
+            # use a small portion otherwise
+            max_batches = None if not self.fast_dev_run else 1
+            for ds_i, dataloader in enumerate(self.val_dataloader):
+                val_out_metrics = self.validate(self.model, dataloader, max_batches, ds_i)
+                self.__add_tqdm_metrics(val_out_metrics)
 
-        if self.progress_bar:
-            # add model specific metrics
-            tqdm_metrics = self.__tng_tqdm_dic
-            self.prog_bar.set_postfix(**tqdm_metrics)
+            # hook
+            if self.__is_function_implemented('on_post_performance_check'):
+                model = self.__get_model()
+                model.on_post_performance_check()
+
+            if self.progress_bar:
+                # add model specific metrics
+                tqdm_metrics = self.__tng_tqdm_dic
+                self.prog_bar.set_postfix(**tqdm_metrics)
 
         # model checkpointing
         if self.proc_rank == 0 and self.checkpoint_callback is not None:

pytorch_lightning/root_module/root_module.py

@@ -36,18 +36,20 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
     def validation_step(self, data_batch, batch_nb):
         """
         return whatever outputs will need to be aggregated in validation_end
+        OPTIONAL
         :param data_batch:
         :return:
         """
-        raise NotImplementedError
+        pass
 
     def validation_end(self, outputs):
         """
         Outputs has the appended output after each validation step
+        OPTIONAL
         :param outputs:
         :return: dic_with_metrics for tqdm
         """
-        raise NotImplementedError
+        pass
 
     def training_step(self, data_batch, batch_nb):
         """
@@ -67,7 +69,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
     @data_loader
     def tng_dataloader(self):
         """
-        Implement a function to load an h5py of this data
+        Implement a PyTorch DataLoader
         :return:
         """
         raise NotImplementedError
@@ -75,18 +77,18 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
     @data_loader
     def test_dataloader(self):
         """
-        Implement a function to load an h5py of this data
+        Implement a PyTorch DataLoader
         :return:
         """
-        raise NotImplementedError
+        return None
 
     @data_loader
     def val_dataloader(self):
         """
-        Implement a function to load an h5py of this data
+        Implement a PyTorch DataLoader
         :return:
         """
-        raise NotImplementedError
+        return None
 
     @classmethod
     def load_from_metrics(cls, weights_path, tags_csv, on_gpu, map_location=None):

pytorch_lightning/testing/__init__.py

@@ -0,0 +1,3 @@
+from .lm_test_module import LightningTestModel
+from .no_val_end_module import NoValEndTestModel
+from .no_val_module import NoValModel

pytorch_lightning/testing/lm_test_module.py

@@ -109,7 +109,7 @@ class LightningTestModel(LightningModule):
         if self.trainer.batch_nb % 2 == 0:
             return loss_val
 
-    def validation_step(self, data_batch, batch_i):
+    def validation_step(self, data_batch, batch_i, dataloader_i):
         """
         Lightning calls this inside the validation loop
         :param data_batch:
@@ -151,6 +151,12 @@ class LightningTestModel(LightningModule):
                 'test_dic': {'val_loss_a': loss_val}
             })
             return output
+        if batch_i % 5 == 0:
+            output = OrderedDict({
+                f'val_loss_{dataloader_i}': loss_val,
+                f'val_acc_{dataloader_i}': val_acc,
+            })
+            return output
 
     def validation_end(self, outputs):
         """

pytorch_lightning/testing/no_val_end_module.py

@@ -0,0 +1,247 @@
+import os
+from collections import OrderedDict
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import optim
+from torch.utils.data import DataLoader
+from torch.utils.data.distributed import DistributedSampler
+from torchvision.datasets import MNIST
+from torchvision import transforms
+from test_tube import HyperOptArgumentParser
+
+from pytorch_lightning.root_module.root_module import LightningModule
+from pytorch_lightning import data_loader
+
+
+class NoValEndTestModel(LightningModule):
+    """
+    Sample model to show how to define a template
+    """
+
+    def __init__(self, hparams, force_remove_distributed_sampler=False):
+        """
+        Pass in parsed HyperOptArgumentParser to the model
+        :param hparams:
+        """
+        # init superclass
+        super(NoValEndTestModel, self).__init__()
+        self.hparams = hparams
+
+        self.batch_size = hparams.batch_size
+
+        # if you specify an example input, the summary will show input/output for each layer
+        self.example_input_array = torch.rand(5, 28 * 28)
+
+        # remove to test warning for dist sampler
+        self.force_remove_distributed_sampler = force_remove_distributed_sampler
+
+        # build model
+        self.__build_model()
+
+    # ---------------------
+    # MODEL SETUP
+    # ---------------------
+    def __build_model(self):
+        """
+        Layout model
+        :return:
+        """
+        self.c_d1 = nn.Linear(in_features=self.hparams.in_features,
+                              out_features=self.hparams.hidden_dim)
+        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
+        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
+
+        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim,
+                              out_features=self.hparams.out_features)
+
+    # ---------------------
+    # TRAINING
+    # ---------------------
+    def forward(self, x):
+        """
+        No special modification required for lightning, define as you normally would
+        :param x:
+        :return:
+        """
+
+        x = self.c_d1(x)
+        x = torch.tanh(x)
+        x = self.c_d1_bn(x)
+        x = self.c_d1_drop(x)
+
+        x = self.c_d2(x)
+        logits = F.log_softmax(x, dim=1)
+
+        return logits
+
+    def loss(self, labels, logits):
+        nll = F.nll_loss(logits, labels)
+        return nll
+
+    def training_step(self, data_batch, batch_i):
+        """
+        Lightning calls this inside the training loop
+        :param data_batch:
+        :return:
+        """
+        # forward pass
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+
+        y_hat = self.forward(x)
+
+        # calculate loss
+        loss_val = self.loss(y, y_hat)
+
+        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
+        if self.trainer.use_dp:
+            loss_val = loss_val.unsqueeze(0)
+
+        # alternate possible outputs to test
+        if self.trainer.batch_nb % 1 == 0:
+            output = OrderedDict({
+                'loss': loss_val,
+                'prog': {'some_val': loss_val * loss_val}
+            })
+            return output
+        if self.trainer.batch_nb % 2 == 0:
+            return loss_val
+
+    def validation_step(self, data_batch, batch_i, dataloader_i):
+        """
+        Lightning calls this inside the validation loop
+        :param data_batch:
+        :return:
+        """
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+        y_hat = self.forward(x)
+
+        loss_val = self.loss(y, y_hat)
+
+        # acc
+        labels_hat = torch.argmax(y_hat, dim=1)
+        val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
+        val_acc = torch.tensor(val_acc)
+
+        if self.on_gpu:
+            val_acc = val_acc.cuda(loss_val.device.index)
+
+        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
+        if self.trainer.use_dp:
+            loss_val = loss_val.unsqueeze(0)
+            val_acc = val_acc.unsqueeze(0)
+
+        # alternate possible outputs to test
+        if batch_i % 1 == 0:
+            output = OrderedDict({
+                'val_loss': loss_val,
+                'val_acc': val_acc,
+            })
+            return output
+        if batch_i % 2 == 0:
+            return val_acc
+
+        if batch_i % 3 == 0:
+            output = OrderedDict({
+                'val_loss': loss_val,
+                'val_acc': val_acc,
+                'test_dic': {'val_loss_a': loss_val}
+            })
+            return output
+
+    def on_tng_metrics(self, logs):
+        logs['some_tensor_to_test'] = torch.rand(1)
+
+    # ---------------------
+    # TRAINING SETUP
+    # ---------------------
+    def configure_optimizers(self):
+        """
+        return whatever optimizers we want here
+        :return: list of optimizers
+        """
+        # try no scheduler for this model (testing purposes)
+        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+
+        # test returning only 1 list instead of 2
+        return [optimizer]
+
+    def __dataloader(self, train):
+        # init data generators
+        transform = transforms.Compose([transforms.ToTensor(),
+                                        transforms.Normalize((0.5,), (1.0,))])
+        dataset = MNIST(root=self.hparams.data_root, train=train,
+                        transform=transform, download=True)
+
+        # when using multi-node we need to add the datasampler
+        train_sampler = None
+        batch_size = self.hparams.batch_size
+
+        try:
+            if self.on_gpu and not self.force_remove_distributed_sampler:
+                train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
+                batch_size = batch_size // self.trainer.world_size  # scale batch size
+        except Exception:
+            pass
+
+        should_shuffle = train_sampler is None
+        loader = DataLoader(
+            dataset=dataset,
+            batch_size=batch_size,
+            shuffle=should_shuffle,
+            sampler=train_sampler
+        )
+
+        return loader
+
+    @data_loader
+    def tng_dataloader(self):
+        return self.__dataloader(train=True)
+
+    @data_loader
+    def val_dataloader(self):
+        return self.__dataloader(train=False)
+
+    @data_loader
+    def test_dataloader(self):
+        return self.__dataloader(train=False)
+
+    @staticmethod
+    def add_model_specific_args(parent_parser, root_dir):  # pragma: no cover
+        """
+        Parameters you define here will be available to your model through self.hparams
+        :param parent_parser:
+        :param root_dir:
+        :return:
+        """
+        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+
+        # param overwrites
+        # parser.set_defaults(gradient_clip=5.0)
+
+        # network params
+        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
+        parser.add_argument('--in_features', default=28 * 28, type=int)
+        parser.add_argument('--out_features', default=10, type=int)
+        # use 500 for CPU, 50000 for GPU to see speed difference
+        parser.add_argument('--hidden_dim', default=50000, type=int)
+
+        # data
+        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
+
+        # training params (opt)
+        parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
+                        options=[0.0001, 0.0005, 0.001, 0.005],
+                        tunable=False)
+        parser.opt_list('--optimizer_name', default='adam', type=str,
+                        options=['adam'], tunable=False)
+
+        # if using 2 nodes with 4 gpus each the batch size here
+        #  (256) will be 256 / (2*8) = 16 per gpu
+        parser.opt_list('--batch_size', default=256 * 8, type=int,
+                        options=[32, 64, 128, 256], tunable=False,
+                        help='batch size will be divided over all gpus being used across all nodes')
+        return parser

pytorch_lightning/testing/no_val_module.py

@@ -0,0 +1,196 @@
+import os
+from collections import OrderedDict
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import optim
+from torch.utils.data import DataLoader
+from torch.utils.data.distributed import DistributedSampler
+from torchvision.datasets import MNIST
+from torchvision import transforms
+from test_tube import HyperOptArgumentParser
+
+from pytorch_lightning.root_module.root_module import LightningModule
+from pytorch_lightning import data_loader
+
+
+class NoValModel(LightningModule):
+    """
+    Sample model to show how to define a template
+    """
+
+    def __init__(self, hparams, force_remove_distributed_sampler=False):
+        """
+        Pass in parsed HyperOptArgumentParser to the model
+        :param hparams:
+        """
+        # init superclass
+        super(NoValModel, self).__init__()
+        self.hparams = hparams
+
+        self.batch_size = hparams.batch_size
+
+        # if you specify an example input, the summary will show input/output for each layer
+        self.example_input_array = torch.rand(5, 28 * 28)
+
+        # remove to test warning for dist sampler
+        self.force_remove_distributed_sampler = force_remove_distributed_sampler
+
+        # build model
+        self.__build_model()
+
+    # ---------------------
+    # MODEL SETUP
+    # ---------------------
+    def __build_model(self):
+        """
+        Layout model
+        :return:
+        """
+        self.c_d1 = nn.Linear(in_features=self.hparams.in_features,
+                              out_features=self.hparams.hidden_dim)
+        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
+        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
+
+        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim,
+                              out_features=self.hparams.out_features)
+
+    # ---------------------
+    # TRAINING
+    # ---------------------
+    def forward(self, x):
+        """
+        No special modification required for lightning, define as you normally would
+        :param x:
+        :return:
+        """
+
+        x = self.c_d1(x)
+        x = torch.tanh(x)
+        x = self.c_d1_bn(x)
+        x = self.c_d1_drop(x)
+
+        x = self.c_d2(x)
+        logits = F.log_softmax(x, dim=1)
+
+        return logits
+
+    def loss(self, labels, logits):
+        nll = F.nll_loss(logits, labels)
+        return nll
+
+    def training_step(self, data_batch, batch_i):
+        """
+        Lightning calls this inside the training loop
+        :param data_batch:
+        :return:
+        """
+        # forward pass
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+
+        y_hat = self.forward(x)
+
+        # calculate loss
+        loss_val = self.loss(y, y_hat)
+
+        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
+        if self.trainer.use_dp:
+            loss_val = loss_val.unsqueeze(0)
+
+        # alternate possible outputs to test
+        if self.trainer.batch_nb % 1 == 0:
+            output = OrderedDict({
+                'loss': loss_val,
+                'prog': {'some_val': loss_val * loss_val}
+            })
+            return output
+        if self.trainer.batch_nb % 2 == 0:
+            return loss_val
+
+    def on_tng_metrics(self, logs):
+        logs['some_tensor_to_test'] = torch.rand(1)
+
+    # ---------------------
+    # TRAINING SETUP
+    # ---------------------
+    def configure_optimizers(self):
+        """
+        return whatever optimizers we want here
+        :return: list of optimizers
+        """
+        # try no scheduler for this model (testing purposes)
+        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+
+        # test returning only 1 list instead of 2
+        return [optimizer]
+
+    def __dataloader(self, train):
+        # init data generators
+        transform = transforms.Compose([transforms.ToTensor(),
+                                        transforms.Normalize((0.5,), (1.0,))])
+        dataset = MNIST(root=self.hparams.data_root, train=train,
+                        transform=transform, download=True)
+
+        # when using multi-node we need to add the datasampler
+        train_sampler = None
+        batch_size = self.hparams.batch_size
+
+        try:
+            if self.on_gpu and not self.force_remove_distributed_sampler:
+                train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
+                batch_size = batch_size // self.trainer.world_size  # scale batch size
+        except Exception:
+            pass
+
+        should_shuffle = train_sampler is None
+        loader = DataLoader(
+            dataset=dataset,
+            batch_size=batch_size,
+            shuffle=should_shuffle,
+            sampler=train_sampler
+        )
+
+        return loader
+
+    @data_loader
+    def tng_dataloader(self):
+        return self.__dataloader(train=True)
+
+    @staticmethod
+    def add_model_specific_args(parent_parser, root_dir):  # pragma: no cover
+        """
+        Parameters you define here will be available to your model through self.hparams
+        :param parent_parser:
+        :param root_dir:
+        :return:
+        """
+        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+
+        # param overwrites
+        # parser.set_defaults(gradient_clip=5.0)
+
+        # network params
+        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
+        parser.add_argument('--in_features', default=28 * 28, type=int)
+        parser.add_argument('--out_features', default=10, type=int)
+        # use 500 for CPU, 50000 for GPU to see speed difference
+        parser.add_argument('--hidden_dim', default=50000, type=int)
+
+        # data
+        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
+
+        # training params (opt)
+        parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
+                        options=[0.0001, 0.0005, 0.001, 0.005],
+                        tunable=False)
+        parser.opt_list('--optimizer_name', default='adam', type=str,
+                        options=['adam'], tunable=False)
+
+        # if using 2 nodes with 4 gpus each the batch size here
+        #  (256) will be 256 / (2*8) = 16 per gpu
+        parser.opt_list('--batch_size', default=256 * 8, type=int,
+                        options=[32, 64, 128, 256], tunable=False,
+                        help='batch size will be divided over all gpus being used across all nodes')
+        return parser

setup.py

@@ -14,7 +14,7 @@ from setuptools import setup, find_packages
 # engineer specific practices
 setup(
     name='pytorch-lightning',
-    version='0.4.3',
+    version='0.4.4',
     description='The Keras for ML researchers using PyTorch',
     author='William Falcon',
     author_email='waf2107@columbia.edu',
@@ -31,7 +31,7 @@ setup(
     install_requires=[
         'torch==1.2.0',
         'tqdm',
-        'test-tube==0.6.8',
+        'test-tube>=0.6.9',
         'pandas>=0.20.3',
     ],
     classifiers=[

tests/test_models.py

@@ -10,7 +10,7 @@ from test_tube import Experiment, SlurmCluster
 
 # sys.path += [os.path.abspath('..'), os.path.abspath('../..')]
 from pytorch_lightning import Trainer
-from pytorch_lightning.testing.lm_test_module import LightningTestModel
+from pytorch_lightning.testing import LightningTestModel, NoValEndTestModel, NoValModel
 from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping
 from pytorch_lightning.utilities.debugging import MisconfigurationException
 from pytorch_lightning.root_module import memory
@@ -26,6 +26,122 @@ np.random.seed(SEED)
 # ------------------------------------------------------------------------
 # TESTS
 # ------------------------------------------------------------------------
+
+def test_early_stopping_cpu_model():
+    """
+    Test each of the trainer options
+    :return:
+    """
+
+    stopping = EarlyStopping(monitor='val_loss')
+    trainer_options = dict(
+        early_stop_callback=stopping,
+        gradient_clip=1.0,
+        overfit_pct=0.20,
+        track_grad_norm=2,
+        print_nan_grads=True,
+        progress_bar=False,
+        experiment=get_exp(),
+        train_percent_check=0.1,
+        val_percent_check=0.1
+    )
+
+    model, hparams = get_model()
+    run_gpu_model_test(trainer_options, model, hparams, on_gpu=False)
+
+    # test freeze on cpu
+    model.freeze()
+    model.unfreeze()
+
+
+def test_no_val_module():
+    """
+    Tests use case where trainer saves the model, and user loads it from tags independently
+    :return:
+    """
+    hparams = get_hparams()
+    model = NoValModel(hparams)
+
+    save_dir = init_save_dir()
+
+    # exp file to get meta
+    exp = get_exp(False)
+    exp.argparse(hparams)
+    exp.save()
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        cluster=SlurmCluster(),
+        experiment=exp,
+        checkpoint_callback=ModelCheckpoint(save_dir)
+    )
+
+    # fit model
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    # traning complete
+    assert result == 1, 'amp + ddp model failed to complete'
+
+    # save model
+    new_weights_path = os.path.join(save_dir, 'save_test.ckpt')
+    trainer.save_checkpoint(new_weights_path)
+
+    # load new model
+    tags_path = exp.get_data_path(exp.name, exp.version)
+    tags_path = os.path.join(tags_path, 'meta_tags.csv')
+    model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
+                                                   tags_csv=tags_path, on_gpu=False)
+    model_2.eval()
+
+    # make prediction
+    clear_save_dir()
+
+
+def test_no_val_end_module():
+    """
+    Tests use case where trainer saves the model, and user loads it from tags independently
+    :return:
+    """
+    hparams = get_hparams()
+    model = NoValEndTestModel(hparams)
+
+    save_dir = init_save_dir()
+
+    # exp file to get meta
+    exp = get_exp(False)
+    exp.argparse(hparams)
+    exp.save()
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        cluster=SlurmCluster(),
+        experiment=exp,
+        checkpoint_callback=ModelCheckpoint(save_dir)
+    )
+
+    # fit model
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    # traning complete
+    assert result == 1, 'amp + ddp model failed to complete'
+
+    # save model
+    new_weights_path = os.path.join(save_dir, 'save_test.ckpt')
+    trainer.save_checkpoint(new_weights_path)
+
+    # load new model
+    tags_path = exp.get_data_path(exp.name, exp.version)
+    tags_path = os.path.join(tags_path, 'meta_tags.csv')
+    model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
+                                                   tags_csv=tags_path, on_gpu=False)
+    model_2.eval()
+
+    # make prediction
+    clear_save_dir()
+
+
 def test_simple_cpu():
     """
     Verify continue training session on CPU
@@ -136,7 +252,7 @@ def test_cpu_restore_training():
         # if model and state loaded correctly, predictions will be good even though we
         # haven't trained with the new loaded model
         trainer.model.eval()
-        run_prediction(trainer.val_dataloader, trainer.model)
+        _ = [run_prediction(dataloader, trainer.model) for dataloader in trainer.val_dataloader]
 
     model.on_sanity_check_start = assert_good_acc
 
@@ -445,33 +561,6 @@ def test_amp_gpu_ddp_slurm_managed():
     clear_save_dir()
 
 
-def test_early_stopping_cpu_model():
-    """
-    Test each of the trainer options
-    :return:
-    """
-
-    stopping = EarlyStopping()
-    trainer_options = dict(
-        early_stop_callback=stopping,
-        gradient_clip=1.0,
-        overfit_pct=0.20,
-        track_grad_norm=2,
-        print_nan_grads=True,
-        progress_bar=False,
-        experiment=get_exp(),
-        train_percent_check=0.1,
-        val_percent_check=0.1
-    )
-
-    model, hparams = get_model()
-    run_gpu_model_test(trainer_options, model, hparams, on_gpu=False)
-
-    # test freeze on cpu
-    model.freeze()
-    model.unfreeze()
-
-
 def test_cpu_model_with_amp():
     """
     Make sure model trains on CPU
@@ -525,6 +614,7 @@ def test_all_features_cpu_model():
         print_nan_grads=True,
         progress_bar=False,
         experiment=get_exp(),
+        accumulate_grad_batches=2,
         max_nb_epochs=1,
         train_percent_check=0.4,
         val_percent_check=0.4
@@ -665,12 +755,43 @@ def test_ddp_sampler_error():
         use_amp=True
     )
 
-    with pytest.raises(MisconfigurationException):
+    with pytest.warns(UserWarning):
         trainer.get_dataloaders(model)
 
     clear_save_dir()
 
 
+def test_multiple_val_dataloader():
+    """
+    Verify multiple val_dataloader
+    :return:
+    """
+    hparams = get_hparams()
+    model = LightningTemplateModel(hparams)
+
+    save_dir = init_save_dir()
+
+    # exp file to get meta
+    trainer_options = dict(
+        max_nb_epochs=1,
+        val_percent_check=0.1,
+        train_percent_check=0.1,
+    )
+
+    # fit model
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    # verify tng completed
+    assert result == 1
+
+    # verify there are 2 val loaders
+    assert len(trainer.val_dataloader) == 2, 'Multiple val_dataloaders not initiated properly'
+
+    # make sure predictions are good for each val set
+    [run_prediction(dataloader, trainer.model) for dataloader in trainer.val_dataloader]
+
+
 # ------------------------------------------------------------------------
 # UTILS
 # ------------------------------------------------------------------------