release v0.5.1.3

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* changed rank 0

* models wait to restore weights

* models wait to restore weights

README.md

@@ -17,6 +17,8 @@
 [![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=latest)](https://pytorch-lightning.readthedocs.io/en/latest)
 [![Gitter](https://badges.gitter.im/PyTorch-Lightning/community.svg)](https://gitter.im/PyTorch-Lightning/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/williamFalcon/pytorch-lightning/blob/master/LICENSE)
+[![Next Release](https://img.shields.io/badge/Next%20Release-Nov%206-<COLOR>.svg)](https://shields.io/)
+
 <!-- 
 removed until codecov badge isn't empy. likely a config error showing nothing on master.
 [![codecov](https://codecov.io/gh/Borda/pytorch-lightning/branch/master/graph/badge.svg)](https://codecov.io/gh/Borda/pytorch-lightning)

docs/Trainer/hooks.md

@@ -65,12 +65,12 @@ You can override this method to adjust how you do the optimizer step for each op
 Called once per optimizer
 ```python
 # DEFAULT
-def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
+def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None):
     optimizer.step()   
     optimizer.zero_grad()   
     
 # Alternating schedule for optimizer steps (ie: GANs)    
-def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
+def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None):
     # update generator opt every 2 steps
     if optimizer_i == 0:
         if batch_nb % 2 == 0 :
@@ -91,7 +91,7 @@ This step allows you to do a lot of non-standard training tricks such as learnin
 
 ```python
 # learning rate warm-up
-def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
+def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None):
     # warm up lr
     if self.trainer.global_step < 500:
         lr_scale = min(1., float(self.trainer.global_step + 1) / 500.)

pytorch_lightning/logging/test_tube_logger.py

@@ -10,11 +10,13 @@ class TestTubeLogger(LightningLoggerBase):
     __test__ = False
 
     def __init__(
-        self, save_dir, name="default", debug=False, version=None, create_git_tag=False
+        self, save_dir, name="default", description=None, debug=False,
+            version=None, create_git_tag=False
     ):
         super().__init__()
         self.save_dir = save_dir
         self.name = name
+        self.description = description
         self.debug = debug
         self._version = version
         self.create_git_tag = create_git_tag
@@ -29,6 +31,7 @@ class TestTubeLogger(LightningLoggerBase):
             name=self.name,
             debug=self.debug,
             version=self.version,
+            description=self.description,
             create_git_tag=self.create_git_tag,
             rank=self.rank,
         )

pytorch_lightning/testing/lm_test_module_base.py

@@ -104,7 +104,8 @@ class LightningTestModelBase(LightningModule):
         if self.trainer.batch_nb % 1 == 0:
             output = OrderedDict({
                 'loss': loss_val,
-                'progress_bar': {'some_val': loss_val * loss_val}
+                'progress_bar': {'some_val': loss_val * loss_val},
+                'log': {'train_some_val': loss_val * loss_val},
             })
 
             return output

pytorch_lightning/testing/lm_test_module_mixins.py

@@ -105,7 +105,7 @@ class LightningValidationMixin(LightningValidationStepMixin):
         val_acc_mean /= len(outputs)
 
         tqdm_dict = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-        results = {'progress_bar': tqdm_dict}
+        results = {'progress_bar': tqdm_dict, 'log': tqdm_dict}
         return results
 
 

pytorch_lightning/trainer/trainer.py

@@ -183,6 +183,7 @@ class Trainer(TrainerIO):
                 version=self.slurm_job_id,
                 name='lightning_logs'
             )
+        self.logger.rank = 0
 
         # configure checkpoint callback
         self.checkpoint_callback = checkpoint_callback
@@ -659,11 +660,12 @@ class Trainer(TrainerIO):
                 """
                 warnings.warn(msg)
 
-            if on_ddp and self.get_val_dataloaders is not None:
+            if on_ddp and self.get_val_dataloaders() is not None:
                 for dataloader in self.get_val_dataloaders():
                     if not isinstance(dataloader.sampler, DistributedSampler):
                         msg = """
                         Your val_dataloader(s) don't use DistributedSampler.
+
                         You're using multiple gpus and multiple nodes without using a
                         DistributedSampler to assign a subset of your data to each process.
                         To silence this warning, pass a DistributedSampler to your DataLoader.
@@ -682,11 +684,12 @@ class Trainer(TrainerIO):
                         warnings.warn(msg)
                         break
 
-            if on_ddp and self.get_test_dataloaders is not None:
+            if on_ddp and self.get_test_dataloaders() is not None:
                 for dataloader in self.get_test_dataloaders():
                     if not isinstance(dataloader.sampler, DistributedSampler):
                         msg = """
                         Your test_dataloader(s) don't use DistributedSampler.
+
                         You're using multiple gpus and multiple nodes without using a
                         DistributedSampler to assign a subset of your data to each process.
                         To silence this warning, pass a DistributedSampler to your DataLoader.
@@ -1157,12 +1160,14 @@ class Trainer(TrainerIO):
     def __metrics_to_scalars(self, metrics):
         new_metrics = {}
         for k, v in metrics.items():
-            if type(v) is torch.Tensor:
+            if isinstance(v, torch.Tensor):
                 v = v.item()
 
             if type(v) is dict:
                 v = self.__metrics_to_scalars(v)
 
+            new_metrics[k] = v
+
         return new_metrics
 
     def __log_vals_blacklist(self):
@@ -1333,7 +1338,6 @@ class Trainer(TrainerIO):
 
                 # track progress bar metrics
                 self.__add_tqdm_metrics(progress_bar_metrics)
-
                 all_log_metrics.append(log_metrics)
 
                 # accumulate loss
@@ -1400,7 +1404,6 @@ class Trainer(TrainerIO):
 
         # collapse all metrics into one dict
         all_log_metrics = {k: v for d in all_log_metrics for k, v in d.items()}
-
         return 0, grad_norm_dic, all_log_metrics
 
     def __run_evaluation(self, test=False):
@@ -1441,7 +1444,6 @@ class Trainer(TrainerIO):
                                          dataloaders,
                                          max_batches,
                                          test)
-
             _, progress_bar_metrics, log_metrics = self.__process_output(eval_results)
 
             # add metrics to prog bar

pytorch_lightning/trainer/trainer_io.py

@@ -5,7 +5,7 @@ import pdb
 from subprocess import call
 
 import torch
-
+import torch.distributed as dist
 from pytorch_lightning.pt_overrides.override_data_parallel import (
     LightningDistributedDataParallel, LightningDataParallel)
 
@@ -35,6 +35,11 @@ class TrainerIO(object):
         # if script called from hpc resubmit, load weights
         self.restore_hpc_weights_if_needed(model)
 
+        # wait for all models to restore weights
+        if self.use_ddp or self.use_ddp2:
+            # wait for all processes to catch up
+            dist.barrier()
+
     def restore_state_if_checkpoint_exists(self, model):
         # do nothing if there's not dir or callback
         no_ckpt_callback = self.checkpoint_callback is None

setup.py

@@ -14,7 +14,7 @@ from setuptools import setup, find_packages
 # engineer specific practices
 setup(
     name='pytorch-lightning',
-    version='0.5.1',
+    version='0.5.1.3',
     description='The Keras for ML researchers using PyTorch',
     author='William Falcon',
     author_email='waf2107@columbia.edu',

tests/debug.py

@@ -14,7 +14,8 @@ from torch.utils.data import DataLoader
 from torchvision.datasets import MNIST
 import numpy as np
 import pdb
-from . import test_models
+# from test_models import assert_ok_test_acc, load_model, \
+#     clear_save_dir, get_test_tube_logger, get_hparams, init_save_dir
 
 
 class CoolModel(pl.LightningModule):
@@ -58,57 +59,55 @@ class CoolModel(pl.LightningModule):
     @pl.data_loader
     def test_dataloader(self):
         return DataLoader(MNIST('path/to/save', train=False), batch_size=32)
+#
+#
+# def main():
+#     """
+#     Make sure DDP + AMP continue training correctly
+#     :return:
+#     """
+#     """
+#     Make sure DDP2 works
+#     :return:
+#     """
+#     hparams = get_hparams()
+#     model = LightningTestModel(hparams)
+#
+#     save_dir = init_save_dir()
+#
+#     # logger file to get meta
+#     logger = get_test_tube_logger(False)
+#     logger.log_hyperparams(hparams)
+#     logger.save()
+#
+#     # logger file to get weights
+#     checkpoint = ModelCheckpoint(save_dir)
+#
+#     trainer_options = dict(
+#         show_progress_bar=True,
+#         max_nb_epochs=1,
+#         train_percent_check=0.4,
+#         val_percent_check=0.2,
+#         checkpoint_callback=checkpoint,
+#         logger=logger,
+#         gpus=[0, 1],
+#         distributed_backend='dp'
+#     )
+#
+#     # fit model
+#     trainer = Trainer(**trainer_options)
+#     result = trainer.fit(model)
+#
+#     # correct result and ok accuracy
+#     assert result == 1, 'training failed to complete'
+#     pretrained_model = load_model(logger.experiment, save_dir, module_class=LightningTestModel)
+#
+#     new_trainer = Trainer(**trainer_options)
+#     new_trainer.test(pretrained_model)
+#
+#     # test we have good test accuracy
+#     assert_ok_test_acc(new_trainer)
+#     clear_save_dir()
 
-
-def main():
-    """
-    Make sure DDP + AMP continue training correctly
-    :return:
-    """
-    """
-    Make sure DDP2 works
-    :return:
-    """
-    hparams = test_models.get_hparams()
-    model = LightningTestModel(hparams)
-
-    save_dir = test_models.init_save_dir()
-
-    # logger file to get meta
-    logger = test_models.get_test_tube_logger(False)
-    logger.log_hyperparams(hparams)
-    logger.save()
-
-    # logger file to get weights
-    checkpoint = ModelCheckpoint(save_dir)
-
-    trainer_options = dict(
-        show_progress_bar=True,
-        max_nb_epochs=1,
-        train_percent_check=0.4,
-        val_percent_check=0.2,
-        checkpoint_callback=checkpoint,
-        logger=logger,
-        gpus=[0, 1],
-        distributed_backend='dp'
-    )
-
-    # fit model
-    trainer = Trainer(**trainer_options)
-    result = trainer.fit(model)
-
-    # correct result and ok accuracy
-    assert result == 1, 'training failed to complete'
-    pretrained_model = test_models.load_model(logger.experiment, save_dir,
-                                              module_class=LightningTestModel)
-
-    new_trainer = Trainer(**trainer_options)
-    new_trainer.test(pretrained_model)
-
-    # test we have good test accuracy
-    test_models.assert_ok_test_acc(new_trainer)
-    test_models.clear_save_dir()
-
-
-if __name__ == '__main__':
-    main()
+# if __name__ == '__main__':
+#     main()

tests/test_models.py

@@ -446,7 +446,7 @@ def test_gradient_accumulation_scheduling():
         assert Trainer(accumulate_grad_batches={1: 2.5, 3: 5})
 
     # test optimizer call freq matches scheduler
-    def optimizer_step(self, epoch_nb, batch_nb, optimizer, optimizer_i):
+    def optimizer_step(self, epoch_nb, batch_nb, optimizer, optimizer_i, second_order_closure=None):
         # only test the first 12 batches in epoch
         if batch_nb < 12:
             if epoch_nb == 0: