indents

This will import the ipywidgets version of tqdm if available. This works nicely in notebooks by not filling up the log.

In the terminal it will use the same old tqdm.

We might also want to consider passing in the tqdm we want as an argument since there may be some edge cases where ipywidgets is available but the interface doesn't support it (e.g. vscode?) or isn't working. In which case people will get a warning message, but may want to configure it themselves.

README.md

@@ -1,6 +1,6 @@
 <div align="center">
 
-![Logo](docs/source/_static/images/lightning_logo_small.png)
+<img src="docs/source/_static/images/lightning_logo.png" width="50" height="50">
 
 # PyTorch Lightning
 
@@ -14,10 +14,10 @@
 [![Coverage](docs/source/_static/images/coverage.svg)](https://github.com/PytorchLightning/pytorch-lightning/tree/master/tests#running-coverage)
 [![CodeFactor](https://www.codefactor.io/repository/github/borda/pytorch-lightning/badge)](https://www.codefactor.io/repository/github/borda/pytorch-lightning)    
 
-[![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=latest)](https://pytorch-lightning.readthedocs.io/en/latest)
+[![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=0.6.0)](https://pytorch-lightning.readthedocs.io/en/0.6.0/)
 [![Slack](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://join.slack.com/t/pytorch-lightning/shared_invite/enQtODU5ODIyNTUzODQwLTFkMDg5Mzc1MDBmNjEzMDgxOTVmYTdhYjA1MDdmODUyOTg2OGQ1ZWZkYTQzODhhNzdhZDA3YmNhMDhlMDY4YzQ)
 [![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/PytorchLightning/pytorch-lightning/blob/master/LICENSE)
-[![Next Release](https://img.shields.io/badge/Next%20Release-Feb%206-<COLOR>.svg)](https://shields.io/)
+[![Next Release](https://img.shields.io/badge/Next%20Release-Mar%2021-<COLOR>.svg)](https://shields.io/)
 
 <!-- 
 removed until codecov badge isn't empy. likely a config error showing nothing on master.
@@ -32,12 +32,9 @@ pip install pytorch-lightning
 ```
 
 ## Docs   
-[jan 20, 2020]   
-
-**[Old docs (some links might be broken)](https://pytorch-lightning.readthedocs.io/en/stable)
-###### As a temporary hack, when you get the 404, replace williamfalcon.github.io with pytorchlightning.github.io.   
-
-**[New docs, CURRENTLY DEBUGING](https://pytorch-lightning.rtfd.io/en/latest)**   
+- [master](https://pytorch-lightning.readthedocs.io/en/latest)   
+- [0.6.0](https://pytorch-lightning.readthedocs.io/en/0.6.0/)
+- [0.5.3.2](https://pytorch-lightning.readthedocs.io/en/0.5.3.2/)
 
 
 ## Demo  

pytorch_lightning/callbacks/pt_callbacks.py

@@ -71,21 +71,23 @@ class EarlyStopping(Callback):
     Stop training when a monitored quantity has stopped improving.
 
     Args:
-        monitor (str): quantity to be monitored.
+        monitor (str): quantity to be monitored. Default: ``'val_loss'``.
         min_delta (float): minimum change in the monitored quantity
             to qualify as an improvement, i.e. an absolute
-            change of less than min_delta, will count as no
-            improvement.
+            change of less than `min_delta`, will count as no
+            improvement. Default: ``0``.
         patience (int): number of epochs with no improvement
-            after which training will be stopped.
-        verbose (bool): verbosity mode.
+            after which training will be stopped. Default: ``0``.
+        verbose (bool): verbosity mode. Default: ``0``.
         mode (str): one of {auto, min, max}. In `min` mode,
             training will stop when the quantity
             monitored has stopped decreasing; in `max`
             mode it will stop when the quantity
             monitored has stopped increasing; in `auto`
             mode, the direction is automatically inferred
-            from the name of the monitored quantity.
+            from the name of the monitored quantity. Default: ``'auto'``.
+        strict (bool): whether to crash the training if `monitor` is
+            not found in the metrics. Default: ``True``.
 
     Example::
 
@@ -97,18 +99,20 @@ class EarlyStopping(Callback):
     """
 
     def __init__(self, monitor='val_loss',
-                 min_delta=0.0, patience=0, verbose=0, mode='auto'):
+                 min_delta=0.0, patience=0, verbose=0, mode='auto', strict=True):
         super(EarlyStopping, self).__init__()
 
         self.monitor = monitor
         self.patience = patience
         self.verbose = verbose
+        self.strict = strict
         self.min_delta = min_delta
         self.wait = 0
         self.stopped_epoch = 0
 
         if mode not in ['auto', 'min', 'max']:
-            logging.info(f'EarlyStopping mode {mode} is unknown, fallback to auto mode.')
+            if self.verbose > 0:
+                logging.info(f'EarlyStopping mode {mode} is unknown, fallback to auto mode.')
             mode = 'auto'
 
         if mode == 'min':
@@ -128,6 +132,22 @@ class EarlyStopping(Callback):
 
         self.on_train_begin()
 
+    def check_metrics(self, logs):
+        monitor_val = logs.get(self.monitor)
+        error_msg = (f'Early stopping conditioned on metric `{self.monitor}`'
+                     f' which is not available. Available metrics are:'
+                     f' `{"`, `".join(list(logs.keys()))}`')
+
+        if monitor_val is None:
+            if self.strict:
+                raise RuntimeError(error_msg)
+            elif self.verbose > 0:
+                warnings.warn(error_msg, RuntimeWarning)
+
+            return False
+
+        return True
+
     def on_train_begin(self, logs=None):
         # Allow instances to be re-used
         self.wait = 0
@@ -135,16 +155,11 @@ class EarlyStopping(Callback):
         self.best = np.Inf if self.monitor_op == np.less else -np.Inf
 
     def on_epoch_end(self, epoch, logs=None):
-        current = logs.get(self.monitor)
         stop_training = False
-        if current is None:
-            warnings.warn(
-                f'Early stopping conditioned on metric `{self.monitor}`'
-                f' which is not available. Available metrics are: {",".join(list(logs.keys()))}',
-                RuntimeWarning)
-            stop_training = True
+        if not self.check_metrics(logs):
             return stop_training
 
+        current = logs.get(self.monitor)
         if self.monitor_op(current - self.min_delta, self.best):
             self.best = current
             self.wait = 0

pytorch_lightning/core/hooks.py

@@ -124,12 +124,13 @@ class ModelHooks(torch.nn.Module):
         """
         pass
 
-    def backward(self, use_amp, loss, optimizer):
+    def backward(self, use_amp, loss, optimizer, optimizer_idx):
         """Override backward with your own implementation if you need to
 
         :param use_amp: Whether amp was requested or not
         :param loss: Loss is already scaled by accumulated grads
         :param optimizer: Current optimizer being used
+        :param optimizer_idx: Index of the current optimizer being used
         :return:
 
         Called to perform backward step.

pytorch_lightning/logging/__init__.py

@@ -76,41 +76,39 @@ from os import environ
 from .base import LightningLoggerBase, rank_zero_only
 from .tensorboard import TensorBoardLogger
 
-all = []
+loggers = ['TensorBoardLogger']
 
 try:
     # needed to prevent ImportError and duplicated logs.
     environ["COMET_DISABLE_AUTO_LOGGING"] = "1"
 
     from .comet import CometLogger
-    all.append('CometLogger')
+    loggers.append('CometLogger')
 except ImportError:
     del environ["COMET_DISABLE_AUTO_LOGGING"]
 
 try:
     from .mlflow import MLFlowLogger
-    all.append('MLFlowLogger')
+    loggers.append('MLFlowLogger')
 except ImportError:
     pass
 
 try:
     from .neptune import NeptuneLogger
-    all.append('NeptuneLogger')
+    loggers.append('NeptuneLogger')
 except ImportError:
     pass
 
-all.append('TensorBoardLogger')
-
 try:
     from .test_tube import TestTubeLogger
-    all.append('TestTubeLogger')
+    loggers.append('TestTubeLogger')
 except ImportError:
     pass
 
 try:
     from .wandb import WandbLogger
-    all.append('WandbLogger')
+    loggers.append('WandbLogger')
 except ImportError:
     pass
 
-__all__ = all
+__all__ = loggers

pytorch_lightning/root_module/__init__.py

@@ -8,4 +8,5 @@ warnings.warn("`root_module` package has been renamed to `core` since v0.6.0"
               " and will be removed in v0.8.0", DeprecationWarning)
 
 from pytorch_lightning.core import (  # noqa: E402
-    decorators, grads, hooks, root_module, memory, model_saving)
+    decorators, grads, hooks, root_module, memory, model_saving
+)

pytorch_lightning/trainer/callback_config.py

@@ -55,10 +55,20 @@ class TrainerCallbackConfigMixin(ABC):
             self.early_stop_callback = EarlyStopping(
                 monitor='val_loss',
                 patience=3,
+                strict=True,
                 verbose=True,
                 mode='min'
             )
             self.enable_early_stop = True
+        elif early_stop_callback is None:
+            self.early_stop_callback = EarlyStopping(
+                monitor='val_loss',
+                patience=3,
+                strict=False,
+                verbose=False,
+                mode='min'
+            )
+            self.enable_early_stop = True
         elif not early_stop_callback:
             self.early_stop_callback = None
             self.enable_early_stop = False

pytorch_lightning/trainer/evaluation_loop.py

@@ -127,7 +127,7 @@ import sys
 from abc import ABC, abstractmethod
 
 import torch
-import tqdm
+from tqdm.auto import tqdm
 
 from pytorch_lightning.utilities.debugging import MisconfigurationException
 
@@ -293,9 +293,9 @@ class TrainerEvaluationLoopMixin(ABC):
         # main progress bar will already be closed when testing so initial position is free
         position = 2 * self.process_position + (not test)
         desc = 'Testing' if test else 'Validating'
-        pbar = tqdm.tqdm(desc=desc, total=max_batches, leave=test, position=position,
-                         disable=not self.show_progress_bar, dynamic_ncols=True,
-                         unit='batch', file=sys.stdout)
+        pbar = tqdm(desc=desc, total=max_batches, leave=test, position=position,
+                    disable=not self.show_progress_bar, dynamic_ncols=True,
+                    unit='batch', file=sys.stdout)
         setattr(self, f'{"test" if test else "val"}_progress_bar', pbar)
 
         # run evaluation

pytorch_lightning/trainer/trainer.py

@@ -7,7 +7,7 @@ import logging
 import torch
 import torch.distributed as dist
 import torch.multiprocessing as mp
-import tqdm
+from tqdm.auto import tqdm
 from torch.optim.optimizer import Optimizer
 
 from pytorch_lightning.trainer.auto_mix_precision import TrainerAMPMixin
@@ -52,7 +52,7 @@ class Trainer(TrainerIOMixin,
             self,
             logger=True,
             checkpoint_callback=True,
-            early_stop_callback=True,
+            early_stop_callback=None,
             default_save_path=None,
             gradient_clip_val=0,
             gradient_clip=None,  # backward compatible, todo: remove in v0.8.0
@@ -121,7 +121,13 @@ class Trainer(TrainerIOMixin,
                     )
 
                     trainer = Trainer(checkpoint_callback=checkpoint_callback)
-            early_stop_callback (:class:`.EarlyStopping`): Callback for early stopping
+            early_stop_callback (:class:`.EarlyStopping`): Callback for early stopping. If
+                set to ``True``, then the default callback monitoring ``'val_loss'`` is created.
+                Will raise an error if ``'val_loss'`` is not found.
+                If set to ``False``, then early stopping will be disabled.
+                If set to ``None``, then the default callback monitoring ``'val_loss'`` is created.
+                If ``'val_loss'`` is not found will work as if early stopping is disabled.
+                Default: ``None``.
                 Example::
                     from pytorch_lightning.callbacks import EarlyStopping
 
@@ -129,7 +135,8 @@ class Trainer(TrainerIOMixin,
                     early_stop_callback = EarlyStopping(
                         monitor='val_loss',
                         patience=3,
-                        verbose=True,
+                        strict=False,
+                        verbose=False,
                         mode='min'
                     )
 
@@ -801,24 +808,29 @@ class Trainer(TrainerIOMixin,
         ref_model.on_train_start()
         if not self.disable_validation and self.num_sanity_val_steps > 0:
             # init progress bars for validation sanity check
-            pbar = tqdm.tqdm(desc='Validation sanity check',
+            pbar = tqdm(desc='Validation sanity check',
                              total=self.num_sanity_val_steps * len(self.get_val_dataloaders()),
                              leave=False, position=2 * self.process_position,
                              disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch')
             self.main_progress_bar = pbar
             # dummy validation progress bar
-            self.val_progress_bar = tqdm.tqdm(disable=True)
+            self.val_progress_bar = tqdm(disable=True)
 
-            self.evaluate(model, self.get_val_dataloaders(), self.num_sanity_val_steps, self.testing)
+            eval_results = self.evaluate(model, self.get_val_dataloaders(),
+                                         self.num_sanity_val_steps, False)
+            _, _, _, callback_metrics, _ = self.process_output(eval_results)
 
             # close progress bars
             self.main_progress_bar.close()
             self.val_progress_bar.close()
 
+            if self.enable_early_stop:
+                self.early_stop_callback.check_metrics(callback_metrics)
+
         # init progress bar
-        pbar = tqdm.tqdm(leave=True, position=2 * self.process_position,
-                         disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch',
-                         file=sys.stdout)
+        pbar = tqdm(leave=True, position=2 * self.process_position,
+                    disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch',
+                    file=sys.stdout)
         self.main_progress_bar = pbar
 
         # clear cache before training

pytorch_lightning/trainer/training_loop.py

@@ -296,6 +296,7 @@ class TrainerTrainLoopMixin(ABC):
             self.current_epoch = epoch
 
             total_val_batches = 0
+            is_val_epoch = False
             if not self.disable_validation:
                 # val can be checked multiple times in epoch
                 is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0
@@ -346,7 +347,8 @@ class TrainerTrainLoopMixin(ABC):
 
             # early stopping
             met_min_epochs = epoch >= self.min_epochs - 1
-            if self.enable_early_stop and (met_min_epochs or self.fast_dev_run):
+            if (self.enable_early_stop and not self.disable_validation and is_val_epoch and
+                    (met_min_epochs or self.fast_dev_run)):
                 should_stop = self.early_stop_callback.on_epoch_end(epoch=epoch,
                                                                     logs=self.callback_metrics)
                 # stop training
@@ -401,6 +403,9 @@ class TrainerTrainLoopMixin(ABC):
             if self.fast_dev_run or should_check_val:
                 self.run_evaluation(test=self.testing)
 
+                if self.enable_early_stop:
+                    self.early_stop_callback.check_metrics(self.callback_metrics)
+
             # when logs should be saved
             should_save_log = (batch_idx + 1) % self.log_save_interval == 0 or early_stop_epoch
             if should_save_log or self.fast_dev_run:
@@ -486,13 +491,14 @@ class TrainerTrainLoopMixin(ABC):
 
                     # backward pass
                     model_ref = self.get_model()
-                    model_ref.backward(self.use_amp, closure_loss, optimizer)
+                    model_ref.backward(self.use_amp, closure_loss, optimizer, opt_idx)
 
                     # track metrics for callbacks
                     all_callback_metrics.append(callback_metrics)
 
                     # track progress bar metrics
                     self.add_tqdm_metrics(progress_bar_metrics)
+                    self.add_tqdm_metrics(progress_bar_metrics)
                     all_log_metrics.append(log_metrics)
 
                     # insert after step hook

tests/test_cpu_models.py

@@ -140,7 +140,8 @@ def test_running_test_without_val(tmpdir):
         val_percent_check=0.2,
         test_percent_check=0.2,
         checkpoint_callback=checkpoint,
-        logger=logger
+        logger=logger,
+        early_stop_callback=False
     )
 
     # fit model
@@ -318,6 +319,7 @@ def test_tbptt_cpu_model(tmpdir):
         truncated_bptt_steps=truncated_bptt_steps,
         val_percent_check=0,
         weights_summary=None,
+        early_stop_callback=False
     )
 
     hparams = tutils.get_hparams()

tests/test_trainer.py

@@ -392,7 +392,7 @@ def test_multiple_test_dataloader(tmpdir):
         default_save_path=tmpdir,
         max_epochs=1,
         val_percent_check=0.1,
-        train_percent_check=0.2,
+        train_percent_check=0.2
     )
 
     # fit model