indents

This will import the ipywidgets version of tqdm if available. This works nicely in notebooks by not filling up the log.

In the terminal it will use the same old tqdm.

We might also want to consider passing in the tqdm we want as an argument since there may be some edge cases where ipywidgets is available but the interface doesn't support it (e.g. vscode?) or isn't working. In which case people will get a warning message, but may want to configure it themselves.

.circleci/config.yml

@@ -16,14 +16,21 @@ references:
         sudo pip install pytest pytest-cov pytest-flake8
         pip install -r ./tests/requirements.txt --user
 
-  tests_format: &tests_format
+  tests: &tests
    run:
-     name: Tests and formating
+     name: Testing
      command: |
        python --version ; pip --version ; pip list
-       py.test pytorch_lightning tests pl_examples -v --doctest-modules --junitxml=test-reports/pytest_junit.xml --flake8
+       py.test pytorch_lightning tests pl_examples -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
      no_output_timeout: 15m
 
+  format: &format
+   run:
+     name: Formatting
+     command: |
+       python --version ; pip --version ; pip list
+       flake8
+
   make_docs: &make_docs
    run:
      name: Make Documentation
@@ -43,6 +50,16 @@ jobs:
       - checkout
       - *make_docs
 
+  Formatting:
+    docker:
+      - image: circleci/python:3.7
+    environment:
+        - TORCH_VERSION: "torch"
+    steps:
+      - checkout
+      - *install_deps
+      - *format
+
   PyTorch:
     docker:
       - image: circleci/python:3.7
@@ -52,7 +69,7 @@ jobs:
       - checkout
 
       - *install_deps
-      - *tests_format
+      - *tests
 
       - store_test_results:
           path: test-reports
@@ -80,11 +97,20 @@ jobs:
       - TORCH_VERSION: "torch>=1.3, <1.4"
     steps: *steps
 
+  PyTorch-v1.4:
+    docker:
+      - image: circleci/python:3.6
+    environment:
+      - TORCH_VERSION: "torch>=1.4, <1.5"
+    steps: *steps
+
 workflows:
   version: 2
   build:
     jobs:
+      - Formatting
       - Build-Docs
       - PyTorch-v1.1
       - PyTorch-v1.2
       - PyTorch-v1.3
+      - PyTorch-v1.4

.github/BECOMING_A_CORE_CONTRIBUTOR.md

@@ -6,7 +6,7 @@ We're currently recruiting for a team of 5 core maintainers.
 As a core maintainer you will have a strong say in the direction of the project. Big changes will require a majority of maintainers to agree.
 
 ### Code of conduct  
-First and foremost, you'll be evaluated against [these core values](https://github.com/williamFalcon/pytorch-lightning/blob/master/.github/CONTRIBUTING.md). Any code we commit or feature we add needs to align with those core values.
+First and foremost, you'll be evaluated against [these core values](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md). Any code we commit or feature we add needs to align with those core values.
 
 ### The bar for joining the team   
 Lightning is being used to solve really hard problems at the top AI labs in the world. As such, the bar for adding team members is extremely high. Candidates must have solid engineering skills, have a good eye for user experience, and must be a power user of Lightning and PyTorch.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,8 +8,8 @@ assignees: ''
 ---
 
 ### Common bugs:
-1. Tensorboard not showing in Jupyter-notebook see [issue 79](https://github.com/williamFalcon/pytorch-lightning/issues/79).    
-2. PyTorch 1.1.0 vs 1.2.0 support [see FAQ](https://github.com/williamFalcon/pytorch-lightning#faq)    
+1. Tensorboard not showing in Jupyter-notebook see [issue 79](https://github.com/PyTorchLightning/pytorch-lightning/issues/79).    
+2. PyTorch 1.1.0 vs 1.2.0 support [see FAQ](https://github.com/PyTorchLightning/pytorch-lightning#faq)    
 
 ## 🐛 Bug
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,7 @@
 # Before submitting
 
 - [ ] Was this discussed/approved via a Github issue? (no need for typos, doc improvements)
-- [ ] Did you read the [contributor guideline](https://github.com/williamFalcon/pytorch-lightning/blob/master/.github/CONTRIBUTING.md)?
+- [ ] Did you read the [contributor guideline](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md)?
 - [ ] Did you make sure to update the docs?   
 - [ ] Did you write any new necessary tests?  
 

.gitignore

@@ -1,27 +1,28 @@
 # project
 .DS_Store
-.data/
 run_configs/
-test_tube_logs/
-test_tube_data/
-datasets/
 model_weights/
 app/models/
 pip-wheel-metadata/
-test_tube_exp/
-tests/tests_tt_dir/
-tests/save_dir
-default/
 lightning_logs/
+
+
+# Test-tube
+test_tube_logs/
+test_tube_data/
+test_tube_exp/
+
+# Documentations
+docs/source/pl_examples*.rst
+docs/source/pytorch_lightning*.rst
 tests/tests/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
-example.py
 timit_data/
-LJSpeech-1.1/
+
 
 # C extensions
 *.so
@@ -30,7 +31,6 @@ LJSpeech-1.1/
 
 # Distribution / packaging
 .Python
-env/
 ide_layouts/
 build/
 develop-eggs/
@@ -42,7 +42,6 @@ lib/
 lib64/
 parts/
 sdist/
-var/
 wheels/
 *.egg-info/
 .installed.cfg
@@ -68,6 +67,9 @@ nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
+tests/tests_tt_dir/
+tests/save_dir
+tests/tests/
 
 # Translations
 *.mo
@@ -85,7 +87,7 @@ instance/
 .scrapy
 
 # Sphinx documentation
-docs/_build/
+docs/build/
 
 # PyBuilder
 target/
@@ -107,6 +109,7 @@ celerybeat-schedule
 
 # virtualenv
 .venv
+env/
 venv/
 ENV/
 
@@ -124,4 +127,6 @@ ENV/
 .mypy_cache/
 
 # data
+.data/
+datasets/
 mnist/

.readthedocs.yml

@@ -20,5 +20,5 @@ formats: all
 python:
   version: 3.7
   install:
-    #- requirements: requirements.txt
     - requirements: docs/requirements.txt
+    #- requirements: requirements.txt

.run_local_tests.sh

@@ -3,6 +3,7 @@ rm -rf _ckpt_*
 rm -rf tests/save_dir*
 rm -rf tests/mlruns_*
 rm -rf tests/cometruns*
+rm -rf tests/wandb*
 rm -rf tests/tests/*
 rm -rf lightning_logs
 coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules

.travis.yml

@@ -33,13 +33,14 @@ matrix:
       python: 3.7
       env: TOXENV=py37
     - os: osx
-      osx_image: xcode9.4
+      # https://blog.travis-ci.com/2019-08-07-extensive-python-testing-on-travis-ci
+      osx_image: xcode10.3
       language: generic
-      env: TOXENV=py36
-      addons:
-        homebrew:
-          # update: true
-          packages: python3.6
+      env: TOXENV=py37
+      #addons:
+      #  homebrew:
+      #    # update: true
+      #    packages: python3.7
       before_install:
         - pip3 install virtualenv
         - virtualenv -p python3 ~/venv

MANIFEST.in

@@ -1,7 +1,6 @@
 # Manifest syntax https://docs.python.org/2/distutils/sourcedist.html
 graft wheelhouse
 
-recursive-include birl *.py
 recursive-exclude __pycache__  *.py[cod] *.orig
 
 # Include the README

README.md

@@ -1,6 +1,6 @@
 <div align="center">
 
-![Logo](docs/source/_static/images/lightning_logo_small.png)
+<img src="docs/source/_static/images/lightning_logo.png" width="50" height="50">
 
 # PyTorch Lightning
 
@@ -9,15 +9,15 @@
 
 [![PyPI Status](https://badge.fury.io/py/pytorch-lightning.svg)](https://badge.fury.io/py/pytorch-lightning)
 [![PyPI Status](https://pepy.tech/badge/pytorch-lightning)](https://pepy.tech/project/pytorch-lightning)
-[![Build Status](https://travis-ci.org/williamFalcon/pytorch-lightning.svg?branch=master)](https://travis-ci.org/williamFalcon/pytorch-lightning)
-[![Build status](https://ci.appveyor.com/api/projects/status/NEW-PROJECT-ID?svg=true)](https://ci.appveyor.com/project/williamFalcon/pytorch-lightning)
-[![Coverage](docs/source/_static/images/coverage.svg)](https://github.com/williamFalcon/pytorch-lightning/tree/master/tests#running-coverage)
+[![Build Status](https://travis-ci.org/PytorchLightning/pytorch-lightning.svg?branch=master)](https://travis-ci.org/PytorchLightning/pytorch-lightning)
+[![Build status](https://ci.appveyor.com/api/projects/status/NEW-PROJECT-ID?svg=true)](https://ci.appveyor.com/project/PytorchLightning/pytorch-lightning)
+[![Coverage](docs/source/_static/images/coverage.svg)](https://github.com/PytorchLightning/pytorch-lightning/tree/master/tests#running-coverage)
 [![CodeFactor](https://www.codefactor.io/repository/github/borda/pytorch-lightning/badge)](https://www.codefactor.io/repository/github/borda/pytorch-lightning)    
 
-[![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=latest)](https://pytorch-lightning.readthedocs.io/en/latest)
+[![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=0.6.0)](https://pytorch-lightning.readthedocs.io/en/0.6.0/)
 [![Slack](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://join.slack.com/t/pytorch-lightning/shared_invite/enQtODU5ODIyNTUzODQwLTFkMDg5Mzc1MDBmNjEzMDgxOTVmYTdhYjA1MDdmODUyOTg2OGQ1ZWZkYTQzODhhNzdhZDA3YmNhMDhlMDY4YzQ)
-[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/williamFalcon/pytorch-lightning/blob/master/LICENSE)
-[![Next Release](https://img.shields.io/badge/Next%20Release-Dec%206-<COLOR>.svg)](https://shields.io/)
+[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/PytorchLightning/pytorch-lightning/blob/master/LICENSE)
+[![Next Release](https://img.shields.io/badge/Next%20Release-Mar%2021-<COLOR>.svg)](https://shields.io/)
 
 <!-- 
 removed until codecov badge isn't empy. likely a config error showing nothing on master.
@@ -32,19 +32,34 @@ pip install pytorch-lightning
 ```
 
 ## Docs   
-**[View the docs here](https://williamfalcon.github.io/pytorch-lightning/)**
+- [master](https://pytorch-lightning.readthedocs.io/en/latest)   
+- [0.6.0](https://pytorch-lightning.readthedocs.io/en/0.6.0/)
+- [0.5.3.2](https://pytorch-lightning.readthedocs.io/en/0.5.3.2/)
+
 
 ## Demo  
 [Copy and run this COLAB!](https://colab.research.google.com/drive/1F_RNcHzTfFuQf-LeKvSlud6x7jXYkG31#scrollTo=HOk9c4_35FKg)
 
 ## What is it?  
-Lightning is a very lightweight wrapper on PyTorch. This means you don't have to learn a new library. To use Lightning, simply refactor your research code into the [LightningModule](https://github.com/williamFalcon/pytorch-lightning#how-do-i-do-use-it) format and Lightning will automate the rest. Lightning guarantees tested, correct, modern best practices for the automated parts.
+Lightning is a very lightweight wrapper on PyTorch that decouples the science code from the engineering code. It's more of a style-guide than a framework. By refactoring your code, we can automate most of the non-research code.  
+
+To use Lightning, simply refactor your research code into the [LightningModule](https://github.com/PytorchLightning/pytorch-lightning#how-do-i-do-use-it) format (the science) and Lightning will automate the rest (the engineering). Lightning guarantees tested, correct, modern best practices for the automated parts.
+
+- If you are a researcher, Lightning is infinitely flexible, you can modify everything down to the way .backward is called or distributed is set up. 
+- If you are a scientist or production team, lightning is very simple to use with best practice defaults.
+
+## What does lightning control for me?   
+
+Everything in Blue!   
+This is how lightning separates the science (red) from the engineering (blue).
+
+![Overview](docs/source/_static/images/pl.gif)
 
 ## How much effort is it to convert?
-You're probably tired of switching frameworks at this point. But it is a very quick process to refactor into the Lightning format. [Check out this tutorial](https://towardsdatascience.com/how-to-refactor-your-pytorch-code-to-get-these-42-benefits-of-pytorch-lighting-6fdd0dc97538)
+You're probably tired of switching frameworks at this point. But it is a very quick process to refactor into the Lightning format (ie: hours). [Check out this tutorial](https://towardsdatascience.com/how-to-refactor-your-pytorch-code-to-get-these-42-benefits-of-pytorch-lighting-6fdd0dc97538)
 
 ## Starting a new project?   
-[Use our seed-project aimed at reproducibility!](https://github.com/williamFalcon/pytorch-lightning-conference-seed)     
+[Use our seed-project aimed at reproducibility!](https://github.com/PytorchLightning/pytorch-lightning-conference-seed)     
 
 ## Why do I want to use lightning?
 Every research project starts the same, a model, a training loop, validation loop, etc. As your research advances, you're likely to need distributed training, 16-bit precision, checkpointing, gradient accumulation, etc.   
@@ -54,27 +69,27 @@ Lightning sets up all the boilerplate state-of-the-art training for you so you c
 ---
  
 ## README Table of Contents        
-- [How do I use it](https://github.com/williamFalcon/pytorch-lightning#how-do-i-do-use-it)     
-- [What lightning automates](https://github.com/williamFalcon/pytorch-lightning#what-does-lightning-control-for-me)    
-- [Tensorboard integration](https://github.com/williamFalcon/pytorch-lightning#tensorboard)    
-- [Lightning features](https://github.com/williamFalcon/pytorch-lightning#lightning-automates-all-of-the-following-each-is-also-configurable)    
-- [Examples](https://github.com/williamFalcon/pytorch-lightning#examples)    
-- [Tutorials](https://github.com/williamFalcon/pytorch-lightning#tutorials)
-- [Contributing](https://github.com/williamFalcon/pytorch-lightning/blob/master/.github/CONTRIBUTING.md)
-- [Bleeding edge install](https://github.com/williamFalcon/pytorch-lightning#bleeding-edge)   
-- [Lightning Design Principles](https://github.com/williamFalcon/pytorch-lightning#lightning-design-principles)   
-- [Asking for help](https://github.com/williamFalcon/pytorch-lightning#asking-for-help)
-- [FAQ](https://github.com/williamFalcon/pytorch-lightning#faq)    
+- [How do I use it](https://github.com/PytorchLightning/pytorch-lightning#how-do-i-do-use-it)     
+- [What lightning automates](https://github.com/PytorchLightning/pytorch-lightning#what-does-lightning-control-for-me)    
+- [Tensorboard integration](https://github.com/PytorchLightning/pytorch-lightning#tensorboard)    
+- [Lightning features](https://github.com/PytorchLightning/pytorch-lightning#lightning-automates-all-of-the-following-each-is-also-configurable)    
+- [Examples](https://github.com/PytorchLightning/pytorch-lightning#examples)    
+- [Tutorials](https://github.com/PytorchLightning/pytorch-lightning#tutorials)
+- [Contributing](https://github.com/PytorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md)
+- [Bleeding edge install](https://github.com/PytorchLightning/pytorch-lightning#bleeding-edge)   
+- [Lightning Design Principles](https://github.com/PytorchLightning/pytorch-lightning#lightning-design-principles)   
+- [Asking for help](https://github.com/PytorchLightning/pytorch-lightning#asking-for-help)
+- [FAQ](https://github.com/PytorchLightning/pytorch-lightning#faq)    
 
 ---
 
 ## How do I do use it?   
-Think about Lightning as refactoring your research code instead of using a new framework. The research code goes into a [LightningModule](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/) which you fit using a Trainer.
+Think about Lightning as refactoring your research code instead of using a new framework. The research code goes into a [LightningModule](https://pytorch-lightning.rtfd.io/en/latest/LightningModule/RequiredTrainerInterface/) which you fit using a Trainer.
 
 The LightningModule defines a *system* such as seq-2-seq, GAN, etc... It can ALSO define a simple classifier such as the example below.     
 
 To use lightning do 2 things:  
-1. [Define a LightningModule](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/)
+1. [Define a LightningModule](https://pytorch-lightning.rtfd.io/en/latest/LightningModule/RequiredTrainerInterface/)
 **WARNING:** This syntax is for version 0.5.0+ where abbreviations were removed.
     ```python
     import os
@@ -116,6 +131,18 @@ To use lightning do 2 things:
             avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
             tensorboard_logs = {'val_loss': avg_loss}
             return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}
+            
+        def test_step(self, batch, batch_idx):
+            # OPTIONAL
+            x, y = batch
+            y_hat = self.forward(x)
+            return {'test_loss': F.cross_entropy(y_hat, y)}
+    
+        def test_end(self, outputs):
+            # OPTIONAL
+            avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()
+            tensorboard_logs = {'test_loss': avg_loss}
+            return {'avg_test_loss': avg_loss, 'log': tensorboard_logs}
     
         def configure_optimizers(self):
             # REQUIRED
@@ -138,7 +165,7 @@ To use lightning do 2 things:
             # OPTIONAL
             return DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=transforms.ToTensor()), batch_size=32)
     ```
-2. Fit with a [trainer](https://williamfalcon.github.io/pytorch-lightning/Trainer/)    
+2. Fit with a [trainer](https://pytorch-lightning.rtfd.io/en/latest/Trainer/)    
     ```python
     from pytorch_lightning import Trainer
     
@@ -179,29 +206,6 @@ When you're all done you can even run the test set separately.
 trainer.test()
 ```
 
-## What does lightning control for me?   
-
-Everything in gray!    
-You define the blue parts using the LightningModule interface:  
-
-![Overview](docs/source/_static/images/overview_flat.jpg)
-
-```python
-# what to do in the training loop
-def training_step(self, batch, batch_idx):
-
-# what to do in the validation loop
-def validation_step(self, batch, batch_idx):
-
-# how to aggregate validation_step outputs
-def validation_end(self, outputs):
-
-# and your dataloaders
-def train_dataloader():
-def val_dataloader():
-def test_dataloader():
-```
-
 **Could be as complex as seq-2-seq + attention**    
 
 ```python
@@ -273,81 +277,26 @@ Lightning also adds a text column with all the hyperparameters for this experime
 
 ![tensorboard-support](docs/source/_static/images/tf_tags.png)
 
-## Lightning automates all of the following ([each is also configurable](https://williamfalcon.github.io/pytorch-lightning/Trainer/)):
-
-#### Checkpointing    
-
-- [Checkpoint callback](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
-- [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
-- [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics) 
-- [Restoring training session](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#restoring-training-session)
-
-#### Computing cluster (SLURM)    
-
-- [Running grid search on a cluster](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#running-grid-search-on-a-cluster) 
-- [Walltime auto-resubmit](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#walltime-auto-resubmit)   
-
-#### Debugging  
-
-- [Fast dev run](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#fast-dev-run)
-- [Inspect gradient norms](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#inspect-gradient-norms)
-- [Log GPU usage](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage)
-- [Make model overfit on subset of data](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data)
-- [Print the parameter count by layer](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer)
-- [Print which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
-- [Print input and output size of every module in system](https://williamfalcon.github.io/pytorch-lightning/LightningModule/properties/#example_input_array)
+## Lightning automates all of the following ([each is also configurable](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.html)):
 
 
-#### Distributed training    
-
-- [Implement Your Own Distributed (DDP) training](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#init_ddp_connection)
-- [16-bit mixed precision](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision)
-- [Multi-GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU)
-- [Multi-node](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node)
-- [Single GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#single-gpu)
-- [Self-balancing architecture](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#self-balancing-architecture)
-
-
-#### Experiment Logging   
-
-- [Display metrics in progress bar](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#display-metrics-in-progress-bar)
-- [Log metric row every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#log-metric-row-every-k-batches)
-- [Process position](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#process-position)
-- [Tensorboard support](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#tensorboard-support)
-- [Save a snapshot of all hyperparameters](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#save-a-snapshot-of-all-hyperparameters) 
-- [Snapshot code for a training run](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#snapshot-code-for-a-training-run) 
-- [Write logs file to csv every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#write-logs-file-to-csv-every-k-batches)
-
-#### Training loop    
-
-- [Accumulate gradients](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients)
-- [Force training for min or max epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs)
-- [Early stopping callback](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#early-stopping)
-- [Force disable early stop](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop)
-- [Gradient Clipping](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping)
-- [Hooks](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/)
-- [Learning rate scheduling](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers)    
-- [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers)
-- [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check)
-- [Step optimizers at arbitrary intervals](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step)
-
-#### Validation loop    
-
-- [Check validation every n epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#check-validation-every-n-epochs)
-- [Hooks](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/)
-- [Set how much of the validation set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-validation-set-to-check)
-- [Set how much of the test set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-test-set-to-check)
-- [Set validation check frequency within 1 training epoch](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-validation-check-frequency-within-1-training-epoch)
-- [Set the number of validation sanity steps](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-the-number-of-validation-sanity-steps)
-
-#### Testing loop  
-- [Run test set](https://williamfalcon.github.io/pytorch-lightning/Trainer/Testing%20loop/)  
+- [Running grid search on a cluster](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.distrib_data_parallel.html)  
+- [Fast dev run](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.utilities.debugging.html)
+- [Logging](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.logging.html)
+- [Implement Your Own Distributed (DDP) training](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.core.lightning.html#pytorch_lightning.core.lightning.LightningModule.configure_ddp)
+- [Multi-GPU & Multi-node](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.distrib_parts.html)
+- [Training loop](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.training_loop.html)
+- [Hooks](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.core.hooks.html)
+- [Configure optimizers](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.core.lightning.html#pytorch_lightning.core.lightning.LightningModule.configure_optimizers)
+- [Validations](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.evaluation_loop.html)
+- [Model saving & Restoring training session](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.training_io.html)
+  
 
 ## Examples   
-- [GAN](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/domain_templates/gan.py)    
-- [MNIST](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/basic_examples)      
-- [Other projects using Lightning](https://github.com/williamFalcon/pytorch-lightning/network/dependents?package_id=UGFja2FnZS0zNzE3NDU4OTM%3D)    
-- [Multi-node](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/multi_node_examples)   
+- [GAN](https://github.com/PytorchLightning/pytorch-lightning/tree/master/pl_examples/domain_templates/gan.py)    
+- [MNIST](https://github.com/PytorchLightning/pytorch-lightning/tree/master/pl_examples/basic_examples)      
+- [Other projects using Lightning](https://github.com/PytorchLightning/pytorch-lightning/network/dependents?package_id=UGFja2FnZS0zNzE3NDU4OTM%3D)    
+- [Multi-node](https://github.com/PytorchLightning/pytorch-lightning/tree/master/pl_examples/multi_node_examples)   
 
 ## Tutorials   
 - [Basic Lightning use](https://towardsdatascience.com/supercharge-your-ai-research-with-pytorch-lightning-337948a99eec)    
@@ -360,8 +309,8 @@ Lightning also adds a text column with all the hyperparameters for this experime
 Welcome to the Lightning community!   
 
 If you have any questions, feel free to:   
-1. [read the docs](https://williamfalcon.github.io/pytorch-lightning/).     
-2. [Search through the issues](https://github.com/williamFalcon/pytorch-lightning/issues?utf8=%E2%9C%93&q=my++question).      
+1. [read the docs](https://pytorch-lightning.rtfd.io/en/latest/).     
+2. [Search through the issues](https://github.com/PytorchLightning/pytorch-lightning/issues?utf8=%E2%9C%93&q=my++question).      
 3. [Ask on stackoverflow](https://stackoverflow.com/questions/ask?guided=false) with the tag pytorch-lightning.   
 
 If no one replies to you quickly enough, feel free to post the stackoverflow link to our Gitter chat!   
@@ -371,7 +320,7 @@ To chat with the rest of us visit our [gitter channel](https://gitter.im/PyTorch
 ---   
 ## FAQ    
 **How do I use Lightning for rapid research?**   
-[Here's a walk-through](https://williamfalcon.github.io/pytorch-lightning/)  
+[Here's a walk-through](https://pytorch-lightning.rtfd.io/en/latest/)  
 
 **Why was Lightning created?**     
 Lightning has 3 goals in mind:
@@ -412,29 +361,29 @@ Nope. Please use anaconda or miniconda.
 If you can't wait for the next release, install the most up to date code with:
 * using GIT (locally clone whole repo with full history)
     ```bash
-    pip install git+https://github.com/williamFalcon/pytorch-lightning.git@master --upgrade
+    pip install git+https://github.com/PytorchLightning/pytorch-lightning.git@master --upgrade
     ```
 * using instant zip (last state of the repo without git history)
     ```bash
-    pip install https://github.com/williamFalcon/pytorch-lightning/archive/master.zip --upgrade
+    pip install https://github.com/PytorchLightning/pytorch-lightning/archive/master.zip --upgrade
     ```
 
 ### Any release installation
 
-You can also install any past release from this repository:
+You can also install any past release `0.X.Y` from this repository:
 ```bash
-pip install https://github.com/williamFalcon/pytorch-lightning/archive/0.4.4.zip --upgrade
+pip install https://github.com/PytorchLightning/pytorch-lightning/archive/0.X.Y.zip --upgrade
 ```
 
 ## Bibtex
 If you want to cite the framework feel free to use this (but only if you loved it 😊):
 ```
 @misc{Falcon2019,
-  author = {Falcon, W.A.},
+  author = {Falcon, W.A. et al.},
   title = {PyTorch Lightning},
   year = {2019},
   publisher = {GitHub},
   journal = {GitHub repository},
-  howpublished = {\url{https://github.com/williamFalcon/pytorch-lightning}}
+  howpublished = {\url{https://github.com/PytorchLightning/pytorch-lightning}}
 }
 ```

docs/requirements.txt

@@ -4,6 +4,7 @@ m2r  # fails with multi-line text
 nbsphinx
 pandoc
 docutils
-git+https://github.com/Borda/lightning_sphinx_theme.git
+git+https://github.com/PytorchLightning/lightning_sphinx_theme.git
 sphinxcontrib-fulltoc
-sphinxcontrib-mockautodoc
+sphinxcontrib-mockautodoc
+pip_shims

docs/source/BECOMING_A_CORE_CONTRIBUTOR.md

@@ -0,0 +1,59 @@
+# How to become a core contributor
+
+Thanks for your interest in joining the Lightning team! We’re a rapidly growing project which is poised to become the go-to framework for DL researchers! 
+We're currently recruiting for a team of 5 core maintainers. 
+
+As a core maintainer you will have a strong say in the direction of the project. Big changes will require a majority of maintainers to agree.
+
+### Code of conduct  
+First and foremost, you'll be evaluated against [these core values](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md). Any code we commit or feature we add needs to align with those core values.
+
+### The bar for joining the team   
+Lightning is being used to solve really hard problems at the top AI labs in the world. As such, the bar for adding team members is extremely high. Candidates must have solid engineering skills, have a good eye for user experience, and must be a power user of Lightning and PyTorch.
+
+With that said, the Lightning team will be diverse and a reflection of an inclusive AI community. You don't have to be an engineer to conntribute! Scientists with great usability intuition and PyTorch ninja skills are welcomed!
+
+### Responsibilities:
+The responsibilities mainly revolve around 3 things.
+
+#### Github issues
+- Here we want to help users have an amazing experience. These range from questions from new people getting into DL to questions from researchers about doing something esoteric with Lightning
+Often, these issues require some sort of bug fix, document clarification or new functionality to be scoped out.
+
+- To become a core member you must resolve at least 10 Github issues which align with the API design goals for Lightning. By the end of these 10 issues I should feel comfortable in the way you answer user questions
+Pleasant/helpful tone.
+
+- Can abstract from that issue or bug into functionality that might solve other related issues or makes the platform more flexible.
+
+- Don’t make users feel like they don’t know what they’re doing. We’re here to help and to make everyone’s experience delightful.
+
+#### Pull requests
+
+- Here we need to ensure the code that enters Lightning is high quality. For each PR we need to:
+- Make sure code coverage does not decrease
+- Documents are updated
+- Code is elegant and simple
+- Code is NOT overly engineered or hard to read
+- Ask yourself, could a non-engineer understand what’s happening here?
+- Make sure new tests are written
+- Is this NECESSARY for Lightning? There are some PRs which are just purely about adding engineering complexity which have no place in Lightning. 
+Guidance
+- Some other PRs are for people who are wanting to get involved and add something unnecessary. We do want their help though! So don’t approve the PR, but direct them to a Github issue that they might be interested in helping with instead!
+- To be considered for core contributor, please review 10 PRs and help the authors land it on master. Once you've finished the review, ping me 
+for a sanity check. At the end of 10 PRs if your PR reviews are inline with expectations described above, then you can merge PRs on your own going forward,
+otherwise we'll do a few more until we're both comfortable :)
+
+#### Project directions
+There are some big decisions which the project must make. For these I expect core contributors to have something meaningful to add if it’s their area of expertise.
+
+#### Diversity
+Lightning should reflect the broader community it serves. As such we should have scientists/researchers from 
+different fields contributing!   
+
+The first 5 core contributors will fit this profile. Thus if you overlap strongly with experiences and expertise as someone else on the team, you might have to wait until the next set of contributors are added.
+
+#### Summary: Requirements to apply
+- Solve 10 Github issues. The goal is to be inline with expectations for solving issues by the last one so you can do them on your own. If not, I might ask you to solve a few more specific ones.
+- Do 10 PR reviews. The goal is to be inline with expectations for solving issues by the last one so you can do them on your own. If not, I might ask you to solve a few more specific ones.
+
+If you want to be considered, ping me on gitter and start [tracking your progress here](https://docs.google.com/spreadsheets/d/15D58gp8DvI0Z6qbbYVRuaWioiwzafcP58-UlbuO_CMU/edit?usp=sharing).

docs/source/CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at waf2107@columbia.edu. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

docs/source/CONTRIBUTING.md

@@ -0,0 +1,53 @@
+# Contributing    
+Welcome to the PyTorch Lightning community! We're building the most advanced research platform on the planet to implement the latest, best practices that the amazing PyTorch team rolls out!   
+
+## Main Core Value: One less thing to remember    
+Simplify the API as much as possible from the user perspective. Any additions or improvements should minimize things the user needs to remember.   
+
+For example: One benefit of the validation_step is that the user doesn't have to remember to set the model to .eval(). This avoids all sorts of subtle errors the user could make.  
+
+## Lightning Design Principles   
+We encourage all sorts of contributions you're interested in adding! When coding for lightning, please follow these principles.    
+#### No PyTorch Interference   
+We don't want to add any abstractions on top of pure PyTorch. This gives researchers all the control they need without having to learn yet another framework.    
+
+#### Simple Internal Code    
+It's useful for users to look at the code and understand very quickly what's happening. Many users won't be engineers. Thus we need to value clear, simple code over condensed ninja moves. While that's super cool, this isn't the project for that :)      
+
+#### Force User Decisions To Best Practices    
+There are 1,000 ways to do something. However, something eventually becomes standard practice that everyone does. Thus we pick one way of doing it and force everyone to do it this way. A good example is accumulated gradients. There are many ways to implement, we just pick one and force users to use that one. A bad forced decision would be to make users use a specific library to do something.    
+
+When something becomes a best practice, we add it to the framework. This likely looks like code in utils or in the model file that everyone keeps adding over and over again across projects. When this happens, bring that code inside the trainer and add a flag for it.
+
+#### Simple External API    
+What makes sense to you may not make sense to others. Create an issue with an API change suggestion and validate that it makes sense for others. Treat code changes how you treat a startup: validate that it's a needed feature, then add if it makes sense for many people. 
+
+#### Backward-compatible API   
+We all hate updating our deep learning packages because we don't want to refactor a bunch of stuff. In Lightning, we make sure every change we make which could break an API is backwards compatible with good deprecation warnings.
+
+You shouldn't be afraid to upgrade Lightning :)
+
+#### Gain User Trust    
+As a researcher you can't have any part of your code going wrong. So, make thorough tests that ensure an implementation of a new trick or subbtle change is correct.    
+
+#### Interoperability  
+Have a favorite feature from other libraries like fast.ai or transformers? Those should just work with lightning as well. Grab your favorite model or learning rate scheduler from your favorite library and run it in Lightning.
+
+## Contribution Types    
+Currently looking for help implementing new features or adding bug fixes. 
+
+A lot of good work has already been done in project mechanics (requirements.txt, setup.py, pep8, badges, ci, etc...) we're in a good state there thanks to all the early contributors (even pre-beta release)!   
+
+## Bug Fixes:  
+1. Submit a github issue.   
+2. Fix it.  
+3. Submit a PR! 
+
+## New Features:  
+1. Submit a github issue.   
+2. We'll agree on the feature scope.     
+3. Submit a PR! (with updated docs and tests 🙃).   
+
+## Coding Styleguide
+1. Test the code with flake8.
+2. Use f-strings.

docs/source/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+# Before submitting
+
+- [ ] Was this discussed/approved via a Github issue? (no need for typos, doc improvements)
+- [ ] Did you read the [contributor guideline](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md)?
+- [ ] Did you make sure to update the docs?   
+- [ ] Did you write any new necessary tests?  
+
+## What does this PR do?
+Fixes # (issue).
+
+## PR review    
+Anyone in the community is free to review the PR once the tests have passed.     
+If we didn't discuss your PR in Github issues there's a high chance it will not be merged.
+
+## Did you have fun?
+Make sure you had fun coding 🙃

docs/source/_static/images/lightning_icon.svg

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   id="svg"
+   version="1.1"
+   width="16.000004"
+   height="15.999986"
+   viewBox="0 0 16.000004 15.999986"
+   sodipodi:docname="lightning_icon.svg"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)">
+  <metadata
+     id="metadata13">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs11" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1920"
+     inkscape:window-height="1028"
+     id="namedview9"
+     showgrid="false"
+     inkscape:zoom="0.59"
+     inkscape:cx="-669.05062"
+     inkscape:cy="373.84245"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg" />
+  <path
+     style="fill:#fbfbfb;fill-rule:evenodd;stroke:none;stroke-width:0.04002798"
+     inkscape:connector-curvature="0"
+     d="m 8.987101,1.723485 c -0.05588,0.03422 -4.121881,4.096544 -4.184645,4.180924 -0.02317,0.0311 -0.04587,0.06016 -0.05044,0.06456 -0.0087,0.0084 -0.07477,0.145063 -0.09679,0.20014 -0.05848,0.146583 -0.05804,0.44387 0.001,0.592413 0.08426,0.21243 0.08826,0.216754 1.576864,1.706274 0.779463,0.779947 1.41719,1.426877 1.41719,1.437604 0,0.0232 -0.253177,0.79848 -0.273873,0.838707 -0.0079,0.0153 -0.01433,0.04087 -0.01433,0.05684 0,0.01597 -0.0059,0.03587 -0.01313,0.04423 -0.0072,0.0084 -0.03678,0.09086 -0.06568,0.18333 -0.02893,0.09246 -0.05904,0.180647 -0.06693,0.195937 -0.0079,0.0153 -0.01437,0.04087 -0.01437,0.05684 0,0.01597 -0.0059,0.03586 -0.01313,0.04423 -0.0072,0.0084 -0.03679,0.09086 -0.06569,0.18333 -0.02893,0.09246 -0.05904,0.180643 -0.06693,0.195937 -0.0079,0.0153 -0.01437,0.04187 -0.01437,0.05908 0,0.0172 -0.0072,0.03574 -0.016,0.04119 -0.0088,0.0054 -0.016,0.02607 -0.016,0.04579 0,0.01973 -0.006,0.04271 -0.0134,0.05108 -0.0074,0.0084 -0.04439,0.112477 -0.08222,0.23136 -0.03787,0.118884 -0.151103,0.461124 -0.251693,0.760534 -0.489984,1.45874 -0.462444,1.36155 -0.413611,1.45938 0.06917,0.138657 0.23128,0.199741 0.358251,0.134974 0.07057,-0.03602 4.143298,-4.099985 4.245368,-4.236242 0.03382,-0.04515 0.09094,-0.165796 0.109916,-0.232123 0.0088,-0.03083 0.0243,-0.08498 0.03442,-0.120363 0.03346,-0.11668 0.0068,-0.361134 -0.0566,-0.520084 C 10.880518,9.229614 10.738898,9.079187 9.372744,7.714673 8.601524,6.944416 7.970523,6.302806 7.970523,6.288916 c 0,-0.01393 0.02817,-0.107833 0.0626,-0.208663 0.03442,-0.100834 0.07881,-0.237367 0.09859,-0.303414 0.0198,-0.06605 0.04207,-0.12693 0.04947,-0.135293 0.0074,-0.0084 0.0135,-0.03133 0.0135,-0.05108 0,-0.01973 0.0072,-0.04035 0.016,-0.04579 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04804 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04804 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02397 0.016,-0.04119 0,-0.0172 0.0065,-0.04379 0.0144,-0.05908 0.0079,-0.0153 0.119204,-0.34484 0.247334,-0.73231 C 9.064507,2.979766 9.220177,2.513319 9.28226,2.330632 9.408267,1.960092 9.41367,1.921146 9.35255,1.826839 9.27225,1.703032 9.099973,1.654399 8.986893,1.723566"
+     id="path0" />
+  <path
+     style="fill:#540c8c;fill-rule:evenodd;stroke:none;stroke-width:0.04002798"
+     inkscape:connector-curvature="0"
+     d="m 0.07719102,0.01733399 c -0.02187,0.0111 -0.04875,0.03799 -0.05984,0.05984 -0.0161,0.03173 -0.01937,1.62421701 -0.01633,7.94479601 l 0.0038,7.905086 0.03647,0.03646 0.03646,0.03647 H 8.00241 15.927073 l 0.03646,-0.03647 0.03647,-0.03646 V 8.002393 0.07773399 l -0.03647,-0.03646 -0.03646,-0.03647 -7.905086,-0.0038 c -6.320579,-0.003 -7.91305298,2.4e-4 -7.94479598,0.01633 M 9.193764,1.668208 c 0.259903,0.09046 0.275193,0.212427 0.09363,0.74628 C 8.845834,3.776859 8.388843,5.102846 7.991127,6.302606 L 9.415644,7.72492 c 1.24415,1.242111 1.51682,1.523547 1.51682,1.565414 0,0.0051 0.0133,0.03987 0.02953,0.07718 0.12913,0.296607 0.0877,0.664983 -0.103314,0.91872 -0.141456,0.187933 -4.207341,4.228478 -4.273468,4.246848 -0.139417,0.03871 -0.248653,-0.006 -0.34324,-0.140417 -0.07665,-0.108996 -0.06985,-0.137256 0.287004,-1.194633 0.34663,-1.101761 0.75901,-2.243218 1.08916,-3.290661 0,-0.0078 -0.636164,-0.650377 -1.413707,-1.427921 C 4.877658,7.152643 4.728155,6.995813 4.673718,6.87361 4.661948,6.84718 4.645988,6.81305 4.638168,6.79776 4.630368,6.78246 4.624038,6.75689 4.624038,6.74092 c 0,-0.01597 -0.0076,-0.03659 -0.01687,-0.04587 -0.02253,-0.02253 -0.02253,-0.436904 0,-0.45944 0.0093,-0.0093 0.01687,-0.0327 0.01687,-0.05204 0,-0.0363 0.06917,-0.178363 0.130414,-0.267907 0.07965,-0.1164 4.221831,-4.237681 4.259458,-4.237921 0.02047,-1.2e-4 0.04803,-0.0072 0.06124,-0.01577 0.03147,-0.02033 0.04415,-0.01967 0.118603,0.0062"
+     id="path1"
+     sodipodi:nodetypes="ccscccccccccccscccccscccccccccsssscccc" />
+</svg>

docs/source/_static/images/lightning_logo-large.svg

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   id="svg"
+   version="1.1"
+   width="400"
+   height="400"
+   viewBox="0, 0, 400,400"
+   sodipodi:docname="lightning_logo.svg"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)">
+  <metadata
+     id="metadata13">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs11" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1920"
+     inkscape:window-height="1028"
+     id="namedview9"
+     showgrid="false"
+     inkscape:zoom="9.44"
+     inkscape:cx="203.07907"
+     inkscape:cy="335.32491"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg" />
+  <path
+     style="fill:#fbfbfb;fill-rule:evenodd;stroke:none"
+     inkscape:connector-curvature="0"
+     d="m 224.6,43.137 c -1.396,0.855 -102.975,102.342 -104.543,104.45 -0.579,0.777 -1.146,1.503 -1.26,1.613 -0.218,0.21 -1.868,3.624 -2.418,5 -1.461,3.662 -1.45,11.089 0.022,14.8 2.105,5.307 2.205,5.415 39.394,42.627 19.473,19.485 35.405,35.647 35.405,35.915 0,0.58 -6.325,19.948 -6.842,20.953 -0.197,0.382 -0.358,1.021 -0.358,1.42 0,0.399 -0.147,0.896 -0.328,1.105 -0.18,0.209 -0.919,2.27 -1.641,4.58 -0.723,2.31 -1.475,4.513 -1.672,4.895 -0.198,0.382 -0.359,1.021 -0.359,1.42 0,0.399 -0.147,0.896 -0.328,1.105 -0.18,0.209 -0.919,2.27 -1.641,4.58 -0.723,2.31 -1.475,4.513 -1.672,4.895 -0.198,0.382 -0.359,1.046 -0.359,1.476 0,0.43 -0.18,0.893 -0.4,1.029 -0.22,0.136 -0.4,0.651 -0.4,1.144 0,0.493 -0.151,1.067 -0.335,1.276 -0.184,0.209 -1.109,2.81 -2.054,5.78 -0.946,2.97 -3.775,11.52 -6.288,19 -12.241,36.443 -11.553,34.015 -10.333,36.459 1.728,3.464 5.778,4.99 8.95,3.372 1.763,-0.9 103.51,-102.428 106.06,-105.832 0.845,-1.128 2.272,-4.142 2.746,-5.799 0.22,-0.77 0.607,-2.123 0.86,-3.007 0.836,-2.915 0.171,-9.022 -1.414,-12.993 -1.493,-3.741 -5.031,-7.499 -39.161,-41.588 C 214.964,173.569 199.2,157.54 199.2,157.193 c 0,-0.348 0.704,-2.694 1.564,-5.213 0.86,-2.519 1.969,-5.93 2.463,-7.58 0.495,-1.65 1.051,-3.171 1.236,-3.38 0.186,-0.209 0.337,-0.783 0.337,-1.276 0,-0.493 0.18,-1.008 0.4,-1.144 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.599 0.4,-1.029 0,-0.43 0.162,-1.094 0.36,-1.476 0.197,-0.382 2.978,-8.615 6.179,-18.295 3.2,-9.68 7.089,-21.333 8.64,-25.897 3.148,-9.257 3.283,-10.23 1.756,-12.586 -2.006,-3.093 -6.31,-4.308 -9.135,-2.58"
+     id="path0" />
+  <path
+     style="fill:#540c8c;fill-rule:evenodd;stroke:none"
+     inkscape:connector-curvature="0"
+     d="M 2.008,0.513 C 1.462,0.79 0.79,1.462 0.513,2.008 0.111,2.801 0.029,42.585 0.105,200.489 L 0.2,397.978 1.111,398.889 2.022,399.8 H 200 397.978 l 0.911,-0.911 0.911,-0.911 V 200 2.022 L 398.889,1.111 397.978,0.2 200.489,0.105 C 42.585,0.029 2.801,0.111 2.008,0.513 m 227.755,41.243 c 6.493,2.26 6.875,5.307 2.339,18.644 -11.0313,34.035452 -22.44803,67.16196 -32.384,97.135 l 35.588,35.533 c 31.082,31.031 37.894,38.062 37.894,39.108 0,0.128 0.332,0.996 0.738,1.928 3.226,7.41 2.191,16.613 -2.581,22.952 -3.534,4.695 -105.11,105.638 -106.762,106.097 -3.483,0.967 -6.212,-0.15 -8.575,-3.508 -1.915,-2.723 -1.745,-3.429 7.17,-29.845 8.65971,-27.52475 18.96205,-56.04122 27.21,-82.209 0,-0.195 -15.893,-16.248 -35.318,-35.673 -33.146,-33.147 -36.881,-37.065 -38.241,-40.118 -0.294,-0.66 -0.693,-1.513 -0.888,-1.895 -0.194,-0.382 -0.353,-1.021 -0.353,-1.42 0,-0.399 -0.189,-0.914 -0.421,-1.146 -0.563,-0.563 -0.563,-10.915 0,-11.478 0.232,-0.232 0.421,-0.817 0.421,-1.3 0,-0.907 1.728,-4.456 3.258,-6.693 C 120.848,144.96 224.33,42 225.27,41.994 c 0.511,-0.003 1.2,-0.181 1.53,-0.394 0.786,-0.508 1.103,-0.491 2.963,0.156"
+     id="path1"
+     sodipodi:nodetypes="ccscccccccccccscccccscccccccccsssscccc" />
+</svg>

docs/source/_static/images/lightning_logo.svg

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   id="svg"
+   version="1.1"
+   width="47.999985"
+   height="47.999943"
+   viewBox="0 0 47.999985 47.999943"
+   sodipodi:docname="lightning_logo.svg"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)">
+  <metadata
+     id="metadata13">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs11" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1920"
+     inkscape:window-height="1028"
+     id="namedview9"
+     showgrid="false"
+     inkscape:zoom="0.59"
+     inkscape:cx="-347.96588"
+     inkscape:cy="389.84243"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg" />
+  <path
+     style="fill:#fbfbfb;fill-rule:evenodd;stroke:none;stroke-width:0.12008391"
+     inkscape:connector-curvature="0"
+     d="m 26.961294,5.1704519 c -0.16764,0.10267 -12.36564,12.2896301 -12.55393,12.5427701 -0.0695,0.0933 -0.13762,0.18048 -0.15131,0.19369 -0.0262,0.0252 -0.22432,0.43519 -0.29036,0.60042 -0.17544,0.43975 -0.17412,1.33161 0.003,1.77724 0.25278,0.63729 0.26479,0.65026 4.73059,5.11882 2.33839,2.33984 4.25157,4.28063 4.25157,4.31281 0,0.0696 -0.75953,2.39544 -0.82162,2.51612 -0.0237,0.0459 -0.043,0.12261 -0.043,0.17052 0,0.0479 -0.0177,0.1076 -0.0394,0.13269 -0.0216,0.0251 -0.11035,0.27259 -0.19705,0.54999 -0.0868,0.27739 -0.17713,0.54194 -0.20078,0.58781 -0.0238,0.0459 -0.0431,0.1226 -0.0431,0.17052 0,0.0479 -0.0177,0.10759 -0.0394,0.13269 -0.0216,0.0251 -0.11036,0.27259 -0.19706,0.54999 -0.0868,0.27739 -0.17712,0.54193 -0.20078,0.58781 -0.0238,0.0459 -0.0431,0.1256 -0.0431,0.17724 0,0.0516 -0.0216,0.10723 -0.048,0.12357 -0.0264,0.0163 -0.048,0.0782 -0.048,0.13737 0,0.0592 -0.0181,0.12813 -0.0402,0.15323 -0.0221,0.0251 -0.13318,0.33743 -0.24666,0.69408 -0.1136,0.35665 -0.45331,1.38337 -0.75508,2.2816 -1.46995,4.37622 -1.38733,4.08465 -1.24083,4.37814 0.2075,0.41597 0.69384,0.59922 1.07475,0.40492 0.21171,-0.10807 12.42989,-12.29995 12.7361,-12.70872 0.10147,-0.13545 0.27283,-0.49739 0.32975,-0.69637 0.0264,-0.0925 0.0729,-0.25493 0.10327,-0.36109 0.10039,-0.35004 0.0205,-1.0834 -0.1698,-1.56025 -0.17928,-0.44923 -0.60414,-0.90051 -4.7026,-4.99405 -2.31366,-2.31077 -4.20666,-4.2356 -4.20666,-4.27727 0,-0.0418 0.0845,-0.3235 0.18781,-0.62599 0.10327,-0.3025 0.23644,-0.7121 0.29577,-0.91024 0.0594,-0.19814 0.1262,-0.38079 0.14842,-0.40588 0.0223,-0.0251 0.0405,-0.094 0.0405,-0.15323 0,-0.0592 0.0216,-0.12105 0.048,-0.13738 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.14411 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.14411 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0719 0.048,-0.12356 0,-0.0516 0.0195,-0.13137 0.0432,-0.17725 0.0237,-0.0459 0.35761,-1.03452 0.742,-2.19693 0.38427,-1.1624101 0.85128,-2.5617501 1.03753,-3.1098101 0.37802,-1.11162 0.39423,-1.22846 0.21087,-1.51138 -0.24089,-0.37142 -0.75773,-0.51732 -1.09697,-0.30982"
+     id="path0" />
+  <path
+     style="fill:#540c8c;fill-rule:evenodd;stroke:none;stroke-width:0.12008391"
+     inkscape:connector-curvature="0"
+     d="m 0.2315739,0.05200186 c -0.0656,0.0333 -0.14626,0.11396 -0.17952,0.17952 -0.0483,0.0952 -0.0581,4.87265004 -0.049,23.83438014 l 0.0114,23.71525 0.1094,0.10939 0.10939,0.1094 h 23.7739701 23.77398 l 0.10939,-0.1094 0.1094,-0.10939 V 24.007172 0.23320186 l -0.1094,-0.10939 -0.10939,-0.1094 -23.71525,-0.0114 c -18.9617301,-0.009 -23.7391501,7.2e-4 -23.8343801,0.049 M 27.581274,5.0046319 c 0.77971,0.27139 0.82558,0.63728 0.28088,2.23884 -1.32468,4.0871101 -2.69565,8.0650701 -3.8888,11.6643501 l 4.27355,4.26694 c 3.73245,3.72633 4.55046,4.57064 4.55046,4.69624 0,0.0154 0.0399,0.11961 0.0886,0.23153 0.38739,0.88982 0.2631,1.99495 -0.30994,2.75616 -0.42437,0.5638 -12.62202,12.68543 -12.8204,12.74054 -0.41825,0.11613 -0.74596,-0.018 -1.02972,-0.42125 -0.22996,-0.32699 -0.20954,-0.41177 0.86101,-3.5839 1.03989,-3.30528 2.27703,-6.72965 3.26748,-9.87198 0,-0.0234 -1.90849,-1.95113 -4.24112,-4.28376 -3.98031,-3.98042 -4.42882,-4.45091 -4.59213,-4.81752 -0.0353,-0.0793 -0.0832,-0.18169 -0.10664,-0.22756 -0.0233,-0.0459 -0.0424,-0.12261 -0.0424,-0.17052 0,-0.0479 -0.0227,-0.10976 -0.0506,-0.13762 -0.0676,-0.0676 -0.0676,-1.31071 0,-1.37832 0.0279,-0.0279 0.0506,-0.0981 0.0506,-0.15611 0,-0.10891 0.20751,-0.53509 0.39124,-0.80372 0.23896,-0.3492 12.66549,-12.7130401 12.77837,-12.7137601 0.0614,-3.6e-4 0.1441,-0.0217 0.18372,-0.0473 0.0944,-0.061 0.13246,-0.059 0.35581,0.0187"
+     id="path1"
+     sodipodi:nodetypes="ccscccccccccccscccccscccccccccsssscccc" />
+</svg>

docs/source/_templates/theme_variables.jinja

@@ -1,17 +1,18 @@
 {%- set external_urls = {
-  'github': 'https://github.com/williamFalcon/pytorch-lightning',
-  'github_issues': 'https://github.com/williamFalcon/pytorch-lightning/issues',
-  'contributing': 'https://github.com/williamFalcon/pytorch-lightning/blob/master/CONTRIBUTING.md',
-  'docs': 'https://williamfalcon.github.io/pytorch-lightning',
+  'github': 'https://github.com/PytorchLightning/pytorch-lightning',
+  'github_issues': 'https://github.com/PytorchLightning/pytorch-lightning/issues',
+  'contributing': 'https://github.com/PytorchLightning/pytorch-lightning/blob/master/CONTRIBUTING.md',
+  'governance': 'https://github.com/PytorchLightning/pytorch-lightning/blob/master/governance.md',
+  'docs': 'https://pytorch-lightning.rtfd.io/en/latest',
   'twitter': 'https://twitter.com/PyTorchLightnin',
   'discuss': 'https://discuss.pytorch.org',
-  'tutorials': 'https://williamfalcon.github.io/pytorch-lightning/',
-  'previous_pytorch_versions': 'https://williamfalcon.github.io/pytorch-lightning/',
-  'home': 'https://williamfalcon.github.io/pytorch-lightning/',
-  'get_started': 'https://williamfalcon.github.io/pytorch-lightning/',
-  'features': 'https://williamfalcon.github.io/pytorch-lightning/',
-  'blog': 'https://williamfalcon.github.io/pytorch-lightning/',
-  'resources': 'https://williamfalcon.github.io/pytorch-lightning/',
-  'support': 'https://williamfalcon.github.io/pytorch-lightning/',
+  'tutorials': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'previous_pytorch_versions': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'home': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'get_started': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'features': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'blog': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'resources': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'support': 'https://pytorch-lightning.rtfd.io/en/latest/',
 }
 -%}

docs/source/callbacks.rst

@@ -0,0 +1,14 @@
+.. role:: hidden
+    :class: hidden-section
+
+Callbacks
+===========
+.. automodule:: pytorch_lightning.callbacks
+   :exclude-members:
+        _del_model,
+        _save_model,
+        on_epoch_end,
+        on_train_end,
+        on_epoch_begin,
+        check_monitor_top_k,
+        on_train_begin,

docs/source/common-cases.rst

@@ -0,0 +1,21 @@
+Multi-gpu (same node) training
+==============================
+
+Multi-node training
+====================
+
+16-bit precision
+=================
+
+gradient clipping
+=================
+
+modifying training via hooks
+=============================
+
+
+
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples

docs/source/conf.py

@@ -62,7 +62,6 @@ version = pytorch_lightning.__version__
 # The full version, including alpha/beta/rc tags
 release = pytorch_lightning.__version__
 
-
 # -- General configuration ---------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -84,6 +83,7 @@ extensions = [
     'sphinx.ext.autosummary',
     'sphinx.ext.napoleon',
     'recommonmark',
+    'sphinx.ext.autosectionlabel',
     # 'm2r',
     'nbsphinx',
 ]
@@ -128,7 +128,6 @@ exclude_patterns = ['*.test_*']
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = None
 
-
 # -- Options for HTML output -------------------------------------------------
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
@@ -151,7 +150,7 @@ html_theme_options = {
     'logo_only': False,
 }
 
-html_logo = '_static/images/lightning_logo_small.png'
+html_logo = '_static/images/lightning_logo-name.svg'
 
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
@@ -174,7 +173,6 @@ html_static_path = ['_static']
 # Output file base name for HTML help builder.
 htmlhelp_basename = project + '-doc'
 
-
 # -- Options for LaTeX output ------------------------------------------------
 
 latex_elements = {
@@ -198,7 +196,6 @@ latex_documents = [
     (master_doc, project + '.tex', project + ' Documentation', author, 'manual'),
 ]
 
-
 # -- Options for manual page output ------------------------------------------
 
 # One entry per manual page. List of tuples
@@ -207,7 +204,6 @@ man_pages = [
     (master_doc, project, project + ' Documentation', [author], 1)
 ]
 
-
 # -- Options for Texinfo output ----------------------------------------------
 
 # Grouping the document tree into Texinfo files. List of tuples
@@ -218,7 +214,6 @@ texinfo_documents = [
      'One line description of project.', 'Miscellaneous'),
 ]
 
-
 # -- Options for Epub output -------------------------------------------------
 
 # Bibliographic Dublin Core info.
@@ -236,7 +231,6 @@ epub_title = project
 # A list of files that should not be packed into the epub file.
 epub_exclude_files = ['search.html']
 
-
 # -- Extension configuration -------------------------------------------------
 
 # -- Options for intersphinx extension ---------------------------------------
@@ -249,7 +243,6 @@ intersphinx_mapping = {'https://docs.python.org/': None}
 # If true, `todo` and `todoList` produce output, else they produce nothing.
 todo_include_todos = True
 
-
 # https://github.com/rtfd/readthedocs.org/issues/1139
 # I use sphinx-apidoc to auto-generate API documentation for my project.
 # Right now I have to commit these auto-generated files to my repository
@@ -302,7 +295,7 @@ with open(os.path.join(PATH_ROOT, 'requirements.txt'), 'r') as fp:
             MOCK_REQUIRE_PACKAGES.append(pkg.rstrip())
 
 # TODO: better parse from package since the import name and package name may differ
-MOCK_MANUAL_PACKAGES = ['torch', 'torchvision', 'sklearn', 'test_tube', 'mlflow', 'comet_ml']
+MOCK_MANUAL_PACKAGES = ['torch', 'torchvision', 'sklearn', 'test_tube', 'mlflow', 'comet_ml', 'wandb', 'neptune']
 autodoc_mock_imports = MOCK_REQUIRE_PACKAGES + MOCK_MANUAL_PACKAGES
 # for mod_name in MOCK_REQUIRE_PACKAGES:
 #     sys.modules[mod_name] = mock.Mock()
@@ -310,7 +303,7 @@ autodoc_mock_imports = MOCK_REQUIRE_PACKAGES + MOCK_MANUAL_PACKAGES
 
 # Options for the linkcode extension
 # ----------------------------------
-github_user = 'williamFalcon'
+github_user = 'PyTorchLightning'
 github_repo = project
 
 

docs/source/examples.rst

@@ -1,8 +1,34 @@
-Examples & Tutorials
-====================
-
-
+GAN
+====
 .. toctree::
    :maxdepth: 3
 
-   pl_examples
+   pl_examples.domain_templates.gan
+
+MNIST
+====
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples.basic_examples.lightning_module_template
+
+Multi-node (ddp) MNIST
+====
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples.multi_node_examples.multi_node_ddp_demo
+
+Multi-node (ddp2) MNIST
+====
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples.multi_node_examples.multi_node_ddp2_demo
+
+Imagenet
+====
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples.full_examples.imagenet.imagenet_example

docs/source/governance.md

@@ -0,0 +1,8 @@
+# Pytorch Lightning Governance | Persons of interest
+
+### Maintainers  
+- William Falcon ([williamFalcon](https://github.com/williamFalcon))   
+- Jirka Borovek ([Borda](https://github.com/Borda))   
+- Nick Eggert ([neggert](https://github.com/neggert))   
+- Jeff Ling ([jeffling](https://github.com/jeffling))   
+- Tullie Murrell ([tullie](https://github.com/tullie))

docs/source/index.rst

@@ -3,33 +3,57 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to PyTorch-Lightning!
+PyTorch-Lightning Documentation
 =============================
 
 .. toctree::
-   :maxdepth: 4
+   :maxdepth: 1
    :name: start
-   :caption: Quick Start
+   :caption: Start Here
 
    new-project
-   examples
 
 .. toctree::
    :maxdepth: 4
    :name: docs
-   :caption: Docs
+   :caption: Python API
 
-   documentation
+   callbacks
+   lightning-module
+   logging
+   trainer
+
+.. toctree::
+   :maxdepth: 1
+   :name: Examples
+   :caption: Examples
+
+   examples
+
+.. toctree::
+   :maxdepth: 1
+   :name: Tutorials
+   :caption: Tutorials
+
+   tutorials
+
+.. toctree::
+   :maxdepth: 1
+   :name: Common Use Cases
+   :caption: Common Use Cases
+
+   common-cases
 
 .. toctree::
    :maxdepth: 1
    :name: community
    :caption: Community
 
+
    CODE_OF_CONDUCT.md
    CONTRIBUTING.md
    BECOMING_A_CORE_CONTRIBUTOR.md
-
+   governance.md
 
 Indices and tables
 ------------------
@@ -37,4 +61,3 @@ Indices and tables
 * :ref:`genindex`
 * :ref:`modindex`
 * :ref:`search`
-

docs/source/lightning-module.rst

@@ -0,0 +1,10 @@
+.. role:: hidden
+    :class: hidden-section
+
+LightningModule
+===========
+.. automodule:: pytorch_lightning.core
+   :exclude-members:
+        _abc_impl,
+        summarize,
+

docs/source/logging.rst

@@ -0,0 +1,12 @@
+.. role:: hidden
+    :class: hidden-section
+
+Logging
+===========
+.. automodule:: pytorch_lightning.logging
+   :exclude-members:
+        _abc_impl,
+        _save_model,
+        on_epoch_end,
+        on_train_end,
+        on_epoch_begin,

docs/source/modules.rst

@@ -0,0 +1,7 @@
+pl_examples
+===========
+
+.. toctree::
+   :maxdepth: 4
+
+   pl_examples

docs/source/new-project.rst

@@ -1,13 +1,13 @@
 Quick Start
 ===========
-To start a new project define two files, a LightningModule and a Trainer file.    
-To illustrate Lightning power and simplicity, here's an example of a typical research flow.    
+| To start a new project define two files, a LightningModule and a Trainer file.
+| To illustrate the power of Lightning and its simplicity, here's an example of a typical research flow.
 
 Case 1: BERT
 ------------
 
-Let's say you're working on something like BERT but want to try different ways of training or even different networks.  
-You would define a single LightningModule and use flags to switch between your different ideas.
+| Let's say you're working on something like BERT but want to try different ways of training or even different networks.
+| You would define a single LightningModule and use flags to switch between your different ideas.
 
 .. code-block:: python
 
@@ -66,6 +66,7 @@ Then you could do rapid research by switching between these two and using the sa
 
 **Notice a few things about this flow:**
 
-1. You're writing pure PyTorch... no unnecessary abstractions or new libraries to learn.   
-2. You get free GPU and 16-bit support without writing any of that code in your model.   
-3. You also get all of the capabilities below (without coding or testing yourself).     
+1. You're writing pure PyTorch... no unnecessary abstractions or new libraries to learn.
+2. You get free GPU and 16-bit support without writing any of that code in your model.
+3. You also get early stopping, multi-gpu training, 16-bit and MUCH more without coding anything!
+

docs/source/trainer.rst

@@ -0,0 +1,21 @@
+.. role:: hidden
+    :class: hidden-section
+
+Trainer
+===========
+.. automodule:: pytorch_lightning.trainer
+   :members: fit, test
+   :exclude-members: 
+        run_pretrain_routine, 
+        _abc_impl, 
+        _Trainer__set_root_gpu, 
+        _Trainer__init_optimizers,
+        _Trainer__parse_gpu_ids,
+        _Trainer__configure_schedulers,
+        data_parallel,
+        num_gpus,
+        slurm_job_id,
+        tng_tqdm_dic,
+        training_tqdm_dict,
+        init_optimizers,
+        configure_schedulers

docs/source/tutorials.rst

@@ -0,0 +1,20 @@
+Refactoring PyTorch into Lightning
+==================================
+`Tutorial <https://towardsdatascience.com/how-to-refactor-your-pytorch-code-to-get-these-42-benefits-of-pytorch-lighting-6fdd0dc97538>`_
+
+Start a research project
+=========================
+`Research seed <https://github.com/PytorchLightning/pytorch-lightning-conference-seed>`_
+
+Basic Lightning use
+====================
+`Tutorial <https://towardsdatascience.com/supercharge-your-ai-research-with-pytorch-lightning-337948a99eec>`_
+
+9 key Lightning  tricks
+========================
+`Tutorial <9 key speed features in Pytorch-Lightning>`_
+
+Multi-node training on SLURM
+=============================
+`Tutorial <https://towardsdatascience.com/trivial-multi-node-training-with-pytorch-lightning-ff75dfb809bd>`_
+

pl_examples/__init__.py

@@ -3,13 +3,13 @@ Template model definition
 -------------------------
 
 In 99% of cases you want to just copy `one of the examples
- <https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples>`_
+ <https://github.com/PyTorchLightning/pytorch-lightning/tree/master/pl_examples>`_
  to start a new lightningModule and change the core of what your model is actually trying to do.
 
 .. code-block:: bash
 
     # get a copy of the module template
-    wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/pl_examples/new_project_templates/lightning_module_template.py  # noqa: E501
+    wget https://raw.githubusercontent.com/PyTorchLightning/pytorch-lightning/master/pl_examples/new_project_templates/lightning_module_template.py  # noqa: E501
 
 
 Trainer Example

pl_examples/basic_examples/lightning_module_template.py

@@ -1,8 +1,8 @@
 """
 Example template for defining a system
 """
-import os
 import logging
+import os
 from argparse import ArgumentParser
 from collections import OrderedDict
 

pl_examples/full_examples/imagenet/imagenet_example.py

@@ -8,20 +8,18 @@ from collections import OrderedDict
 
 import torch
 import torch.backends.cudnn as cudnn
-import torch.nn.parallel
 import torch.nn.functional as F
+import torch.nn.parallel
 import torch.optim as optim
 import torch.optim.lr_scheduler as lr_scheduler
 import torch.utils.data
 import torch.utils.data.distributed
-
-import torchvision.transforms as transforms
-import torchvision.models as models
 import torchvision.datasets as datasets
+import torchvision.models as models
+import torchvision.transforms as transforms
 
 import pytorch_lightning as pl
 
-
 # pull out resnet names from torchvision models
 MODEL_NAMES = sorted(
     name for name in models.__dict__
@@ -36,9 +34,12 @@ class ImageNetLightningModel(pl.LightningModule):
         self.hparams = hparams
         self.model = models.__dict__[self.hparams.arch](pretrained=self.hparams.pretrained)
 
+    def forward(self, x):
+        return self.model(x)
+
     def training_step(self, batch, batch_idx):
         images, target = batch
-        output = self.model(images)
+        output = self.forward(images)
         loss_val = F.cross_entropy(output, target)
         acc1, acc5 = self.__accuracy(output, target, topk=(1, 5))
 
@@ -61,7 +62,7 @@ class ImageNetLightningModel(pl.LightningModule):
 
     def validation_step(self, batch, batch_idx):
         images, target = batch
-        output = self.model(images)
+        output = self.forward(images)
         loss_val = F.cross_entropy(output, target)
         acc1, acc5 = self.__accuracy(output, target, topk=(1, 5))
 
@@ -134,7 +135,7 @@ class ImageNetLightningModel(pl.LightningModule):
             std=[0.229, 0.224, 0.225],
         )
 
-        train_dir = os.path.join(self.hparams.data, 'train')
+        train_dir = os.path.join(self.hparams.data_path, 'train')
         train_dataset = datasets.ImageFolder(
             train_dir,
             transforms.Compose([
@@ -164,7 +165,7 @@ class ImageNetLightningModel(pl.LightningModule):
             mean=[0.485, 0.456, 0.406],
             std=[0.229, 0.224, 0.225],
         )
-        val_dir = os.path.join(self.hparams.data, 'val')
+        val_dir = os.path.join(self.hparams.data_path, 'val')
         val_loader = torch.utils.data.DataLoader(
             datasets.ImageFolder(val_dir, transforms.Compose([
                 transforms.Resize(256),
@@ -187,7 +188,7 @@ class ImageNetLightningModel(pl.LightningModule):
                                  ' (default: resnet18)')
         parser.add_argument('--epochs', default=90, type=int, metavar='N',
                             help='number of total epochs to run')
-        parser.add_argument('--seed', type=int, default=None,
+        parser.add_argument('--seed', type=int, default=42,
                             help='seed for initializing training. ')
         parser.add_argument('-b', '--batch-size', default=256, type=int,
                             metavar='N',
@@ -216,7 +217,7 @@ def get_args():
                                help='how many gpus')
     parent_parser.add_argument('--distributed-backend', type=str, default='dp', choices=('dp', 'ddp', 'ddp2'),
                                help='supports three options dp, ddp, ddp2')
-    parent_parser.add_argument('--use-16bit', dest='use-16bit', action='store_true',
+    parent_parser.add_argument('--use-16bit', dest='use_16bit', action='store_true',
                                help='if true uses 16 bit precision')
     parent_parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',
                                help='evaluate model on validation set')

pytorch_lightning/__init__.py

@@ -1,11 +1,11 @@
 """Package info"""
 
-__version__ = '0.5.3.2'
+__version__ = '0.6.0'
 __author__ = 'William Falcon et al.'
 __author_email__ = 'waf2107@columbia.edu'
 __license__ = 'Apache-2.0'
 __copyright__ = 'Copyright (c) 2018-2019, %s.' % __author__
-__homepage__ = 'https://github.com/williamFalcon/pytorch-lightning'
+__homepage__ = 'https://github.com/PyTorchLightning/pytorch-lightning'
 # this has to be simple string, see: https://github.com/pypa/twine/issues/522
 __docs__ = "PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers." \
            " Scale your models. Write less boilerplate."
@@ -21,7 +21,7 @@ except NameError:
 
 if __LIGHTNING_SETUP__:
     import sys
-    sys.stderr.write('Partial import of skimage during the build process.\n')
+    sys.stderr.write('Partial import of torchlightning during the build process.\n')
     # We are not importing the rest of the scikit during the build
     # process, as it may not be compiled yet
 else:

pytorch_lightning/callbacks/pt_callbacks.py

@@ -1,3 +1,9 @@
+"""
+Callbacks
+====================================
+Callbacks supported by Lightning
+"""
+
 import os
 import shutil
 import logging
@@ -8,26 +14,7 @@ from pytorch_lightning.overrides.data_parallel import LightningDistributedDataPa
 
 
 class Callback(object):
-    """Abstract base class used to build new callbacks.
-
-    # Properties
-        * params: dict. Training parameters
-            (eg. verbosity, batch size, number of epochs...).
-            Reference of the model being trained.
-
-    The `logs` dictionary that callback methods take as argument will contain keys
-     for quantities relevant to the current batch or epoch.
-    Currently, the `.fit()` method of the `Sequential` model class will include the following
-     quantities in the `logs` that it passes to its callbacks:
-    * on_epoch_end: logs include `acc` and `loss`, and
-            optionally include `val_loss`
-            (if validation is enabled in `fit`), and `val_acc`
-            (if validation and accuracy monitoring are enabled).
-    * on_batch_begin: logs include `size`,
-            the number of samples in the current batch.
-    * on_batch_end: logs include `loss`, and optionally `acc`
-            (if accuracy monitoring is enabled).
-
+    r"""Abstract base class used to build new callbacks.
     """
 
     def __init__(self):
@@ -43,12 +30,30 @@ class Callback(object):
         self.model = model
 
     def on_epoch_begin(self, epoch, logs=None):
+        """
+        called when the epoch begins
+
+        Args:
+            epoch (int): current epoch
+            logs (dict): key-value pairs of quantities to monitor
+
+            Example:
+
+                on_epoch_begin(epoch=2, logs={'val_loss': 0.2})
+        """
         pass
 
     def on_epoch_end(self, epoch, logs=None):
         pass
 
     def on_batch_begin(self, batch, logs=None):
+        """
+        called when the batch starts.
+
+        Args:
+            batch (Tensor): current batch tensor
+            logs (dict): key-value pairs of quantities to monitor
+        """
         pass
 
     def on_batch_end(self, batch, logs=None):
@@ -62,40 +67,52 @@ class Callback(object):
 
 
 class EarlyStopping(Callback):
-    """Stop training when a monitored quantity has stopped improving.
+    r"""
+    Stop training when a monitored quantity has stopped improving.
 
-    # Arguments
-        monitor: quantity to be monitored.
-        min_delta: minimum change in the monitored quantity
+    Args:
+        monitor (str): quantity to be monitored. Default: ``'val_loss'``.
+        min_delta (float): minimum change in the monitored quantity
             to qualify as an improvement, i.e. an absolute
-            change of less than min_delta, will count as no
-            improvement.
-        patience: number of epochs with no improvement
-            after which training will be stopped.
-        verbose: verbosity mode.
-        mode: one of {auto, min, max}. In `min` mode,
+            change of less than `min_delta`, will count as no
+            improvement. Default: ``0``.
+        patience (int): number of epochs with no improvement
+            after which training will be stopped. Default: ``0``.
+        verbose (bool): verbosity mode. Default: ``0``.
+        mode (str): one of {auto, min, max}. In `min` mode,
             training will stop when the quantity
             monitored has stopped decreasing; in `max`
             mode it will stop when the quantity
             monitored has stopped increasing; in `auto`
             mode, the direction is automatically inferred
-            from the name of the monitored quantity.
+            from the name of the monitored quantity. Default: ``'auto'``.
+        strict (bool): whether to crash the training if `monitor` is
+            not found in the metrics. Default: ``True``.
 
+    Example::
+
+        from pytorch_lightning import Trainer
+        from pytorch_lightning.callbacks import EarlyStopping
+
+        early_stopping = EarlyStopping('val_loss')
+        Trainer(early_stop_callback=early_stopping)
     """
 
     def __init__(self, monitor='val_loss',
-                 min_delta=0.0, patience=0, verbose=0, mode='auto'):
+                 min_delta=0.0, patience=0, verbose=0, mode='auto', strict=True):
         super(EarlyStopping, self).__init__()
 
         self.monitor = monitor
         self.patience = patience
         self.verbose = verbose
+        self.strict = strict
         self.min_delta = min_delta
         self.wait = 0
         self.stopped_epoch = 0
 
         if mode not in ['auto', 'min', 'max']:
-            logging.info(f'EarlyStopping mode {mode} is unknown, fallback to auto mode.')
+            if self.verbose > 0:
+                logging.info(f'EarlyStopping mode {mode} is unknown, fallback to auto mode.')
             mode = 'auto'
 
         if mode == 'min':
@@ -115,6 +132,22 @@ class EarlyStopping(Callback):
 
         self.on_train_begin()
 
+    def check_metrics(self, logs):
+        monitor_val = logs.get(self.monitor)
+        error_msg = (f'Early stopping conditioned on metric `{self.monitor}`'
+                     f' which is not available. Available metrics are:'
+                     f' `{"`, `".join(list(logs.keys()))}`')
+
+        if monitor_val is None:
+            if self.strict:
+                raise RuntimeError(error_msg)
+            elif self.verbose > 0:
+                warnings.warn(error_msg, RuntimeWarning)
+
+            return False
+
+        return True
+
     def on_train_begin(self, logs=None):
         # Allow instances to be re-used
         self.wait = 0
@@ -122,16 +155,11 @@ class EarlyStopping(Callback):
         self.best = np.Inf if self.monitor_op == np.less else -np.Inf
 
     def on_epoch_end(self, epoch, logs=None):
-        current = logs.get(self.monitor)
         stop_training = False
-        if current is None:
-            warnings.warn(
-                f'Early stopping conditioned on metric `{self.monitor}`'
-                f' which is not available. Available metrics are: {",".join(list(logs.keys()))}',
-                RuntimeWarning)
-            stop_training = True
+        if not self.check_metrics(logs):
             return stop_training
 
+        current = logs.get(self.monitor)
         if self.monitor_op(current - self.min_delta, self.best):
             self.best = current
             self.wait = 0
@@ -150,20 +178,22 @@ class EarlyStopping(Callback):
 
 
 class ModelCheckpoint(Callback):
-    """Save the model after every epoch.
+    r"""
 
-    The `filepath` can contain named formatting options,
-    which will be filled the value of `epoch` and
-    keys in `logs` (passed in `on_epoch_end`).
-    For example: if `filepath` is `weights.{epoch:02d}-{val_loss:.2f}.hdf5`,
-    then the model checkpoints will be saved with the epoch number and
-    the validation loss in the filename.
+    Save the model after every epoch.
 
-    # Arguments
-        filepath: string, path to save the model file.
-        monitor: quantity to monitor.
-        verbose: verbosity mode, 0 or 1.
-        save_top_k: if `save_top_k == k`,
+    Args:
+        filepath (str): path to save the model file.
+            Can contain named formatting options to be auto-filled.
+
+            Example::
+
+                # save epoch and val_loss in name
+                ModelCheckpoint(filepath='{epoch:02d}-{val_loss:.2f}.hdf5')
+                # saves file like: /path/epoch_2-val_loss_0.2.hdf5
+        monitor (str): quantity to monitor.
+        verbose (bool): verbosity mode, 0 or 1.
+        save_top_k (int): if `save_top_k == k`,
             the best k models according to
             the quantity monitored will be saved.
             if `save_top_k == 0`, no models are saved.
@@ -172,7 +202,7 @@ class ModelCheckpoint(Callback):
             if `save_top_k >= 2` and the callback is called multiple
             times inside an epoch, the name of the saved file will be
             appended with a version count starting with `v0`.
-        mode: one of {auto, min, max}.
+        mode (str): one of {auto, min, max}.
             If `save_top_k != 0`, the decision
             to overwrite the current save file is made
             based on either the maximization or the
@@ -180,11 +210,20 @@ class ModelCheckpoint(Callback):
             this should be `max`, for `val_loss` this should
             be `min`, etc. In `auto` mode, the direction is
             automatically inferred from the name of the monitored quantity.
-        save_weights_only: if True, then only the model's weights will be
+        save_weights_only (bool): if True, then only the model's weights will be
             saved (`model.save_weights(filepath)`), else the full model
             is saved (`model.save(filepath)`).
-        period: Interval (number of epochs) between checkpoints.
+        period (int): Interval (number of epochs) between checkpoints.
 
+    Example::
+
+        from pytorch_lightning import Trainer
+        from pytorch_lightning.callbacks import ModelCheckpoint
+
+        checkpoint_callback = ModelCheckpoint(filepath='my_path')
+        Trainer(checkpoint_callback=checkpoint_callback)
+
+        # saves checkpoints to my_path whenever 'val_loss' has a new min
     """
 
     def __init__(self, filepath, monitor='val_loss', verbose=0,
@@ -330,11 +369,20 @@ class ModelCheckpoint(Callback):
 
 
 class GradientAccumulationScheduler(Callback):
-    """Change gradient accumulation factor according to scheduling.
+    r"""
+    Change gradient accumulation factor according to scheduling.
 
-    # Arguments
-        scheduling: dict, scheduling in format {epoch: accumulation_factor}
+    Args:
+        scheduling (dict): scheduling in format {epoch: accumulation_factor}
 
+    Example::
+
+        from pytorch_lightning import Trainer
+        from pytorch_lightning.callbacks import GradientAccumulationScheduler
+
+        # at epoch 5 start accumulating every 2 batches
+        accumulator = GradientAccumulationScheduler(scheduling: {5: 2})
+        Trainer(accumulate_grad_batches=accumulator)
     """
 
     def __init__(self, scheduling: dict):

pytorch_lightning/core/__init__.py

@@ -1,17 +1,16 @@
 """
-Lightning Module interface
-==========================
+A LightningModule is a strict superclass of torch.nn.Module but provides an interface to standardize
+the "ingredients" for a research or production system.
 
-A lightning module is a strict superclass of nn.Module, it provides a standard interface
- for the trainer to interact with the model.
+- The model/system definition (__init__)
+- The model/system computations (forward)
+- What happens in the training loop (training_step, training_end)
+- What happens in the validation loop (validation_step, validation_end)
+- What happens in the test loop (test_step, test_end)
+- What optimizers to use (configure_optimizers)
+- What data to use (train_dataloader, val_dataloader, test_dataloader)
 
-The easiest thing to do is copy the minimal example below and modify accordingly.
-
-Otherwise, to Define a Lightning Module, implement the following methods:
-
-
-Minimal example
----------------
+Most methods are optional. Here's a minimal example.
 
 .. code-block:: python
 
@@ -28,14 +27,12 @@ Minimal example
 
         def __init__(self):
             super(CoolModel, self).__init__()
-            # not the best model...
             self.l1 = torch.nn.Linear(28 * 28, 10)
 
         def forward(self, x):
             return torch.relu(self.l1(x.view(x.size(0), -1)))
 
         def training_step(self, batch, batch_idx):
-            # REQUIRED
             x, y = batch
             y_hat = self.forward(x)
             return {'loss': F.cross_entropy(y_hat, y)}
@@ -48,8 +45,8 @@ Minimal example
 
         def validation_end(self, outputs):
             # OPTIONAL
-            avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
-            return {'avg_val_loss': avg_loss}
+            val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean()
+            return {'val_loss': val_loss_mean}
 
         def test_step(self, batch, batch_idx):
             # OPTIONAL
@@ -59,8 +56,8 @@ Minimal example
 
         def test_end(self, outputs):
             # OPTIONAL
-            avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()
-            return {'avg_test_loss': avg_loss}
+            test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean()
+            return {'test_loss': test_loss_mean}
 
         def configure_optimizers(self):
             # REQUIRED
@@ -85,66 +82,19 @@ Minimal example
             return DataLoader(MNIST(os.getcwd(), train=False, download=True,
                               transform=transforms.ToTensor()), batch_size=32)
 
-
-How do these methods fit into the broader training?
----------------------------------------------------
-
-The LightningModule interface is on the right. Each method corresponds
- to a part of a research project. Lightning automates everything not in blue.
-
-.. figure::  docs/source/_static/images/overview_flat.jpg
-   :align:   center
-
-   Overview.
-
-
-Optional Methods
-----------------
-
-**add_model_specific_args**
+Once you've defined the LightningModule, fit  it using a trainer.
 
 .. code-block:: python
+   trainer = pl.Trainer()
+   model = CoolModel()
 
-    @staticmethod
-    def add_model_specific_args(parent_parser, root_dir)
+   trainer.fit(model)
 
-Lightning has a list of default argparse commands.
- This method is your chance to add or modify commands specific to your model.
- The `hyperparameter argument parser
-  <https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser>`_
- is available anywhere in your model by calling self.hparams.
-
-**Return**
-An argument parser
-
-**Example**
-
-.. code-block:: python
-
-    @staticmethod
-    def add_model_specific_args(parent_parser, root_dir):
-        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
-
-        # param overwrites
-        # parser.set_defaults(gradient_clip_val=5.0)
-
-        # network params
-        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
-        parser.add_argument('--in_features', default=28*28)
-        parser.add_argument('--out_features', default=10)
-        # use 500 for CPU, 50000 for GPU to see speed difference
-        parser.add_argument('--hidden_dim', default=50000)
-
-        # data
-        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
-
-        # training params (opt)
-        parser.opt_list('--learning_rate', default=0.001, type=float,
-                        options=[0.0001, 0.0005, 0.001, 0.005], tunable=False)
-        parser.opt_list('--batch_size', default=256, type=int,
-                        options=[32, 64, 128, 256], tunable=False)
-        parser.opt_list('--optimizer_name', default='adam', type=str,
-                        options=['adam'], tunable=False)
-        return parser
+Check out this
+`COLAB <https://colab.research.google.com/drive/1F_RNcHzTfFuQf-LeKvSlud6x7jXYkG31#scrollTo=HOk9c4_35FKg>`_
+for a live demo.
 
 """
+from .lightning import LightningModule
+
+__all__ = ['LightningModule']

pytorch_lightning/core/decorators.py

@@ -1,4 +1,5 @@
 import traceback
+from functools import wraps
 
 
 def data_loader(fn):
@@ -8,6 +9,7 @@ def data_loader(fn):
     :return:
     """
 
+    wraps(fn)
     attr_name = '_lazy_' + fn.__name__
 
     def _get_data_loader(self):

pytorch_lightning/core/grads.py

@@ -10,7 +10,7 @@ class GradInformation(nn.Module):
     def grad_norm(self, norm_type):
         results = {}
         total_norm = 0
-        for i, p in enumerate(self.parameters()):
+        for name, p in self.named_parameters():
             if p.requires_grad:
                 try:
                     param_norm = p.grad.data.norm(norm_type)
@@ -18,7 +18,7 @@ class GradInformation(nn.Module):
                     norm = param_norm ** (1 / norm_type)
 
                     grad = round(norm.data.cpu().numpy().flatten()[0], 3)
-                    results['grad_{}_norm_{}'.format(norm_type, i)] = grad
+                    results['grad_{}_norm_{}'.format(norm_type, name)] = grad
                 except Exception:
                     # this param had no grad
                     pass

pytorch_lightning/core/hooks.py

@@ -1,5 +1,6 @@
 """
-# Hooks
+Hooks
+=====
 
 There are cases when you might want to do something different at different parts of the training/validation loop.
  To enable a hook, simply override the method in your LightningModule and the trainer will call it at the correct time.
@@ -123,12 +124,13 @@ class ModelHooks(torch.nn.Module):
         """
         pass
 
-    def backward(self, use_amp, loss, optimizer):
+    def backward(self, use_amp, loss, optimizer, optimizer_idx):
         """Override backward with your own implementation if you need to
 
         :param use_amp: Whether amp was requested or not
         :param loss: Loss is already scaled by accumulated grads
         :param optimizer: Current optimizer being used
+        :param optimizer_idx: Index of the current optimizer being used
         :return:
 
         Called to perform backward step.

pytorch_lightning/core/lightning.py

@@ -1,110 +1,114 @@
+import collections
+import logging
 import os
 import warnings
-import collections
+from abc import ABC, abstractmethod
 from argparse import Namespace
 
+
+import pandas as pd
 import torch
 import torch.distributed as dist
-
+#
 from pytorch_lightning.core.decorators import data_loader
 from pytorch_lightning.core.grads import GradInformation
 from pytorch_lightning.core.hooks import ModelHooks
-from pytorch_lightning.core.memory import ModelSummary
 from pytorch_lightning.core.saving import ModelIO
-from pytorch_lightning.trainer.trainer_io import load_hparams_from_tags_csv
-import logging
+from pytorch_lightning.core.memory import ModelSummary
 from pytorch_lightning.overrides.data_parallel import LightningDistributedDataParallel
 
 
-class LightningModule(GradInformation, ModelIO, ModelHooks):
-    """
-    A LightningModule has the following properties which you can access at any time
-
-    **logger**
-    A reference to the logger you passed into trainer.
-    Passing a logger is optional. If you don't pass one in, Lightning will create one
-     for you automatically. This logger saves logs to `/os.getcwd()/lightning_logs`::
-
-        Trainer(logger=your_logger)
-
-
-    Call it from anywhere in your LightningModule to add metrics, images, etc...
-     whatever your logger supports.
-
-    Here is an example using the TestTubeLogger (which is a wrapper
-     on 'PyTorch SummaryWriter <https://pytorch.org/docs/stable/tensorboard.html>`_
-     with versioned folder structure).
-
-    .. code-block:: python
-
-        # if logger is a tensorboard logger or TestTubeLogger
-        self.logger.experiment.add_embedding(...)
-        self.logger.experiment.log({'val_loss': 0.9})
-        self.logger.experiment.add_scalars(...)
-
-
-    **trainer**
-    Last resort access to any state the trainer has.
-     Changing certain properties here could affect your training run.
-
-    .. code-block:: python
-
-        self.trainer.optimizers
-        self.trainer.current_epoch
-        ...
-
-    Debugging
-    ---------
-
-    The LightningModule also offers these tricks to help debug.
-
-    **example_input_array**
-
-    In the LightningModule init, you can set a dummy tensor for this property
-    to get a print out of sizes coming into and out of every layer.
-
-    .. code-block:: python
-
-        def __init__(self):
-            # put the dimensions of the first input to your system
-            self.example_input_array = torch.rand(5, 28 * 28)
-
-
-    """
-
+class LightningModule(ABC, GradInformation, ModelIO, ModelHooks):
     def __init__(self, *args, **kwargs):
         super(LightningModule, self).__init__(*args, **kwargs)
 
         #: Current dtype
         self.dtype = torch.FloatTensor
+
         self.exp_save_path = None
+
         #: The current epoch
         self.current_epoch = 0
+
         #: Total training batches seen across all epochs
         self.global_step = 0
+
         self.loaded_optimizer_states_dict = {}
+
+        #: Pointer to the trainer object
         self.trainer = None
+
+        #: Pointer to the logger object
         self.logger = None
         self.example_input_array = None
 
-        # track if gpu was requested for checkpointing
         #: True if your model is currently running on GPUs.
         #: Useful to set flags around the LightningModule for different CPU vs GPU behavior.
         self.on_gpu = False
+
+        #: True if using dp
         self.use_dp = False
+
+        #: True if using ddp
         self.use_ddp = False
+
+        #: True if using ddp2
         self.use_ddp2 = False
+
+        #: True if using amp
         self.use_amp = False
 
+    @abstractmethod
     def forward(self, *args, **kwargs):
-        """
-        Expand model in into whatever you need.
-        Also need to return the target
-        :param x:
-        :return:
-        """
-        raise NotImplementedError
+        r"""
+        Same as torch.nn.Module.forward(), however in Lightning you want this to define
+        the  operations you want to use for prediction (ie: on a server or as a feature extractor).
 
+        Normally you'd call self.forward() from your training_step() method. This makes it easy to write a complex
+        system for training with the outputs you'd want in a prediction setting.
+
+        Args:
+            x (tensor): Whatever  you decide to define in the forward method
+
+        Return:
+            Predicted output
+
+        Example
+        -------
+
+        .. code-block:: python
+
+            # example if we were using this model as a feature extractor
+            def forward(self, x):
+                feature_maps = self.convnet(x)
+                return feature_maps
+
+            def training_step(self, batch, batch_idx):
+                x, y = batch
+                feature_maps = self.forward(x)
+                logits = self.classifier(feature_maps)
+
+                # ...
+                return loss
+
+            # splitting it this way allows model to be used a feature extractor
+            model = MyModelAbove()
+
+            inputs = server.get_request()
+            results = model(inputs)
+            server.write_results(results)
+
+            # -------------
+            # This is in stark contrast to torch.nn.Module where normally you would have this:
+            def forward(self, batch):
+                x, y = batch
+                feature_maps = self.convnet(x)
+                logits = self.classifier(feature_maps)
+                return logits
+
+        """
+
+    @abstractmethod
     def training_step(self, *args, **kwargs):
         """return loss, dict with metrics for tqdm
 
@@ -133,7 +137,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
 
                 logger_logs = {'training_loss': loss} # optional (MUST ALL BE TENSORS)
 
-                # if using TestTubeLogger or TensorboardLogger you can nest scalars
+                # if using TestTubeLogger or TensorBoardLogger you can nest scalars
                 logger_logs = {'losses': logger_logs} # optional (MUST ALL BE TENSORS)
 
                 output = {
@@ -169,7 +173,6 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         You can also return a -1 instead of a dict to stop the current loop. This is useful
          if you want to break out of the current training epoch early.
         """
-        raise NotImplementedError
 
     def training_end(self, *args, **kwargs):
         """return loss, dict with metrics for tqdm
@@ -246,12 +249,21 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         pass
 
     def validation_step(self, *args, **kwargs):
-        """return whatever outputs will need to be aggregated in validation_end
+        r"""
 
-        :param batch: The output of your dataloader. A tensor, tuple or list
-        :param int batch_idx: Integer displaying which batch this is
-        :param int dataloader_idx: Integer displaying which dataloader this is (only if multiple val datasets used)
-        :return dict: Dict or OrderedDict - passed to the validation_end step
+        This is the validation loop. It is called for each batch of the validation set.
+        Whatever is returned from here will be passed in as a list on validation_end.
+        In this step you'd normally generate examples or calculate anything of interest such as accuracy.
+
+        Args:
+            batch (torch.nn.Tensor | (Tensor, Tensor) | [Tensor, Tensor]): The output of your dataloader.
+                A tensor, tuple or list
+            batch_idx (int): The index of this batch
+            dataloader_idx (int): The index of the dataloader that produced this batch (only if multiple
+                val datasets used)
+
+        Return:
+            Dict or OrderedDict - passed to the validation_end step
 
         .. code-block:: python
 
@@ -261,14 +273,6 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
             # if you have multiple val dataloaders:
             def validation_step(self, batch, batch_idx, dataloader_idxdx)
 
-        If you don't need to validate you don't need to implement this method.
-         In this step you'd normally generate examples or calculate anything of interest such as accuracy.
-
-        When the validation_step is called, the model has been put in eval mode and PyTorch gradients
-         have been disabled. At the end of validation, model goes back to training mode and gradients are enabled.
-
-        The dict you return here will be available in the `validation_end` method.
-
         Example
         -------
 
@@ -310,7 +314,10 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
             def validation_step(self, batch, batch_idx, dataset_idx):
                 # dataset_idx tells you which dataset this is.
 
-        The `dataset_idx` corresponds to the order of datasets returned in `val_dataloader`.
+        .. note:: If you don't need to validate you don't need to implement this method.
+
+        .. note:: When the validation_step is called, the model has been put in eval mode and PyTorch gradients
+            have been disabled. At the end of validation, model goes back to training mode and gradients are enabled.
         """
         pass
 
@@ -521,20 +528,27 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         pass
 
     def configure_ddp(self, model, device_ids):
-        """Override to init DDP in a different way or use your own wrapper.
+        r"""
 
-        :param model:
-        :param device_ids:
-        :return: DDP wrapped model
+        Override to init DDP in your own way or with your own wrapper.
+        The only requirements are that:
 
-        Overwrite to define your own DDP implementation init.
-        The only requirement is that:
         1. On a validation batch the call goes to model.validation_step.
         2. On a training batch the call goes to model.training_step.
         3. On a testing batch, the call goes to model.test_step
 
+        Args:
+            model (LightningModule): the LightningModule currently being optimized
+            device_ids (list): the list of GPU ids
+
+        Return:
+            DDP wrapped model
+
+        Example
+        -------
         .. code-block:: python
 
+            # default implementation used in Trainer
             def configure_ddp(self, model, device_ids):
                 # Lightning DDP simply routes to test_step, val_step, etc...
                 model = LightningDistributedDataParallel(
@@ -554,11 +568,17 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         return model
 
     def init_ddp_connection(self, proc_rank, world_size):
-        """Connect all procs in the world using the env:// init
-        Use the first node as the root address
+        r"""
 
-        Override to init DDP in your own way.
+        Override to define your custom way of setting up a distributed environment.
 
+        Lightning's implementation uses env:// init by default and sets the first node as root.
+
+        Args:
+            proc_rank (int): The current process rank within the node.
+            world_size (int): Number of GPUs being use across all nodes. (num_nodes*nb_gpu_nodes).
+        Example
+        -------
         .. code-block:: python
 
             def init_ddp_connection(self):
@@ -589,7 +609,11 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
 
                 root_node = self.trainer.resolve_root_node_address(root_node)
                 os.environ['MASTER_ADDR'] = root_node
-                dist.init_process_group('nccl', rank=self.proc_rank, world_size=self.world_size)
+                dist.init_process_group(
+                    'nccl',
+                    rank=self.proc_rank,
+                    world_size=self.world_size
+                )
 
         """
         # use slurm job id for the port number
@@ -602,7 +626,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
             # all ports should be in the 10k+ range
             default_port = int(default_port) + 15000
 
-        except Exception as e:
+        except Exception:
             default_port = 12910
 
         # if user gave a port number, use that one instead
@@ -622,19 +646,24 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         dist.init_process_group('nccl', rank=proc_rank, world_size=world_size)
 
     def configure_apex(self, amp, model, optimizers, amp_level):
-        """
+        r"""
         Override to init AMP your own way
         Must return a model and list of optimizers
-        :param amp:
-        :param model:
-        :param optimizers:
-        :param amp_level:
-        :return: Apex wrapped model and optimizers
 
-        Overwrite to define your own Apex implementation init.
+        Args:
+            amp (object): pointer to amp library object
+            model (LightningModule): pointer to current lightningModule
+            optimizers (list): list of optimizers passed in configure_optimizers()
+            amp_level (str): AMP mode chosen ('O1', 'O2', etc...)
 
+        Return:
+            Apex wrapped model and optimizers
+
+        Example
+        -------
         .. code-block:: python
 
+            # Default implementation used by Trainer.
             def configure_apex(self, amp, model, optimizers, amp_level):
                 model, optimizers = amp.initialize(
                     model, optimizers, opt_level=amp_level,
@@ -648,23 +677,17 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
 
         return model, optimizers
 
+    @abstractmethod
     def configure_optimizers(self):
-        """Return a list of optimizers and a list of schedulers (could be empty)
+        r"""
+        This is where you choose what optimizers and learning-rate schedulers to use in your optimization.
+        Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple.
 
-        :return: any of these 3 options:
+        Return: any of these 3 options:
             - Single optimizer
             - List or Tuple - List of optimizers
             - Two lists - The first list has multiple optimizers, the second a list of learning-rate schedulers
 
-        Set up as many optimizers and (optionally) learning rate schedulers as you need.
-         Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple.
-         Lightning will call .backward() and .step() on each one in every epoch.
-         If you use 16 bit precision it will also handle that.
-
-        .. note:: If you use multiple optimizers, training_step will have an additional `optimizer_idx` parameter.
-
-        .. note:: If you use LBFGS lightning handles the closure function automatically for you.
-
         Example
         -------
 
@@ -688,27 +711,39 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
                 discriminator_sched = CosineAnnealing(discriminator_opt, T_max=10)
                 return [generator_opt, disriminator_opt], [discriminator_sched]
 
-        If you need to control how often those optimizers step or override the default .step() schedule,
-         override the `optimizer_step` hook.
+        .. note:: Lightning calls .backward() and .step() on each optimizer and learning rate scheduler as needed.
+
+        .. note:: If you use 16-bit precision (use_amp=True), Lightning will automatically
+            handle the optimizers for you.
+
+        .. note:: If you use multiple optimizers, training_step will have an additional `optimizer_idx` parameter.
+
+        .. note:: If you use LBFGS lightning handles the closure function automatically for you
+
+        .. note:: If you use multiple optimizers, gradients will be calculated only
+            for the parameters of current optimizer at each training step.
+
+        .. note:: If you need to control how often those optimizers step or override the default .step() schedule,
+            override the `optimizer_step` hook.
+
 
         """
-        raise NotImplementedError
 
     def optimizer_step(self, epoch, batch_idx, optimizer, optimizer_idx, second_order_closure=None):
-        """Do something instead of the standard optimizer behavior
+        r"""
 
-        :param int epoch:
-        :param int batch_idx:
-        :param optimizer:
-        :param optimizer_idx:
-        :param second_order_closure: closure for second order methods
-        :return:
+        Override this method to adjust the default way the Trainer calls each optimizer. By default, Lightning
+        calls .step() and zero_grad() as shown in the example once per optimizer.
 
-        Calls `.step()` and `.zero_grad` for each optimizer.
-        You can override this method to adjust how you do the optimizer step for each optimizer
-
-        Called once per optimizer
+        Args:
+            epoch (int): Current epoch
+            batch_idx (int): Index of current batch
+            optimizer (torch.nn.Optimizer): A PyTorch optimizer
+            optimizer_idx (int): If you used multiple optimizers this indexes into that list
+            second_order_closure (int): closure for second order methods
 
+        Example
+        -------
         .. code-block:: python
 
             # DEFAULT
@@ -734,7 +769,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
                 # add as many optimizers as you want
 
 
-        This step allows you to do a lot of non-standard training tricks such as learning-rate warm-up:
+        Here's another example showing how to use this for more advanced things such as learning-rate warm-up:
 
         .. code-block:: python
 
@@ -760,18 +795,22 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         optimizer.zero_grad()
 
     def tbptt_split_batch(self, batch, split_size):
-        """
-        Return list of batch splits. Each split will be passed to forward_step to enable truncated
-        back propagation through time. The default implementation splits root level Tensors and
-        Sequences at dim=1 (i.e. time dim). It assumes that each time dim is the same length.
+        r"""
 
-        :param batch:
-        :param split_size:
-        :return:
+        When using truncated backpropagation through time, each batch must be split along the time dimension.
+        Lightning handles this by default, but  for custom behavior override this function.
 
-        Called in the training loop after on_batch_start if `truncated_bptt_steps > 0`.
-         Each returned batch split is passed separately to training_step(...).
+        Args:
+            batch (torch.nn.Tensor): Current batch
+            split_size (int): How big the split  is
 
+        Return:
+            list of batch splits. Each split will be passed to forward_step to enable truncated
+            back propagation through time. The default implementation splits root level Tensors and
+            Sequences at dim=1 (i.e. time dim). It assumes that each time dim is the same length.
+
+        Example
+        -------
         .. code-block:: python
 
             def tbptt_split_batch(self, batch, split_size):
@@ -791,6 +830,10 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
                   splits.append(batch_split)
 
               return splits
+
+        .. note:: Called in the training loop after on_batch_start if `truncated_bptt_steps > 0`.
+            Each returned batch split is passed separately to training_step(...).
+
         """
         time_dims = [len(x[0]) for x in batch if isinstance(
             x, torch.Tensor) or isinstance(x, collections.Sequence)]
@@ -815,6 +858,7 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         return splits
 
     @data_loader
+    @abstractmethod
     def train_dataloader(self):
         """Implement a PyTorch DataLoader
 
@@ -842,7 +886,6 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
 
 
         """
-        raise NotImplementedError
 
     @data_loader
     def tng_dataloader(self):
@@ -850,25 +893,20 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
 
         .. warning:: Deprecated in v0.5.0. use train_dataloader instead.
         """
-        try:
-            output = self.tng_dataloader()
-            warnings.warn("`tng_dataloader` has been renamed to `train_dataloader` since v0.5.0"
-                          " and will be removed in v0.8.0", DeprecationWarning)
-            return output
-        except NotImplementedError:
-            raise NotImplementedError
+        output = self.train_dataloader()
+        warnings.warn("`tng_dataloader` has been renamed to `train_dataloader` since v0.5.0"
+                      " and will be removed in v0.8.0", DeprecationWarning)
+        return output
 
     @data_loader
     def test_dataloader(self):
-        """Implement a PyTorch DataLoader.
-
-        :return: PyTorch DataLoader
-
-        If you don't need a test dataset and a test_step, you don't need to implement this method.
+        r"""
 
         Called by lightning during test loop. Make sure to use the @pl.data_loader decorator,
-         this ensures not calling this function until the data are needed.
-         If you want to change the data during every epoch DON'T use the data_loader decorator.
+        this ensures not calling this function until the data are needed.
+
+        Return:
+            PyTorch DataLoader
 
         Example
         -------
@@ -887,20 +925,22 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
 
                 return loader
 
+        .. note:: If you don't need a test dataset and a test_step, you don't need to implement this method.
+
+        .. note:: If you want to change the data during every epoch DON'T use the data_loader decorator.
+
         """
         return None
 
     @data_loader
     def val_dataloader(self):
-        """Implement a PyTorch DataLoader.
-
-        :return: PyTorch DataLoader or list of PyTorch Dataloaders.
-
-        If you don't need a validation dataset and a validation_step, you don't need to implement this method.
+        r"""
 
         Called by lightning during validation loop. Make sure to use the @pl.data_loader decorator,
-         this ensures not calling this function until the data are needed.
-         If you want to change the data during every epoch DON'T use the data_loader decorator.
+        this ensures not calling this function until the data are needed.
+
+        Return:
+            PyTorch DataLoader
 
         Example
         -------
@@ -924,30 +964,67 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
             def val_dataloader(self):
                 return [loader_a, loader_b, ..., loader_n]
 
-        In the case where you return multiple `val_dataloaders`, the `validation_step`
-         will have an arguement `dataset_idx` which matches the order here.
+        Example
+        -------
+
+        .. code-block:: python
+
+            @pl.data_loader
+            def val_dataloader(self):
+                transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+                dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True)
+                loader = torch.utils.data.DataLoader(
+                    dataset=dataset,
+                    batch_size=self.hparams.batch_size,
+                    shuffle=True
+                )
+
+                return loader
+
+            # can also return multiple dataloaders
+            @pl.data_loader
+            def val_dataloader(self):
+                return [loader_a, loader_b, ..., loader_n]
+
+        .. note:: If you don't need a validation dataset and a validation_step, you don't need to implement this method.
+
+        .. note:: If you want to change the data during every epoch DON'T use the data_loader decorator.
+
+        .. note:: In the case where you return multiple `val_dataloaders`, the `validation_step`
+            will have an argument `dataset_idx` which matches the order here.
         """
         return None
 
     @classmethod
-    def load_from_metrics(cls, weights_path, tags_csv):
-        """Primary way of loading model from csv weights path.
+    def load_from_metrics(cls, weights_path, tags_csv, map_location=None):
+        r"""
 
-        :param str weights_path: Path to a PyTorch checkpoint
-        :param str tags_csv: Path to meta_tags.csv file generated by the test-tube Experiment
-        :param dict map_location: A dictionary mapping saved weight GPU devices to new GPU devices
-            for mapping storage {'cuda:1':'cuda:0'}
-        :return: The pretrained LightningModule
+        You should use `load_from_checkpoint` instead!
+        However, if your .ckpt weights don't have the hyperparameters saved, use this method  to pass
+        in a .csv with the hparams you'd like to use. These will  be converted  into a argparse.Namespace
+        and passed into  your LightningModule for use.
 
-        If you're using test tube, there is an alternate method which uses the meta_tags.csv
-        file from test-tube to rebuild the model. The meta_tags.csv file can be found in the
-        test-tube experiment save_dir.
+        Args:
 
+            weights_path (str): Path to a PyTorch checkpoint
+            tags_csv (str): Path to a .csv with two columns (key, value) as in this
+                Example::
+                    key,value
+                    drop_prob,0.2
+                    batch_size,32
+
+            map_location (dict): A dictionary mapping saved weight GPU devices to new
+                GPU devices (example: {'cuda:1':'cuda:0'})
+        Return:
+            LightningModule with loaded weights
+
+        Example
+        -------
         .. code-block:: python
 
             pretrained_model = MyLightningModule.load_from_metrics(
                 weights_path='/path/to/pytorch_checkpoint.ckpt',
-                tags_csv='/path/to/test_tube/experiment/version/meta_tags.csv',
+                tags_csv='/path/to/hparams_file.csv',
                 on_gpu=True,
                 map_location=None
             )
@@ -956,28 +1033,15 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
             pretrained_model.eval()
             pretrained_model.freeze()
             y_hat = pretrained_model(x)
-
-        This is the easiest/fastest way which loads hyperparameters and weights from a checkpoint,
-        such as the one saved by the `ModelCheckpoint` callback
-
-        .. code-block:: python
-
-            pretrained_model = MyLightningModule.load_from_checkpoint(
-                checkpoint_path='/path/to/pytorch_checkpoint.ckpt'
-            )
-
-            # predict
-            pretrained_model.eval()
-            pretrained_model.freeze()
-            y_hat = pretrained_model(x)
-
         """
+
         hparams = load_hparams_from_tags_csv(tags_csv)
         hparams.__setattr__('on_gpu', False)
 
-        # load on CPU only to avoid OOM issues
-        # then its up to user to put back on GPUs
-        checkpoint = torch.load(weights_path, map_location=lambda storage, loc: storage)
+        if map_location is not None:
+            checkpoint = torch.load(weights_path, map_location=map_location)
+        else:
+            checkpoint = torch.load(weights_path, map_location=lambda storage, loc: storage)
 
         # load the state_dict on the model automatically
         model = cls(hparams)
@@ -989,17 +1053,64 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         return model
 
     @classmethod
-    def load_from_checkpoint(cls, checkpoint_path):
-        """
-        Primary way of loading model from a checkpoint
-        :param checkpoint_path:
-        :param map_location: dic for mapping storage {'cuda:1':'cuda:0'}
-        :return:
+    def load_from_checkpoint(cls, checkpoint_path, map_location=None):
+        r"""
+
+        Primary way of loading model from a checkpoint. When Lightning saves a checkpoint
+        it  stores  the hyperparameters in the checkpoint if you initialized your  LightningModule
+        with an argument  called `hparams` which is a Namespace or dictionary of hyperparameters
+
+            Example
+            -------
+            .. code-block:: python
+
+                # --------------
+                # Case 1
+                # when using Namespace (output of using Argparse to parse command line arguments)
+                from argparse import Namespace
+                hparams = Namespace(**{'learning_rate': 0.1})
+
+                model = MyModel(hparams)
+
+                class MyModel(pl.LightningModule):
+                    def __init__(self, hparams):
+                        self.learning_rate = hparams.learning_rate
+
+                # --------------
+                # Case 2
+                # when using a dict
+                model = MyModel({'learning_rate': 0.1})
+
+                class MyModel(pl.LightningModule):
+                    def __init__(self, hparams):
+                        self.learning_rate = hparams['learning_rate']
+
+        Args:
+            checkpoint_path (str): Path to checkpoint.
+            map_location (dic): If your checkpoint saved from a GPU model and you now load on CPUs
+                or a different number of GPUs, use this to map to the new setup.
+
+        Return:
+            LightningModule with loaded weights.
+
+        Example
+        -------
+        .. code-block:: python
+
+            # load weights without mapping
+            MyLightningModule.load_from_checkpoint('path/to/checkpoint.ckpt')
+
+            # load weights mapping all weights from GPU 1 to GPU 0
+            map_location = {'cuda:1':'cuda:0'}
+            MyLightningModule.load_from_checkpoint('path/to/checkpoint.ckpt', map_location=map_location)
+
         """
 
-        # load on CPU only to avoid OOM issues
-        # then its up to user to put back on GPUs
-        checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
+        if map_location is not None:
+            checkpoint = torch.load(checkpoint_path, map_location=map_location)
+        else:
+            checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
+
         try:
             ckpt_hparams = checkpoint['hparams']
         except KeyError:
@@ -1023,8 +1134,11 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         logging.info('\n' + model_summary.__str__())
 
     def freeze(self):
-        """Freeze all params for inference
+        r"""
+        Freeze all params for inference
 
+        Example
+        -------
         .. code-block:: python
 
             model = MyLightningModule(...)
@@ -1051,13 +1165,13 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         self.train()
 
     def on_load_checkpoint(self, checkpoint):
-        """
+        r"""
+        Called by lightning to restore your model.
+        If you saved something with **on_save_checkpoint** this is your chance to restore this.
 
-        :param checkpoint:
+        Args:
+            checkpoint (dict): Loaded checkpoint
 
-        Called by lightning to restore your model. Lighting auto-restores global step, epoch, etc...
-         It also restores the model state_dict.
-         If you saved something with **on_save_checkpoint** this is your chance to restore this.
 
         Example
         -------
@@ -1068,17 +1182,19 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
                 # 99% of the time you don't need to implement this method
                 self.something_cool_i_want_to_save = checkpoint['something_cool_i_want_to_save']
 
+        .. note:: Lighting auto-restores global step, epoch, and all training state including amp scaling.
+            No need for you to restore anything regarding training.
         """
         pass
 
     def on_save_checkpoint(self, checkpoint):
-        """
+        r"""
 
-        :param checkpoint:
+        Called by lightning when saving a  checkpoint  to give you a chance to store anything else you
+        might want to  save
 
-        Called by lightning to checkpoint your model. Lightning saves the training state
-         (current epoch, global_step, etc) and also saves the model state_dict.
-         If you want to save anything else, use this method to add your own key-value pair.
+        Args:
+            checkpoint (dic): Checkpoint to be saved
 
         Example
         -------
@@ -1089,5 +1205,37 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
                 # 99% of use cases you don't need to implement this method
                 checkpoint['something_cool_i_want_to_save'] = my_cool_pickable_object
 
+        .. note:: Lighting saves all aspects of training (epoch, global step, etc...) including amp scaling. No need
+            for you to store anything about training.
+
         """
         pass
+
+
+def load_hparams_from_tags_csv(tags_csv):
+    if not os.path.isfile(tags_csv):
+        logging.warning(f'Missing Tags: {tags_csv}.')
+        return Namespace()
+
+    tags_df = pd.read_csv(tags_csv)
+    dic = tags_df.to_dict(orient='records')
+    ns_dict = {row['key']: convert(row['value']) for row in dic}
+    ns = Namespace(**ns_dict)
+    return ns
+
+
+def convert(val):
+    constructors = [int, float, str]
+
+    if type(val) is str:
+        if val.lower() == 'true':
+            return True
+        if val.lower() == 'false':
+            return False
+
+    for c in constructors:
+        try:
+            return c(val)
+        except ValueError:
+            pass
+    return val

pytorch_lightning/core/memory.py

@@ -3,13 +3,14 @@ Generates a summary of a model's layers and dimensionality
 '''
 
 import gc
+import logging
 import os
 import subprocess
+from subprocess import PIPE
 
 import numpy as np
 import pandas as pd
 import torch
-import logging
 
 
 class ModelSummary(object):
@@ -235,7 +236,8 @@ def get_gpu_memory_map():
             '--format=csv,nounits,noheader',
         ],
         encoding='utf-8',
-        capture_output=True,
+        # capture_output=True,          # valid for python version >=3.7
+        stdout=PIPE, stderr=PIPE,       # for backward compatibility with python version 3.6
         check=True)
     # Convert lines into a dictionary
     gpu_memory = [int(x) for x in result.stdout.strip().split(os.linesep)]

pytorch_lightning/core/model_saving.py

@@ -1,10 +1,10 @@
 """
-.. warning:: `model_saving` module has been renamed to `saving` since v0.5.3 and will be removed in v0.8.0
+.. warning:: `model_saving` module has been renamed to `saving` since v0.6.0 and will be removed in v0.8.0
 """
 
 import warnings
 
-warnings.warn("`model_saving` module has been renamed to `saving` since v0.5.3"
+warnings.warn("`model_saving` module has been renamed to `saving` since v0.6.0"
               " and will be removed in v0.8.0", DeprecationWarning)
 
 from pytorch_lightning.core.saving import ModelIO  # noqa: E402

pytorch_lightning/core/root_module.py

@@ -1,10 +1,8 @@
 """
-.. warning:: `root_module` module has been renamed to `lightning` since v0.5.3 and will be removed in v0.8.0
+.. warning:: `root_module` module has been renamed to `lightning` since v0.6.0 and will be removed in v0.8.0
 """
 
 import warnings
 
-warnings.warn("`root_module` module has been renamed to `lightning` since v0.5.3"
+warnings.warn("`root_module` module has been renamed to `lightning` since v0.6.0"
               " and will be removed in v0.8.0", DeprecationWarning)
-
-from pytorch_lightning.core.lightning import LightningModule  # noqa: E402

pytorch_lightning/logging/__init__.py

@@ -1,36 +1,20 @@
 """
-Lighting offers options for logging information about model, gpu usage, etc,
- via several different logging frameworks. It also offers printing options for training monitoring.
-
-**default_save_path**
-
-Lightning sets a default TestTubeLogger and CheckpointCallback for you which log to
-`os.getcwd()` by default. To modify the logging path you can set::
-
-    Trainer(default_save_path='/your/path/to/save/checkpoints')
-
-
-If you need more custom behavior (different paths for both, different metrics, etc...)
- from the logger and the checkpointCallback, pass in your own instances as explained below.
-
-Setting up logging
-------------------
-
-The trainer inits a default logger for you (TestTubeLogger). All logs will
-go to the current working directory under a folder named `os.getcwd()/lightning_logs`.
-
-If you want to modify the default logging behavior even more, pass in a logger
- (which should inherit from `LightningBaseLogger`).
+Lightning supports most popular logging frameworks (Tensorboard, comet, weights and biases, etc...).
+To use a logger, simply pass it into the trainer.
 
 .. code-block:: python
+    from pytorch_lightning import logging
 
-    my_logger = MyLightningLogger(...)
-    trainer = Trainer(logger=my_logger)
+    # lightning uses tensorboard by default
+    tb_logger = logging.TensorBoardLogger()
+    trainer = Trainer(logger=tb_logger)
 
+    # or choose from any of the others such as MLFlow, Comet, Neptune, Wandb
+    comet_logger = logging.CometLogger()
+    trainer = Trainer(logger=comet_logger)
 
-The path in this logger will overwrite `default_save_path`.
-
-Lightning supports several common experiment tracking frameworks out of the box
+.. note:: All loggers log by default to `os.getcwd()`. To change the path without creating a logger set
+    Trainer(default_save_path='/your/path/to/save/checkpoints')
 
 Custom logger
 -------------
@@ -73,7 +57,7 @@ a pull request to add it to Lighting!
 Using loggers
 -------------
 
-You can call the logger anywhere from your LightningModule by doing:
+Call the logger anywhere from your LightningModule by doing:
 
 .. code-block:: python
 
@@ -84,106 +68,47 @@ You can call the logger anywhere from your LightningModule by doing:
     def any_lightning_module_function_or_hook(...):
         self.logger.experiment.add_histogram(...)
 
-Display metrics in progress bar
--------------------------------
-
-.. code-block:: python
-
-    # DEFAULT
-    trainer = Trainer(show_progress_bar=True)
-
-Log metric row every k batches
-------------------------------
-
-Every k batches lightning will make an entry in the metrics log
-
-.. code-block:: python
-
-    # DEFAULT (ie: save a .csv log file every 10 batches)
-    trainer = Trainer(row_log_interval=10)
-
-Log GPU memory
---------------
-
-Logs GPU memory when metrics are logged.
-
-.. code-block:: python
-
-    # DEFAULT
-    trainer = Trainer(log_gpu_memory=None)
-
-    # log only the min/max utilization
-    trainer = Trainer(log_gpu_memory='min_max')
-
-    # log all the GPU memory (if on DDP, logs only that node)
-    trainer = Trainer(log_gpu_memory='all')
-
-Process position
-----------------
-
-When running multiple models on the same machine we want to decide which progress bar to use.
- Lightning will stack progress bars according to this value.
-
-.. code-block:: python
-
-    # DEFAULT
-    trainer = Trainer(process_position=0)
-
-    # if this is the second model on the node, show the second progress bar below
-    trainer = Trainer(process_position=1)
-
-
-Save a snapshot of all hyperparameters
---------------------------------------
-
-Automatically log hyperparameters stored in the `hparams` attribute as an `argparse.Namespace`
-
-.. code-block:: python
-
-    class MyModel(pl.Lightning):
-        def __init__(self, hparams):
-            self.hparams = hparams
-
-        ...
-
-    args = parser.parse_args()
-    model = MyModel(args)
-
-    logger = TestTubeLogger(...)
-    t = Trainer(logger=logger)
-    trainer.fit(model)
-
-Write logs file to csv every k batches
---------------------------------------
-
-Every k batches, lightning will write the new logs to disk
-
-.. code-block:: python
-
-    # DEFAULT (ie: save a .csv log file every 100 batches)
-    trainer = Trainer(log_save_interval=100)
-
+Supported Loggers
+-----------------
 """
-
 from os import environ
-from .base import LightningLoggerBase, rank_zero_only
 
+from .base import LightningLoggerBase, rank_zero_only
 from .tensorboard import TensorBoardLogger
 
-try:
-    from .test_tube import TestTubeLogger
-except ImportError:
-    pass
-
-try:
-    from .mlflow import MLFlowLogger
-except ImportError:
-    pass
+loggers = ['TensorBoardLogger']
 
 try:
     # needed to prevent ImportError and duplicated logs.
     environ["COMET_DISABLE_AUTO_LOGGING"] = "1"
 
     from .comet import CometLogger
+    loggers.append('CometLogger')
 except ImportError:
     del environ["COMET_DISABLE_AUTO_LOGGING"]
+
+try:
+    from .mlflow import MLFlowLogger
+    loggers.append('MLFlowLogger')
+except ImportError:
+    pass
+
+try:
+    from .neptune import NeptuneLogger
+    loggers.append('NeptuneLogger')
+except ImportError:
+    pass
+
+try:
+    from .test_tube import TestTubeLogger
+    loggers.append('TestTubeLogger')
+except ImportError:
+    pass
+
+try:
+    from .wandb import WandbLogger
+    loggers.append('WandbLogger')
+except ImportError:
+    pass
+
+__all__ = loggers

pytorch_lightning/logging/comet.py

@@ -1,52 +1,3 @@
-"""
-Log using `comet <https://www.comet.ml>`_
-
-Comet logger can be used in either online or offline mode.
-To log in online mode, CometLogger requries an API key:
-
-.. code-block:: python
-
-    from pytorch_lightning.logging import CometLogger
-    # arguments made to CometLogger are passed on to the comet_ml.Experiment class
-    comet_logger = CometLogger(
-        api_key=os.environ["COMET_KEY"],
-        workspace=os.environ["COMET_WORKSPACE"], # Optional
-        project_name="default_project", # Optional
-        rest_api_key=os.environ["COMET_REST_KEY"], # Optional
-        experiment_name="default" # Optional
-    )
-    trainer = Trainer(logger=comet_logger)
-
-To log in offline mode, CometLogger requires a path to a local directory:
-
-.. code-block:: python
-
-    from pytorch_lightning.logging import CometLogger
-    # arguments made to CometLogger are passed on to the comet_ml.Experiment class
-    comet_logger = CometLogger(
-        save_dir=".",
-        workspace=os.environ["COMET_WORKSPACE"], # Optional
-        project_name="default_project", # Optional
-        rest_api_key=os.environ["COMET_REST_KEY"], # Optional
-        experiment_name="default" # Optional
-    )
-    trainer = Trainer(logger=comet_logger)
-
-
-Use the logger anywhere in you LightningModule as follows:
-
-.. code-block:: python
-
-    def train_step(...):
-        # example
-        self.logger.experiment.whatever_comet_ml_supports(...)
-
-    def any_lightning_module_function_or_hook(...):
-        self.logger.experiment.whatever_comet_ml_supports(...)
-
-
-"""
-
 from logging import getLogger
 
 try:
@@ -71,18 +22,54 @@ logger = getLogger(__name__)
 class CometLogger(LightningLoggerBase):
     def __init__(self, api_key=None, save_dir=None, workspace=None,
                  rest_api_key=None, project_name=None, experiment_name=None, **kwargs):
-        """Initialize a Comet.ml logger.
+        r"""
+
+        Log using `comet <https://www.comet.ml>`_.
+
         Requires either an API Key (online mode) or a local directory path (offline mode)
 
-        :param str api_key: Required in online mode. API key, found on Comet.ml
-        :param str save_dir: Required in offline mode. The path for the directory to save local comet logs
-        :param str workspace: Optional. Name of workspace for this user
-        :param str project_name: Optional. Send your experiment to a specific project.
-        Otherwise will be sent to Uncategorized Experiments.
-        If project name does not already exists Comet.ml will create a new project.
-        :param str rest_api_key: Optional. Rest API key found in Comet.ml settings.
-        This is used to determine version number
-        :param str experiment_name: Optional. String representing the name for this particular experiment on Comet.ml
+        .. code-block:: python
+
+            # ONLINE MODE
+            from pytorch_lightning.logging import CometLogger
+
+            # arguments made to CometLogger are passed on to the comet_ml.Experiment class
+            comet_logger = CometLogger(
+                api_key=os.environ["COMET_KEY"],
+                workspace=os.environ["COMET_WORKSPACE"], # Optional
+                project_name="default_project", # Optional
+                rest_api_key=os.environ["COMET_REST_KEY"], # Optional
+                experiment_name="default" # Optional
+            )
+            trainer = Trainer(logger=comet_logger)
+
+
+        .. code-block:: python
+
+            # OFFLINE MODE
+            from pytorch_lightning.logging import CometLogger
+
+            # arguments made to CometLogger are passed on to the comet_ml.Experiment class
+            comet_logger = CometLogger(
+                save_dir=".",
+                workspace=os.environ["COMET_WORKSPACE"], # Optional
+                project_name="default_project", # Optional
+                rest_api_key=os.environ["COMET_REST_KEY"], # Optional
+                experiment_name="default" # Optional
+            )
+            trainer = Trainer(logger=comet_logger)
+
+        Args:
+            api_key (str): Required in online mode. API key, found on Comet.ml
+            save_dir (str): Required in offline mode. The path for the directory to save local comet logs
+            workspace (str): Optional. Name of workspace for this user
+            project_name (str): Optional. Send your experiment to a specific project.
+            Otherwise will be sent to Uncategorized Experiments.
+            If project name does not already exists Comet.ml will create a new project.
+            rest_api_key (str): Optional. Rest API key found in Comet.ml settings.
+                This is used to determine version number
+            experiment_name (str): Optional. String representing the name for this particular experiment on Comet.ml
+
         """
         super().__init__()
         self._experiment = None
@@ -124,6 +111,15 @@ class CometLogger(LightningLoggerBase):
 
     @property
     def experiment(self):
+        r"""
+
+        Actual comet object. To use comet features do the following.
+
+        Example::
+
+            self.logger.experiment.some_comet_function()
+
+        """
         if self._experiment is not None:
             return self._experiment
 

pytorch_lightning/logging/comet_logger.py

@@ -1,10 +1,10 @@
 """
-.. warning:: `comet_logger` module has been renamed to `comet` since v0.5.3 and will be removed in v0.8.0
+.. warning:: `comet_logger` module has been renamed to `comet` since v0.6.0 and will be removed in v0.8.0
 """
 
 import warnings
 
-warnings.warn("`comet_logger` module has been renamed to `comet` since v0.5.3"
+warnings.warn("`comet_logger` module has been renamed to `comet` since v0.6.0"
               " and will be removed in v0.8.0", DeprecationWarning)
 
 from pytorch_lightning.logging.comet import CometLogger  # noqa: E402

pytorch_lightning/logging/mlflow.py

@@ -39,6 +39,15 @@ logger = getLogger(__name__)
 
 class MLFlowLogger(LightningLoggerBase):
     def __init__(self, experiment_name, tracking_uri=None, tags=None):
+        r"""
+
+        Logs using MLFlow
+
+        Args:
+            experiment_name (str): The name of the experiment
+            tracking_uri (str): where this should track
+            tags (dict): todo this param
+        """
         super().__init__()
         self._mlflow_client = mlflow.tracking.MlflowClient(tracking_uri)
         self.experiment_name = experiment_name
@@ -47,6 +56,15 @@ class MLFlowLogger(LightningLoggerBase):
 
     @property
     def experiment(self):
+        r"""
+
+        Actual mlflow object. To use mlflow features do the following.
+
+        Example::
+
+            self.logger.experiment.some_mlflow_function()
+
+        """
         return self._mlflow_client
 
     @property
@@ -59,7 +77,7 @@ class MLFlowLogger(LightningLoggerBase):
         if expt:
             self._expt_id = expt.experiment_id
         else:
-            logger.warning(f"Experiment with name f{self.experiment_name} not found. Creating it.")
+            logger.warning(f"Experiment with name {self.experiment_name} not found. Creating it.")
             self._expt_id = self._mlflow_client.create_experiment(name=self.experiment_name)
 
         run = self._mlflow_client.create_run(experiment_id=self._expt_id, tags=self.tags)

pytorch_lightning/logging/mlflow_logger.py

@@ -1,10 +1,10 @@
 """
-.. warning:: `mlflow_logger` module has been renamed to `mlflow` since v0.5.3 and will be removed in v0.8.0
+.. warning:: `mlflow_logger` module has been renamed to `mlflow` since v0.6.0 and will be removed in v0.8.0
 """
 
 import warnings
 
-warnings.warn("`mlflow_logger` module has been renamed to `mlflow` since v0.5.3"
+warnings.warn("`mlflow_logger` module has been renamed to `mlflow` since v0.6.0"
               " and will be removed in v0.8.0", DeprecationWarning)
 
 from pytorch_lightning.logging.mlflow import MLFlowLogger  # noqa: E402

pytorch_lightning/logging/neptune.py

@@ -0,0 +1,286 @@
+"""
+Log using `neptune <https://www.neptune.ml>`_
+
+Neptune logger can be used in the online mode or offline (silent) mode.
+To log experiment data in online mode, NeptuneLogger requries an API key:
+
+.. code-block:: python
+
+    from pytorch_lightning.logging import NeptuneLogger
+    # arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class
+
+    neptune_logger = NeptuneLogger(
+        api_key=os.environ["NEPTUNE_API_TOKEN"],
+        project_name="USER_NAME/PROJECT_NAME",
+        experiment_name="default", # Optional,
+        params={"max_epochs": 10}, # Optional,
+        tags=["pytorch-lightning","mlp"] # Optional,
+    )
+    trainer = Trainer(max_epochs=10, logger=neptune_logger)
+
+Use the logger anywhere in you LightningModule as follows:
+
+.. code-block:: python
+
+    def train_step(...):
+        # example
+        self.logger.experiment.log_metric("acc_train", acc_train) # log metrics
+        self.logger.experiment.log_image("worse_predictions", prediction_image) # log images
+        self.logger.experiment.log_artifact("model_checkpoint.pt", prediction_image) # log model checkpoint
+        self.logger.experiment.whatever_neptune_supports(...)
+
+    def any_lightning_module_function_or_hook(...):
+        self.logger.experiment.log_metric("acc_train", acc_train) # log metrics
+        self.logger.experiment.log_image("worse_predictions", prediction_image) # log images
+        self.logger.experiment.log_artifact("model_checkpoint.pt", prediction_image) # log model checkpoint
+        self.logger.experiment.whatever_neptune_supports(...)
+
+
+"""
+
+from logging import getLogger
+
+try:
+    import neptune
+except ImportError:
+    raise ImportError('Missing neptune package. Run `pip install neptune-client`')
+
+from torch import is_tensor
+
+# from .base import LightningLoggerBase, rank_zero_only
+from pytorch_lightning.logging.base import LightningLoggerBase, rank_zero_only
+
+logger = getLogger(__name__)
+
+
+class NeptuneLogger(LightningLoggerBase):
+    def __init__(self, api_key=None, project_name=None, offline_mode=False,
+                 experiment_name=None, upload_source_files=None,
+                 params=None, properties=None, tags=None, **kwargs):
+        r"""
+
+        Initialize a neptune.ml logger.
+
+        .. note:: Requires either an API Key (online mode) or a local directory path (offline mode)
+
+        .. code-block:: python
+
+            # ONLINE MODE
+            from pytorch_lightning.logging import NeptuneLogger
+            # arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class
+
+            neptune_logger = NeptuneLogger(
+                api_key=os.environ["NEPTUNE_API_TOKEN"],
+                project_name="USER_NAME/PROJECT_NAME",
+                experiment_name="default", # Optional,
+                params={"max_epochs": 10}, # Optional,
+                tags=["pytorch-lightning","mlp"] # Optional,
+            )
+            trainer = Trainer(max_epochs=10, logger=neptune_logger)
+
+        .. code-block:: python
+
+            # OFFLINE MODE
+            from pytorch_lightning.logging import NeptuneLogger
+            # arguments made to NeptuneLogger are passed on to the neptune.experiments.Experiment class
+
+            neptune_logger = NeptuneLogger(
+                project_name="USER_NAME/PROJECT_NAME",
+                experiment_name="default", # Optional,
+                params={"max_epochs": 10}, # Optional,
+                tags=["pytorch-lightning","mlp"] # Optional,
+            )
+            trainer = Trainer(max_epochs=10, logger=neptune_logger)
+
+        Args:
+            api_key (str | None): Required in online mode. Neputne API token, found on https://neptune.ml.
+                Read how to get your API key
+                https://docs.neptune.ml/python-api/tutorials/get-started.html#copy-api-token.
+            project_name (str): Required in online mode. Qualified name of a project in a form of
+               "namespace/project_name" for example "tom/minst-classification".
+               If None, the value of NEPTUNE_PROJECT environment variable will be taken.
+               You need to create the project in https://neptune.ml first.
+            offline_mode (bool): Optional default False. If offline_mode=True no logs will be send to neptune.
+               Usually used for debug purposes.
+            experiment_name (str|None): Optional. Editable name of the experiment.
+               Name is displayed in the experiment’s Details (Metadata section) and in experiments view as a column.
+            upload_source_files (list|None): Optional. List of source files to be uploaded.
+               Must be list of str or single str. Uploaded sources are displayed in the experiment’s Source code tab.
+               If None is passed, Python file from which experiment was created will be uploaded.
+               Pass empty list ([]) to upload no files. Unix style pathname pattern expansion is supported.
+               For example, you can pass '*.py' to upload all python source files from the current directory.
+               For recursion lookup use '**/*.py' (for Python 3.5 and later). For more information see glob library.
+            params (dict|None): Optional. Parameters of the experiment. After experiment creation params are read-only.
+               Parameters are displayed in the experiment’s Parameters section and each key-value pair can be
+               viewed in experiments view as a column.
+            properties (dict|None): Optional default is {}. Properties of the experiment.
+               They are editable after experiment is created. Properties are displayed in the experiment’s Details and
+               each key-value pair can be viewed in experiments view as a column.
+            tags (list|None): Optional default []. Must be list of str. Tags of the experiment.
+               They are editable after experiment is created (see: append_tag() and remove_tag()).
+               Tags are displayed in the experiment’s Details and can be viewed in experiments view as a column.
+        """
+        super().__init__()
+        self.api_key = api_key
+        self.project_name = project_name
+        self.offline_mode = offline_mode
+        self.experiment_name = experiment_name
+        self.upload_source_files = upload_source_files
+        self.params = params
+        self.properties = properties
+        self.tags = tags
+        self._experiment = None
+        self._kwargs = kwargs
+
+        if offline_mode:
+            self.mode = "offline"
+            neptune.init(project_qualified_name='dry-run/project',
+                         backend=neptune.OfflineBackend())
+        else:
+            self.mode = "online"
+            neptune.init(api_token=self.api_key,
+                         project_qualified_name=self.project_name)
+
+        logger.info(f"NeptuneLogger was initialized in {self.mode} mode")
+
+    @property
+    def experiment(self):
+        r"""
+
+        Actual neptune object. To use neptune features do the following.
+
+        Example::
+
+            self.logger.experiment.some_neptune_function()
+
+        """
+
+        if self._experiment is not None:
+            return self._experiment
+        else:
+            self._experiment = neptune.create_experiment(name=self.experiment_name,
+                                                         params=self.params,
+                                                         properties=self.properties,
+                                                         tags=self.tags,
+                                                         upload_source_files=self.upload_source_files,
+                                                         **self._kwargs)
+        return self._experiment
+
+    @rank_zero_only
+    def log_hyperparams(self, params):
+        for key, val in vars(params).items():
+            self.experiment.set_property(f"param__{key}", val)
+
+    @rank_zero_only
+    def log_metrics(self, metrics, step=None):
+        """Log metrics (numeric values) in Neptune experiments
+
+        :param float metric: Dictionary with metric names as keys and measured quanties as values
+        :param int|None step: Step number at which the metrics should be recorded, must be strictly increasing
+
+        """
+
+        for key, val in metrics.items():
+            if is_tensor(val):
+                val = val.cpu().detach()
+
+            if step is None:
+                self.experiment.log_metric(key, val)
+            else:
+                self.experiment.log_metric(key, x=step, y=val)
+
+    @rank_zero_only
+    def finalize(self, status):
+        self.experiment.stop()
+
+    @property
+    def name(self):
+        if self.mode == "offline":
+            return "offline-name"
+        else:
+            return self.experiment.name
+
+    @property
+    def version(self):
+        if self.mode == "offline":
+            return "offline-id-1234"
+        else:
+            return self.experiment.id
+
+    @rank_zero_only
+    def log_metric(self, metric_name, metric_value, step=None):
+        """Log metrics (numeric values) in Neptune experiments
+
+        :param str metric_name:  The name of log, i.e. mse, loss, accuracy.
+        :param str metric_value: The value of the log (data-point).
+        :param int|None step: Step number at which the metrics should be recorded, must be strictly increasing
+
+        """
+        if step is None:
+            self.experiment.log_metric(metric_name, metric_value)
+        else:
+            self.experiment.log_metric(metric_name, x=step, y=metric_value)
+
+    @rank_zero_only
+    def log_text(self, log_name, text, step=None):
+        """Log text data in Neptune experiment
+
+        :param str log_name:  The name of log, i.e. mse, my_text_data, timing_info.
+        :param str text: The value of the log (data-point).
+        :param int|None step: Step number at which the metrics should be recorded, must be strictly increasing
+
+        """
+        if step is None:
+            self.experiment.log_metric(log_name, text)
+        else:
+            self.experiment.log_metric(log_name, x=step, y=text)
+
+    @rank_zero_only
+    def log_image(self, log_name, image, step=None):
+        """Log image data in Neptune experiment
+
+        :param str log_name: The name of log, i.e. bboxes, visualisations, sample_images.
+        :param str|PIL.Image|matplotlib.figure.Figure image: The value of the log (data-point).
+           Can be one of the following types: PIL image, matplotlib.figure.Figure, path to image file (str)
+        :param int|None step: Step number at which the metrics should be recorded, must be strictly increasing
+
+        """
+        if step is None:
+            self.experiment.log_image(log_name, image)
+        else:
+            self.experiment.log_image(log_name, x=step, y=image)
+
+    @rank_zero_only
+    def log_artifact(self, artifact, destination=None):
+        """Save an artifact (file) in Neptune experiment storage.
+
+        :param str artifact: A path to the file in local filesystem.
+        :param str|None destination: Optional default None.
+           A destination path. If None is passed, an artifact file name will be used.
+
+        """
+        self.experiment.log_artifact(artifact, destination)
+
+    @rank_zero_only
+    def set_property(self, key, value):
+        """Set key-value pair as Neptune experiment property.
+
+        :param str key: Property key.
+        :param obj value: New value of a property.
+
+        """
+        self.experiment.set_property(key, value)
+
+    @rank_zero_only
+    def append_tags(self, tags):
+        """appends tags to neptune experiment
+
+        :param str|tuple|list(str) tags: Tags to add to the current experiment.
+           If str is passed, singe tag is added.
+           If multiple - comma separated - str are passed, all of them are added as tags.
+           If list of str is passed, all elements of the list are added as tags.
+
+        """
+        if not isinstance(tags, (list, set, tuple)):
+            tags = [tags]  # make it as an iterable is if it is not yet
+        self.experiment.append_tags(*tags)

pytorch_lightning/logging/tensorboard.py

@@ -1,20 +1,25 @@
 import os
 from warnings import warn
+from argparse import Namespace
+from pkg_resources import parse_version
 
 import torch
-from pkg_resources import parse_version
+import pandas as pd
 from torch.utils.tensorboard import SummaryWriter
 
 from .base import LightningLoggerBase, rank_zero_only
 
 
 class TensorBoardLogger(LightningLoggerBase):
-    r"""Log to local file system in TensorBoard format
+    r"""
+
+    Log to local file system in TensorBoard format
 
     Implemented using :class:`torch.utils.tensorboard.SummaryWriter`. Logs are saved to
     `os.path.join(save_dir, name, version)`
 
-    :example:
+    Example
+    --------
 
     .. code-block:: python
 
@@ -22,14 +27,15 @@ class TensorBoardLogger(LightningLoggerBase):
         trainer = Trainer(logger=logger)
         trainer.train(model)
 
-    :param str save_dir: Save directory
-    :param str name: Experiment name. Defaults to "default".
-    :param int version: Experiment version. If version is not specified the logger inspects the save
+    Args:
+        save_dir (str): Save directory
+        name (str): Experiment name. Defaults to "default".
+        version (int): Experiment version. If version is not specified the logger inspects the save
         directory for existing versions, then automatically assigns the next available version.
-    :param \**kwargs: Other arguments are passed directly to the :class:`SummaryWriter` constructor.
-
+        \**kwargs  (dict): Other arguments are passed directly to the :class:`SummaryWriter` constructor.
 
     """
+    NAME_CSV_TAGS = 'meta_tags.csv'
 
     def __init__(self, save_dir, name="default", version=None, **kwargs):
         super().__init__()
@@ -38,41 +44,50 @@ class TensorBoardLogger(LightningLoggerBase):
         self._version = version
 
         self._experiment = None
+        self.tags = {}
         self.kwargs = kwargs
 
     @property
     def experiment(self):
-        """The underlying :class:`torch.utils.tensorboard.SummaryWriter`.
+        r"""
 
-        :rtype: torch.utils.tensorboard.SummaryWriter
-        """
+         Actual tensorboard object. To use tensorboard features do the following.
+
+         Example::
+
+             self.logger.experiment.some_tensorboard_function()
+
+         """
         if self._experiment is not None:
             return self._experiment
 
         root_dir = os.path.join(self.save_dir, self.name)
         os.makedirs(root_dir, exist_ok=True)
-        log_dir = os.path.join(root_dir, str(self.version))
+        log_dir = os.path.join(root_dir, "version_" + str(self.version))
         self._experiment = SummaryWriter(log_dir=log_dir, **self.kwargs)
         return self._experiment
 
     @rank_zero_only
     def log_hyperparams(self, params):
+        if params is None:
+            return
+
+        # in case converting from namespace
+        if isinstance(params, Namespace):
+            params = vars(params)
+        params = dict(params)
+
         if parse_version(torch.__version__) < parse_version("1.3.0"):
             warn(
                 f"Hyperparameter logging is not available for Torch version {torch.__version__}."
                 " Skipping log_hyperparams. Upgrade to Torch 1.3.0 or above to enable"
                 " hyperparameter logging."
             )
-            # TODO: some alternative should be added
-            return
-        try:
-            # in case converting from namespace, todo: rather test if it is namespace
-            params = vars(params)
-        except TypeError:
-            pass
-        if params is not None:
+        else:
             # `add_hparams` requires both - hparams and metric
-            self.experiment.add_hparams(hparam_dict=dict(params), metric_dict={})
+            self.experiment.add_hparams(hparam_dict=params, metric_dict={})
+        # some alternative should be added
+        self.tags.update(params)
 
     @rank_zero_only
     def log_metrics(self, metrics, step=None):
@@ -89,6 +104,17 @@ class TensorBoardLogger(LightningLoggerBase):
             # you are using PT version (<v1.2) which does not have implemented flush
             self.experiment._get_file_writer().flush()
 
+        # create a preudo standard path ala test-tube
+        dir_path = os.path.join(self.save_dir, self.name, 'version_%s' % self.version)
+        if not os.path.isdir(dir_path):
+            dir_path = self.save_dir
+        # prepare the file path
+        meta_tags_path = os.path.join(dir_path, self.NAME_CSV_TAGS)
+        # save the metatags file
+        df = pd.DataFrame({'key': list(self.tags.keys()),
+                           'value': list(self.tags.values())})
+        df.to_csv(meta_tags_path, index=False)
+
     @rank_zero_only
     def finalize(self, status):
         self.save()
@@ -105,9 +131,11 @@ class TensorBoardLogger(LightningLoggerBase):
 
     def _get_next_version(self):
         root_dir = os.path.join(self.save_dir, self.name)
-        existing_versions = [
-            int(d) for d in os.listdir(root_dir) if os.path.isdir(os.path.join(root_dir, d)) and d.isdigit()
-        ]
+        existing_versions = []
+        for d in os.listdir(root_dir):
+            if os.path.isdir(os.path.join(root_dir, d)) and d.startswith("version_"):
+                existing_versions.append(int(d.split("_")[1]))
+
         if len(existing_versions) == 0:
             return 0
         else:

pytorch_lightning/logging/test_tube.py

@@ -38,6 +38,33 @@ from .base import LightningLoggerBase, rank_zero_only
 
 
 class TestTubeLogger(LightningLoggerBase):
+    r"""
+
+    Log to local file system in TensorBoard format but using a nicer folder structure.
+
+    Implemented using :class:`torch.utils.tensorboard.SummaryWriter`. Logs are saved to
+    `os.path.join(save_dir, name, version)`
+
+    Example
+    --------
+
+    .. code-block:: python
+
+        logger = TestTubeLogger("tt_logs", name="my_exp_name")
+        trainer = Trainer(logger=logger)
+        trainer.train(model)
+
+    Args:
+        save_dir (str): Save directory
+        name (str): Experiment name. Defaults to "default".
+        description (str): A short snippet about this experiment
+        debug (bool): If True, it doesn't log anything
+        version (int): Experiment version. If version is not specified the logger inspects the save
+        directory for existing versions, then automatically assigns the next available version.
+        create_git_tag (bool): If True creates a git tag to save the code used in this experiment
+
+    """
+
     __test__ = False
 
     def __init__(
@@ -55,6 +82,16 @@ class TestTubeLogger(LightningLoggerBase):
 
     @property
     def experiment(self):
+        r"""
+
+          Actual test-tube object. To use test-tube features do the following.
+
+          Example::
+
+              self.logger.experiment.some_test_tube_function()
+
+          """
+
         if self._experiment is not None:
             return self._experiment
 
@@ -98,8 +135,9 @@ class TestTubeLogger(LightningLoggerBase):
     def close(self):
         # TODO: HACK figure out where this is being set to true
         self.experiment.debug = self.debug
-        exp = self.experiment
-        exp.close()
+        if not self.debug:
+            exp = self.experiment
+            exp.close()
 
     @property
     def rank(self):

pytorch_lightning/logging/test_tube_logger.py

@@ -1,10 +1,10 @@
 """
-.. warning:: `test_tube_logger` module has been renamed to `test_tube` since v0.5.3 and will be removed in v0.8.0
+.. warning:: `test_tube_logger` module has been renamed to `test_tube` since v0.6.0 and will be removed in v0.8.0
 """
 
 import warnings
 
-warnings.warn("`test_tube_logger` module has been renamed to `test_tube` since v0.5.3"
+warnings.warn("`test_tube_logger` module has been renamed to `test_tube` since v0.6.0"
               " and will be removed in v0.8.0", DeprecationWarning)
 
 from pytorch_lightning.logging.test_tube import TestTubeLogger  # noqa: E402

pytorch_lightning/logging/wandb.py

@@ -0,0 +1,103 @@
+import os
+
+try:
+    import wandb
+except ImportError:
+    raise ImportError('Missing wandb package.')
+
+from .base import LightningLoggerBase, rank_zero_only
+
+
+class WandbLogger(LightningLoggerBase):
+    """
+    Logger for W&B.
+
+    Args:
+        name (str): display name for the run.
+        save_dir (str): path where data is saved.
+        offline (bool): run offline (data can be streamed later to wandb servers).
+        id or version (str): sets the version, mainly used to resume a previous run.
+        anonymous (bool): enables or explicitly disables anonymous logging.
+        project (str): the name of the project to which this run will belong.
+        tags (list of str): tags associated with this run.
+
+    Example
+    --------
+    .. code-block:: python
+
+        from pytorch_lightning.logging import WandbLogger
+        from pytorch_lightning import Trainer
+
+        wandb_logger = WandbLogger()
+        trainer = Trainer(logger=wandb_logger)
+    """
+
+    def __init__(self, name=None, save_dir=None, offline=False, id=None, anonymous=False,
+                 version=None, project=None, tags=None, experiment=None):
+        super().__init__()
+        self._name = name
+        self._save_dir = save_dir
+        self._anonymous = "allow" if anonymous else None
+        self._id = version or id
+        self._tags = tags
+        self._project = project
+        self._experiment = experiment
+        self._offline = offline
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+        # cannot be pickled
+        state['_experiment'] = None
+        # args needed to reload correct experiment
+        state['_id'] = self.experiment.id
+        return state
+
+    @property
+    def experiment(self):
+        r"""
+
+          Actual wandb object. To use wandb features do the following.
+
+          Example::
+
+              self.logger.experiment.some_wandb_function()
+
+          """
+        if self._experiment is None:
+            if self._offline:
+                os.environ["WANDB_MODE"] = "dryrun"
+            self._experiment = wandb.init(
+                name=self._name, dir=self._save_dir, project=self._project, anonymous=self._anonymous,
+                id=self._id, resume="allow", tags=self._tags)
+        return self._experiment
+
+    def watch(self, model, log="gradients", log_freq=100):
+        wandb.watch(model, log, log_freq)
+
+    @rank_zero_only
+    def log_hyperparams(self, params):
+        self.experiment.config.update(params)
+
+    @rank_zero_only
+    def log_metrics(self, metrics, step=None):
+        metrics["global_step"] = step
+        self.experiment.log(metrics)
+
+    def save(self):
+        pass
+
+    @rank_zero_only
+    def finalize(self, status='success'):
+        try:
+            exit_code = 0 if status == 'success' else 1
+            wandb.join(exit_code)
+        except TypeError:
+            wandb.join()
+
+    @property
+    def name(self):
+        return self.experiment.project_name()
+
+    @property
+    def version(self):
+        return self.experiment.id

pytorch_lightning/overrides/override_data_parallel.py

@@ -1,11 +1,11 @@
 """
-.. warning:: `override_data_parallel` module has been renamed to `data_parallel` since v0.5.3
+.. warning:: `override_data_parallel` module has been renamed to `data_parallel` since v0.6.0
  and will be removed in v0.8.0
 """
 
 import warnings
 
-warnings.warn("`override_data_parallel` module has been renamed to `data_parallel` since v0.5.3"
+warnings.warn("`override_data_parallel` module has been renamed to `data_parallel` since v0.6.0"
               " and will be removed in v0.8.0", DeprecationWarning)
 
 from pytorch_lightning.overrides.data_parallel import (  # noqa: E402

pytorch_lightning/pt_overrides/__init__.py

@@ -1,10 +1,10 @@
 """
-.. warning:: `pt_overrides` package has been renamed to `overrides` since v0.5.3 and will be removed in v0.8.0
+.. warning:: `pt_overrides` package has been renamed to `overrides` since v0.6.0 and will be removed in v0.8.0
 """
 
 import warnings
 
-warnings.warn("`pt_overrides` package has been renamed to `overrides` since v0.5.3"
+warnings.warn("`pt_overrides` package has been renamed to `overrides` since v0.6.0"
               " and will be removed in v0.8.0", DeprecationWarning)
 
 from pytorch_lightning.overrides import override_data_parallel  # noqa: E402

pytorch_lightning/root_module/__init__.py

@@ -1,11 +1,12 @@
 """
-.. warning:: `root_module` package has been renamed to `core` since v0.5.3 and will be removed in v0.8.0
+.. warning:: `root_module` package has been renamed to `core` since v0.6.0 and will be removed in v0.8.0
 """
 
 import warnings
 
-warnings.warn("`root_module` package has been renamed to `core` since v0.5.3"
+warnings.warn("`root_module` package has been renamed to `core` since v0.6.0"
               " and will be removed in v0.8.0", DeprecationWarning)
 
 from pytorch_lightning.core import (  # noqa: E402
-    decorators, grads, hooks, root_module, memory, model_saving)
+    decorators, grads, hooks, root_module, memory, model_saving
+)

pytorch_lightning/testing/model_base.py

@@ -9,6 +9,7 @@ from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
 from torchvision import transforms
 from torchvision.datasets import MNIST
+
 try:
     from test_tube import HyperOptArgumentParser
 except ImportError:

pytorch_lightning/trainer/__init__.py

@@ -1,9 +1,17 @@
 """
-# Trainer
 
-The lightning trainer abstracts best practices for running a training, val, test routine.
- It calls parts of your model when it wants to hand over full control and otherwise makes
- training assumptions which are now standard practice in AI research.
+The trainer de-couples the engineering code (16-bit, early stopping, GPU distribution, etc...) from the
+science code (GAN, BERT, your project, etc...). It uses many assumptions which are best practices in
+AI research today.
+
+The trainer automates all parts of training except:
+
+- what happens in training , test, val loop
+- where the data come from
+- which optimizers to use
+- how to do the computations
+
+The Trainer delegates those calls to your LightningModule which defines how to do those parts.
 
 This is the basic use of the trainer:
 
@@ -11,9 +19,11 @@ This is the basic use of the trainer:
 
     from pytorch_lightning import Trainer
 
-    model = LightningTemplate()
+    model = MyLightningModule()
 
     trainer = Trainer()
     trainer.fit(model)
-
 """
+
+from .trainer import Trainer
+__all__ = ['Trainer']

pytorch_lightning/trainer/auto_mix_precision.py

@@ -1,3 +1,4 @@
+
 from abc import ABC
 
 try:

pytorch_lightning/trainer/callback_config.py

@@ -2,7 +2,7 @@ import os
 from abc import ABC
 
 from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping
-from pytorch_lightning.logging import TestTubeLogger
+from pytorch_lightning.logging import TensorBoardLogger
 
 
 class TrainerCallbackConfigMixin(ABC):
@@ -55,10 +55,20 @@ class TrainerCallbackConfigMixin(ABC):
             self.early_stop_callback = EarlyStopping(
                 monitor='val_loss',
                 patience=3,
+                strict=True,
                 verbose=True,
                 mode='min'
             )
             self.enable_early_stop = True
+        elif early_stop_callback is None:
+            self.early_stop_callback = EarlyStopping(
+                monitor='val_loss',
+                patience=3,
+                strict=False,
+                verbose=False,
+                mode='min'
+            )
+            self.enable_early_stop = True
         elif not early_stop_callback:
             self.early_stop_callback = None
             self.enable_early_stop = False
@@ -69,7 +79,7 @@ class TrainerCallbackConfigMixin(ABC):
         # configure logger
         if logger is True:
             # default logger
-            self.logger = TestTubeLogger(
+            self.logger = TensorBoardLogger(
                 save_dir=self.default_save_path,
                 version=self.slurm_job_id,
                 name='lightning_logs'

pytorch_lightning/trainer/data_loading.py

@@ -2,6 +2,7 @@ import warnings
 from abc import ABC
 
 import torch.distributed as dist
+
 try:
     # loading for pyTorch 1.3
     from torch.utils.data import IterableDataset
@@ -36,6 +37,15 @@ class TrainerDataLoadingMixin(ABC):
         self.shown_warnings = None
         self.val_check_interval = None
 
+    def _percent_range_check(self, name):
+        value = getattr(self, name)
+        msg = f"`{name}` must lie in the range [0.0, 1.0], but got {value:.3f}."
+        if name == "val_check_interval":
+            msg += " If you want to disable validation set `val_percent_check` to 0.0 instead."
+
+        if not 0. <= value <= 1.:
+            raise ValueError(msg)
+
     def init_train_dataloader(self, model):
         """
         Dataloaders are provided by the model
@@ -48,6 +58,8 @@ class TrainerDataLoadingMixin(ABC):
         if EXIST_ITER_DATASET and isinstance(self.get_train_dataloader().dataset, IterableDataset):
             self.num_training_batches = float('inf')
         else:
+            self._percent_range_check('train_percent_check')
+
             self.num_training_batches = len(self.get_train_dataloader())
             self.num_training_batches = int(self.num_training_batches * self.train_percent_check)
 
@@ -56,7 +68,14 @@ class TrainerDataLoadingMixin(ABC):
         # otherwise, it checks in [0, 1.0] % range of a training epoch
         if isinstance(self.val_check_interval, int):
             self.val_check_batch = self.val_check_interval
+            if self.val_check_batch > self.num_training_batches:
+                raise ValueError(
+                    f"`val_check_interval` ({self.val_check_interval}) must be less than or equal "
+                    f"to the number of the training batches ({self.num_training_batches}). "
+                    f"If you want to disable validation set `val_percent_check` to 0.0 instead.")
         else:
+            self._percent_range_check('val_check_interval')
+
             self.val_check_batch = int(self.num_training_batches * self.val_check_interval)
             self.val_check_batch = max(1, self.val_check_batch)
 
@@ -89,13 +108,15 @@ class TrainerDataLoadingMixin(ABC):
         :return:
         """
         self.get_val_dataloaders = model.val_dataloader
+        self.num_val_batches = 0
 
         # determine number of validation batches
         # val datasets could be none, 1 or 2+
         if self.get_val_dataloaders() is not None:
+            self._percent_range_check('val_percent_check')
+
             self.num_val_batches = sum(len(dataloader) for dataloader in self.get_val_dataloaders())
             self.num_val_batches = int(self.num_val_batches * self.val_percent_check)
-            self.num_val_batches = max(1, self.num_val_batches)
 
         on_ddp = self.use_ddp or self.use_ddp2
         if on_ddp and self.get_val_dataloaders() is not None:
@@ -134,10 +155,11 @@ class TrainerDataLoadingMixin(ABC):
 
         # determine number of test batches
         if self.get_test_dataloaders() is not None:
+            self._percent_range_check('test_percent_check')
+
             len_sum = sum(len(dataloader) for dataloader in self.get_test_dataloaders())
             self.num_test_batches = len_sum
             self.num_test_batches = int(self.num_test_batches * self.test_percent_check)
-            self.num_test_batches = max(1, self.num_test_batches)
 
         on_ddp = self.use_ddp or self.use_ddp2
         if on_ddp and self.get_test_dataloaders() is not None:
@@ -208,6 +230,10 @@ class TrainerDataLoadingMixin(ABC):
         self.val_percent_check = val_percent_check
         self.test_percent_check = test_percent_check
         if overfit_pct > 0:
+            if overfit_pct > 1:
+                raise ValueError(f"`overfit_pct` must be not greater than 1.0, but got "
+                                 f"{overfit_pct:.3f}.")
+
             self.train_percent_check = overfit_pct
             self.val_percent_check = overfit_pct
             self.test_percent_check = overfit_pct

pytorch_lightning/trainer/distrib_data_parallel.py

@@ -113,9 +113,9 @@ When the script starts again, Lightning will:
 
 """
 
+import logging
 import os
 import re
-import logging
 import warnings
 from abc import ABC, abstractmethod
 

pytorch_lightning/trainer/distrib_parts.py

@@ -276,7 +276,7 @@ Instead of manually building SLURM scripts, you can use the
 
 Here is an example where you run a grid search of 9 combinations of hyperparams.
  The full examples are `here
- <https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/new_project_templates/multi_node_examples>`_.
+ <https://github.com/PyTorchLightning/pytorch-lightning/tree/master/pl_examples/new_project_templates/multi_node_examples>`_.
 
 .. code-block:: python
 

pytorch_lightning/trainer/evaluation_loop.py

@@ -1,5 +1,6 @@
 """
-# Validation loop
+Validation loop
+===============
 
 The lightning validation loop handles everything except the actual computations of your model.
 To decide what will happen in your validation loop, define the `validation_step` function.
@@ -122,11 +123,11 @@ In this second case, the options you pass to trainer will be used when running
 
 """
 
+import sys
 from abc import ABC, abstractmethod
 
 import torch
-import sys
-import tqdm
+from tqdm.auto import tqdm
 
 from pytorch_lightning.utilities.debugging import MisconfigurationException
 
@@ -266,76 +267,67 @@ class TrainerEvaluationLoopMixin(ABC):
 
     def run_evaluation(self, test=False):
         # when testing make sure user defined a test step
-        can_run_test_step = False
+        if test and not (self.is_overriden('test_step') and self.is_overriden('test_end')):
+            m = '''You called `.test()` without defining model's `.test_step()` or `.test_end()`.
+                    Please define and try again'''
+            raise MisconfigurationException(m)
+
+        # hook
+        model = self.get_model()
+        model.on_pre_performance_check()
+
+        # select dataloaders
         if test:
-            can_run_test_step = self.is_overriden('test_step') and self.is_overriden('test_end')
-            if not can_run_test_step:
-                m = '''You called .test() without defining a test step or test_end.
-                Please define and try again'''
-                raise MisconfigurationException(m)
+            dataloaders = self.get_test_dataloaders()
+            max_batches = self.num_test_batches
+        else:
+            # val
+            dataloaders = self.get_val_dataloaders()
+            max_batches = self.num_val_batches
 
-        # validate only if model has validation_step defined
-        # test only if test_step or validation_step are defined
-        run_val_step = self.is_overriden('validation_step')
+        # cap max batches to 1 when using fast_dev_run
+        if self.fast_dev_run:
+            max_batches = 1
 
-        if run_val_step or can_run_test_step:
+        # init validation or test progress bar
+        # main progress bar will already be closed when testing so initial position is free
+        position = 2 * self.process_position + (not test)
+        desc = 'Testing' if test else 'Validating'
+        pbar = tqdm(desc=desc, total=max_batches, leave=test, position=position,
+                    disable=not self.show_progress_bar, dynamic_ncols=True,
+                    unit='batch', file=sys.stdout)
+        setattr(self, f'{"test" if test else "val"}_progress_bar', pbar)
 
-            # hook
-            model = self.get_model()
-            model.on_pre_performance_check()
+        # run evaluation
+        eval_results = self.evaluate(self.model,
+                                     dataloaders,
+                                     max_batches,
+                                     test)
+        _, prog_bar_metrics, log_metrics, callback_metrics, _ = self.process_output(
+            eval_results)
 
-            # select dataloaders
-            if test:
-                dataloaders = self.get_test_dataloaders()
-                max_batches = self.num_test_batches
-            else:
-                # val
-                dataloaders = self.get_val_dataloaders()
-                max_batches = self.num_val_batches
+        # add metrics to prog bar
+        self.add_tqdm_metrics(prog_bar_metrics)
 
-            # cap max batches to 1 when using fast_dev_run
-            if self.fast_dev_run:
-                max_batches = 1
+        # log metrics
+        self.log_metrics(log_metrics, {})
 
-            # init validation or test progress bar
-            # main progress bar will already be closed when testing so initial position is free
-            position = 2 * self.process_position + (not test)
-            desc = 'Testing' if test else 'Validating'
-            pbar = tqdm.tqdm(desc=desc, total=max_batches, leave=test, position=position,
-                             disable=not self.show_progress_bar, dynamic_ncols=True,
-                             unit='batch', file=sys.stdout)
-            setattr(self, f'{"test" if test else "val"}_progress_bar', pbar)
+        # track metrics for callbacks
+        self.callback_metrics.update(callback_metrics)
 
-            # run evaluation
-            eval_results = self.evaluate(self.model,
-                                         dataloaders,
-                                         max_batches,
-                                         test)
-            _, prog_bar_metrics, log_metrics, callback_metrics, _ = self.process_output(
-                eval_results)
+        # hook
+        model.on_post_performance_check()
 
-            # add metrics to prog bar
-            self.add_tqdm_metrics(prog_bar_metrics)
+        # add model specific metrics
+        tqdm_metrics = self.training_tqdm_dict
+        if not test:
+            self.main_progress_bar.set_postfix(**tqdm_metrics)
 
-            # log metrics
-            self.log_metrics(log_metrics, {})
-
-            # track metrics for callbacks
-            self.callback_metrics.update(callback_metrics)
-
-            # hook
-            model.on_post_performance_check()
-
-            # add model specific metrics
-            tqdm_metrics = self.training_tqdm_dict
-            if not test:
-                self.main_progress_bar.set_postfix(**tqdm_metrics)
-
-            # close progress bar
-            if test:
-                self.test_progress_bar.close()
-            else:
-                self.val_progress_bar.close()
+        # close progress bar
+        if test:
+            self.test_progress_bar.close()
+        else:
+            self.val_progress_bar.close()
 
         # model checkpointing
         if self.proc_rank == 0 and self.checkpoint_callback is not None and not test:

pytorch_lightning/trainer/trainer.py

@@ -1,16 +1,13 @@
-"""
-The trainer handles all the logic for running a val loop, training loop, distributing, etc.. .
-"""
-
 import os
 import sys
 import warnings
 import logging
 
+
 import torch
 import torch.distributed as dist
 import torch.multiprocessing as mp
-import tqdm
+from tqdm.auto import tqdm
 from torch.optim.optimizer import Optimizer
 
 from pytorch_lightning.trainer.auto_mix_precision import TrainerAMPMixin
@@ -22,11 +19,12 @@ from pytorch_lightning.trainer.distrib_parts import (
     parse_gpu_ids,
     determine_root_gpu_device
 )
+
 from pytorch_lightning.trainer.evaluation_loop import TrainerEvaluationLoopMixin
 from pytorch_lightning.trainer.logging import TrainerLoggingMixin
 from pytorch_lightning.trainer.model_hooks import TrainerModelHooksMixin
+from pytorch_lightning.trainer.training_io import TrainerIOMixin
 from pytorch_lightning.trainer.training_loop import TrainerTrainLoopMixin
-from pytorch_lightning.trainer.trainer_io import TrainerIOMixin
 from pytorch_lightning.trainer.training_tricks import TrainerTrainingTricksMixin
 from pytorch_lightning.utilities.debugging import MisconfigurationException
 
@@ -50,12 +48,11 @@ class Trainer(TrainerIOMixin,
               TrainerTrainLoopMixin,
               TrainerCallbackConfigMixin,
               ):
-
     def __init__(
             self,
             logger=True,
             checkpoint_callback=True,
-            early_stop_callback=True,
+            early_stop_callback=None,
             default_save_path=None,
             gradient_clip_val=0,
             gradient_clip=None,  # backward compatible, todo: remove in v0.8.0
@@ -92,88 +89,390 @@ class Trainer(TrainerIOMixin,
             truncated_bptt_steps=None,
             resume_from_checkpoint=None,
     ):
+        r"""
+
+        Customize every aspect of training via flags
+
+        Args:
+            logger (:class:`.Logger`): Logger for experiment tracking.
+                Example::
+                    from pytorch_lightning.logging import TensorBoardLogger
+
+                    # default logger used by trainer
+                    logger = TensorBoardLogger(
+                        save_dir=os.getcwd(),
+                        version=self.slurm_job_id,
+                        name='lightning_logs'
+                    )
+
+                    Trainer(logger=logger)
+            checkpoint_callback (:class:`CheckpointCallback`): Callback for checkpointing.
+                Example::
+                    from pytorch_lightning.callbacks import ModelCheckpoint
+
+                    # default used by the Trainer
+                    checkpoint_callback = ModelCheckpoint(
+                        filepath=os.getcwd(),
+                        save_best_only=True,
+                        verbose=True,
+                        monitor='val_loss',
+                        mode='min',
+                        prefix=''
+                    )
+
+                    trainer = Trainer(checkpoint_callback=checkpoint_callback)
+            early_stop_callback (:class:`.EarlyStopping`): Callback for early stopping. If
+                set to ``True``, then the default callback monitoring ``'val_loss'`` is created.
+                Will raise an error if ``'val_loss'`` is not found.
+                If set to ``False``, then early stopping will be disabled.
+                If set to ``None``, then the default callback monitoring ``'val_loss'`` is created.
+                If ``'val_loss'`` is not found will work as if early stopping is disabled.
+                Default: ``None``.
+                Example::
+                    from pytorch_lightning.callbacks import EarlyStopping
+
+                    # default used by the Trainer
+                    early_stop_callback = EarlyStopping(
+                        monitor='val_loss',
+                        patience=3,
+                        strict=False,
+                        verbose=False,
+                        mode='min'
+                    )
+
+                    trainer = Trainer(early_stop_callback=early_stop_callback)
+            default_save_path (str): Default path for logs and weights when no logger/ckpt_callback passed
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(default_save_path=os.getcwd())
+            gradient_clip_val (float): 0 means don't clip.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(gradient_clip_val=0.0)
+            gradient_clip (int):
+                .. deprecated:: 0.5.0
+                    Use `gradient_clip_val` instead. Will remove 0.8.0.
+
+            process_position (int): orders the tqdm bar when running multiple models on same machine.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(process_position=0)
+
+            num_nodes (int): number of GPU nodes for distributed training.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(num_nodes=1)
+
+                    # to train on 8 nodes
+                    trainer = Trainer(num_nodes=8)
+
+            nb_gpu_nodes (int):
+                .. deprecated:: 0.5.0
+                    Use `num_nodes` instead. Will remove 0.8.0.
+
+            gpus (list|str|int): Which GPUs to train on.
+                Example::
+                    # default used by the Trainer (ie: train on CPU)
+                    trainer = Trainer(gpus=None)
+
+                    # int: train on 2 gpus
+                    trainer = Trainer(gpus=2)
+
+                    # list: train on GPUs 1, 4 (by bus ordering)
+                    trainer = Trainer(gpus=[1, 4])
+                    trainer = Trainer(gpus='1, 4') # equivalent
+
+                    # -1: train on all gpus
+                    trainer = Trainer(gpus=-1)
+                    trainer = Trainer(gpus='-1') # equivalent
+
+                    # combine with num_nodes to train on multiple GPUs across nodes
+                    trainer = Trainer(gpus=2, num_nodes=4) # uses 8 gpus in total
+
+            log_gpu_memory (str): None, 'min_max', 'all'. Might slow performance
+                because it uses the output of nvidia-smi.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(log_gpu_memory=None)
+
+                    # log all the GPUs (on master node only)
+                    trainer = Trainer(log_gpu_memory='all')
+
+                    # log only the min and max memory on the master node
+                    trainer = Trainer(log_gpu_memory='min_max')
+
+            show_progress_bar (bool): If true shows tqdm progress bar
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(show_progress_bar=True)
+
+            overfit_pct (float): uses this much data of all datasets.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(overfit_pct=0.0)
+
+                    # use only 1% of the train, test, val datasets
+                    trainer = Trainer(overfit_pct=0.01)
+
+            track_grad_norm (int): -1 no tracking. Otherwise tracks that norm
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(track_grad_norm=-1)
+
+                    # track the 2-norm
+                    trainer = Trainer(track_grad_norm=2)
+
+            check_val_every_n_epoch (int): check val every n train epochs
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(check_val_every_n_epoch=1)
+
+                    # run val loop every 10 training epochs
+                    trainer = Trainer(check_val_every_n_epoch=10)
+
+            fast_dev_run (bool): runs 1 batch of train, test  and val to find any bugs (ie: a sort of unit test).
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(fast_dev_run=False)
+
+                    # runs 1 train, val, test  batch and program ends
+                    trainer = Trainer(fast_dev_run=True)
+
+            accumulate_grad_batches (int|dict): Accumulates grads every k batches or as set up in the dict.
+                Example::
+                    # default used by the Trainer (no accumulation)
+                    trainer = Trainer(accumulate_grad_batches=1)
+
+                    # accumulate every 4 batches (effective batch size is batch*4)
+                    trainer = Trainer(accumulate_grad_batches=4)
+
+                    # no accumulation for epochs 1-4. accumulate 3 for epochs 5-10. accumulate 20 after that
+                    trainer = Trainer(accumulate_grad_batches={5: 3, 10: 20})
+
+            max_epochs (int): Stop training once this number of epochs is reached
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(max_epochs=1000)
+
+            max_nb_epochs (int):
+                .. deprecated:: 0.5.0
+                    Use `max_epochs` instead. Will remove 0.8.0.
+
+            min_epochs (int): Force training for at least these many epochs
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(min_epochs=1)
+
+            min_nb_epochs (int):
+                .. deprecated:: 0.5.0
+                    Use `min_nb_epochs` instead. Will remove 0.8.0.
+
+            train_percent_check (int): How much of training dataset to check.
+                Useful when debugging or testing something that happens at the end of an epoch.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(train_percent_check=1.0)
+
+                    # run through only 25% of the training set each epoch
+                    trainer = Trainer(train_percent_check=0.25)
+
+            val_percent_check (int): How much of validation dataset to check.
+                Useful when debugging or testing something that happens at the end of an epoch.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(val_percent_check=1.0)
+
+                    # run through only 25% of the validation set each epoch
+                    trainer = Trainer(val_percent_check=0.25)
+
+            test_percent_check (int): How much of test dataset to check.
+                Useful when debugging or testing something that happens at the end of an epoch.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(test_percent_check=1.0)
+
+                    # run through only 25% of the test set each epoch
+                    trainer = Trainer(test_percent_check=0.25)
+
+            val_check_interval (float|int): How often within one training epoch to check the validation set
+                If float, % of tng epoch. If int, check every n batch
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(val_check_interval=1.0)
+
+                    # check validation set 4 times during a training epoch
+                    trainer = Trainer(val_check_interval=0.25)
+
+                    # check validation set every 1000 training batches
+                    # use this when using iterableDataset and your dataset has no length
+                    # (ie: production cases with streaming data)
+                    trainer = Trainer(val_check_interval=1000)
+
+            log_save_interval (int): Writes logs to disk this often
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(log_save_interval=100)
+
+            row_log_interval (int): How often to add logging rows (does not write to disk)
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(row_log_interval=10)
+
+            add_row_log_interval (int):
+                .. deprecated:: 0.5.0
+                    Use `row_log_interval` instead. Will remove 0.8.0.
+
+            distributed_backend (str): The distributed backend to use.
+                Options: 'dp', 'ddp', 'ddp2'.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(distributed_backend=None)
+
+                    # dp = DataParallel (split a batch onto k gpus on same machine).
+                    trainer = Trainer(gpus=2, distributed_backend='dp')
+
+                    # ddp = DistributedDataParallel
+                    # Each gpu trains by itself on a subset of the data.
+                    # Gradients sync across all gpus and all machines.
+                    trainer = Trainer(gpus=2, num_nodes=2, distributed_backend='ddp')
+
+                    # ddp2 = DistributedDataParallel + dp
+                    # behaves like dp on every node
+                    # syncs gradients across nodes like ddp
+                    # useful for things like increasing the number of negative samples
+                    trainer = Trainer(gpus=2, num_nodes=2, distributed_backend='ddp2')
+
+            use_amp (bool): If true uses apex for 16bit precision
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(use_amp=False)
+
+            print_nan_grads (bool): Prints gradients with nan values
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(print_nan_grads=False)
+
+            weights_summary (str): Prints a summary of the weights when training begins.
+                Options: 'full', 'top', None.
+                Example::
+                    # default used by the Trainer (ie: print all weights)
+                    trainer = Trainer(weights_summary='full')
+
+                    # print only the top level modules
+                    trainer = Trainer(weights_summary='top')
+
+                    # don't print a summary
+                    trainer = Trainer(weights_summary=None)
+
+            weights_save_path (str): Where to save weights if specified.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(weights_save_path=os.getcwd())
+
+                    # save to your custom path
+                    trainer = Trainer(weights_save_path='my/path')
+
+                    # if checkpoint callback used, then overrides the weights path
+                    # **NOTE: this saves weights to some/path NOT my/path
+                    checkpoint_callback = ModelCheckpoint(filepath='some/path')
+                    trainer = Trainer(
+                        checkpoint_callback=checkpoint_callback,
+                        weights_save_path='my/path'
+                    )
+
+            amp_level (str): The optimization level to use (O1, O2, etc...).
+                Check nvidia docs for level (https://nvidia.github.io/apex/amp.html#opt-levels)
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(amp_level='O1')
+
+            num_sanity_val_steps (int): Sanity check runs n batches of val before starting the training routine.
+                This catches any bugs in your validation without having to wait for the first validation check.
+                The Trainer uses 5 steps by default. Turn it off or modify it here.
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(num_sanity_val_steps=5)
+
+                    # turn it off
+                    trainer = Trainer(num_sanity_val_steps=0)
+
+            nb_sanity_val_steps (int):
+                .. deprecated:: 0.5.0
+                    Use `num_sanity_val_steps` instead. Will remove 0.8.0.
+
+            truncated_bptt_steps (int): Truncated back prop breaks performs backprop every k steps of
+                a much longer sequence If this is enabled, your batches will automatically get truncated
+                and the trainer will apply Truncated Backprop to it. Make sure your batches have a sequence
+                dimension. (`Williams et al. "An efficient gradient-based algorithm for on-line training of
+                recurrent network trajectories."
+                <http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.56.7941&rep=rep1&type=pdf>`_)
+                Example::
+                    # default used by the Trainer (ie: disabled)
+                    trainer = Trainer(truncated_bptt_steps=None)
+
+                    # backprop every 5 steps in a batch
+                    trainer = Trainer(truncated_bptt_steps=5)
+
+            resume_from_checkpoint (str): To resume training from a specific checkpoint pass in the path here.k
+                Example::
+                    # default used by the Trainer
+                    trainer = Trainer(resume_from_checkpoint=None)
+
+                    # resume from a specific checkpoint
+                    trainer = Trainer(resume_from_checkpoint='some/path/to/my_checkpoint.ckpt')
         """
+        #
+        # .. warning:: Following arguments become deprecated and they will be removed in v0.8.0:
+        #     - `nb_sanity_val_steps`
 
-        :param logger: Logger for experiment tracking
-        :param checkpoint_callback: Callback for checkpointing
-        :param early_stop_callback: Callback for early stopping
-        :param str default_save_path: Default path for logs+weights if no logger/ckpt_callback passed
-        :param int gradient_clip_val: 0 means don't clip.
-        :param int gradient_clip: 0 means don't clip. Deprecated.
-        :param process_position: shown in the tqdm bar
-        :param int num_nodes: number of GPU nodes
-        :param list|str|int gpus: int. (ie: 2 gpus) OR list to specify which GPUs [0, 1] OR '0,1'
-            OR '-1' / -1 to use all available gpus
-        :param str log_gpu_memory: None, 'min_max', 'all'
-        :param bool show_progress_bar: If true shows tqdm bar
-        :param float overfit_pct: uses this much of all datasets
-        :param int track_grad_norm: -1 no tracking. Otherwise tracks that norm
-        :param int check_val_every_n_epoch: check val every n train epochs
-        :param bool fast_dev_run: runs full iteration over everything to find bugs
-        :param int accumulate_grad_batches: Accumulates grads every k batches
-        :param int max_epochs:
-        :param int min_epochs:
-        :param int train_percent_check: How much of train set to check
-        :param int val_percent_check: How much of val set to check
-        :param int test_percent_check: How much of test set to check
-        :param float|int val_check_interval: If float, % of tng epoch. If int, check every n batch
-        :param int log_save_interval: Writes logs to disk this often
-        :param int row_log_interval: How often to add logging rows
-        :param int add_row_log_interval: How often to add logging rows. Deprecated.
-        :param str distributed_backend: Options: 'dp', 'ddp', 'ddp2'.
-        :param bool use_amp: If true uses apex for 16bit precision
-        :param bool print_nan_grads: Prints nan gradients
-        :param str weights_summary: Options: 'full', 'top', None to not print.
-        :param bool weights_save_path: Where to save weights if on cluster
-        :param str amp_level: Check nvidia docs for level
-        :param int num_sanity_val_steps: How many val steps before a full train loop.
-        :param int truncated_bptt_steps: Enables multiple backward passes for each batch.
-
-        .. warning:: Following arguments become deprecated and they will be removed in v0.8.0:
-            - `gradient_clip`,
-            - `nb_gpu_nodes`,
-            - `max_nb_epochs`,
-            - `min_nb_epochs`,
-            - `add_row_log_interval`,
-            - `nb_sanity_val_steps`
-
-        """
         # Transfer params
-        if nb_gpu_nodes is not None:  # Backward compatibility
+        # Backward compatibility
+        if nb_gpu_nodes is not None:
             warnings.warn("`nb_gpu_nodes` has renamed to `num_nodes` since v0.5.0"
                           " and will be removed in v0.8.0", DeprecationWarning)
             if not num_nodes:  # in case you did not set the proper value
                 num_nodes = nb_gpu_nodes
         self.num_gpu_nodes = num_nodes
+
         self.log_gpu_memory = log_gpu_memory
-        if gradient_clip is not None:  # Backward compatibility
+
+        # Backward compatibility
+        if gradient_clip is not None:
             warnings.warn("`gradient_clip` has renamed to `gradient_clip_val` since v0.5.0"
                           " and will be removed in v0.8.0", DeprecationWarning)
             if not gradient_clip_val:  # in case you did not set the proper value
                 gradient_clip_val = gradient_clip
         self.gradient_clip_val = gradient_clip_val
+
         self.check_val_every_n_epoch = check_val_every_n_epoch
         self.track_grad_norm = track_grad_norm
         self.on_gpu = True if (gpus and torch.cuda.is_available()) else False
         self.process_position = process_position
         self.weights_summary = weights_summary
-        if max_nb_epochs is not None:  # Backward compatibility
+
+        # Backward compatibility
+        if max_nb_epochs is not None:
             warnings.warn("`max_nb_epochs` has renamed to `max_epochs` since v0.5.0"
                           " and will be removed in v0.8.0", DeprecationWarning)
             if not max_epochs:  # in case you did not set the proper value
                 max_epochs = max_nb_epochs
         self.max_epochs = max_epochs
-        if min_nb_epochs is not None:  # Backward compatibility
+
+        # Backward compatibility
+        if min_nb_epochs is not None:
             warnings.warn("`min_nb_epochs` has renamed to `min_epochs` since v0.5.0"
                           " and will be removed in v0.8.0", DeprecationWarning)
             if not min_epochs:  # in case you did not set the proper value
                 min_epochs = min_nb_epochs
         self.min_epochs = min_epochs
-        if nb_sanity_val_steps is not None:  # Backward compatibility
+
+        # Backward compatibility
+        if nb_sanity_val_steps is not None:
             warnings.warn("`nb_sanity_val_steps` has renamed to `num_sanity_val_steps` since v0.5.0"
                           " and will be removed in v0.8.0", DeprecationWarning)
             if not num_sanity_val_steps:  # in case you did not set the proper value
                 num_sanity_val_steps = nb_sanity_val_steps
+
         self.num_sanity_val_steps = num_sanity_val_steps
         self.print_nan_grads = print_nan_grads
         self.truncated_bptt_steps = truncated_bptt_steps
@@ -213,6 +512,7 @@ class Trainer(TrainerIOMixin,
         # training state
         self.model = None
         self.testing = False
+        self.disable_validation = False
         self.lr_schedulers = []
         self.optimizers = None
         self.global_step = 0
@@ -261,8 +561,9 @@ class Trainer(TrainerIOMixin,
         # logging
         self.log_save_interval = log_save_interval
         self.val_check_interval = val_check_interval
+
+        # backward compatibility
         if add_row_log_interval is not None:
-            # backward compatibility
             warnings.warn("`add_row_log_interval` has renamed to `row_log_interval` since v0.5.0"
                           " and will be removed in v0.8.0", DeprecationWarning)
             if not row_log_interval:  # in case you did not set the proper value
@@ -370,6 +671,16 @@ class Trainer(TrainerIOMixin,
     # MODEL TRAINING
     # -----------------------------
     def fit(self, model):
+        r"""
+        Runs the full optimization routine.
+
+        Example::
+
+            trainer = Trainer()
+            model = LightningModule()
+
+            trainer.fit()
+        """
         # when using multi-node or DDP within a node start each module in a separate process
         if self.use_ddp2:
             task = int(os.environ['SLURM_LOCALID'])
@@ -486,30 +797,40 @@ class Trainer(TrainerIOMixin,
             self.run_evaluation(test=True)
             return
 
+        # check if we should run validation during training
+        self.disable_validation = ((self.num_val_batches == 0 or
+                                   not self.is_overriden('validation_step')) and
+                                   not self.fast_dev_run)
+
         # run tiny validation (if validation defined)
         # to make sure program won't crash during val
         ref_model.on_sanity_check_start()
         ref_model.on_train_start()
-        if self.get_val_dataloaders() is not None and self.num_sanity_val_steps > 0:
+        if not self.disable_validation and self.num_sanity_val_steps > 0:
             # init progress bars for validation sanity check
-            pbar = tqdm.tqdm(desc='Validation sanity check',
+            pbar = tqdm(desc='Validation sanity check',
                              total=self.num_sanity_val_steps * len(self.get_val_dataloaders()),
                              leave=False, position=2 * self.process_position,
                              disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch')
             self.main_progress_bar = pbar
             # dummy validation progress bar
-            self.val_progress_bar = tqdm.tqdm(disable=True)
+            self.val_progress_bar = tqdm(disable=True)
 
-            self.evaluate(model, self.get_val_dataloaders(), self.num_sanity_val_steps, self.testing)
+            eval_results = self.evaluate(model, self.get_val_dataloaders(),
+                                         self.num_sanity_val_steps, False)
+            _, _, _, callback_metrics, _ = self.process_output(eval_results)
 
             # close progress bars
             self.main_progress_bar.close()
             self.val_progress_bar.close()
 
+            if self.enable_early_stop:
+                self.early_stop_callback.check_metrics(callback_metrics)
+
         # init progress bar
-        pbar = tqdm.tqdm(leave=True, position=2 * self.process_position,
-                         disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch',
-                         file=sys.stdout)
+        pbar = tqdm(leave=True, position=2 * self.process_position,
+                    disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch',
+                    file=sys.stdout)
         self.main_progress_bar = pbar
 
         # clear cache before training
@@ -520,6 +841,29 @@ class Trainer(TrainerIOMixin,
         self.train()
 
     def test(self, model=None):
+        r"""
+
+        Separates from fit to make sure you never run on your test set until you want to.
+
+        Args:
+            model (LightningModule): The model to test.
+
+        Example::
+
+            # Option 1
+            # run test after fitting
+            trainer = Trainer()
+            model = LightningModule()
+
+            trainer.fit()
+            trainer.test()
+
+            # Option 2
+            # run test from a loaded model
+            model = LightningModule.load_from_checkpoint('path/to/checkpoint.ckpt')
+            trainer = Trainer()
+            trainer.test(model)
+        """
         self.testing = True
         if model is not None:
             self.fit(model)

pytorch_lightning/trainer/training_io.py

@@ -89,13 +89,14 @@ At a rough level, here's what happens inside Trainer :py:mod:`pytorch_lightning.
 
 """
 
+import logging
 import os
 import re
 import signal
 import warnings
-from subprocess import call
-import logging
 from abc import ABC
+from subprocess import call
+from argparse import Namespace
 
 import torch
 import torch.distributed as dist
@@ -255,17 +256,34 @@ class TrainerIOMixin(ABC):
     # --------------------
     # MODEL SAVE CHECKPOINT
     # --------------------
+    def _atomic_save(self, checkpoint, filepath):
+        """Saves a checkpoint atomically, avoiding the creation of incomplete checkpoints.
+
+        This will create a temporary checkpoint with a suffix of ``.part``, then copy it to the final location once
+        saving is finished.
+
+        Args:
+            checkpoint (object): The object to save.
+                Built to be used with the ``dump_checkpoint`` method, but can deal with anything which ``torch.save``
+                accepts.
+            filepath (str|pathlib.Path): The path to which the checkpoint will be saved.
+                This points to the file that the checkpoint will be stored in.
+        """
+        tmp_path = str(filepath) + ".part"
+        torch.save(checkpoint, tmp_path)
+        os.replace(tmp_path, filepath)
+
     def save_checkpoint(self, filepath):
         checkpoint = self.dump_checkpoint()
 
         # do the actual save
         try:
-            torch.save(checkpoint, filepath)
+            self._atomic_save(checkpoint, filepath)
         except AttributeError:
             if 'hparams' in checkpoint:
                 del checkpoint['hparams']
 
-            torch.save(checkpoint, filepath)
+            self._atomic_save(checkpoint, filepath)
 
     def restore(self, checkpoint_path, on_gpu):
 
@@ -413,12 +431,12 @@ class TrainerIOMixin(ABC):
         # do the actual save
         # TODO: fix for anything with multiprocess DP, DDP, DDP2
         try:
-            torch.save(checkpoint, filepath)
+            self._atomic_save(checkpoint, filepath)
         except AttributeError:
             if 'hparams' in checkpoint:
                 del checkpoint['hparams']
 
-            torch.save(checkpoint, filepath)
+            self._atomic_save(checkpoint, filepath)
 
         return filepath
 
@@ -458,33 +476,3 @@ class TrainerIOMixin(ABC):
             ckpt_vs.append(int(name))
 
         return max(ckpt_vs)
-
-
-def load_hparams_from_tags_csv(tags_csv):
-    from argparse import Namespace
-    import pandas as pd
-
-    tags_df = pd.read_csv(tags_csv)
-    dic = tags_df.to_dict(orient='records')
-
-    ns_dict = {row['key']: convert(row['value']) for row in dic}
-
-    ns = Namespace(**ns_dict)
-    return ns
-
-
-def convert(val):
-    constructors = [int, float, str]
-
-    if type(val) is str:
-        if val.lower() == 'true':
-            return True
-        if val.lower() == 'false':
-            return False
-
-    for c in constructors:
-        try:
-            return c(val)
-        except ValueError:
-            pass
-    return val

pytorch_lightning/trainer/training_loop.py

@@ -152,9 +152,9 @@ When this flag is enabled each batch is split into sequences of size truncated_b
 
 """
 
-import inspect
-from abc import ABC, abstractmethod
+import copy
 import warnings
+from abc import ABC, abstractmethod
 
 import numpy as np
 
@@ -184,6 +184,7 @@ class TrainerTrainLoopMixin(ABC):
         self.num_training_batches = None
         self.val_check_batch = None
         self.num_val_batches = None
+        self.disable_validation = None
         self.fast_dev_run = None
         self.is_iterable_train_dataloader = None
         self.main_progress_bar = None
@@ -294,14 +295,17 @@ class TrainerTrainLoopMixin(ABC):
             model.current_epoch = epoch
             self.current_epoch = epoch
 
-            # val can be checked multiple times in epoch
-            is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0
-            val_checks_per_epoch = self.num_training_batches // self.val_check_batch
-            val_checks_per_epoch = val_checks_per_epoch if is_val_epoch else 0
+            total_val_batches = 0
+            is_val_epoch = False
+            if not self.disable_validation:
+                # val can be checked multiple times in epoch
+                is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0
+                val_checks_per_epoch = self.num_training_batches // self.val_check_batch
+                val_checks_per_epoch = val_checks_per_epoch if is_val_epoch else 0
+                total_val_batches = self.num_val_batches * val_checks_per_epoch
 
             # total batches includes multiple val checks
-            self.total_batches = (self.num_training_batches +
-                                  self.num_val_batches * val_checks_per_epoch)
+            self.total_batches = self.num_training_batches + total_val_batches
             self.batch_loss_value = 0  # accumulated grads
 
             if self.fast_dev_run:
@@ -343,13 +347,15 @@ class TrainerTrainLoopMixin(ABC):
 
             # early stopping
             met_min_epochs = epoch >= self.min_epochs - 1
-            if self.enable_early_stop and (met_min_epochs or self.fast_dev_run):
+            if (self.enable_early_stop and not self.disable_validation and is_val_epoch and
+                    (met_min_epochs or self.fast_dev_run)):
                 should_stop = self.early_stop_callback.on_epoch_end(epoch=epoch,
                                                                     logs=self.callback_metrics)
                 # stop training
                 stop = should_stop and met_min_epochs
                 if stop:
                     self.main_progress_bar.close()
+                    model.on_train_end()
                     return
 
         self.main_progress_bar.close()
@@ -367,6 +373,10 @@ class TrainerTrainLoopMixin(ABC):
 
         # run epoch
         for batch_idx, batch in enumerate(self.get_train_dataloader()):
+            # stop epoch if we limited the number of training batches
+            if batch_idx >= self.num_training_batches:
+                break
+
             self.batch_idx = batch_idx
 
             model = self.get_model()
@@ -386,12 +396,16 @@ class TrainerTrainLoopMixin(ABC):
             # ---------------
             is_val_check_batch = (batch_idx + 1) % self.val_check_batch == 0
             can_check_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0
-            should_check_val = ((is_val_check_batch or early_stop_epoch) and can_check_epoch)
+            should_check_val = (not self.disable_validation and can_check_epoch and
+                                (is_val_check_batch or early_stop_epoch))
 
             # fast_dev_run always forces val checking after train batch
             if self.fast_dev_run or should_check_val:
                 self.run_evaluation(test=self.testing)
 
+                if self.enable_early_stop:
+                    self.early_stop_callback.check_metrics(self.callback_metrics)
+
             # when logs should be saved
             should_save_log = (batch_idx + 1) % self.log_save_interval == 0 or early_stop_epoch
             if should_save_log or self.fast_dev_run:
@@ -413,11 +427,6 @@ class TrainerTrainLoopMixin(ABC):
             if early_stop_epoch or self.fast_dev_run:
                 break
 
-            # stop epoch if we limited the number of training batches
-            met_batch_limit = batch_idx >= self.num_training_batches
-            if met_batch_limit:
-                break
-
         # epoch end hook
         if self.is_function_implemented('on_epoch_end'):
             model = self.get_model()
@@ -455,6 +464,14 @@ class TrainerTrainLoopMixin(ABC):
 
             # call training_step once per optimizer
             for opt_idx, optimizer in enumerate(self.optimizers):
+                # make sure only the gradients of the current optimizer's paramaters are calculated
+                # in the training step to prevent dangling gradients in multiple-optimizer setup.
+                if len(self.optimizers) > 1:
+                    for param in self.get_model().parameters():
+                        param.requires_grad = False
+                    for group in optimizer.param_groups:
+                        for param in group['params']:
+                            param.requires_grad = True
 
                 # wrap the forward step in a closure so second order methods work
                 def optimizer_closure():
@@ -474,13 +491,14 @@ class TrainerTrainLoopMixin(ABC):
 
                     # backward pass
                     model_ref = self.get_model()
-                    model_ref.backward(self.use_amp, closure_loss, optimizer)
+                    model_ref.backward(self.use_amp, closure_loss, optimizer, opt_idx)
 
                     # track metrics for callbacks
                     all_callback_metrics.append(callback_metrics)
 
                     # track progress bar metrics
                     self.add_tqdm_metrics(progress_bar_metrics)
+                    self.add_tqdm_metrics(progress_bar_metrics)
                     all_log_metrics.append(log_metrics)
 
                     # insert after step hook
@@ -576,7 +594,7 @@ class TrainerTrainLoopMixin(ABC):
             gpu_id = 0
             if isinstance(self.data_parallel_device_ids, list):
                 gpu_id = self.data_parallel_device_ids[0]
-            batch = self.transfer_batch_to_gpu(batch.copy(), gpu_id)
+            batch = self.transfer_batch_to_gpu(copy.copy(batch), gpu_id)
             args[0] = batch
             output = self.model.training_step(*args)
 

pytorch_lightning/trainer/training_tricks.py

@@ -1,7 +1,8 @@
+import logging
 from abc import ABC, abstractmethod
 
 import torch
-import logging
+
 from pytorch_lightning.callbacks import GradientAccumulationScheduler
 
 

requirements.txt

@@ -2,7 +2,7 @@ scikit-learn>=0.20.2
 tqdm>=4.35.0
 numpy>=1.16.4
 torch>=1.1
-torchvision>=0.4.0
+torchvision>=0.4.0, < 0.5  # the 0.5. has some issues with torch JIT
 pandas>=0.24  # lower version do not support py3.7
-test-tube>=0.7.5
-future>=0.17.1  # required for buildins in setup.py
+tensorboard>=1.14
+future>=0.17.1  # required for builtins in setup.py

setup.py

@@ -44,7 +44,7 @@ setup(
     author=pytorch_lightning.__author__,
     author_email=pytorch_lightning.__author_email__,
     url=pytorch_lightning.__homepage__,
-    download_url='https://github.com/williamFalcon/pytorch-lightning',
+    download_url='https://github.com/PyTorchLightning/pytorch-lightning',
     license=pytorch_lightning.__license__,
     packages=find_packages(exclude=['tests']),
 

tests/README.md

@@ -10,7 +10,7 @@ run on a 2-GPU machine to validate the full test-suite.
 
 To run all tests do the following:
 ```bash
-git clone https://github.com/williamFalcon/pytorch-lightning
+git clone https://github.com/PyTorchLightning/pytorch-lightning
 cd pytorch-lightning
 
 # install module locally

tests/conftest.py

@@ -0,0 +1,22 @@
+import pytest
+
+import torch.multiprocessing as mp
+
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "spawn: spawn test in a separate process using torch.multiprocessing.spawn")
+
+
+def wrap(i, fn, args):
+    return fn(*args)
+
+
+@pytest.mark.tryfirst
+def pytest_pyfunc_call(pyfuncitem):
+    if pyfuncitem.get_closest_marker("spawn"):
+        testfunction = pyfuncitem.obj
+        funcargs = pyfuncitem.funcargs
+        testargs = tuple([funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames])
+
+        mp.spawn(wrap, (testfunction, testargs))
+        return True

tests/requirements.txt

@@ -5,7 +5,11 @@ pytest>=3.0.5
 pytest-cov
 flake8
 check-manifest
-# test_tube  # already installed in main req.
+test-tube>=0.7.5
 mlflow
 comet_ml
-twine==1.13.0
+wandb
+sacred
+neptune-client
+twine==1.13.0
+pillow<7.0.0

tests/test_amp.py

@@ -32,6 +32,7 @@ def test_amp_single_gpu(tmpdir):
     tutils.run_model_test(trainer_options, model)
 
 
+@pytest.mark.spawn
 def test_no_amp_single_gpu(tmpdir):
     """Make sure DDP + AMP work."""
     tutils.reset_seed()
@@ -51,8 +52,10 @@ def test_no_amp_single_gpu(tmpdir):
         use_amp=True
     )
 
-    with pytest.raises((MisconfigurationException, ModuleNotFoundError)):
-        tutils.run_model_test(trainer_options, model)
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    assert result == 1
 
 
 def test_amp_gpu_ddp(tmpdir):
@@ -78,6 +81,7 @@ def test_amp_gpu_ddp(tmpdir):
     tutils.run_model_test(trainer_options, model)
 
 
+@pytest.mark.spawn
 def test_amp_gpu_ddp_slurm_managed(tmpdir):
     """Make sure DDP + AMP work."""
     if not tutils.can_run_gpu_test():
@@ -124,26 +128,6 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir):
     assert trainer.resolve_root_node_address('abc[23-24]') == 'abc23'
     assert trainer.resolve_root_node_address('abc[23-24, 45-40, 40]') == 'abc23'
 
-    # test model loading with a map_location
-    pretrained_model = tutils.load_model(logger.experiment, trainer.checkpoint_callback.filepath)
-
-    # test model preds
-    for dataloader in trainer.get_test_dataloaders():
-        tutils.run_prediction(dataloader, pretrained_model)
-
-    if trainer.use_ddp:
-        # on hpc this would work fine... but need to hack it for the purpose of the test
-        trainer.model = pretrained_model
-        trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers()
-
-    # test HPC loading / saving
-    trainer.hpc_save(tmpdir, logger)
-    trainer.hpc_load(tmpdir, on_gpu=True)
-
-    # test freeze on gpu
-    model.freeze()
-    model.unfreeze()
-
 
 def test_cpu_model_with_amp(tmpdir):
     """Make sure model trains on CPU."""
@@ -165,6 +149,7 @@ def test_cpu_model_with_amp(tmpdir):
         tutils.run_model_test(trainer_options, model, on_gpu=False)
 
 
+@pytest.mark.spawn
 def test_amp_gpu_dp(tmpdir):
     """Make sure DP + AMP work."""
     tutils.reset_seed()
@@ -180,8 +165,11 @@ def test_amp_gpu_dp(tmpdir):
         distributed_backend='dp',
         use_amp=True
     )
-    with pytest.raises(MisconfigurationException):
-        tutils.run_model_test(trainer_options, model, hparams)
+
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    assert result == 1
 
 
 if __name__ == '__main__':

tests/test_cpu_models.py

@@ -29,7 +29,7 @@ def test_early_stopping_cpu_model(tmpdir):
         show_progress_bar=True,
         logger=tutils.get_test_tube_logger(tmpdir),
         train_percent_check=0.1,
-        val_percent_check=0.1
+        val_percent_check=0.1,
     )
 
     model, hparams = tutils.get_model()
@@ -51,7 +51,7 @@ def test_lbfgs_cpu_model(tmpdir):
         show_progress_bar=False,
         weights_summary='top',
         train_percent_check=1.0,
-        val_percent_check=0.2
+        val_percent_check=0.2,
     )
 
     model, hparams = tutils.get_model(use_test_model=True, lbfgs=True)
@@ -70,7 +70,7 @@ def test_default_logger_callbacks_cpu_model(tmpdir):
         print_nan_grads=True,
         show_progress_bar=False,
         train_percent_check=0.01,
-        val_percent_check=0.01
+        val_percent_check=0.01,
     )
 
     model, hparams = tutils.get_model()
@@ -97,7 +97,7 @@ def test_running_test_after_fitting(tmpdir):
     trainer_options = dict(
         default_save_path=tmpdir,
         show_progress_bar=False,
-        max_epochs=1,
+        max_epochs=4,
         train_percent_check=0.4,
         val_percent_check=0.2,
         test_percent_check=0.2,
@@ -114,7 +114,7 @@ def test_running_test_after_fitting(tmpdir):
     trainer.test()
 
     # test we have good test accuracy
-    tutils.assert_ok_test_acc(trainer)
+    tutils.assert_ok_model_acc(trainer)
 
 
 def test_running_test_without_val(tmpdir):
@@ -140,7 +140,8 @@ def test_running_test_without_val(tmpdir):
         val_percent_check=0.2,
         test_percent_check=0.2,
         checkpoint_callback=checkpoint,
-        logger=logger
+        logger=logger,
+        early_stop_callback=False
     )
 
     # fit model
@@ -152,7 +153,7 @@ def test_running_test_without_val(tmpdir):
     trainer.test()
 
     # test we have good test accuracy
-    tutils.assert_ok_test_acc(trainer)
+    tutils.assert_ok_model_acc(trainer)
 
 
 def test_single_gpu_batch_parse():
@@ -318,6 +319,7 @@ def test_tbptt_cpu_model(tmpdir):
         truncated_bptt_steps=truncated_bptt_steps,
         val_percent_check=0,
         weights_summary=None,
+        early_stop_callback=False
     )
 
     hparams = tutils.get_hparams()

tests/test_gpu_models.py

@@ -183,7 +183,7 @@ def test_multi_gpu_none_backend(tmpdir):
         gpus='-1'
     )
 
-    with pytest.raises(MisconfigurationException):
+    with pytest.warns(UserWarning):
         tutils.run_model_test(trainer_options, model)
 
 

tests/test_logging.py

@@ -193,6 +193,72 @@ def test_comet_pickle(tmpdir, monkeypatch):
     trainer2.logger.log_metrics({"acc": 1.0})
 
 
+def test_wandb_logger(tmpdir):
+    """Verify that basic functionality of wandb logger works."""
+    tutils.reset_seed()
+
+    from pytorch_lightning.logging import WandbLogger
+
+    wandb_dir = os.path.join(tmpdir, "wandb")
+    logger = WandbLogger(save_dir=wandb_dir, anonymous=True)
+
+
+def test_neptune_logger(tmpdir):
+    """Verify that basic functionality of neptune logger works."""
+    tutils.reset_seed()
+
+    from pytorch_lightning.logging import NeptuneLogger
+
+    hparams = tutils.get_hparams()
+    model = LightningTestModel(hparams)
+    logger = NeptuneLogger(offline_mode=True)
+
+    trainer_options = dict(
+        default_save_path=tmpdir,
+        max_epochs=1,
+        train_percent_check=0.01,
+        logger=logger
+    )
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    print('result finished')
+    assert result == 1, "Training failed"
+
+
+def test_wandb_pickle(tmpdir):
+    """Verify that pickling trainer with wandb logger works."""
+    tutils.reset_seed()
+
+    from pytorch_lightning.logging import WandbLogger
+    wandb_dir = str(tmpdir)
+    logger = WandbLogger(save_dir=wandb_dir, anonymous=True)
+    assert logger is not None
+
+
+def test_neptune_pickle(tmpdir):
+    """Verify that pickling trainer with neptune logger works."""
+    tutils.reset_seed()
+
+    from pytorch_lightning.logging import NeptuneLogger
+
+    # hparams = tutils.get_hparams()
+    # model = LightningTestModel(hparams)
+
+    logger = NeptuneLogger(offline_mode=True)
+
+    trainer_options = dict(
+        default_save_path=tmpdir,
+        max_epochs=1,
+        logger=logger
+    )
+
+    trainer = Trainer(**trainer_options)
+    pkl_bytes = pickle.dumps(trainer)
+    trainer2 = pickle.loads(pkl_bytes)
+    trainer2.logger.log_metrics({"acc": 1.0})
+
+
 def test_tensorboard_logger(tmpdir):
     """Verify that basic functionality of Tensorboard logger works."""
 
@@ -230,8 +296,8 @@ def test_tensorboard_automatic_versioning(tmpdir):
     """Verify that automatic versioning works"""
 
     root_dir = tmpdir.mkdir("tb_versioning")
-    root_dir.mkdir("0")
-    root_dir.mkdir("1")
+    root_dir.mkdir("version_0")
+    root_dir.mkdir("version_1")
 
     logger = TensorBoardLogger(save_dir=tmpdir, name="tb_versioning")
 
@@ -242,9 +308,9 @@ def test_tensorboard_manual_versioning(tmpdir):
     """Verify that manual versioning works"""
 
     root_dir = tmpdir.mkdir("tb_versioning")
-    root_dir.mkdir("0")
-    root_dir.mkdir("1")
-    root_dir.mkdir("2")
+    root_dir.mkdir("version_0")
+    root_dir.mkdir("version_1")
+    root_dir.mkdir("version_2")
 
     logger = TensorBoardLogger(save_dir=tmpdir, name="tb_versioning", version=1)
 
@@ -309,7 +375,7 @@ def test_custom_logger(tmpdir):
 
     trainer_options = dict(
         max_epochs=1,
-        train_percent_check=0.01,
+        train_percent_check=0.05,
         logger=logger,
         default_save_path=tmpdir
     )

tests/test_restore_models.py

@@ -41,12 +41,11 @@ def test_running_test_pretrained_model_ddp(tmpdir):
     trainer = Trainer(**trainer_options)
     result = trainer.fit(model)
 
-    exp = logger.experiment
-    logging.info(os.listdir(exp.get_data_path(exp.name, exp.version)))
+    logging.info(os.listdir(tutils.get_data_path(logger, path_dir=tmpdir)))
 
     # correct result and ok accuracy
     assert result == 1, 'training failed to complete'
-    pretrained_model = tutils.load_model(logger.experiment,
+    pretrained_model = tutils.load_model(logger,
                                          trainer.checkpoint_callback.filepath,
                                          module_class=LightningTestModel)
 
@@ -87,14 +86,14 @@ def test_running_test_pretrained_model(tmpdir):
     # correct result and ok accuracy
     assert result == 1, 'training failed to complete'
     pretrained_model = tutils.load_model(
-        logger.experiment, trainer.checkpoint_callback.filepath, module_class=LightningTestModel
+        logger, trainer.checkpoint_callback.filepath, module_class=LightningTestModel
     )
 
     new_trainer = Trainer(**trainer_options)
     new_trainer.test(pretrained_model)
 
     # test we have good test accuracy
-    tutils.assert_ok_test_acc(new_trainer)
+    tutils.assert_ok_model_acc(new_trainer)
 
 
 def test_load_model_from_checkpoint(tmpdir):
@@ -106,10 +105,10 @@ def test_load_model_from_checkpoint(tmpdir):
 
     trainer_options = dict(
         show_progress_bar=False,
-        max_epochs=1,
+        max_epochs=2,
         train_percent_check=0.4,
         val_percent_check=0.2,
-        checkpoint_callback=True,
+        checkpoint_callback=ModelCheckpoint(tmpdir, save_top_k=-1),
         logger=False,
         default_save_path=tmpdir,
     )
@@ -120,9 +119,12 @@ def test_load_model_from_checkpoint(tmpdir):
 
     # correct result and ok accuracy
     assert result == 1, 'training failed to complete'
-    pretrained_model = LightningTestModel.load_from_checkpoint(
-        os.path.join(trainer.checkpoint_callback.filepath, "_ckpt_epoch_0.ckpt")
-    )
+
+    # load last checkpoint
+    last_checkpoint = os.path.join(trainer.checkpoint_callback.filepath, "_ckpt_epoch_1.ckpt")
+    if not os.path.isfile(last_checkpoint):
+        last_checkpoint = os.path.join(trainer.checkpoint_callback.filepath, "_ckpt_epoch_0.ckpt")
+    pretrained_model = LightningTestModel.load_from_checkpoint(last_checkpoint)
 
     # test that hparams loaded correctly
     for k, v in vars(hparams).items():
@@ -132,7 +134,7 @@ def test_load_model_from_checkpoint(tmpdir):
     new_trainer.test(pretrained_model)
 
     # test we have good test accuracy
-    tutils.assert_ok_test_acc(new_trainer)
+    tutils.assert_ok_model_acc(new_trainer)
 
 
 def test_running_test_pretrained_model_dp(tmpdir):
@@ -153,7 +155,7 @@ def test_running_test_pretrained_model_dp(tmpdir):
 
     trainer_options = dict(
         show_progress_bar=True,
-        max_epochs=1,
+        max_epochs=4,
         train_percent_check=0.4,
         val_percent_check=0.2,
         checkpoint_callback=checkpoint,
@@ -168,7 +170,7 @@ def test_running_test_pretrained_model_dp(tmpdir):
 
     # correct result and ok accuracy
     assert result == 1, 'training failed to complete'
-    pretrained_model = tutils.load_model(logger.experiment,
+    pretrained_model = tutils.load_model(logger,
                                          trainer.checkpoint_callback.filepath,
                                          module_class=LightningTestModel)
 
@@ -176,7 +178,7 @@ def test_running_test_pretrained_model_dp(tmpdir):
     new_trainer.test(pretrained_model)
 
     # test we have good test accuracy
-    tutils.assert_ok_test_acc(new_trainer)
+    tutils.assert_ok_model_acc(new_trainer)
 
 
 def test_dp_resume(tmpdir):
@@ -269,12 +271,12 @@ def test_cpu_restore_training(tmpdir):
     logger = tutils.get_test_tube_logger(tmpdir, False, version=test_logger_version)
 
     trainer_options = dict(
-        max_epochs=2,
+        max_epochs=8,
         val_check_interval=0.50,
         val_percent_check=0.2,
         train_percent_check=0.2,
         logger=logger,
-        checkpoint_callback=ModelCheckpoint(tmpdir)
+        checkpoint_callback=ModelCheckpoint(tmpdir, save_top_k=-1)
     )
 
     # fit model
@@ -358,7 +360,7 @@ def test_model_saving_loading(tmpdir):
     trainer.save_checkpoint(new_weights_path)
 
     # load new model
-    tags_path = logger.experiment.get_data_path(logger.experiment.name, logger.experiment.version)
+    tags_path = tutils.get_data_path(logger, path_dir=tmpdir)
     tags_path = os.path.join(tags_path, 'meta_tags.csv')
     model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
                                                    tags_csv=tags_path)
@@ -369,6 +371,5 @@ def test_model_saving_loading(tmpdir):
     new_pred = model_2(x)
     assert torch.all(torch.eq(pred_before_saving, new_pred)).item() == 1
 
-
 # if __name__ == '__main__':
 #     pytest.main([__file__])

tests/test_trainer.py

@@ -15,7 +15,7 @@ from pytorch_lightning.testing import (
     LightningValidationMultipleDataloadersMixin,
     LightningTestMultipleDataloadersMixin,
 )
-from pytorch_lightning.trainer import trainer_io
+from pytorch_lightning.core.lightning import load_hparams_from_tags_csv
 from pytorch_lightning.trainer.logging import TrainerLoggingMixin
 
 
@@ -51,7 +51,7 @@ def test_no_val_module(tmpdir):
     trainer.save_checkpoint(new_weights_path)
 
     # load new model
-    tags_path = logger.experiment.get_data_path(logger.experiment.name, logger.experiment.version)
+    tags_path = tutils.get_data_path(logger, path_dir=tmpdir)
     tags_path = os.path.join(tags_path, 'meta_tags.csv')
     model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
                                                    tags_csv=tags_path)
@@ -89,7 +89,7 @@ def test_no_val_end_module(tmpdir):
     trainer.save_checkpoint(new_weights_path)
 
     # load new model
-    tags_path = logger.experiment.get_data_path(logger.experiment.name, logger.experiment.version)
+    tags_path = tutils.get_data_path(logger, path_dir=tmpdir)
     tags_path = os.path.join(tags_path, 'meta_tags.csv')
     model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
                                                    tags_csv=tags_path)
@@ -184,10 +184,9 @@ def test_loading_meta_tags(tmpdir):
     logger.save()
 
     # load tags
-    tags_path = logger.experiment.get_data_path(
-        logger.experiment.name, logger.experiment.version
-    ) + '/meta_tags.csv'
-    tags = trainer_io.load_hparams_from_tags_csv(tags_path)
+    path_expt_dir = tutils.get_data_path(logger, path_dir=tmpdir)
+    tags_path = os.path.join(path_expt_dir, 'meta_tags.csv')
+    tags = load_hparams_from_tags_csv(tags_path)
 
     assert tags.batch_size == 32 and tags.hidden_dim == 1000
 
@@ -393,7 +392,7 @@ def test_multiple_test_dataloader(tmpdir):
         default_save_path=tmpdir,
         max_epochs=1,
         val_percent_check=0.1,
-        train_percent_check=0.1,
+        train_percent_check=0.2
     )
 
     # fit model

tests/utils.py

@@ -1,5 +1,4 @@
 import os
-import shutil
 import warnings
 from argparse import Namespace
 
@@ -8,13 +7,9 @@ import torch
 
 from pl_examples import LightningTemplateModel
 from pytorch_lightning import Trainer
-from pytorch_lightning.callbacks import (
-    ModelCheckpoint,
-)
-from pytorch_lightning.logging import TestTubeLogger
-from pytorch_lightning.testing import (
-    LightningTestModel,
-)
+from pytorch_lightning.callbacks import ModelCheckpoint
+from pytorch_lightning.logging import TestTubeLogger, TensorBoardLogger
+from pytorch_lightning.testing import LightningTestModel
 
 # generate a list of random seeds for each test
 RANDOM_PORTS = list(np.random.randint(12000, 19000, 1000))
@@ -22,6 +17,7 @@ ROOT_SEED = 1234
 torch.manual_seed(ROOT_SEED)
 np.random.seed(ROOT_SEED)
 RANDOM_SEEDS = list(np.random.randint(0, 10000, 1000))
+ROOT_PATH = os.path.abspath(os.path.dirname(__file__))
 
 
 def run_model_test_no_loggers(trainer_options, model, min_acc=0.50):
@@ -35,8 +31,9 @@ def run_model_test_no_loggers(trainer_options, model, min_acc=0.50):
     assert result == 1, 'amp + ddp model failed to complete'
 
     # test model loading
-    pretrained_model = load_model(trainer.logger.experiment,
-                                  trainer.checkpoint_callback.filepath)
+    pretrained_model = load_model(trainer.logger,
+                                  trainer.checkpoint_callback.filepath,
+                                  path_expt=trainer_options.get('default_save_path'))
 
     # test new model accuracy
     for dataloader in model.test_dataloader():
@@ -69,7 +66,7 @@ def run_model_test(trainer_options, model, on_gpu=True):
     assert result == 1, 'amp + ddp model failed to complete'
 
     # test model loading
-    pretrained_model = load_model(logger.experiment, trainer.checkpoint_callback.filepath)
+    pretrained_model = load_model(logger, trainer.checkpoint_callback.filepath)
 
     # test new model accuracy
     [run_prediction(dataloader, pretrained_model) for dataloader in model.test_dataloader()]
@@ -127,10 +124,28 @@ def get_test_tube_logger(save_dir, debug=True, version=None):
     return logger
 
 
-def load_model(exp, root_weights_dir, module_class=LightningTemplateModel):
+def get_data_path(expt_logger, path_dir=None):
+    # some calls contain only experiment not complete logger
+    expt = expt_logger.experiment if hasattr(expt_logger, 'experiment') else expt_logger
+    # each logger has to have these attributes
+    name, version = expt_logger.name, expt_logger.version
+    # only the test-tube experiment has such attribute
+    if hasattr(expt, 'get_data_path'):
+        return expt.get_data_path(name, version)
+    # the other experiments...
+    if not path_dir:
+        path_dir = ROOT_PATH
+    path_expt = os.path.join(path_dir, name, 'version_%s' % version)
+    # try if the new sub-folder exists, typical case for test-tube
+    if not os.path.isdir(path_expt):
+        path_expt = path_dir
+    return path_expt
+
+
+def load_model(exp, root_weights_dir, module_class=LightningTemplateModel, path_expt=None):
     # load trained model
-    tags_path = exp.get_data_path(exp.name, exp.version)
-    tags_path = os.path.join(tags_path, 'meta_tags.csv')
+    path_expt_dir = get_data_path(exp, path_dir=path_expt)
+    tags_path = os.path.join(path_expt_dir, TensorBoardLogger.NAME_CSV_TAGS)
 
     checkpoints = [x for x in os.listdir(root_weights_dir) if '.ckpt' in x]
     weights_dir = os.path.join(root_weights_dir, checkpoints[0])
@@ -168,16 +183,10 @@ def run_prediction(dataloader, trained_model, dp=False, min_acc=0.50):
     assert acc > min_acc, f'this model is expected to get > {min_acc} in test set (it got {acc})'
 
 
-def assert_ok_val_acc(trainer):
+def assert_ok_model_acc(trainer, key='test_acc', thr=0.4):
     # this model should get 0.80+ acc
-    acc = trainer.training_tqdm_dict['val_acc']
-    assert acc > 0.50, f'model failed to get expected 0.50 validation accuracy. Got: {acc}'
-
-
-def assert_ok_test_acc(trainer):
-    # this model should get 0.80+ acc
-    acc = trainer.training_tqdm_dict['test_acc']
-    assert acc > 0.50, f'model failed to get expected 0.50 validation accuracy. Got: {acc}'
+    acc = trainer.training_tqdm_dict[key]
+    assert acc > thr, f'Model failed to get expected {thr} accuracy. {key} = {acc}'
 
 
 def can_run_gpu_test():
@@ -193,9 +202,9 @@ def can_run_gpu_test():
 
 
 def reset_seed():
-    SEED = RANDOM_SEEDS.pop()
-    torch.manual_seed(SEED)
-    np.random.seed(SEED)
+    seed = RANDOM_SEEDS.pop()
+    torch.manual_seed(seed)
+    np.random.seed(seed)
 
 
 def set_random_master_port():
@@ -203,9 +212,8 @@ def set_random_master_port():
     os.environ['MASTER_PORT'] = str(port)
 
 
-def init_checkpoint_callback(logger):
-    exp = logger.experiment
-    exp_path = exp.get_data_path(exp.name, exp.version)
+def init_checkpoint_callback(logger, path_dir=None):
+    exp_path = get_data_path(logger, path_dir=path_dir)
     ckpt_dir = os.path.join(exp_path, 'checkpoints')
     checkpoint = ModelCheckpoint(ckpt_dir)
     return checkpoint