release v0.4.6

* added smarter optimizer options

* added smarter optimizer options

* added smarter optimizer options tests

* added smarter optimizer options tests

* added smarter optimizer options tests

* added smarter optimizer options tests

* added smarter optimizer options tests

* added smarter optimizer options tests

* added smarter optimizer options tests

* added single gpu data transfer recursive

* added single gpu data transfer recursive

* added single gpu data transfer recursive

* added single gpu data transfer recursive

* added single gpu data transfer recursive

.codecov.yml

@@ -0,0 +1,44 @@
+#see https://github.com/codecov/support/wiki/Codecov-Yaml
+codecov:
+  notify:
+    require_ci_to_pass: yes
+
+coverage:
+  precision: 0  # 2 = xx.xx%, 0 = xx%
+  round: nearest # how coverage is rounded: down/up/nearest
+  range: 40...100 # custom range of coverage colors from red -> yellow -> green
+  status:
+    # https://codecov.readme.io/v1.0/docs/commit-status
+    project:
+      default:
+        against: auto
+        target: 99% # specify the target coverage for each commit status
+        threshold: 20% # allow this little decrease on project
+        # https://github.com/codecov/support/wiki/Filtering-Branches
+        # branches: master
+        if_ci_failed: error
+    # https://github.com/codecov/support/wiki/Patch-Status
+    patch:
+      default:
+        against: auto
+        target: 40% # specify the target "X%" coverage to hit
+        # threshold: 50% # allow this much decrease on patch
+    changes: false
+
+parsers:
+  gcov:
+    branch_detection:
+      conditional: true
+      loop: true
+      macro: false
+      method: false
+  javascript:
+    enable_partials: false
+
+comment:
+  layout: header, diff
+  require_changes: false
+  behavior: default  # update if exists else create new
+  # branches: *
+
+

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,36 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+### Common bugs:
+1. Tensorboard not showing in Jupyter-notebook see [issue 79](https://github.com/williamFalcon/pytorch-lightning/issues/79).    
+2. PyTorch 1.1.0 vs 1.2.0 support [see FAQ](https://github.com/williamFalcon/pytorch-lightning#faq)    
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (please complete the following information):**
+ - OS: [e.g. iOS]
+ - Browser [e.g. chrome, safari]
+ - Version [e.g. 22]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement, help wanted
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/how-to-question.md

@@ -0,0 +1,26 @@
+---
+name: How to question
+about: Asking how-to questions
+title: ''
+labels: question
+assignees: ''
+
+---
+
+### Before asking:   
+1. search the issues.   
+2. search the docs.    
+
+If you still can't find what you need:     
+#### What is your question?    
+
+#### Code    
+Please paste a code snippet if your question requires it!   
+
+#### What have you tried?    
+
+#### What's your environment?   
+- conda version (no venv)   
+- PyTorch version   
+- Lightning version   
+- Test-tube version

.github/ISSUE_TEMPLATE/typos-and-doc-fixes.md

@@ -0,0 +1,17 @@
+---
+name: Typos and doc fixes
+about: Typos and doc fixes
+title: ''
+labels: typo
+assignees: ''
+
+---
+
+For typos and doc fixes, please go ahead and:
+
+1. Create an issue.
+2. Fix the typo.   
+3. Submit a PR.
+
+
+Thanks!

.gitignore

@@ -10,6 +10,8 @@ app/models/
 pip-wheel-metadata/
 test_tube_exp/
 tests/tests_tt_dir/
+tests/save_dir
+default/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

.readthedocs.yml

@@ -16,4 +16,4 @@ formats: all
 python:
   version: 3.7
   install:
-    - requirements: docs/doc_requirements.txt
+    - requirements: docs/requirements.txt

.travis.yml

@@ -1,16 +1,46 @@
+# vim ft=yaml
+
+# After changing this file, check it on:
+#   http://yaml-online-parser.appspot.com/
+
+# See doc/travis_notes.txt for some guidelines
+
+# this file is *not* meant to cover or endorse the use of travis, but rather to
+# help confirm pull requests to this project.
+
+dist: xenial  # Ubuntu 16.04
+
+env:
+  global:
+    - DISPLAY=""
+
 language: python
-python:
+
-  - "3.7"
+matrix:
-# command to install dependencies
+  include:
+    - python: 3.6
+      env: TOXENV=py36
+    - python: 3.7
+      env: TOXENV=py37
+
+# See http://docs.travis-ci.com/user/caching/#pip-cache
 cache: pip
+
 install:
-  - pip install -e .
   - pip install -r requirements.txt
-  - pip install -U numpy
+  - pip install -r ./tests/requirements.txt
+  - pip --version ; pip list
 
-# keep build from timing out
-dist: xenial
-
-# command to run tests
 script:
-  - py.test # or py.test for Python versions 3.5 and below
+  # integration
+  - tox --sitepackages
+  - python setup.py install --dry-run
+
+after_success:
+  - coverage report
+  # disable auto coverage bc it isn't accurate since it misses gpu code.
+  # to get coverage, run local and push results
+  # - codecov
+
+notifications:
+  email: false

CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at waf2107@columbia.edu. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -0,0 +1,37 @@
+# Contributing    
+Welcome to the PyTorch Lightning community! We're building the most advanced research platform on the planet to implement the latest, best practices that the amazing PyTorch team rolls out!  
+
+## Lightning Design Principles   
+We encourage all sorts of contributions you're interested in adding! When coding for lightning, please follow these principles.    
+
+#### No PyTorch interference   
+We don't want to add any abstractions on top of pure PyTorch. This gives researchers all the control they need without having to learn yet another framework.    
+
+#### Simple Internal Code    
+It's useful for users to look at the code and understand very quickly what's happening. Many users won't be engineers. Thus we need to value clear, simple code over condensed ninja moves. While that's super cool, this isn't the project for that :)      
+
+#### Force User Decisions To Best Practices    
+There are 1,000 ways to do something. However, something eventually becomes standard practice that everyone does. Thus we pick one way of doing it and force everyone to do it this way. A good example is accumulated gradients. There are many ways to implement, we just pick one and force users to use that one. A bad forced decision would be to make users use a specific library to do something.    
+
+When something becomes a best practice, we add it to the framework. This likely looks like code in utils or in the model file that everyone keeps adding over and over again across projects. When this happens, bring that code inside the trainer and add a flag for it.
+
+#### Simple External API    
+What makes sense to you may not make sense to others. Create an issue with an API change suggestion and validate that it makes sense for others. Treat code changes how you treat a startup: validate that it's a needed feature, then add if it makes sense for many people.    
+
+#### Gain User Trust    
+As a researcher you can't have any part of your code going wrong. So, make thorough tests that ensure an implementation of a new trick or subbtle change is correct.    
+
+## Contribution types    
+Currently looking for help implementing new features or adding bug fixes. 
+
+A lot of good work has already been done in project mechanics (requirements.txt, setup.py, pep8, badges, ci, etc...) we're in a good state there thanks to all the early contributors (even pre-beta release)!   
+
+## Bug fixes:  
+1. Submit a github issue.   
+2. Fix it.  
+3. Submit a PR! 
+
+## New Features:  
+1. Submit a github issue.   
+2. We'll agree on the feature scope.     
+3. Submit a PR! (with updated docs and tests 🙃).   

COPYING

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2019 William Falcon
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MANIFEST.in

@@ -1,9 +1,37 @@
-graft docs
+# Manifest syntax https://docs.python.org/2/distutils/sourcedist.html
+graft wheelhouse
 
-include COPYING
+recursive-include birl *.py
-include AUTHORS
+recursive-exclude __pycache__  *.py[cod] *.orig
 
-recursive-include src/einsteinpy/tests *.py *.html
+# Include the README
+include *.md
 
-prune docs/source/examples/.ipynb_checkpoints
+# Include the license file
-global-exclude *.py[cod] __pycache__ *.so *.dylib
+include LICENSE
+
+exclude *.sh
+exclude *.toml
+exclude *.svg    
+recursive-include examples *.py
+recursive-include pytorch_lightning *.py
+
+# exclude tests from package
+recursive-exclude tests *
+exclude tests
+
+# Exclude the documentation files
+recursive-exclude docs *
+exclude docs
+
+# Include the Requirements
+include requirements.txt
+
+# Exclude build configs
+exclude *.yml
+
+prune .git
+prune .github
+prune notebook*
+prune temp*
+prune test*

README.md

@@ -1,24 +1,28 @@
-<p align="center">
+<div align="center">
-  <a href="https://williamfalcon.github.io/pytorch-lightning/">
-    <img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/lightning_logo.png" width="50">
-  </a>
-</p>
-<h3 align="center">
-  PyTorch Lightning
-</h3>
-<p align="center">
-  The PyTorch Keras for ML researchers. More control. Less boilerplate.    
-</p>
 
-<p align="center">
+![Logo](./docs/source/_static/lightning_logo_small.png)
-  <a href="https://badge.fury.io/py/pytorch-lightning"><img src="https://badge.fury.io/py/pytorch-lightning.svg" alt="PyPI version" height="18"></a>
-  <a href="https://pepy.tech/project/pytorch-lightning"><img src="https://pepy.tech/badge/pytorch-lightning" alt="PyPI version" height="18"></a>
-  <a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/tests"><img src="https://github.com/williamFalcon/pytorch-lightning/blob/master/coverage.svg"></a>
-  <a href="https://travis-ci.org/williamFalcon/pytorch-lightning"><img src="https://travis-ci.org/williamFalcon/pytorch-lightning.svg?branch=master"></a>
-  <a href="https://williamfalcon.github.io/pytorch-lightning/"><img src="https://readthedocs.org/projects/pytorch-lightning/badge/?version=latest"></a>
-  <a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/COPYING"><img src="https://img.shields.io/badge/License-MIT-yellow.svg"></a>
-</p>   
 
+# PyTorch Lightning
+
+**The PyTorch Keras for ML researchers. More control. Less boilerplate.**
+
+
+[![PyPI Status](https://badge.fury.io/py/pytorch-lightning.svg)](https://badge.fury.io/py/pytorch-lightning)
+[![PyPI Status](https://pepy.tech/badge/pytorch-lightning)](https://pepy.tech/project/pytorch-lightning)
+[![Build Status](https://travis-ci.org/williamFalcon/pytorch-lightning.svg?branch=master)](https://travis-ci.org/williamFalcon/pytorch-lightning)
+[![Build status](https://ci.appveyor.com/api/projects/status/rum89d7hq8l1kfye?svg=true)](https://ci.appveyor.com/project/Borda/pytorch-lightning)
+[![Coverage](https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/coverage.svg)](https://github.com/williamFalcon/pytorch-lightning/tree/master/tests#running-coverage)
+[![CodeFactor](https://www.codefactor.io/repository/github/borda/pytorch-lightning/badge)](https://www.codefactor.io/repository/github/borda/pytorch-lightning)
+[![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=latest)](https://pytorch-lightning.readthedocs.io/en/latest)
+[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/williamFalcon/pytorch-lightning/blob/master/LICENSE)
+<!-- 
+removed until codecov badge isn't empy. likely a config error showing nothing on master.
+[![codecov](https://codecov.io/gh/Borda/pytorch-lightning/branch/master/graph/badge.svg)](https://codecov.io/gh/Borda/pytorch-lightning)
+-->
+
+</div>
+
+Simple installation from PyPI
 ```bash
 pip install pytorch-lightning  
 ```
@@ -27,18 +31,37 @@ pip install pytorch-lightning
 **[View the docs here](https://williamfalcon.github.io/pytorch-lightning/)**
 
 ## What is it?  
-Lightning defers training and validation loop logic to you. It guarantees correct, modern best practices for the core training logic.
+Lightning is a very lightweight wrapper on PyTorch. This means you don't have to learn a new library. It defers core training and validation logic to you and automates the rest. It guarantees tested, correct, modern best practices for the automated parts.
 
 
 ## Why do I want to use lightning?
-When starting a new project the last thing you want to do is recode a training loop, model loading/saving, distributed training, when to validate, etc... You're likely to spend a long time ironing out all the bugs without even getting to the core of your research.
+When starting a new project the last thing you want to do is recode a training loop, multi-cluster training, 16-bit precision, early-stopping, model loading/saving, when to validate, etc... You're likely to spend a long time ironing out all the bugs without even getting to the core of your research.
 
-With lightning, you guarantee those parts of your code work so you can focus on what the meat of the research: Data and training, validation loop logic. Don't worry about multiple gpus or speeding up your code, lightning will do that for you!
+With lightning, you guarantee those parts of your code work so you can focus on what the meat of the research: The data and the training/validation loop logic. 
 
+Don't worry about training on multiple gpus or speeding up your code, lightning will do that for you!
+
+---   
+## README Table of Contents        
+- [How do I use it](https://github.com/williamFalcon/pytorch-lightning#how-do-i-do-use-it)     
+- [What lightning automates](https://github.com/williamFalcon/pytorch-lightning#what-does-lightning-control-for-me)    
+- [Tensorboard integration](https://github.com/williamFalcon/pytorch-lightning#tensorboard)    
+- [Lightning features](https://github.com/williamFalcon/pytorch-lightning#lightning-automates-all-of-the-following-each-is-also-configurable)    
+- [Demos](https://github.com/williamFalcon/pytorch-lightning#demo)    
+- [Tutorials](https://github.com/williamFalcon/pytorch-lightning#tutorials)
+- [Contributing](https://github.com/williamFalcon/pytorch-lightning/blob/master/CONTRIBUTING.md)    
+- [Bleeding edge install](https://github.com/williamFalcon/pytorch-lightning#bleeding-edge)   
+- [Lightning Design Principles](https://github.com/williamFalcon/pytorch-lightning#lightning-design-principles)
+- [FAQ](https://github.com/williamFalcon/pytorch-lightning#faq)    
+
+---   
 ## How do I do use it?   
+The research code goes into a [LightningModule]((https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/)) which you fit using a Trainer.    
+
+Think of the LightningModule as a *system* such as seq-2-seq, GAN, etc... However, the LightningModule can ALSO just be a simple classifier such as the example below.     
 
 To use lightning do 2 things:  
-1. [Define a LightningModel](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/)         
+1. [Define a LightningModule](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/)         
 ```python
 import os
 import torch
@@ -47,60 +70,75 @@ from torch.utils.data import DataLoader
 from torchvision.datasets import MNIST
 import torchvision.transforms as transforms
 
-import pytorch_lightning as ptl
+import pytorch_lightning as pl
 
-class CoolModel(ptl.LightningModule):
+class CoolSystem(pl.LightningModule):
 
     def __init__(self):
-        super(CoolModel, self).__init__()
+        super(CoolSystem, self).__init__()
         # not the best model...
         self.l1 = torch.nn.Linear(28 * 28, 10)
 
     def forward(self, x):
         return torch.relu(self.l1(x.view(x.size(0), -1)))
 
-    def my_loss(self, y_hat, y):
-        return F.cross_entropy(y_hat, y)
-
     def training_step(self, batch, batch_nb):
+        # REQUIRED
         x, y = batch
         y_hat = self.forward(x)
-        return {'loss': self.my_loss(y_hat, y)}
+        return {'loss': F.cross_entropy(y_hat, y)}
 
     def validation_step(self, batch, batch_nb):
+        # OPTIONAL
         x, y = batch
         y_hat = self.forward(x)
-        return {'val_loss': self.my_loss(y_hat, y)}
+        return {'val_loss': F.cross_entropy(y_hat, y)}
 
     def validation_end(self, outputs):
+        # OPTIONAL
         avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
         return {'avg_val_loss': avg_loss}
 
     def configure_optimizers(self):
+        # REQUIRED
         return [torch.optim.Adam(self.parameters(), lr=0.02)]
 
-    @ptl.data_loader
+    @pl.data_loader
     def tng_dataloader(self):
+        # REQUIRED
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 
-    @ptl.data_loader
+    @pl.data_loader
     def val_dataloader(self):
+        # OPTIONAL
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 
-    @ptl.data_loader
+    @pl.data_loader
     def test_dataloader(self):
+        # OPTIONAL
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 ```
 
 2. Fit with a [trainer](https://williamfalcon.github.io/pytorch-lightning/Trainer/)    
 ```python
 from pytorch_lightning import Trainer
-from test_tube import Experiment
 
-model = CoolModel()
+model = CoolSystem()
+
+# most basic trainer, uses good defaults
+trainer = Trainer()    
+trainer.fit(model)   
+```     
+
+Or with tensorboard logger and some options turned on such as multi-gpu, etc...    
+```python   
+from test_tube import Experiment    
+
+# PyTorch summarywriter with a few bells and whistles    
 exp = Experiment(save_dir=os.getcwd())
 
 # train on cpu using only 10% of the data (for demo purposes)
+# pass in experiment for automatic tensorboard logging.    
 trainer = Trainer(experiment=exp, max_nb_epochs=1, train_percent_check=0.1)
 
 # train on 4 gpus
@@ -113,14 +151,16 @@ trainer = Trainer(experiment=exp, max_nb_epochs=1, train_percent_check=0.1)
 trainer.fit(model)
 
 # view tensorflow logs 
-print(f'View tensorboard logs by running\ntensorboard --logdir {os.getcwd()}')
+print('View tensorboard logs by running\ntensorboard --logdir %s' % os.getcwd())
 print('and going to http://localhost:6006 on your browser')
-```
+```    
 
+## What does lightning control for me?   
 
-## What does lightning control for me?
+Everything in gray!    
-Everything!    
+You define the blue parts using the LightningModule interface:  
-Except for these 6 core functions which you define:    
+
+![Ouverview](./docs/source/_static/overview_flat.jpg)
 
 ```{.python}
 # what to do in the training loop
@@ -148,7 +188,7 @@ def training_step(self, data_batch, batch_nb):
     # define your own forward and loss calculation
     hidden_states = self.encoder(x)
      
-    # even as complex as a seq-2seq + attn model
+    # even as complex as a seq-2-seq + attn model
     # (this is just a toy, non-working example to illustrate)
     start_token = '<SOS>'
     last_hidden = torch.zeros(...)
@@ -202,19 +242,11 @@ def validation_end(self, outputs):
 ## Tensorboard    
 Lightning is fully integrated with tensorboard.   
 
-<p align="center">
+![tensorboard-support](./docs/source/_static/tf_loss.png)
-  <a href="https://williamfalcon.github.io/pytorch-lightning/">
-    <img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/tf_loss.png" width="900px">
-  </a>
-</p>
 
 Lightning also adds a text column with all the hyperparameters for this experiment.      
 
-<p align="center">
+![tensorboard-support](./docs/source/_static/tf_tags.png)
-  <a href="https://williamfalcon.github.io/pytorch-lightning/">
-        <img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/tf_tags.png" width="900px">
-  </a>
-</p>
 
 Simply note the path you set for the Experiment    
 ``` {.python}   
@@ -238,6 +270,7 @@ tensorboard --logdir /some/path
 
 - [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
 - [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics) 
+- [Restoring training session](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#restoring-training-session)
 
 ###### Computing cluster (SLURM)    
 
@@ -251,7 +284,8 @@ tensorboard --logdir /some/path
 - [Log GPU usage](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage)
 - [Make model overfit on subset of data](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data)
 - [Print the parameter count by layer](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer)
-- [Pring which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
+- [Print which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
+- [Print input and output size of every module in system](https://williamfalcon.github.io/pytorch-lightning/LightningModule/properties/#example_input_array)
 
 
 ###### Distributed training    
@@ -283,6 +317,7 @@ tensorboard --logdir /some/path
 - [Learning rate scheduling](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers)    
 - [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers)
 - [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check)
+- [Step optimizers at arbitrary intervals](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step)
 
 ###### Validation loop    
 
@@ -294,7 +329,6 @@ tensorboard --logdir /some/path
 - [Set the number of validation sanity steps](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-the-number-of-validation-sanity-steps)
 
 
-
 ## Demo
 ```bash
 # install lightning
@@ -302,7 +336,8 @@ pip install pytorch-lightning
 
 # clone lightning for the demo
 git clone https://github.com/williamFalcon/pytorch-lightning.git
-cd pytorch_lightning/examples/new_project_templates/
+cd pytorch-lightning
+cd examples/new_project_templates/
 
 # all of the following demos use the SAME model to show no modification needs to be made to your code
 
@@ -316,6 +351,46 @@ python single_gpu_node_template.py --gpus "0,1"
 python multi_node_cluster_template.py --nb_gpu_nodes 4 --gpus '0,1,2,3,4,5,6,7'
 ```
 
+## Tutorials   
+- [Basic Lightning use](https://towardsdatascience.com/supercharge-your-ai-research-with-pytorch-lightning-337948a99eec)    
+- [9 key speed features in Pytorch-Lightning](https://towardsdatascience.com/9-tips-for-training-lightning-fast-neural-networks-in-pytorch-8e63a502f565)    
+- [SLURM, multi-node training with Lightning](https://towardsdatascience.com/trivial-multi-node-training-with-pytorch-lightning-ff75dfb809bd)     
+
+---   
+## FAQ    
+**Why was Lightning created?**     
+Lightning has 3 goals in mind:
+1. Maximal flexibility while abstracting out the common boilerplate across research projects.   
+2. Reproducibility. If all projects use the LightningModule template, it will be much much easier to understand what's going on and where to look! It will also mean every implementation follows a standard format.   
+3. Democratizing PyTorch power user features. Distributed training? 16-bit? know you need them but don't want to take the time to implement? All good... these come built into Lightning.    
+
+**How does Lightning compare with Ignite and fast.ai?**     
+[Here's a thorough comparison](https://medium.com/@_willfalcon/pytorch-lightning-vs-pytorch-ignite-vs-fast-ai-61dc7480ad8a).    
+
+**Is this another library I have to learn?**    
+Nope! We use pure Pytorch everywhere and don't add unecessary abstractions!   
+
+**Are there plans to support Python 2?**          
+Nope.     
+
+**Are there plans to support virtualenv?**    
+Nope. Please use anaconda or miniconda.    
+
+**Which PyTorch versions do you support?**    
+##### PyTorch 1.1.0       
+```bash    
+# install pytorch 1.1.0 using the official instructions   
+
+# install test-tube 0.6.7.6 which supports 1.1.0   
+pip install test-tube==0.6.7.6   
+
+# install latest Lightning version without upgrading deps    
+pip install -U --no-deps pytorch-lightning
+```     
+
+##### PyTorch 1.2.0   
+Install via pip as normal   
+
 ## Bleeding edge
 If you can't wait for the next release, install the most up to date code with:  
 ```bash

appveyor.yml

@@ -0,0 +1,64 @@
+# https://www.appveyor.com/docs/appveyor-yml/
+environment:
+
+  # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+  # /E:ON and /V:ON options are not enabled in the batch script interpreter
+  # See: http://stackoverflow.com/a/13751649/163740
+  CMD_IN_ENV: "cmd /E:ON /V:ON /C obvci_appveyor_python_build_env.cmd"
+
+  matrix:
+    # Pre-installed Python versions, which Appveyor may upgrade to
+    # a later point release.
+    # See: http://www.appveyor.com/docs/installed-software#python
+
+
+    # - PYTHON: "C:\\Python35-x64"
+    #   PYTHON_VERSION: "3.5.x"
+    #   PYTHON_ARCH: "64"
+    #  TOXENV: "py35"
+
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "64"
+      TOXENV: "py36"
+      PIP_PYVER: "36"
+
+    - PYTHON: "C:\\Python37-x64"
+      PYTHON_VERSION: "3.7.x"
+      PYTHON_ARCH: "64"
+      TOXENV: "py37"
+      PIP_PYVER: "37"
+
+build: off
+
+# https://www.appveyor.com/docs/build-cache/
+cache:
+  - C:\ProgramData\chocolatey\bin -> appveyor.yml
+  - C:\ProgramData\chocolatey\lib -> appveyor.yml
+  - '%LOCALAPPDATA%\pip\Cache -> appveyor.yml'
+
+# scripts that run after cloning repository
+install:
+  # If there is a newer build queued for the same PR, cancel this one.
+  # The AppVeyor 'rollout builds' option is supposed to serve the same
+  # purpose but it is problematic because it tends to cancel builds pushed
+  # directly to master instead of just PR builds (or the converse).
+  - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%path%
+  - pip install -U --user pip
+  - pip install -r requirements.txt -f https://download.pytorch.org/whl/torch_stable.html
+  - pip install -r ./tests/requirements.txt
+
+# scripts to run before tests (working directory and environment changes are persisted from the previous steps such as "before_build")
+before_test:
+  - python --version
+  - pip --version
+  - pip list
+  - dir
+
+# to run your custom scripts instead of automatic tests
+test_script:
+  - tox --sitepackages --parallel auto
+
+on_success:
+  - coverage report
+  # - codecov

docs/LightningModule/RequiredTrainerInterface.md

@@ -9,22 +9,20 @@ Otherwise, to Define a Lightning Module, implement the following methods:
 
 **Required**:  
 
-- [training_step](RequiredTrainerInterface.md#training_step)   
+- [training_step](RequiredTrainerInterface.md#training_step)      
-- [validation_step](RequiredTrainerInterface.md#validation_step)
+- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)    
-- [validation_end](RequiredTrainerInterface.md#validation_end)
+- [configure_optimizers](RequiredTrainerInterface.md#configure_optimizers)    
-    
-- [configure_optimizers](RequiredTrainerInterface.md#configure_optimizers)
-
-- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)
-- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)
-- [test_dataloader](RequiredTrainerInterface.md#test_dataloader)
 
 **Optional**:   
 
-- [on_save_checkpoint](RequiredTrainerInterface.md#on_save_checkpoint)
+- [validation_step](RequiredTrainerInterface.md#validation_step)    
-- [on_load_checkpoint](RequiredTrainerInterface.md#on_load_checkpoint)
+- [validation_end](RequiredTrainerInterface.md#validation_end)    
-- [update_tng_log_metrics](RequiredTrainerInterface.md#update_tng_log_metrics)
+- [val_dataloader](RequiredTrainerInterface.md#val_dataloader)    
-- [add_model_specific_args](RequiredTrainerInterface.md#add_model_specific_args)
+- [test_dataloader](RequiredTrainerInterface.md#test_dataloader)    
+- [on_save_checkpoint](RequiredTrainerInterface.md#on_save_checkpoint)    
+- [on_load_checkpoint](RequiredTrainerInterface.md#on_load_checkpoint)    
+- [update_tng_log_metrics](RequiredTrainerInterface.md#update_tng_log_metrics)    
+- [add_model_specific_args](RequiredTrainerInterface.md#add_model_specific_args)    
 
 ---
 ### Minimal example   
@@ -36,9 +34,9 @@ from torch.utils.data import DataLoader
 from torchvision.datasets import MNIST
 import torchvision.transforms as transforms
 
-import pytorch_lightning as ptl
+import pytorch_lightning as pl
 
-class CoolModel(ptl.LightningModule):
+class CoolModel(pl.LightningModule):
 
     def __init__(self):
         super(CoolModel, self).__init__()
@@ -48,40 +46,53 @@ class CoolModel(ptl.LightningModule):
     def forward(self, x):
         return torch.relu(self.l1(x.view(x.size(0), -1)))
 
-    def my_loss(self, y_hat, y):
-        return F.cross_entropy(y_hat, y)
-
     def training_step(self, batch, batch_nb):
+        # REQUIRED
         x, y = batch
         y_hat = self.forward(x)
-        return {'loss': self.my_loss(y_hat, y)}
+        return {'loss': F.cross_entropy(y_hat, y)(y_hat, y)}
 
     def validation_step(self, batch, batch_nb):
+        # OPTIONAL
         x, y = batch
         y_hat = self.forward(x)
-        return {'val_loss': self.my_loss(y_hat, y)}
+        return {'val_loss': F.cross_entropy(y_hat, y)(y_hat, y)}
 
     def validation_end(self, outputs):
+        # OPTIONAL
         avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
         return {'avg_val_loss': avg_loss}
 
     def configure_optimizers(self):
+        # REQUIRED
         return [torch.optim.Adam(self.parameters(), lr=0.02)]
 
-    @ptl.data_loader
+    @pl.data_loader
     def tng_dataloader(self):
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 
-    @ptl.data_loader
+    @pl.data_loader
     def val_dataloader(self):
+        # OPTIONAL
+        # can also return a list of val dataloaders
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 
-    @ptl.data_loader
+    @pl.data_loader
     def test_dataloader(self):
+        # OPTIONAL
         return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
 ```
+---   
+### How do these methods fit into the broader training?     
+The LightningModule interface is on the right. Each method corresponds to a part of a research project. Lightning automates everything not in blue.    
 
----
+<p align="center">
+  <a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/overview_flat.jpg">
+    <img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/overview_flat.jpg" height="900px">
+  </a>
+</p>   
+
+## Required Methods    
 
 ### training_step
 
@@ -125,17 +136,96 @@ def training_step(self, data_batch, batch_nb):
     
     # return a dict
     return output
+```    
+
+If you define multiple optimizers, this step will also be called with an additional ```optimizer_idx``` param.    
+``` {.python}
+# Multiple optimizers (ie: GANs)     
+def training_step(self, data_batch, batch_nb, optimizer_idx):
+    if optimizer_idx == 0:
+        # do training_step with encoder
+    if optimizer_idx == 1:
+        # do training_step with decoder    
+```    
+
+--- 
+### tng_dataloader 
+
+``` {.python}
+@pl.data_loader
+def tng_dataloader(self)
+```
+Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.
+
+##### Return
+PyTorch DataLoader
+
+**Example**
+
+``` {.python}
+@pl.data_loader
+def tng_dataloader(self):
+    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+    dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True)
+    loader = torch.utils.data.DataLoader(
+        dataset=dataset,
+        batch_size=self.hparams.batch_size,
+        shuffle=True
+    )
+    return loader
 ```
 
----
+--- 
+### configure_optimizers 
+
+``` {.python}
+def configure_optimizers(self)
+```
+
+Set up as many optimizers and (optionally) learning rate schedulers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. 
+Lightning will call .backward() and .step() on each one in every epoch.  If you use 16 bit precision it will also handle that.
+
+**Note:** If you use multiple optimizers, training_step will have an additional ```optimizer_idx``` parameter.    
+
+
+
+##### Return
+List or Tuple - List of optimizers with an optional second list of learning-rate schedulers
+
+**Example**
+
+``` {.python}
+# most cases
+def configure_optimizers(self):
+    opt = Adam(self.parameters(), lr=0.01)
+    return [opt]
+    
+# gan example, with scheduler for discriminator
+def configure_optimizers(self):
+    generator_opt = Adam(self.model_gen.parameters(), lr=0.01)
+    disriminator_opt = Adam(self.model_disc.parameters(), lr=0.02)
+    discriminator_sched = CosineAnnealing(discriminator_opt, T_max=10)
+    return [generator_opt, disriminator_opt], [discriminator_sched]
+```  
+
+If you need to control how often those optimizers step or override the default .step() schedule, override 
+the [optimizer_step](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step) hook.   
+
+## Optional Methods    
 
 ### validation_step
 
 ``` {.python}
-def validation_step(self, data_batch, batch_nb)
+def validation_step(self, data_batch, batch_nb)   
+
+# if have multiple val dataloaders:  
+def validation_step(self, data_batch, batch_nb, dataloader_idx)
 ```
+**OPTIONAL**    
+If you don't need to validate you don't need to implement this method.    
+
+In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes, calculate accuracy, or save example outputs (using self.experiment or whatever you want). Really, anything you want.       
 
-In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model.
 This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation.
 
 **Params**   
@@ -144,6 +234,7 @@ This is most likely the same as your training_step. But unlike training step, th
 |---|---|
 |  data_batch | The output of your dataloader. A tensor, tuple or list  |
 |  batch_nb | Integer displaying which batch this is  |
+|  dataloader_i | Integer displaying which dataloader this is (only if multiple val datasets used)  |
 
 **Return**   
 
@@ -154,6 +245,7 @@ This is most likely the same as your training_step. But unlike training step, th
 **Example**
 
 ``` {.python}
+# CASE 1: A single validation dataset
 def validation_step(self, data_batch, batch_nb):
     x, y, z = data_batch
     
@@ -174,16 +266,29 @@ def validation_step(self, data_batch, batch_nb):
     
     # return an optional dict
     return output
-```
+```   
+
+If you pass in multiple validation datasets, validation_step will have an additional argument.
+
+```python
+# CASE 2: multiple validation datasets
+def validation_step(self, data_batch, batch_nb, dataset_idx):
+    # dataset_idx tells you which dataset this is.   
+```   
+
+The ```dataset_idx``` corresponds to the order of datasets returned in ```val_dataloader```.    
 
 --- 
 ### validation_end
 
 ``` {.python}
 def validation_end(self, outputs)
-```
+```   
+If you didn't define a validation_step, this won't be called.       
 
-Called at the end of the validation loop with the output of each validation_step.
+Called at the end of the validation loop with the output of each validation_step.  Called once per validation dataset.   
+
+The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything.    
 
 **Params**    
 
@@ -218,36 +323,6 @@ def validation_end(self, outputs):
     return tqdm_dic
 ```
 
---- 
-### configure_optimizers 
-
-``` {.python}
-def configure_optimizers(self)
-```
-
-Set up as many optimizers and (optionally) learning rate schedulers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. 
-Lightning will call .backward() and .step() on each one in every epoch.  If you use 16 bit precision it will also handle that.
-
-
-##### Return
-List or Tuple - List of optimizers with an optional second list of learning-rate schedulers
-
-**Example**
-
-``` {.python}
-# most cases
-def configure_optimizers(self):
-    opt = Adam(self.parameters(), lr=0.01)
-    return [opt]
-    
-# gan example, with scheduler for discriminator
-def configure_optimizers(self):
-    generator_opt = Adam(self.model_gen.parameters(), lr=0.01)
-    disriminator_opt = Adam(self.model_disc.parameters(), lr=0.02)
-    discriminator_sched = CosineAnnealing(discriminator_opt, T_max=10)
-    return [generator_opt, disriminator_opt], [discriminator_sched]
-```
-
 --- 
 ### on_save_checkpoint 
 
@@ -290,49 +365,25 @@ def on_load_checkpoint(self, checkpoint):
     self.something_cool_i_want_to_save = checkpoint['something_cool_i_want_to_save']
 ```
 
---- 
-### tng_dataloader 
-
-``` {.python}
-@ptl.data_loader
-def tng_dataloader(self)
-```
-Called by lightning during training loop. Make sure to use the @ptl.data_loader decorator, this ensures not calling this function until the data are needed.
-
-##### Return
-PyTorch DataLoader
-
-**Example**
-
-``` {.python}
-@ptl.data_loader
-def tng_dataloader(self):
-    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
-    dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True)
-    loader = torch.utils.data.DataLoader(
-        dataset=dataset,
-        batch_size=self.hparams.batch_size,
-        shuffle=True
-    )
-    return loader
-```
-
 --- 
 ### val_dataloader 
 
 ``` {.python}
-@ptl.data_loader
+@pl.data_loader
 def tng_dataloader(self)
 ```
-Called by lightning during validation loop. Make sure to use the @ptl.data_loader decorator, this ensures not calling this function until the data are needed.
+**OPTIONAL**    
+If you don't need a validation dataset and a validation_step, you don't need to implement this method.    
+
+Called by lightning during validation loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.   
 
 ##### Return
-PyTorch DataLoader
+PyTorch DataLoader or list of PyTorch Dataloaders.    
 
 **Example**
 
 ``` {.python}
-@ptl.data_loader
+@pl.data_loader
 def val_dataloader(self):
     transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
     dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True)
@@ -343,16 +394,27 @@ def val_dataloader(self):
     )
     
     return loader
+
+# can also return multiple dataloaders   
+@pl.data_loader
+def val_dataloader(self):
+    return [loader_a, loader_b, ..., loader_n]   
 ```
 
+In the case where you return multiple val_dataloaders, the validation_step will have an arguement ```dataset_idx```
+which matches the order here.    
+
 --- 
 ### test_dataloader 
 
 ``` {.python}
-@ptl.data_loader
+@pl.data_loader
 def test_dataloader(self)
 ```
-Called by lightning during test loop. Make sure to use the @ptl.data_loader decorator, this ensures not calling this function until the data are needed.
+**OPTIONAL**    
+If you don't need a test dataset and a test_step, you don't need to implement this method.    
+
+Called by lightning during test loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.
 
 ##### Return
 PyTorch DataLoader
@@ -360,7 +422,7 @@ PyTorch DataLoader
 **Example**
 
 ``` {.python}
-@ptl.data_loader
+@pl.data_loader
 def test_dataloader(self):
     transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
     dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True)

docs/LightningModule/properties.md

@@ -38,3 +38,17 @@ self.trainer.current_epoch
 ...   
 ```   
 
+## Debugging   
+The LightningModule also offers these tricks to help debug.   
+
+---   
+#### example_input_array    
+In the LightningModule init, you can set a dummy tensor for this property
+to get a print out of sizes coming into and out of every layer.   
+```python
+def __init__(self):
+    # put the dimensions of the first input to your system
+    self.example_input_array = torch.rand(5, 28 * 28)
+```    
+
+

docs/Trainer/Checkpointing.md

@@ -1,11 +1,11 @@
-Lightning can automate saving and loading checkpoints.
+i Lightning can automate saving and loading checkpoints.
 
 ---
 ### Model saving
 To enable checkpointing, define the checkpoint callback and give it to the trainer.
 
 ``` {.python}
-from pytorch_lightning.utils.pt_callbacks import ModelCheckpoint
+from pytorch_lightning.callbacks import ModelCheckpoint
 
 checkpoint_callback = ModelCheckpoint(
     filepath='/path/to/store/weights.ckpt',
@@ -18,5 +18,51 @@ checkpoint_callback = ModelCheckpoint(
 trainer = Trainer(checkpoint_callback=checkpoint_callback)
 ```
 
+---
+### Restoring training session 
+You might want to not only load a model but also continue training it. Use this method to
+restore the trainer state as well. This will continue from the epoch and global step you last left off.  
+However, the dataloaders will start from the first batch again (if you shuffled it shouldn't matter).   
 
+Lightning will restore the session if you pass an experiment with the same version and there's a saved checkpoint.   
+``` {.python}
+from test_tube import Experiment
 
+exp = Experiment(version=a_previous_version_with_a_saved_checkpoint)
+trainer = Trainer(experiment=exp)
+
+# this fit call loads model weights and trainer state
+# the trainer continues seamlessly from where you left off
+# without having to do anything else.
+trainer.fit(model)
+```
+
+The trainer restores:  
+- global_step    
+- current_epoch    
+- All optimizers    
+- All lr_schedulers    
+- Model weights
+
+You can even change the logic of your model as long as the weights and "architecture" of 
+the system isn't different. If you add a layer, for instance, it might not work.   
+
+At a rough level, here's [what happens inside Trainer](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/root_module/model_saving.py#L63):   
+```python
+
+self.global_step = checkpoint['global_step']
+self.current_epoch = checkpoint['epoch']
+
+# restore the optimizers
+optimizer_states = checkpoint['optimizer_states']
+for optimizer, opt_state in zip(self.optimizers, optimizer_states):
+    optimizer.load_state_dict(opt_state)
+
+# restore the lr schedulers
+lr_schedulers = checkpoint['lr_schedulers']
+for scheduler, lrs_state in zip(self.lr_schedulers, lr_schedulers):
+    scheduler.load_state_dict(lrs_state)
+
+# uses the model you passed into trainer        
+model.load_state_dict(checkpoint['state_dict'])
+```    

docs/Trainer/Distributed training.md

@@ -23,6 +23,22 @@ have configuration issues depending on your cluster.
 
 For a deeper understanding of what lightning is doing, feel free to read [this guide](https://medium.com/@_willfalcon/9-tips-for-training-lightning-fast-neural-networks-in-pytorch-8e63a502f565).   
 
+---
+#### Distributed and 16-bit precision.    
+Due to an issue with apex and DistributedDataParallel (PyTorch and NVIDIA issue), Lightning does
+not allow 16-bit and DP training. We tried to get this to work, but it's an issue on their end.   
+
+Below are the possible configurations we support.    
+
+| 1 GPU  | 1+ GPUs  | DP  | DDP  | 16-bit  | command |
+|---|---|---|---|---|---|
+| Y  |   |   |   |  | ```Trainer(gpus=[0])``` |
+| Y  |   |   |   | Y | ```Trainer(gpus=[0], use_amp=True)``` |
+|   | Y | Y |   |   | ```Trainer(gpus=[0, ...])``` |
+|   | Y |  | Y  |  | ```Trainer(gpus=[0, ...], distributed_backend='ddp')``` |
+|   | Y |  | Y  | Y | ```Trainer(gpus=[0, ...], distributed_backend='ddp', use_amp=True)``` |
+
+
 ---
 #### CUDA flags   
 CUDA flags make certain GPUs visible to your script. 
@@ -94,7 +110,7 @@ cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo')
 cluster.add_command('export NCCL_DEBUG=INFO')
 
 # setting a master port here is a good idea.
-cluster.add_command(f'export MASTER_PORT={PORT}')
+cluster.add_command('export MASTER_PORT=%r' % PORT)
 
 # good to load the latest NCCL version
 cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0'])

docs/Trainer/Logging.md

@@ -33,7 +33,7 @@ trainer = Trainer(process_position=1)
 Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use.
 Give lightning a test-tube Experiment object to automate this for you.
 ``` {.python}
-from test-tube import Experiment
+from test_tube import Experiment
 
 exp = Experiment(...)
 Trainer(experiment=exp)
@@ -44,7 +44,7 @@ Trainer(experiment=exp)
 Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag.
 Give lightning a test-tube Experiment object to automate this for you.
 ``` {.python}
-from test-tube import Experiment
+from test_tube import Experiment
 
 exp = Experiment(create_git_tag=True)
 Trainer(experiment=exp)
@@ -52,6 +52,15 @@ Trainer(experiment=exp)
 
 ---
 ### Tensorboard support   
+In the LightningModule you can access the experiment logger by doing:
+```python
+self.experiment
+
+# add image
+# Look at PyTorch SummaryWriter docs for what you can do.   
+self.experiment.add_image(...)
+```
+
 The experiment object is a strict subclass of PyTorch SummaryWriter. However, this class
 also snapshots every detail about the experiment (data folder paths, code, hyperparams),
 and allows you to visualize it using tensorboard.

docs/Trainer/Training Loop.md

@@ -28,15 +28,18 @@ trainer = Trainer(enable_early_stop=True)
 ```
 
 ---
-#### Gradient Clipping 
+#### Gradient Clipping
-Use this to turn off early stopping and run training to the [max_epoch](#force-training-for-min-or-max-epochs)
+Gradient clipping may be enabled to avoid exploding gradients.
+Specifically, this will [clip the gradient norm computed over all model parameters *together*](https://pytorch.org/docs/stable/nn.html#torch.nn.utils.clip_grad_norm_).
+
 ``` {.python}
 # DEFAULT (ie: don't clip)
 trainer = Trainer(gradient_clip=0)
+
+# clip gradients with norm above 0.5
+trainer = Trainer(gradient_clip=0.5)
 ```
 
-
-
 ---
 #### Inspect gradient norms
 Looking at grad norms can help you figure out where training might be going wrong.

docs/Trainer/hooks.md

@@ -18,7 +18,7 @@ def on_epoch_start(self):
 ```
 
 ---
-#### on_batch_end
+#### on_epoch_end
 Called in the training loop at the very end of the epoch.   
 ```python
 def on_epoch_end(self):
@@ -33,6 +33,14 @@ def on_batch_start(self):
     # do something when the batch starts
 ```
 
+---
+#### on_batch_end
+Called in the training loop after the batch.   
+```python
+def on_batch_end(self):
+    # do something when the batch ends 
+```
+
 ---
 #### on_pre_performance_check
 Called at the very beginning of the validation loop.   
@@ -59,6 +67,36 @@ def on_tng_metrics(self, metrics):
     # do something before validation end
 ```
 
+---   
+#### optimizer_step 
+Calls .step() and .zero_grad for each optimizer.  
+You can override this method to adjust how you do the optimizer step for each optimizer
+
+Called once per optimizer
+```python
+# DEFAULT
+def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
+    optimizer.step()   
+    optimizer.zero_grad()   
+    
+# Alternating schedule for optimizer steps (ie: GANs)    
+def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
+    # update generator opt every 2 steps
+    if optimizer_i == 0:
+        if batch_nb % 2 == 0 :
+            optimizer.step()
+            optimizer.zero_grad()
+   
+    # update discriminator opt every 4 steps
+    if optimizer_i == 1:
+        if batch_nb % 4 == 0 :
+            optimizer.step()
+            optimizer.zero_grad()    
+    
+    # ...
+    # add as many optimizers as you want 
+```
+
 ---
 #### on_before_zero_grad
 Called in the training loop after taking an optimizer step and before zeroing grads.

docs/Trainer/index.md

@@ -19,42 +19,44 @@ But of course the fun is in all the advanced things it can do:
 
 **Checkpointing**    
 
-- Model saving
+- [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
-- Model loading 
+- [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics) 
+- [Restoring training session](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#restoring-training-session)
 
 **Computing cluster (SLURM)**    
 
-- [Running grid search on a cluster](SLURM%20Managed%20Cluster/#running-grid-search-on-a-cluster)
+- [Running grid search on a cluster](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#running-grid-search-on-a-cluster) 
-- [Walltime auto-resubmit](SLURM%20Managed%20Cluster/#walltime-auto-resubmit)   
+- [Walltime auto-resubmit](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#walltime-auto-resubmit)   
 
 **Debugging**  
 
-- [Fast dev run](Debugging/#fast-dev-run)
+- [Fast dev run](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#fast-dev-run)
-- [Inspect gradient norms](Debugging/#inspect-gradient-norms)
+- [Inspect gradient norms](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#inspect-gradient-norms)
-- [Log GPU usage](Debugging/#Log-gpu-usage)
+- [Log GPU usage](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage)
-- [Make model overfit on subset of data](Debugging/#make-model-overfit-on-subset-of-data)
+- [Make model overfit on subset of data](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data)
-- [Print the parameter count by layer](Debugging/#print-the-parameter-count-by-layer)
+- [Print the parameter count by layer](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer)
-- [Pring which gradients are nan](Debugging/#print-which-gradients-are-nan)
+- [Pring which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
+- [Print input and output size of every module in system](https://williamfalcon.github.io/pytorch-lightning/LightningModule/properties/#example_input_array)
 
 
 **Distributed training**    
 
-- [16-bit mixed precision](Distributed%20training/#16-bit-mixed-precision)
+- [16-bit mixed precision](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision)
-- [Multi-GPU](Distributed%20training/#Multi-GPU)
+- [Multi-GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU)
-- [Multi-node](Distributed%20training/#Multi-node)
+- [Multi-node](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node)
-- [Single GPU](Distributed%20training/#single-gpu)
+- [Single GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#single-gpu)
-- [Self-balancing architecture](Distributed%20training/#self-balancing-architecture)
+- [Self-balancing architecture](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#self-balancing-architecture)
 
 
 **Experiment Logging**   
 
-- [Display metrics in progress bar](Logging/#display-metrics-in-progress-bar)
+- [Display metrics in progress bar](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#display-metrics-in-progress-bar)
-- [Log metric row every k batches](Logging/#log-metric-row-every-k-batches)
+- [Log metric row every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#log-metric-row-every-k-batches)
-- [Process position](Logging/#process-position)
+- [Process position](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#process-position)
 - [Tensorboard support](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#tensorboard-support)
-- [Save a snapshot of all hyperparameters](Logging/#save-a-snapshot-of-all-hyperparameters) 
+- [Save a snapshot of all hyperparameters](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#save-a-snapshot-of-all-hyperparameters) 
-- [Snapshot code for a training run](Logging/#snapshot-code-for-a-training-run) 
+- [Snapshot code for a training run](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#snapshot-code-for-a-training-run) 
-- [Write logs file to csv every k batches](Logging/#write-logs-file-to-csv-every-k-batches)
+- [Write logs file to csv every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#write-logs-file-to-csv-every-k-batches)
 
 **Training loop**    
 
@@ -62,16 +64,17 @@ But of course the fun is in all the advanced things it can do:
 - [Force training for min or max epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs)
 - [Force disable early stop](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop)
 - [Gradient Clipping](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping)
-- [Hooks](hooks)
+- [Hooks](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/)
 - [Learning rate scheduling](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers)
 - [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers)
 - [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check)
+- [Step optimizers at arbitrary intervals](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step)
 
 **Validation loop**    
 
-- [Check validation every n epochs](Validation%20Loop/#check-validation-every-n-epochs)
+- [Check validation every n epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#check-validation-every-n-epochs)
-- [Hooks](hooks)
+- [Hooks](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/)
-- [Set how much of the validation set to check](Validation%20Loop/#set-how-much-of-the-validation-set-to-check)
+- [Set how much of the validation set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-validation-set-to-check)
-- [Set how much of the test set to check](Validation%20Loop/#set-how-much-of-the-test-set-to-check)
+- [Set how much of the test set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-test-set-to-check)
-- [Set validation check frequency within 1 training epoch](Validation%20Loop/#set-validation-check-frequency-within-1-training-epoch)
+- [Set validation check frequency within 1 training epoch](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-validation-check-frequency-within-1-training-epoch)
-- [Set the number of validation sanity steps](Validation%20Loop/#set-the-number-of-validation-sanity-steps)
+- [Set the number of validation sanity steps](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-the-number-of-validation-sanity-steps)

docs/examples/Examples.md

@@ -3,7 +3,7 @@ In 99% of cases you want to just copy [this template](https://github.com/william
 
 ```bash
 # get a copy of the module template
-wget https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py
+wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/examples/new_project_templates/lightning_module_template.py
 ```
 
 ---    

docs/index.md

@@ -9,9 +9,9 @@ one could be a seq-2-seq model, both (optionally) ran by the same trainer file.
 
 1. [MNIST LightningModule](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#minimal-example) 
 2. [Trainer](https://williamfalcon.github.io/pytorch-lightning/Trainer/)
-    - [Basic CPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/examples/new_project_templates/single_cpu_template.py) 
+    - [Basic CPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/single_cpu_template.py) 
-    - [Multi-GPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/examples/new_project_templates/single_gpu_node_template.py)
+    - [Multi-GPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/single_gpu_node_template.py)
-    - [GPU cluster Trainer Template](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/examples/new_project_templates/multi_node_cluster_template.py)
+    - [GPU cluster Trainer Template](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/multi_node_cluster_template.py)
 
 ###### Docs shortcuts
 - [LightningModule](LightningModule/RequiredTrainerInterface/)  
@@ -28,6 +28,7 @@ one could be a seq-2-seq model, both (optionally) ran by the same trainer file.
 
 - [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
 - [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics) 
+- [Restoring training session](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#restoring-training-session)
 
 ###### Computing cluster (SLURM)    
 
@@ -42,6 +43,7 @@ one could be a seq-2-seq model, both (optionally) ran by the same trainer file.
 - [Make model overfit on subset of data](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data)
 - [Print the parameter count by layer](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer)
 - [Pring which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
+- [Print input and output size of every module in system](https://williamfalcon.github.io/pytorch-lightning/LightningModule/properties/#example_input_array)
 
 
 ###### Distributed training    
@@ -73,6 +75,7 @@ one could be a seq-2-seq model, both (optionally) ran by the same trainer file.
 - [Learning rate scheduling](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers)
 - [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers)
 - [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check)
+- [Step optimizers at arbitrary intervals](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step)
 
 ###### Validation loop    
 

docs/requirements.txt

@@ -1 +1,2 @@
 mkdocs-material==4.4.0
+mkdocs==1.0.4

examples/__init__.py

@@ -1 +1,5 @@
-from .new_project_templates.lightning_module_template import LightningTemplateModel
+from .new_project_templates.lightning_module_template import LightningTemplateModel
+
+__all__ = [
+    'LightningTemplateModel'
+]

examples/new_project_templates/lightning_module_template.py

@@ -1,3 +1,6 @@
+"""
+Example template for defining a system
+"""
 import os
 from collections import OrderedDict
 import torch.nn as nn
@@ -10,7 +13,7 @@ from torch import optim
 from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
 
-import pytorch_lightning as ptl
+import pytorch_lightning as pl
 from pytorch_lightning.root_module.root_module import LightningModule
 
 
@@ -44,11 +47,13 @@ class LightningTemplateModel(LightningModule):
         Layout model
         :return:
         """
-        self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
+        self.c_d1 = nn.Linear(in_features=self.hparams.in_features,
+                              out_features=self.hparams.hidden_dim)
         self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
         self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
 
-        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)
+        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim,
+                              out_features=self.hparams.out_features)
 
     # ---------------------
     # TRAINING
@@ -146,12 +151,23 @@ class LightningTemplateModel(LightningModule):
         val_loss_mean = 0
         val_acc_mean = 0
         for output in outputs:
-            val_loss_mean += output['val_loss']
+            val_loss = output['val_loss']
-            val_acc_mean += output['val_acc']
+
+            # reduce manually when using dp
+            if self.trainer.use_dp:
+                val_loss = torch.mean(val_loss)
+            val_loss_mean += val_loss
+
+            # reduce manually when using dp
+            val_acc = output['val_acc']
+            if self.trainer.use_dp:
+                val_acc_mean = torch.mean(val_acc)
+
+            val_acc_mean += val_acc_mean
 
         val_loss_mean /= len(outputs)
         val_acc_mean /= len(outputs)
-        tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
+        tqdm_dic = {'val_loss': val_loss_mean, 'val_acc': val_acc_mean}
         return tqdm_dic
 
     # ---------------------
@@ -168,8 +184,10 @@ class LightningTemplateModel(LightningModule):
 
     def __dataloader(self, train):
         # init data generators
-        transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+        transform = transforms.Compose([transforms.ToTensor(),
-        dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)
+                                        transforms.Normalize((0.5,), (1.0,))])
+        dataset = MNIST(root=self.hparams.data_root, train=train,
+                        transform=transform, download=True)
 
         # when using multi-node we need to add the datasampler
         train_sampler = None
@@ -179,7 +197,7 @@ class LightningTemplateModel(LightningModule):
             if self.on_gpu:
                 train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
                 batch_size = batch_size // self.trainer.world_size  # scale batch size
-        except Exception as e:
+        except Exception:
             pass
 
         should_shuffle = train_sampler is None
@@ -192,23 +210,23 @@ class LightningTemplateModel(LightningModule):
 
         return loader
 
-    @ptl.data_loader
+    @pl.data_loader
     def tng_dataloader(self):
         print('tng data loader called')
         return self.__dataloader(train=True)
 
-    @ptl.data_loader
+    @pl.data_loader
     def val_dataloader(self):
         print('val data loader called')
         return self.__dataloader(train=False)
 
-    @ptl.data_loader
+    @pl.data_loader
     def test_dataloader(self):
         print('test data loader called')
         return self.__dataloader(train=False)
 
     @staticmethod
-    def add_model_specific_args(parent_parser, root_dir): # pragma: no cover
+    def add_model_specific_args(parent_parser, root_dir):  # pragma: no cover
         """
         Parameters you define here will be available to your model through self.hparams
         :param parent_parser:
@@ -221,19 +239,25 @@ class LightningTemplateModel(LightningModule):
         # parser.set_defaults(gradient_clip=5.0)
 
         # network params
-        parser.add_argument('--in_features', default=28*28, type=int)
+        parser.add_argument('--in_features', default=28 * 28, type=int)
         parser.add_argument('--out_features', default=10, type=int)
-        parser.add_argument('--hidden_dim', default=50000, type=int) # use 500 for CPU, 50000 for GPU to see speed difference
+        # use 500 for CPU, 50000 for GPU to see speed difference
+        parser.add_argument('--hidden_dim', default=50000, type=int)
+        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
 
         # data
         parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
 
         # training params (opt)
-        parser.opt_list('--learning_rate', default=0.001*8, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
+        parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
+                        options=[0.0001, 0.0005, 0.001, 0.005],
                         tunable=False)
-        parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
+        parser.opt_list('--optimizer_name', default='adam', type=str,
+                        options=['adam'], tunable=False)
 
-        # if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu
+        # if using 2 nodes with 4 gpus each the batch size here
-        parser.opt_list('--batch_size', default=256*8, type=int, options=[32, 64, 128, 256], tunable=False,
+        #  (256) will be 256 / (2*8) = 16 per gpu
-                        help='batch size will be divided over all the gpus being used across all nodes')
+        parser.opt_list('--batch_size', default=256 * 8, type=int,
+                        options=[32, 64, 128, 256], tunable=False,
+                        help='batch size will be divided over all gpus being used across all nodes')
         return parser

examples/new_project_templates/multi_node_cluster_template.py

@@ -1,31 +1,21 @@
+"""
+Multi-node example (GPU)
+"""
 import os
-import sys
 import numpy as np
 from time import sleep
 import torch
 
 from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
 from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
 from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
 
+from examples.new_project_templates.lightning_module_template import LightningTemplateModel
+
 SEED = 2334
 torch.manual_seed(SEED)
 np.random.seed(SEED)
 
-# ---------------------
-# DEFINE MODEL HERE
-# ---------------------
-from lightning_module_template import LightningTemplateModel
-# ---------------------
-
-"""
-Allows training by using command line arguments
-Run by:
-# TYPE YOUR RUN COMMAND HERE
-"""
-
 
 def main_local(hparams):
     main(hparams, None, None)
@@ -122,8 +112,10 @@ def optimize_on_cluster(hyperparams):
     cluster.add_command('source activate lightning')
 
     # run only on 32GB voltas
-    cluster.add_slurm_cmd(cmd='constraint', value='volta32gb', comment='use 32gb gpus')
+    cluster.add_slurm_cmd(cmd='constraint', value='volta32gb',
-    cluster.add_slurm_cmd(cmd='partition', value=hyperparams.gpu_partition, comment='use 32gb gpus')
+                          comment='use 32gb gpus')
+    cluster.add_slurm_cmd(cmd='partition', value=hyperparams.gpu_partition,
+                          comment='use 32gb gpus')
 
     # run hopt
     # creates and submits jobs to slurm
@@ -150,15 +142,23 @@ if __name__ == '__main__':
     parent_parser.add_argument('--gpu_partition', type=str, help='consult your cluster manual')
 
     # TODO: make 1 param
-    parent_parser.add_argument('--per_experiment_nb_gpus', type=int, help='how many gpus to use in a node')
+    parent_parser.add_argument('--per_experiment_nb_gpus', type=int,
-    parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node')
+                               help='how many gpus to use in a node')
+    parent_parser.add_argument('--gpus', type=str, default='-1',
+                               help='how many gpus to use in the node')
 
-    parent_parser.add_argument('--nb_gpu_nodes', type=int, default=1, help='how many nodes to use in a cluster')
+    parent_parser.add_argument('--nb_gpu_nodes', type=int, default=1,
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
+                               help='how many nodes to use in a cluster')
-    parent_parser.add_argument('--slurm_log_path', type=str, default=slurm_out_dir, help='where to save slurm meta')
+    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir,
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
+                               help='where to save logs')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
+    parent_parser.add_argument('--slurm_log_path', type=str, default=slurm_out_dir,
-    parent_parser.add_argument('--nb_hopt_trials', type=int, default=1, help='how many grid search trials to run')
+                               help='where to save slurm meta')
+    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir,
+                               help='where to save model')
+    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a',
+                               help='test tube exp name')
+    parent_parser.add_argument('--nb_hopt_trials', type=int, default=1,
+                               help='how many grid search trials to run')
 
     # allow model to overwrite or extend args
     parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)

examples/new_project_templates/single_cpu_template.py

@@ -1,24 +1,20 @@
 """
-Runs a model on a single node across N-gpus.
+Runs a model on a single node on CPU only..
 """
 import os
-import sys
 import numpy as np
-from time import sleep
 import torch
 
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
+from test_tube import HyperOptArgumentParser, Experiment
 from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
 from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
 
+from examples.new_project_templates.lightning_module_template import LightningTemplateModel
+
 SEED = 2334
 torch.manual_seed(SEED)
 np.random.seed(SEED)
 
-from lightning_module_template import LightningTemplateModel
-
 
 def main(hparams):
     """
@@ -94,9 +90,12 @@ if __name__ == '__main__':
     parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
 
     # gpu args
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
+    parent_parser.add_argument('--test_tube_save_path', type=str,
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
+                               default=test_tube_dir, help='where to save logs')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
+    parent_parser.add_argument('--model_save_path', type=str,
+                               default=checkpoint_dir, help='where to save model')
+    parent_parser.add_argument('--experiment_name', type=str,
+                               default='pt_lightning_exp_a', help='test tube exp name')
 
     # allow model to overwrite or extend args
     parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
@@ -106,5 +105,5 @@ if __name__ == '__main__':
     # RUN TRAINING
     # ---------------------
     # run on HPC cluster
-    print(f'RUNNING ON CPU')
+    print('RUNNING ON CPU')
     main(hyperparams)

examples/new_project_templates/single_gpu_node_16bit_template.py

@@ -1,24 +1,20 @@
 """
-Runs a model on a single node across N-gpus.
+16-bit single node, CPU example
 """
 import os
-import sys
 import numpy as np
-from time import sleep
 import torch
 
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
+from test_tube import HyperOptArgumentParser, Experiment
 from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
 from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
 
+from examples.new_project_templates.lightning_module_template import LightningTemplateModel
+
 SEED = 2334
 torch.manual_seed(SEED)
 np.random.seed(SEED)
 
-from lightning_module_template import LightningTemplateModel
-
 
 def main(hparams):
     """
@@ -96,10 +92,15 @@ if __name__ == '__main__':
     parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
 
     # gpu args
-    parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node. -1 uses all the gpus on the node')
+    parent_parser.add_argument('--gpus', type=str, default='-1',
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
+                               help='how many gpus to use in the node.'
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
+                                    'value -1 uses all the gpus on the node')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
+    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir,
+                               help='where to save logs')
+    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir,
+                               help='where to save model')
+    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a',
+                               help='test tube exp name')
 
     # allow model to overwrite or extend args
     parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
@@ -109,5 +110,5 @@ if __name__ == '__main__':
     # RUN TRAINING
     # ---------------------
     # run on HPC cluster
-    print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
+    print('RUNNING INTERACTIVE MODE ON GPUS. gpu ids: %i' % hyperparams.gpus)
     main(hyperparams)

examples/new_project_templates/single_gpu_node_ddp_template.py

@@ -2,23 +2,19 @@
 Runs a model on a single node across N-gpus.
 """
 import os
-import sys
 import numpy as np
-from time import sleep
 import torch
 
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
+from test_tube import HyperOptArgumentParser, Experiment
 from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
 from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
 
+from examples.new_project_templates.lightning_module_template import LightningTemplateModel
+
 SEED = 2334
 torch.manual_seed(SEED)
 np.random.seed(SEED)
 
-from lightning_module_template import LightningTemplateModel
-
 
 def main(hparams):
     """
@@ -75,6 +71,7 @@ def main(hparams):
         checkpoint_callback=checkpoint,
         early_stop_callback=early_stop,
         gpus=hparams.gpus,
+        distributed_backend='ddp'
     )
 
     # ------------------------
@@ -95,10 +92,15 @@ if __name__ == '__main__':
     parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
 
     # gpu args
-    parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node. -1 uses all the gpus on the node')
+    parent_parser.add_argument('--gpus', type=str, default='-1',
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
+                               help='how many gpus to use in the node.'
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
+                                    ' value -1 uses all the gpus on the node')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
+    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir,
+                               help='where to save logs')
+    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir,
+                               help='where to save model')
+    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a',
+                               help='test tube exp name')
 
     # allow model to overwrite or extend args
     parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
@@ -108,5 +110,5 @@ if __name__ == '__main__':
     # RUN TRAINING
     # ---------------------
     # run on HPC cluster
-    print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
+    print('RUNNING INTERACTIVE MODE ON GPUS. gpu ids: %i' % hyperparams.gpus)
     main(hyperparams)

examples/new_project_templates/single_gpu_node_dp_template.py

@@ -1,24 +1,20 @@
 """
-Runs a model on a single node across N-gpus.
+Runs a model on a single node across N-gpus using dataParallel
 """
 import os
-import sys
 import numpy as np
-from time import sleep
 import torch
 
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
+from test_tube import HyperOptArgumentParser, Experiment
 from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
 from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
 
+from examples.new_project_templates.lightning_module_template import LightningTemplateModel
+
 SEED = 2334
 torch.manual_seed(SEED)
 np.random.seed(SEED)
 
-from lightning_module_template import LightningTemplateModel
-
 
 def main(hparams):
     """
@@ -95,10 +91,15 @@ if __name__ == '__main__':
     parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
 
     # gpu args
-    parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node. -1 uses all the gpus on the node')
+    parent_parser.add_argument('--gpus', type=str, default='-1',
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
+                               help='how many gpus to use in the node.'
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
+                                    ' value -1 uses all the gpus on the node')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
+    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir,
+                               help='where to save logs')
+    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir,
+                               help='where to save model')
+    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a',
+                               help='test tube exp name')
 
     # allow model to overwrite or extend args
     parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
@@ -108,5 +109,5 @@ if __name__ == '__main__':
     # RUN TRAINING
     # ---------------------
     # run on HPC cluster
-    print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
+    print('RUNNING INTERACTIVE MODE ON GPUS. gpu ids: %i' % hyperparams.gpus)
     main(hyperparams)

examples/new_project_templates/trainer_cpu_template.py

@@ -3,9 +3,10 @@ import sys
 
 from test_tube import HyperOptArgumentParser, Experiment
 from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
+from pytorch_lightning.utilities.arg_parse import add_default_args
 from pytorch_lightning.callbacks.pt_callbacks import EarlyStopping, ModelCheckpoint
-from docs.source.examples.example_model import ExampleModel
+
+from examples.new_project_templates.lightning_module_template import LightningTemplateModel
 
 
 def main(hparams):
@@ -28,7 +29,7 @@ def main(hparams):
     exp.save()
 
     # build model
-    model = ExampleModel(hparams)
+    model = LightningTemplateModel(hparams)
 
     # callbacks
     early_stop = EarlyStopping(
@@ -66,7 +67,7 @@ if __name__ == '__main__':
     add_default_args(parent_parser, root_dir)
 
     # allow model to overwrite or extend args
-    parser = ExampleModel.add_model_specific_args(parent_parser)
+    parser = LightningTemplateModel.add_model_specific_args(parent_parser)
     hyperparams = parser.parse_args()
 
     # train model

examples/templates/gan.py

@@ -0,0 +1,178 @@
+"""
+To run this template just do:  
+python gan.py   
+
+After a few epochs, launch tensorboard to see the images being generated at every batch.   
+
+tensorboard --logdir default
+"""
+from argparse import ArgumentParser
+import os
+import numpy as np
+
+import torchvision
+import torchvision.transforms as transforms
+from torchvision.datasets import MNIST
+
+from torch.utils.data import DataLoader
+
+import torch.nn as nn
+import torch.nn.functional as F
+import torch
+
+import pytorch_lightning as pl
+from test_tube import Experiment
+
+
+class Generator(nn.Module):
+    def __init__(self, latent_dim, img_shape):
+        super(Generator, self).__init__()
+        self.img_shape = img_shape
+
+        def block(in_feat, out_feat, normalize=True):
+            layers = [nn.Linear(in_feat, out_feat)]
+            if normalize:
+                layers.append(nn.BatchNorm1d(out_feat, 0.8))
+            layers.append(nn.LeakyReLU(0.2, inplace=True))
+            return layers
+
+        self.model = nn.Sequential(
+            *block(latent_dim, 128, normalize=False),
+            *block(128, 256),
+            *block(256, 512),
+            *block(512, 1024),
+            nn.Linear(1024, int(np.prod(img_shape))),
+            nn.Tanh()
+        )
+
+    def forward(self, z):
+        img = self.model(z)
+        img = img.view(img.size(0), *self.img_shape)
+        return img
+
+
+class Discriminator(nn.Module):
+    def __init__(self, img_shape):
+        super(Discriminator, self).__init__()
+
+        self.model = nn.Sequential(
+            nn.Linear(int(np.prod(img_shape)), 512),
+            nn.LeakyReLU(0.2, inplace=True),
+            nn.Linear(512, 256),
+            nn.LeakyReLU(0.2, inplace=True),
+            nn.Linear(256, 1),
+            nn.Sigmoid(),
+        )
+
+    def forward(self, img):
+        img_flat = img.view(img.size(0), -1)
+        validity = self.model(img_flat)
+
+        return validity
+
+
+class GAN(pl.LightningModule):
+
+    def __init__(self, hparams):
+        super(GAN, self).__init__()
+        self.hparams = hparams
+
+        # networks
+        mnist_shape = (1, 28, 28)
+        self.generator = Generator(latent_dim=hparams.latent_dim, img_shape=mnist_shape)
+        self.discriminator = Discriminator(img_shape=mnist_shape)
+
+        # cache for generated images
+        self.generated_imgs = None
+
+    def forward(self, z):
+        return self.generator(z)
+
+    def adversarial_loss(self, y_hat, y):
+        return F.binary_cross_entropy(y_hat, y)
+
+    def training_step(self, batch, batch_nb, optimizer_i):
+        imgs, _ = batch
+
+        # train generator
+        if optimizer_i == 0:
+            # sample noise
+            z = torch.randn(imgs.shape[0], self.hparams.latent_dim)
+
+            # match gpu device (or keep as cpu)
+            if self.on_gpu:
+                z = z.cuda(imgs.device.index)
+
+            # generate images
+            self.generated_imgs = self.forward(z)
+
+            # log sampled images
+            sample_imgs = self.generated_imgs[:6]
+            grid = torchvision.utils.make_grid(sample_imgs)
+            self.experiment.add_image('generated_images', grid, 0)
+
+            # ground truth result (ie: all fake)
+            valid = torch.ones(imgs.size(0), 1)
+
+            # adversarial loss is binary cross-entropy
+            g_loss = self.adversarial_loss(self.discriminator(self.generated_imgs), valid)
+
+            return g_loss
+
+        # train discriminator
+        if optimizer_i == 1:
+            # Measure discriminator's ability to classify real from generated samples
+
+            # how well can it label as real?
+            valid = torch.ones(imgs.size(0), 1)
+            real_loss = self.adversarial_loss(self.discriminator(imgs), valid)
+
+            # how well can it label as fake?
+            fake = torch.zeros(imgs.size(0), 1)
+            fake_loss = self.adversarial_loss(self.discriminator(self.generated_imgs.detach()), fake)
+
+            # discriminator loss is the average of these
+            d_loss = (real_loss + fake_loss) / 2
+
+            return d_loss
+
+    def configure_optimizers(self):
+        lr = self.hparams.lr
+        b1 = self.hparams.b1
+        b2 = self.hparams.b2
+
+        opt_g = torch.optim.Adam(self.generator.parameters(), lr=lr, betas=(b1, b2))
+        opt_d = torch.optim.Adam(self.discriminator.parameters(), lr=lr, betas=(b1, b2))
+        return [opt_g, opt_d], []
+
+    @pl.data_loader
+    def tng_dataloader(self):
+        transform = transforms.Compose([transforms.ToTensor(),
+                                        transforms.Normalize([0.5], [0.5])])
+        dataset = MNIST(os.getcwd(), train=True, download=True, transform=transform)
+        return DataLoader(dataset, batch_size=self.hparams.batch_size)
+
+
+def main(hparams):
+    # save tensorboard logs
+    exp = Experiment(save_dir=os.getcwd())
+
+    # init model
+    model = GAN(hparams)
+
+    # fit trainer on CPU
+    trainer = pl.Trainer(experiment=exp, max_nb_epochs=200)
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument("--batch_size", type=int, default=64, help="size of the batches")
+    parser.add_argument("--lr", type=float, default=0.0002, help="adam: learning rate")
+    parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient")
+    parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of first order momentum of gradient")
+    parser.add_argument("--latent_dim", type=int, default=100, help="dimensionality of the latent space")
+
+    hparams = parser.parse_args()
+
+    main(hparams)

mkdocs.yml

@@ -8,3 +8,8 @@ site_description: 'Documentation for PyTorch LightningModule, the researcher ver
 
 dev_addr: '0.0.0.0:8000'
 #google_analytics: ['UA-aasd', 'sitename']
+
+markdown_extensions:
+  - codehilite:
+      guess_lang: false
+      linenums: true

pytorch_lightning/__init__.py

@@ -1,3 +1,9 @@
-from .models import Trainer
+from .models.trainer import Trainer
 from .root_module.root_module import LightningModule
-from .root_module.decorators import data_loader
+from .root_module.decorators import data_loader
+
+__all__ = [
+    'Trainer',
+    'LightningModule',
+    'data_loader',
+]

pytorch_lightning/callbacks/__init__.py

@@ -1 +1,6 @@
-from .pt_callbacks import EarlyStopping, ModelCheckpoint
+from .pt_callbacks import EarlyStopping, ModelCheckpoint
+
+__all__ = [
+    'EarlyStopping',
+    'ModelCheckpoint',
+]

pytorch_lightning/callbacks/pt_callbacks.py

@@ -1,5 +1,8 @@
+import os
+import shutil
+
 import numpy as np
-import os, shutil
+
 from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel
 
 
@@ -119,9 +122,9 @@ class EarlyStopping(Callback):
         current = logs.get(self.monitor)
         stop_training = False
         if current is None:
-            print('Early stopping conditioned on metric `%s` ''which is not available. Available metrics are: %s' %
+            print('Early stopping conditioned on metric `%s` '
-                (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning
+                  'which is not available. Available metrics are: %s' %
-            )
+                  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning)
             exit(-1)
 
         if self.monitor_op(current - self.min_delta, self.best):
@@ -185,8 +188,7 @@ class ModelCheckpoint(Callback):
 
         if mode not in ['auto', 'min', 'max']:
             print('ModelCheckpoint mode %s is unknown, '
-                          'fallback to auto mode.' % (mode),
+                  'fallback to auto mode.' % (mode), RuntimeWarning)
-                          RuntimeWarning)
             mode = 'auto'
 
         if mode == 'min':
@@ -230,8 +232,8 @@ class ModelCheckpoint(Callback):
             if self.save_best_only:
                 current = logs.get(self.monitor)
                 if current is None:
-                    print('Can save best model only with %s available, '
+                    print('Can save best model only with %s available,'
-                                  'skipping.' % (self.monitor), RuntimeWarning)
+                          ' skipping.' % (self.monitor), RuntimeWarning)
                 else:
                     if self.monitor_op(current, self.best):
                         if self.verbose > 0:
@@ -260,4 +262,3 @@ if __name__ == '__main__':
         print(loss)
         if should_stop:
             break
-

pytorch_lightning/examples/new_project_templates/single_gpu_template.py

@@ -1,112 +0,0 @@
-"""
-Runs a model on a single node across N-gpus.
-"""
-import os
-import sys
-import numpy as np
-from time import sleep
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
-from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-from lightning_module_template import LightningTemplateModel
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    print('loading model...')
-    model = LightningTemplateModel(hparams)
-    print('model built')
-
-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-        gpus=hparams.gpus,
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # dirs
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-
-    # although we user hyperOptParser, we are using it only as argparse right now
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # gpu args
-    parent_parser.add_argument('--gpus', type=str, default='0', help='how many gpus to use in the node. -1 uses all the gpus on the node')
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
-    main(hyperparams)

pytorch_lightning/models/__init__.py

@@ -1 +0,0 @@
-from .trainer import Trainer

pytorch_lightning/pt_overrides/override_data_parallel.py

@@ -6,7 +6,6 @@ from itertools import chain
 import threading
 import torch
 from torch.cuda._utils import _get_device_index
-import pdb
 
 
 def _find_tensors(obj):  # pragma: no cover
@@ -64,7 +63,6 @@ class LightningDataParallel(DataParallel):
         outputs = self.parallel_apply(replicas, inputs, kwargs)
         return self.gather(outputs, self.output_device)
 
-
     def parallel_apply(self, replicas, inputs, kwargs):
         return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
 

pytorch_lightning/root_module/grads.py

@@ -1,10 +1,9 @@
-import numpy as np
-from torch import nn
-
 """
 Module to describe gradients
 """
 
+from torch import nn
+
 
 class GradInformation(nn.Module):
 
@@ -18,12 +17,13 @@ class GradInformation(nn.Module):
                     total_norm += param_norm ** norm_type
                     norm = param_norm ** (1 / norm_type)
 
-                    results['grad_{}_norm_{}'.format(norm_type, i)] = round(norm.data.cpu().numpy().flatten()[0], 3)
+                    grad = round(norm.data.cpu().numpy().flatten()[0], 3)
-                except Exception as e:
+                    results['grad_{}_norm_{}'.format(norm_type, i)] = grad
+                except Exception:
                     # this param had no grad
                     pass
 
         total_norm = total_norm ** (1. / norm_type)
-        results['grad_{}_norm_total'.format(norm_type)] = round(total_norm.data.cpu().numpy().flatten()[0], 3)
+        grad = round(total_norm.data.cpu().numpy().flatten()[0], 3)
+        results['grad_{}_norm_total'.format(norm_type)] = grad
         return results
-

pytorch_lightning/root_module/hooks.py

@@ -1,6 +1,15 @@
 import torch
 
+
 class ModelHooks(torch.nn.Module):
+
+    def on_sanity_check_start(self):
+        """
+        Called before starting validate
+        :return:
+        """
+        pass
+
     def on_batch_start(self, data_batch):
         pass
 
@@ -42,4 +51,3 @@ class ModelHooks(torch.nn.Module):
         :return:
         """
         pass
-

pytorch_lightning/root_module/memory.py

@@ -1,14 +1,14 @@
-import torch
-import gc
-import subprocess
-import numpy as np
-import pandas as pd
-
-
 '''
 Generates a summary of a model's layers and dimensionality
 '''
 
+import gc
+
+import torch
+import subprocess
+import numpy as np
+import pandas as pd
+
 
 class ModelSummary(object):
 
@@ -94,7 +94,7 @@ class ModelSummary(object):
         mods = list(self.model.modules())
         sizes = []
 
-        for i in range(1,len(mods)):
+        for i in range(1, len(mods)):
             m = mods[i]
             p = list(m.parameters())
             modsz = []
@@ -127,7 +127,7 @@ class ModelSummary(object):
         if self.model.example_input_array is not None:
             cols.extend(['In_sizes', 'Out_sizes'])
 
-        df = pd.DataFrame(np.zeros( (len(self.layer_names), len(cols))))
+        df = pd.DataFrame(np.zeros((len(self.layer_names), len(cols))))
         df.columns = cols
 
         df['Name'] = self.layer_names
@@ -152,16 +152,16 @@ class ModelSummary(object):
         self.make_summary()
 
 
-def print_mem_stack(): # pragma: no cover
+def print_mem_stack():  # pragma: no cover
     for obj in gc.get_objects():
         try:
             if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
                 print(type(obj), obj.size())
-        except Exception as e:
+        except Exception:
             pass
 
 
-def count_mem_items(): # pragma: no cover
+def count_mem_items():  # pragma: no cover
     nb_params = 0
     nb_tensors = 0
     for obj in gc.get_objects():
@@ -172,7 +172,7 @@ def count_mem_items(): # pragma: no cover
                     nb_params += 1
                 else:
                     nb_tensors += 1
-        except Exception as e:
+        except Exception:
             pass
 
     return nb_params, nb_tensors
@@ -196,6 +196,6 @@ def get_gpu_memory_map():
     gpu_memory = [int(x) for x in result.strip().split('\n')]
     gpu_memory_map = {}
     for k, v in zip(range(len(gpu_memory)), gpu_memory):
-        k = f'gpu_{k}'
+        k = 'gpu_%i' % k
         gpu_memory_map[k] = v
     return gpu_memory_map

pytorch_lightning/root_module/model_saving.py

@@ -1,8 +1,10 @@
-import torch
 import os
 import re
-import pdb
+
-from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel
+import torch
+
+from pytorch_lightning.pt_overrides.override_data_parallel import (
+    LightningDistributedDataParallel, LightningDataParallel)
 
 
 class ModelIO(object):
@@ -44,7 +46,8 @@ class ModelIO(object):
 class TrainerIO(object):
 
     def __get_model(self):
-        is_dp_module = type(self.model) is LightningDistributedDataParallel or type(self.model) is LightningDataParallel
+        is_dp_module = isinstance(self.model, (LightningDistributedDataParallel,
+                                               LightningDataParallel))
         model = self.model.module if is_dp_module else self.model
         return model
 
@@ -57,6 +60,22 @@ class TrainerIO(object):
         # do the actual save
         torch.save(checkpoint, filepath)
 
+    def restore(self, checkpoint_path, on_gpu):
+
+        if on_gpu:
+            checkpoint = torch.load(checkpoint_path)
+        else:
+            checkpoint = torch.load(checkpoint_path, map_location=lambda storage, loc: storage)
+
+        # load training state (affects trainer only)
+        self.restore_training_state(checkpoint)
+
+        # load model state
+        model = self.__get_model()
+
+        # load the state_dict on the model automatically
+        model.load_state_dict(checkpoint['state_dict'])
+
     def dump_checkpoint(self):
 
         checkpoint = {
@@ -77,7 +96,7 @@ class TrainerIO(object):
             optimizer_states.append(optimizer.state_dict())
 
         checkpoint['optimizer_states'] = optimizer_states
-        
+
         # save lr schedulers
         lr_schedulers = []
         for i, scheduler in enumerate(self.lr_schedulers):
@@ -102,13 +121,16 @@ class TrainerIO(object):
             return
 
         # allow test tube to handle model check pointing automatically
-        self.cluster.set_checkpoint_save_function(
+        # only if proc 0 so we don't trigger world_size resubmits
-            self.hpc_save,
+        if self.proc_rank == 0:
-            kwargs={
+            self.cluster.set_checkpoint_save_function(
-                'folderpath': self.checkpoint_callback.filepath,
+                self.hpc_save,
-                'experiment': self.experiment
+                kwargs={
-            }
+                    'folderpath': self.checkpoint_callback.filepath,
-        )
+                    'experiment': self.experiment
+                }
+            )
+
         self.cluster.set_checkpoint_load_function(
             self.hpc_load,
             kwargs={
@@ -138,7 +160,7 @@ class TrainerIO(object):
         optimizer_states = checkpoint['optimizer_states']
         for optimizer, opt_state in zip(self.optimizers, optimizer_states):
             optimizer.load_state_dict(opt_state)
-        
+
         # restore the lr schedulers
         lr_schedulers = checkpoint['lr_schedulers']
         for scheduler, lrs_state in zip(self.lr_schedulers, lr_schedulers):
@@ -194,15 +216,15 @@ class TrainerIO(object):
         # call model hook
         model.on_hpc_load(checkpoint)
 
-    def max_ckpt_in_folder(self, path):
+    def max_ckpt_in_folder(self, path, name_key='ckpt_'):
         files = os.listdir(path)
-        files = [x for x in files if 'ckpt_' in x]
+        files = [x for x in files if name_key in x]
         if len(files) == 0:
             return 0
 
         ckpt_vs = []
         for name in files:
-            name = name.split('ckpt_')[-1]
+            name = name.split(name_key)[-1]
             name = re.sub('[^0-9]', '', name)
             ckpt_vs.append(int(name))
 

pytorch_lightning/root_module/root_module.py

@@ -1,4 +1,5 @@
 import torch
+
 from pytorch_lightning.root_module.memory import ModelSummary
 from pytorch_lightning.root_module.grads import GradInformation
 from pytorch_lightning.root_module.model_saving import ModelIO, load_hparams_from_tags_csv
@@ -32,29 +33,33 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         """
         raise NotImplementedError
 
-    def validation_step(self, data_batch, batch_nb):
+    def training_step(self, *args, **kwargs):
         """
-        return whatever outputs will need to be aggregated in validation_end
+        return loss, dict with metrics for tqdm
-        :param data_batch:
+        :param called with batch, batch_nb
+        additional: optimizer_i if multiple optimizers used
         :return:
         """
         raise NotImplementedError
 
+    def validation_step(self, *args, **kwargs):
+        """
+        return whatever outputs will need to be aggregated in validation_end
+        OPTIONAL
+        :param called with batch, batch_nb
+        additional: dataset_i if multiple val datasets used
+        :return:
+        """
+        pass
+
     def validation_end(self, outputs):
         """
         Outputs has the appended output after each validation step
+        OPTIONAL
         :param outputs:
         :return: dic_with_metrics for tqdm
         """
-        raise NotImplementedError
+        pass
-
-    def training_step(self, data_batch, batch_nb):
-        """
-        return loss, dict with metrics for tqdm
-        :param data_batch:
-        :return:
-        """
-        raise NotImplementedError
 
     def configure_optimizers(self):
         """
@@ -63,10 +68,24 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
         """
         raise NotImplementedError
 
+    def optimizer_step(self, epoch_nb, batch_nb, optimizer, optimizer_i):
+        """
+        Do something instead of the standard optimizer behavior
+        :param epoch_nb:
+        :param batch_nb:
+        :param optimizer:
+        :param optimizer_i:
+        :return:
+        """
+        optimizer.step()
+
+        # clear gradients
+        optimizer.zero_grad()
+
     @data_loader
     def tng_dataloader(self):
         """
-        Implement a function to load an h5py of this data
+        Implement a PyTorch DataLoader
         :return:
         """
         raise NotImplementedError
@@ -74,18 +93,18 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
     @data_loader
     def test_dataloader(self):
         """
-        Implement a function to load an h5py of this data
+        Implement a PyTorch DataLoader
         :return:
         """
-        raise NotImplementedError
+        return None
 
     @data_loader
     def val_dataloader(self):
         """
-        Implement a function to load an h5py of this data
+        Implement a PyTorch DataLoader
         :return:
         """
-        raise NotImplementedError
+        return None
 
     @classmethod
     def load_from_metrics(cls, weights_path, tags_csv, on_gpu, map_location=None):
@@ -128,6 +147,3 @@ class LightningModule(GradInformation, ModelIO, ModelHooks):
     def unfreeze(self):
         for param in self.parameters():
             param.requires_grad = True
-
-
-

pytorch_lightning/testing/__init__.py

@@ -0,0 +1,3 @@
+from .lm_test_module import LightningTestModel
+from .no_val_end_module import NoValEndTestModel
+from .no_val_module import NoValModel

pytorch_lightning/testing/lm_test_module.py

@@ -1,17 +1,18 @@
 import os
 from collections import OrderedDict
-import torch.nn as nn
+
-from torchvision.datasets import MNIST
-import torchvision.transforms as transforms
 import torch
+import torch.nn as nn
 import torch.nn.functional as F
-from test_tube import HyperOptArgumentParser
 from torch import optim
 from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
+from torchvision.datasets import MNIST
+from torchvision import transforms
+from test_tube import HyperOptArgumentParser
 
 from pytorch_lightning.root_module.root_module import LightningModule
-import pytorch_lightning as ptl
+from pytorch_lightning import data_loader
 
 
 class LightningTestModel(LightningModule):
@@ -47,11 +48,13 @@ class LightningTestModel(LightningModule):
         Layout model
         :return:
         """
-        self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
+        self.c_d1 = nn.Linear(in_features=self.hparams.in_features,
+                              out_features=self.hparams.hidden_dim)
         self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
         self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
 
-        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)
+        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim,
+                              out_features=self.hparams.out_features)
 
     # ---------------------
     # TRAINING
@@ -106,7 +109,7 @@ class LightningTestModel(LightningModule):
         if self.trainer.batch_nb % 2 == 0:
             return loss_val
 
-    def validation_step(self, data_batch, batch_i):
+    def validation_step(self, data_batch, batch_i, dataloader_i):
         """
         Lightning calls this inside the validation loop
         :param data_batch:
@@ -132,22 +135,28 @@ class LightningTestModel(LightningModule):
             val_acc = val_acc.unsqueeze(0)
 
         # alternate possible outputs to test
-        if self.trainer.batch_nb % 1 == 0:
+        if batch_i % 1 == 0:
             output = OrderedDict({
                 'val_loss': loss_val,
                 'val_acc': val_acc,
             })
             return output
-        if self.trainer.batch_nb % 2 == 0:
+        if batch_i % 2 == 0:
             return val_acc
 
-        if self.trainer.batch_nb % 3 == 0:
+        if batch_i % 3 == 0:
             output = OrderedDict({
                 'val_loss': loss_val,
                 'val_acc': val_acc,
                 'test_dic': {'val_loss_a': loss_val}
             })
             return output
+        if batch_i % 5 == 0:
+            output = OrderedDict({
+                f'val_loss_{dataloader_i}': loss_val,
+                f'val_acc_{dataloader_i}': val_acc,
+            })
+            return output
 
     def validation_end(self, outputs):
         """
@@ -186,12 +195,14 @@ class LightningTestModel(LightningModule):
         optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
 
         # test returning only 1 list instead of 2
-        return [optimizer]
+        return optimizer
 
     def __dataloader(self, train):
         # init data generators
-        transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+        transform = transforms.Compose([transforms.ToTensor(),
-        dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)
+                                        transforms.Normalize((0.5,), (1.0,))])
+        dataset = MNIST(root=self.hparams.data_root, train=train,
+                        transform=transform, download=True)
 
         # when using multi-node we need to add the datasampler
         train_sampler = None
@@ -201,7 +212,7 @@ class LightningTestModel(LightningModule):
             if self.on_gpu and not self.force_remove_distributed_sampler:
                 train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
                 batch_size = batch_size // self.trainer.world_size  # scale batch size
-        except Exception as e:
+        except Exception:
             pass
 
         should_shuffle = train_sampler is None
@@ -214,20 +225,20 @@ class LightningTestModel(LightningModule):
 
         return loader
 
-    @ptl.data_loader
+    @data_loader
     def tng_dataloader(self):
         return self.__dataloader(train=True)
 
-    @ptl.data_loader
+    @data_loader
     def val_dataloader(self):
-        return self.__dataloader(train=False)
+        return [self.__dataloader(train=False), self.__dataloader(train=False)]
 
-    @ptl.data_loader
+    @data_loader
     def test_dataloader(self):
         return self.__dataloader(train=False)
 
     @staticmethod
-    def add_model_specific_args(parent_parser, root_dir):
+    def add_model_specific_args(parent_parser, root_dir):  # pragma: no cover
         """
         Parameters you define here will be available to your model through self.hparams
         :param parent_parser:
@@ -241,19 +252,24 @@ class LightningTestModel(LightningModule):
 
         # network params
         parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
-        parser.add_argument('--in_features', default=28*28, type=int)
+        parser.add_argument('--in_features', default=28 * 28, type=int)
         parser.add_argument('--out_features', default=10, type=int)
-        parser.add_argument('--hidden_dim', default=50000, type=int) # use 500 for CPU, 50000 for GPU to see speed difference
+        # use 500 for CPU, 50000 for GPU to see speed difference
+        parser.add_argument('--hidden_dim', default=50000, type=int)
 
         # data
         parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
 
         # training params (opt)
-        parser.opt_list('--learning_rate', default=0.001*8, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
+        parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
+                        options=[0.0001, 0.0005, 0.001, 0.005],
                         tunable=False)
-        parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
+        parser.opt_list('--optimizer_name', default='adam', type=str,
+                        options=['adam'], tunable=False)
 
-        # if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu
+        # if using 2 nodes with 4 gpus each the batch size here
-        parser.opt_list('--batch_size', default=256*8, type=int, options=[32, 64, 128, 256], tunable=False,
+        #  (256) will be 256 / (2*8) = 16 per gpu
-                        help='batch size will be divided over all the gpus being used across all nodes')
+        parser.opt_list('--batch_size', default=256 * 8, type=int,
+                        options=[32, 64, 128, 256], tunable=False,
+                        help='batch size will be divided over all gpus being used across all nodes')
         return parser

pytorch_lightning/testing/no_val_end_module.py

@@ -0,0 +1,247 @@
+import os
+from collections import OrderedDict
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import optim
+from torch.utils.data import DataLoader
+from torch.utils.data.distributed import DistributedSampler
+from torchvision.datasets import MNIST
+from torchvision import transforms
+from test_tube import HyperOptArgumentParser
+
+from pytorch_lightning.root_module.root_module import LightningModule
+from pytorch_lightning import data_loader
+
+
+class NoValEndTestModel(LightningModule):
+    """
+    Sample model to show how to define a template
+    """
+
+    def __init__(self, hparams, force_remove_distributed_sampler=False):
+        """
+        Pass in parsed HyperOptArgumentParser to the model
+        :param hparams:
+        """
+        # init superclass
+        super(NoValEndTestModel, self).__init__()
+        self.hparams = hparams
+
+        self.batch_size = hparams.batch_size
+
+        # if you specify an example input, the summary will show input/output for each layer
+        self.example_input_array = torch.rand(5, 28 * 28)
+
+        # remove to test warning for dist sampler
+        self.force_remove_distributed_sampler = force_remove_distributed_sampler
+
+        # build model
+        self.__build_model()
+
+    # ---------------------
+    # MODEL SETUP
+    # ---------------------
+    def __build_model(self):
+        """
+        Layout model
+        :return:
+        """
+        self.c_d1 = nn.Linear(in_features=self.hparams.in_features,
+                              out_features=self.hparams.hidden_dim)
+        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
+        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
+
+        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim,
+                              out_features=self.hparams.out_features)
+
+    # ---------------------
+    # TRAINING
+    # ---------------------
+    def forward(self, x):
+        """
+        No special modification required for lightning, define as you normally would
+        :param x:
+        :return:
+        """
+
+        x = self.c_d1(x)
+        x = torch.tanh(x)
+        x = self.c_d1_bn(x)
+        x = self.c_d1_drop(x)
+
+        x = self.c_d2(x)
+        logits = F.log_softmax(x, dim=1)
+
+        return logits
+
+    def loss(self, labels, logits):
+        nll = F.nll_loss(logits, labels)
+        return nll
+
+    def training_step(self, data_batch, batch_i):
+        """
+        Lightning calls this inside the training loop
+        :param data_batch:
+        :return:
+        """
+        # forward pass
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+
+        y_hat = self.forward(x)
+
+        # calculate loss
+        loss_val = self.loss(y, y_hat)
+
+        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
+        if self.trainer.use_dp:
+            loss_val = loss_val.unsqueeze(0)
+
+        # alternate possible outputs to test
+        if self.trainer.batch_nb % 1 == 0:
+            output = OrderedDict({
+                'loss': loss_val,
+                'prog': {'some_val': loss_val * loss_val}
+            })
+            return output
+        if self.trainer.batch_nb % 2 == 0:
+            return loss_val
+
+    def validation_step(self, data_batch, batch_nb):
+        """
+        Lightning calls this inside the validation loop
+        :param data_batch:
+        :return:
+        """
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+        y_hat = self.forward(x)
+
+        loss_val = self.loss(y, y_hat)
+
+        # acc
+        labels_hat = torch.argmax(y_hat, dim=1)
+        val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
+        val_acc = torch.tensor(val_acc)
+
+        if self.on_gpu:
+            val_acc = val_acc.cuda(loss_val.device.index)
+
+        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
+        if self.trainer.use_dp:
+            loss_val = loss_val.unsqueeze(0)
+            val_acc = val_acc.unsqueeze(0)
+
+        # alternate possible outputs to test
+        if batch_nb % 1 == 0:
+            output = OrderedDict({
+                'val_loss': loss_val,
+                'val_acc': val_acc,
+            })
+            return output
+        if batch_nb % 2 == 0:
+            return val_acc
+
+        if batch_nb % 3 == 0:
+            output = OrderedDict({
+                'val_loss': loss_val,
+                'val_acc': val_acc,
+                'test_dic': {'val_loss_a': loss_val}
+            })
+            return output
+
+    def on_tng_metrics(self, logs):
+        logs['some_tensor_to_test'] = torch.rand(1)
+
+    # ---------------------
+    # TRAINING SETUP
+    # ---------------------
+    def configure_optimizers(self):
+        """
+        return whatever optimizers we want here
+        :return: list of optimizers
+        """
+        # try no scheduler for this model (testing purposes)
+        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+
+        # test returning only 1 list instead of 2
+        return [optimizer]
+
+    def __dataloader(self, train):
+        # init data generators
+        transform = transforms.Compose([transforms.ToTensor(),
+                                        transforms.Normalize((0.5,), (1.0,))])
+        dataset = MNIST(root=self.hparams.data_root, train=train,
+                        transform=transform, download=True)
+
+        # when using multi-node we need to add the datasampler
+        train_sampler = None
+        batch_size = self.hparams.batch_size
+
+        try:
+            if self.on_gpu and not self.force_remove_distributed_sampler:
+                train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
+                batch_size = batch_size // self.trainer.world_size  # scale batch size
+        except Exception:
+            pass
+
+        should_shuffle = train_sampler is None
+        loader = DataLoader(
+            dataset=dataset,
+            batch_size=batch_size,
+            shuffle=should_shuffle,
+            sampler=train_sampler
+        )
+
+        return loader
+
+    @data_loader
+    def tng_dataloader(self):
+        return self.__dataloader(train=True)
+
+    @data_loader
+    def val_dataloader(self):
+        return self.__dataloader(train=False)
+
+    @data_loader
+    def test_dataloader(self):
+        return self.__dataloader(train=False)
+
+    @staticmethod
+    def add_model_specific_args(parent_parser, root_dir):  # pragma: no cover
+        """
+        Parameters you define here will be available to your model through self.hparams
+        :param parent_parser:
+        :param root_dir:
+        :return:
+        """
+        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+
+        # param overwrites
+        # parser.set_defaults(gradient_clip=5.0)
+
+        # network params
+        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
+        parser.add_argument('--in_features', default=28 * 28, type=int)
+        parser.add_argument('--out_features', default=10, type=int)
+        # use 500 for CPU, 50000 for GPU to see speed difference
+        parser.add_argument('--hidden_dim', default=50000, type=int)
+
+        # data
+        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
+
+        # training params (opt)
+        parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
+                        options=[0.0001, 0.0005, 0.001, 0.005],
+                        tunable=False)
+        parser.opt_list('--optimizer_name', default='adam', type=str,
+                        options=['adam'], tunable=False)
+
+        # if using 2 nodes with 4 gpus each the batch size here
+        #  (256) will be 256 / (2*8) = 16 per gpu
+        parser.opt_list('--batch_size', default=256 * 8, type=int,
+                        options=[32, 64, 128, 256], tunable=False,
+                        help='batch size will be divided over all gpus being used across all nodes')
+        return parser

pytorch_lightning/testing/no_val_module.py

@@ -0,0 +1,196 @@
+import os
+from collections import OrderedDict
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch import optim
+from torch.utils.data import DataLoader
+from torch.utils.data.distributed import DistributedSampler
+from torchvision.datasets import MNIST
+from torchvision import transforms
+from test_tube import HyperOptArgumentParser
+
+from pytorch_lightning.root_module.root_module import LightningModule
+from pytorch_lightning import data_loader
+
+
+class NoValModel(LightningModule):
+    """
+    Sample model to show how to define a template
+    """
+
+    def __init__(self, hparams, force_remove_distributed_sampler=False):
+        """
+        Pass in parsed HyperOptArgumentParser to the model
+        :param hparams:
+        """
+        # init superclass
+        super(NoValModel, self).__init__()
+        self.hparams = hparams
+
+        self.batch_size = hparams.batch_size
+
+        # if you specify an example input, the summary will show input/output for each layer
+        self.example_input_array = torch.rand(5, 28 * 28)
+
+        # remove to test warning for dist sampler
+        self.force_remove_distributed_sampler = force_remove_distributed_sampler
+
+        # build model
+        self.__build_model()
+
+    # ---------------------
+    # MODEL SETUP
+    # ---------------------
+    def __build_model(self):
+        """
+        Layout model
+        :return:
+        """
+        self.c_d1 = nn.Linear(in_features=self.hparams.in_features,
+                              out_features=self.hparams.hidden_dim)
+        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
+        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
+
+        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim,
+                              out_features=self.hparams.out_features)
+
+    # ---------------------
+    # TRAINING
+    # ---------------------
+    def forward(self, x):
+        """
+        No special modification required for lightning, define as you normally would
+        :param x:
+        :return:
+        """
+
+        x = self.c_d1(x)
+        x = torch.tanh(x)
+        x = self.c_d1_bn(x)
+        x = self.c_d1_drop(x)
+
+        x = self.c_d2(x)
+        logits = F.log_softmax(x, dim=1)
+
+        return logits
+
+    def loss(self, labels, logits):
+        nll = F.nll_loss(logits, labels)
+        return nll
+
+    def training_step(self, data_batch, batch_i):
+        """
+        Lightning calls this inside the training loop
+        :param data_batch:
+        :return:
+        """
+        # forward pass
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+
+        y_hat = self.forward(x)
+
+        # calculate loss
+        loss_val = self.loss(y, y_hat)
+
+        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
+        if self.trainer.use_dp:
+            loss_val = loss_val.unsqueeze(0)
+
+        # alternate possible outputs to test
+        if self.trainer.batch_nb % 1 == 0:
+            output = OrderedDict({
+                'loss': loss_val,
+                'prog': {'some_val': loss_val * loss_val}
+            })
+            return output
+        if self.trainer.batch_nb % 2 == 0:
+            return loss_val
+
+    def on_tng_metrics(self, logs):
+        logs['some_tensor_to_test'] = torch.rand(1)
+
+    # ---------------------
+    # TRAINING SETUP
+    # ---------------------
+    def configure_optimizers(self):
+        """
+        return whatever optimizers we want here
+        :return: list of optimizers
+        """
+        # try no scheduler for this model (testing purposes)
+        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+
+        # test returning only 1 list instead of 2
+        return [optimizer]
+
+    def __dataloader(self, train):
+        # init data generators
+        transform = transforms.Compose([transforms.ToTensor(),
+                                        transforms.Normalize((0.5,), (1.0,))])
+        dataset = MNIST(root=self.hparams.data_root, train=train,
+                        transform=transform, download=True)
+
+        # when using multi-node we need to add the datasampler
+        train_sampler = None
+        batch_size = self.hparams.batch_size
+
+        try:
+            if self.on_gpu and not self.force_remove_distributed_sampler:
+                train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
+                batch_size = batch_size // self.trainer.world_size  # scale batch size
+        except Exception:
+            pass
+
+        should_shuffle = train_sampler is None
+        loader = DataLoader(
+            dataset=dataset,
+            batch_size=batch_size,
+            shuffle=should_shuffle,
+            sampler=train_sampler
+        )
+
+        return loader
+
+    @data_loader
+    def tng_dataloader(self):
+        return self.__dataloader(train=True)
+
+    @staticmethod
+    def add_model_specific_args(parent_parser, root_dir):  # pragma: no cover
+        """
+        Parameters you define here will be available to your model through self.hparams
+        :param parent_parser:
+        :param root_dir:
+        :return:
+        """
+        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+
+        # param overwrites
+        # parser.set_defaults(gradient_clip=5.0)
+
+        # network params
+        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
+        parser.add_argument('--in_features', default=28 * 28, type=int)
+        parser.add_argument('--out_features', default=10, type=int)
+        # use 500 for CPU, 50000 for GPU to see speed difference
+        parser.add_argument('--hidden_dim', default=50000, type=int)
+
+        # data
+        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
+
+        # training params (opt)
+        parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
+                        options=[0.0001, 0.0005, 0.001, 0.005],
+                        tunable=False)
+        parser.opt_list('--optimizer_name', default='adam', type=str,
+                        options=['adam'], tunable=False)
+
+        # if using 2 nodes with 4 gpus each the batch size here
+        #  (256) will be 256 / (2*8) = 16 per gpu
+        parser.opt_list('--batch_size', default=256 * 8, type=int,
+                        options=[32, 64, 128, 256], tunable=False,
+                        help='batch size will be divided over all gpus being used across all nodes')
+        return parser

pytorch_lightning/utilities/arg_parse.py

@@ -1,31 +1,48 @@
-import pdb
+"""
+List of default args which mught be useful for all the available flags
+Might need to update with the new flags
+"""
+
+import os
+
 
 def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None):
 
     # tng, test, val check intervals
-    parser.add_argument('--eval_test_set', dest='eval_test_set', action='store_true', help='true = run test set also')
+    parser.add_argument('--eval_test_set', dest='eval_test_set', action='store_true',
-    parser.add_argument('--check_val_every_n_epoch', default=1, type=int, help='check val every n epochs')
+                        help='true = run test set also')
+    parser.add_argument('--check_val_every_n_epoch', default=1, type=int,
+                        help='check val every n epochs')
     parser.opt_list('--accumulate_grad_batches', default=1, type=int, tunable=False,
-                    help='accumulates gradients k times before applying update. Simulates huge batch size')
+                    help='accumulates gradients k times before applying update.'
+                         ' Simulates huge batch size')
     parser.add_argument('--max_nb_epochs', default=200, type=int, help='cap epochs')
     parser.add_argument('--min_nb_epochs', default=2, type=int, help='min epochs')
-    parser.add_argument('--train_percent_check', default=1.0, type=float, help='how much of tng set to check')
+    parser.add_argument('--train_percent_check', default=1.0, type=float,
-    parser.add_argument('--val_percent_check', default=1.0, type=float, help='how much of val set to check')
+                        help='how much of tng set to check')
-    parser.add_argument('--test_percent_check', default=1.0, type=float, help='how much of test set to check')
+    parser.add_argument('--val_percent_check', default=1.0, type=float,
+                        help='how much of val set to check')
+    parser.add_argument('--test_percent_check', default=1.0, type=float,
+                        help='how much of test set to check')
 
-    parser.add_argument('--val_check_interval', default=0.95, type=float, help='how much within 1 epoch to check val')
+    parser.add_argument('--val_check_interval', default=0.95, type=float,
-    parser.add_argument('--log_save_interval', default=100, type=int, help='how many batches between log saves')
+                        help='how much within 1 epoch to check val')
-    parser.add_argument('--add_log_row_interval', default=100, type=int, help='add log every k batches')
+    parser.add_argument('--log_save_interval', default=100, type=int,
+                        help='how many batches between log saves')
+    parser.add_argument('--add_log_row_interval', default=100, type=int,
+                        help='add log every k batches')
 
     # early stopping
     parser.add_argument('--disable_early_stop', dest='enable_early_stop', action='store_false')
     parser.add_argument('--early_stop_metric', default='val_acc', type=str)
     parser.add_argument('--early_stop_mode', default='min', type=str)
-    parser.add_argument('--early_stop_patience', default=3, type=int, help='number of epochs until stop')
+    parser.add_argument('--early_stop_patience', default=3, type=int,
+                        help='number of epochs until stop')
 
     # gradient handling
     parser.add_argument('--gradient_clip', default=-1, type=int)
-    parser.add_argument('--track_grad_norm', default=-1, type=int, help='if > 0, will track this grad norm')
+    parser.add_argument('--track_grad_norm', default=-1, type=int,
+                        help='if > 0, will track this grad norm')
 
     # model saving
     parser.add_argument('--model_save_path', default=root_dir + '/model_weights')
@@ -41,7 +58,8 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None
     # test_tube settings
     parser.add_argument('-en', '--tt_name', default='pt_test')
     parser.add_argument('-td', '--tt_description', default='pytorch lightning test')
-    parser.add_argument('--tt_save_path', default=root_dir + '/test_tube_logs', help='logging dir')
+    parser.add_argument('--tt_save_path', default=os.path.join(root_dir, 'test_tube_logs'),
+                        help='logging dir')
     parser.add_argument('--enable_single_run', dest='single_run', action='store_true')
     parser.add_argument('--nb_hopt_trials', default=1, type=int)
     parser.add_argument('--log_stdout', dest='log_stdout', action='store_true')
@@ -52,25 +70,30 @@ def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None
     parser.add_argument('--default_tensor_type', default='torch.cuda.FloatTensor', type=str)
     parser.add_argument('--use_amp', dest='use_amp', action='store_true')
     parser.add_argument('--check_grad_nans', dest='check_grad_nans', action='store_true')
-    parser.add_argument('--amp_level', default='O2',type=str)
+    parser.add_argument('--amp_level', default='O2', type=str)
-
 
     # run on hpc
     parser.add_argument('--on_cluster', dest='on_cluster', action='store_true')
 
     # FAST training
     # use these settings to make sure network has no bugs without running a full dataset
-    parser.add_argument('--fast_dev_run', dest='fast_dev_run', default=False, action='store_true', help='runs validation after 1 tng step')
+    parser.add_argument('--fast_dev_run', dest='fast_dev_run', default=False, action='store_true',
-    parser.add_argument('--enable_tqdm', dest='enable_tqdm', default=False, action='store_true', help='false removes the prog bar')
+                        help='runs validation after 1 tng step')
-    parser.add_argument('--overfit', default=-1, type=float, help='% of dataset to use with this option. float, or -1 for none')
+    parser.add_argument('--enable_tqdm', dest='enable_tqdm', default=False, action='store_true',
+                        help='false removes the prog bar')
+    parser.add_argument('--overfit', default=-1, type=float,
+                        help='% of dataset to use with this option. float, or -1 for none')
 
     # debug args
     if rand_seed is not None:
         parser.add_argument('--random_seed', default=rand_seed, type=int)
 
-    parser.add_argument('--interactive', dest='interactive', action='store_true', help='runs on gpu without cluster')
+    parser.add_argument('--interactive', dest='interactive', action='store_true',
-    parser.add_argument('--debug', dest='debug', action='store_true', help='enables/disables test tube')
+                        help='runs on gpu without cluster')
-    parser.add_argument('--local', dest='local', action='store_true', help='enables local tng')
+    parser.add_argument('--debug', dest='debug', action='store_true',
+                        help='enables/disables test tube')
+    parser.add_argument('--local', dest='local', action='store_true',
+                        help='enables local tng')
 
     # optimizer
-    parser.add_argument('--lr_scheduler_milestones', default=None, type=str)
+    parser.add_argument('--lr_scheduler_milestones', default=None, type=str)

pytorch_lightning/utilities/debugging.py

@@ -1,5 +1,2 @@
-import pdb
-import sys
-
 class MisconfigurationException(Exception):
-    pass
+    pass

requirements.txt

@@ -1,9 +1,7 @@
-coverage==4.5.3
-mkdocs==1.0.4
-pytest==5.0.1
 scikit-learn==0.20.2
 tqdm==4.32.1
 twine==1.13.0
 numpy==1.16.4
 torch>=1.1.0
-torchvision==0.3.0
+torchvision>=0.3.0
+pandas

setup.cfg

@@ -31,6 +31,8 @@ exclude_lines =
     print(traceback.print_exc())
     return *
     raise Exception
+    raise *
+    except *
     warnings
     print
     raise RuntimeError
@@ -42,6 +44,8 @@ omit =
     pytorch_lightning/callbacks/pt_callbacks.py
     tests/test_models.py
     pytorch_lightning/testing_models/lm_test_module.py
+    pytorch_lightning/utilities/arg_parse.py
+    examples/templates
 
 [flake8]
 ignore = E731,W504,F401,F841

setup.py

@@ -1,29 +1,58 @@
 #!/usr/bin/env python
 
+# Always prefer setuptools over distutils
 from setuptools import setup, find_packages
 
 # https://packaging.python.org/guides/single-sourcing-package-version/
 
 # http://blog.ionelmc.ro/2014/05/25/python-packaging/
+
+# https://packaging.python.org/discussions/install-requires-vs-requirements /
+# keep the meta-data here for simplicity in reading this file... it's not obvious
+# what happens and to non-engineers they won't know to look in init ...
+# the goal of the project is simplicity for researchers, don't want to add too much
+# engineer specific practices
 setup(
-    name="pytorch-lightning",
+    name='pytorch-lightning',
-    version='0.3.6.7',
+    version='0.4.6',
-    description="The Keras for ML researchers using PyTorch",
+    description='The Keras for ML researchers using PyTorch',
-    author="William Falcon",
+    author='William Falcon',
-    author_email="waf2107@columbia.edu",
+    author_email='waf2107@columbia.edu',
-    url="https://github.com/williamFalcon/pytorch-lightning",
+    url='https://github.com/williamFalcon/pytorch-lightning',
-    download_url="https://github.com/williamFalcon/pytorch-lightning",
+    download_url='https://github.com/williamFalcon/pytorch-lightning',
-    license="MIT",
+    license='Apache-2',
-    keywords=["deep learning", "pytorch", "AI"],
-    python_requires=">=3.5",
-    install_requires=[
-        "torch>=1.1.0",
-        "tqdm",
-        "test-tube>=0.6.7.4",
-    ],
     packages=find_packages(),
-    long_description=open("README.md", encoding="utf-8").read(),
+    long_description=open('README.md', encoding='utf-8').read(),
     long_description_content_type='text/markdown',
     include_package_data=True,
     zip_safe=False,
+    keywords=['deep learning', 'pytorch', 'AI'],
+    python_requires='>=3.6',
+    install_requires=[
+        'torch==1.2.0',
+        'tqdm',
+        'test-tube>=0.6.9',
+        'pandas>=0.20.3',
+    ],
+    classifiers=[
+        'Environment :: Console',
+        'Natural Language :: English',
+        # How mature is this project? Common values are
+        #   3 - Alpha, 4 - Beta, 5 - Production/Stable
+        'Development Status :: 4 - Beta',
+        # Indicate who your project is intended for
+        'Intended Audience :: Developers',
+        'Topic :: Scientific/Engineering :: Artificial Intelligence',
+        'Topic :: Scientific/Engineering :: Image Recognition',
+        'Topic :: Scientific/Engineering :: Information Analysis',
+        # Pick your license as you wish
+        'License :: OSI Approved :: BSD License',
+        'Operating System :: OS Independent',
+        # Specify the Python versions you support here. In particular, ensure
+        # that you indicate whether you support Python 2, Python 3 or both.
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+    ],
 )

tests/README.md

@@ -17,7 +17,7 @@ pip install -e .
 pip install -r requirements.txt
 
 # run tests
-py.test
+py.test -v
 ```
 
 To test models that require GPU make sure to run the above command on a GPU machine.
@@ -43,16 +43,21 @@ For each set up it also tests:
 5. simulated load from HPC signal.
 
 ## Running Coverage   
+Make sure to run coverage on a GPU machine with at least 2 GPUs and NVIDIA apex installed. 
 
 ```bash
 cd pytorch-lightning
 
 # generate coverage 
 pip install coverage
-coverage run tests/test_models.py   
+coverage run --source pytorch_lightning -m py.test pytorch_lightning tests examples -v --doctest-modules
 
 # print coverage stats
-coverage report -m   
+coverage report -m
+
+# exporting resulys
+coverage xml
+codecov -t 17327163-8cca-4a5d-86c8-ca5f2ef700bc  -v
 ```
 
 

tests/debug.py

@@ -1,24 +1,19 @@
-import pytest
 from pytorch_lightning import Trainer
-from pytorch_lightning.examples.new_project_templates.lightning_module_template import LightningTemplateModel
+from examples import LightningTemplateModel
 from argparse import Namespace
 from test_tube import Experiment
 from pytorch_lightning.callbacks import ModelCheckpoint
-import numpy as np
-import warnings
-import torch
 import os
 import shutil
-import pdb
 
-import pytorch_lightning as ptl
+import pytorch_lightning as pl
 import torch
 from torch.nn import functional as F
 from torch.utils.data import DataLoader
 from torchvision.datasets import MNIST
 
 
-class CoolModel(ptl.LightningModule):
+class CoolModel(pl.LightningModule):
 
     def __init(self):
         super(CoolModel, self).__init__()
@@ -48,15 +43,15 @@ class CoolModel(ptl.LightningModule):
     def configure_optimizers(self):
         return [torch.optim.Adam(self.parameters(), lr=0.02)]
 
-    @ptl.data_loader
+    @pl.data_loader
     def tng_dataloader(self):
         return DataLoader(MNIST('path/to/save', train=True), batch_size=32)
 
-    @ptl.data_loader
+    @pl.data_loader
     def val_dataloader(self):
         return DataLoader(MNIST('path/to/save', train=False), batch_size=32)
 
-    @ptl.data_loader
+    @pl.data_loader
     def test_dataloader(self):
         return DataLoader(MNIST('path/to/save', train=False), batch_size=32)
 
@@ -66,8 +61,8 @@ def get_model():
     root_dir = os.path.dirname(os.path.realpath(__file__))
     hparams = Namespace(**{'drop_prob': 0.2,
                            'batch_size': 32,
-                           'in_features': 28*28,
+                           'in_features': 28 * 28,
-                           'learning_rate': 0.001*8,
+                           'learning_rate': 0.001 * 8,
                            'optimizer_name': 'adam',
                            'data_root': os.path.join(root_dir, 'mnist'),
                            'out_features': 10,
@@ -112,7 +107,8 @@ def load_model(exp, save_dir):
     checkpoints = [x for x in os.listdir(save_dir) if '.ckpt' in x]
     weights_dir = os.path.join(save_dir, checkpoints[0])
 
-    trained_model = LightningTemplateModel.load_from_metrics(weights_path=weights_dir, tags_csv=tags_path, on_gpu=True)
+    trained_model = LightningTemplateModel.load_from_metrics(weights_path=weights_dir,
+                                                             tags_csv=tags_path, on_gpu=True)
 
     assert trained_model is not None, 'loading model failed'
 
@@ -137,7 +133,7 @@ def run_prediction(dataloader, trained_model):
 
     print(val_acc)
 
-    assert val_acc > 0.70, f'this model is expected to get > 0.7 in test set (it got {val_acc})'
+    assert val_acc > 0.70, 'this model is expected to get > 0.7 in test set (it got %f)' % val_acc
 
 
 def main():

tests/requirements.txt

@@ -0,0 +1,8 @@
+tox
+coverage
+codecov
+pytest>=3.0.5
+pytest-cov
+flake8
+check-manifest
+test_tube

tests/test_models.py

@@ -1,20 +1,22 @@
-import pytest
+import os
-from pytorch_lightning import Trainer
+import shutil
-from pytorch_lightning.examples.new_project_templates.lightning_module_template import LightningTemplateModel
+import warnings
-from pytorch_lightning.testing_models.lm_test_module import LightningTestModel
 from argparse import Namespace
+
+import pytest
+import numpy as np
+import torch
 from test_tube import Experiment, SlurmCluster
+
+# sys.path += [os.path.abspath('..'), os.path.abspath('../..')]
+from pytorch_lightning import Trainer
+from pytorch_lightning.testing import LightningTestModel, NoValEndTestModel, NoValModel
 from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping
-from pytorch_lightning.utils.debugging import MisconfigurationException
+from pytorch_lightning.utilities.debugging import MisconfigurationException
 from pytorch_lightning.root_module import memory
 from pytorch_lightning.models.trainer import reduce_distributed_output
 from pytorch_lightning.root_module import model_saving
-import numpy as np
+from examples import LightningTemplateModel
-import warnings
-import torch
-import os
-import shutil
-import pdb
 
 SEED = 2334
 torch.manual_seed(SEED)
@@ -24,16 +26,321 @@ np.random.seed(SEED)
 # ------------------------------------------------------------------------
 # TESTS
 # ------------------------------------------------------------------------
+
+def test_optimizer_return_options():
+
+    trainer = Trainer()
+    model, hparams = get_model()
+
+    # single optimizer
+    opt_a = torch.optim.Adam(model.parameters(), lr=0.002)
+    opt_b = torch.optim.SGD(model.parameters(), lr=0.002)
+    optim, lr_sched = trainer.init_optimizers(opt_a)
+    assert len(optim) == 1 and len(lr_sched) == 0
+
+    # opt tuple
+    opts = (opt_a, opt_b)
+    optim, lr_sched = trainer.init_optimizers(opts)
+    assert len(optim) == 2 and optim[0] == opts[0] and optim[1] == opts[1]
+    assert len(lr_sched) == 0
+
+    # opt list
+    opts = [opt_a, opt_b]
+    optim, lr_sched = trainer.init_optimizers(opts)
+    assert len(optim) == 2 and optim[0] == opts[0] and optim[1] == opts[1]
+    assert len(lr_sched) == 0
+
+    # opt tuple of lists
+    opts = ([opt_a], ['lr_scheduler'])
+    optim, lr_sched = trainer.init_optimizers(opts)
+    assert len(optim) == 1 and len(lr_sched) == 1
+    assert optim[0] == opts[0][0] and lr_sched[0] == 'lr_scheduler'
+
+
+def test_single_gpu_batch_parse():
+    if not torch.cuda.is_available():
+        warnings.warn('test_amp_gpu_ddp cannot run.'
+                      'Rerun on a GPU node to run this test')
+        return
+    if not torch.cuda.device_count() > 1:
+        warnings.warn('test_amp_gpu_ddp cannot run.'
+                      'Rerun on a node with 2+ GPUs to run this test')
+        return
+
+    trainer = Trainer()
+
+    # batch is just a tensor
+    batch = torch.rand(2, 3)
+    batch = trainer.transfer_batch_to_gpu(batch, 0)
+    assert batch.device.index == 0 and batch.type() == 'torch.cuda.FloatTensor'
+
+    # tensor list
+    batch = [torch.rand(2, 3), torch.rand(2, 3)]
+    batch = trainer.transfer_batch_to_gpu(batch, 0)
+    assert batch[0].device.index == 0 and batch[0].type() == 'torch.cuda.FloatTensor'
+    assert batch[1].device.index == 0 and batch[1].type() == 'torch.cuda.FloatTensor'
+
+    # tensor list of lists
+    batch = [[torch.rand(2, 3), torch.rand(2, 3)]]
+    batch = trainer.transfer_batch_to_gpu(batch, 0)
+    assert batch[0][0].device.index == 0 and batch[0][0].type() == 'torch.cuda.FloatTensor'
+    assert batch[0][1].device.index == 0 and batch[0][1].type() == 'torch.cuda.FloatTensor'
+
+    # tensor dict
+    batch = [{'a': torch.rand(2, 3), 'b': torch.rand(2, 3)}]
+    batch = trainer.transfer_batch_to_gpu(batch, 0)
+    assert batch[0]['a'].device.index == 0 and batch[0]['a'].type() == 'torch.cuda.FloatTensor'
+    assert batch[0]['b'].device.index == 0 and batch[0]['b'].type() == 'torch.cuda.FloatTensor'
+
+
+def test_early_stopping_cpu_model():
+    """
+    Test each of the trainer options
+    :return:
+    """
+
+    stopping = EarlyStopping(monitor='val_loss')
+    trainer_options = dict(
+        early_stop_callback=stopping,
+        gradient_clip=1.0,
+        overfit_pct=0.20,
+        track_grad_norm=2,
+        print_nan_grads=True,
+        progress_bar=False,
+        experiment=get_exp(),
+        train_percent_check=0.1,
+        val_percent_check=0.1
+    )
+
+    model, hparams = get_model()
+    run_gpu_model_test(trainer_options, model, hparams, on_gpu=False)
+
+    # test freeze on cpu
+    model.freeze()
+    model.unfreeze()
+
+
+def test_no_val_module():
+    """
+    Tests use case where trainer saves the model, and user loads it from tags independently
+    :return:
+    """
+    hparams = get_hparams()
+    model = NoValModel(hparams)
+
+    save_dir = init_save_dir()
+
+    # exp file to get meta
+    exp = get_exp(False)
+    exp.argparse(hparams)
+    exp.save()
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        cluster=SlurmCluster(),
+        experiment=exp,
+        checkpoint_callback=ModelCheckpoint(save_dir)
+    )
+
+    # fit model
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    # traning complete
+    assert result == 1, 'amp + ddp model failed to complete'
+
+    # save model
+    new_weights_path = os.path.join(save_dir, 'save_test.ckpt')
+    trainer.save_checkpoint(new_weights_path)
+
+    # load new model
+    tags_path = exp.get_data_path(exp.name, exp.version)
+    tags_path = os.path.join(tags_path, 'meta_tags.csv')
+    model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
+                                                   tags_csv=tags_path, on_gpu=False)
+    model_2.eval()
+
+    # make prediction
+    clear_save_dir()
+
+
+def test_no_val_end_module():
+    """
+    Tests use case where trainer saves the model, and user loads it from tags independently
+    :return:
+    """
+    hparams = get_hparams()
+    model = NoValEndTestModel(hparams)
+
+    save_dir = init_save_dir()
+
+    # exp file to get meta
+    exp = get_exp(False)
+    exp.argparse(hparams)
+    exp.save()
+
+    trainer_options = dict(
+        max_nb_epochs=1,
+        cluster=SlurmCluster(),
+        experiment=exp,
+        checkpoint_callback=ModelCheckpoint(save_dir)
+    )
+
+    # fit model
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    # traning complete
+    assert result == 1, 'amp + ddp model failed to complete'
+
+    # save model
+    new_weights_path = os.path.join(save_dir, 'save_test.ckpt')
+    trainer.save_checkpoint(new_weights_path)
+
+    # load new model
+    tags_path = exp.get_data_path(exp.name, exp.version)
+    tags_path = os.path.join(tags_path, 'meta_tags.csv')
+    model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
+                                                   tags_csv=tags_path, on_gpu=False)
+    model_2.eval()
+
+    # make prediction
+    clear_save_dir()
+
+
+def test_simple_cpu():
+    """
+    Verify continue training session on CPU
+    :return:
+    """
+    hparams = get_hparams()
+    model = LightningTestModel(hparams)
+
+    save_dir = init_save_dir()
+
+    # exp file to get meta
+    trainer_options = dict(
+        max_nb_epochs=1,
+        val_percent_check=0.1,
+        train_percent_check=0.1,
+    )
+
+    # fit model
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    # traning complete
+    assert result == 1, 'amp + ddp model failed to complete'
+
+    clear_save_dir()
+
+
+def test_amp_single_gpu():
+    """
+    Make sure DDP + AMP work
+    :return:
+    """
+    if not torch.cuda.is_available():
+        warnings.warn('test_amp_gpu_ddp cannot run.'
+                      'Rerun on a GPU node to run this test')
+        return
+    if not torch.cuda.device_count() > 1:
+        warnings.warn('test_amp_gpu_ddp cannot run.'
+                      'Rerun on a node with 2+ GPUs to run this test')
+        return
+
+    hparams = get_hparams()
+    model = LightningTestModel(hparams)
+
+    trainer_options = dict(
+        progress_bar=True,
+        max_nb_epochs=1,
+        gpus=[0],
+        distributed_backend='dp',
+        use_amp=True
+    )
+
+    run_gpu_model_test(trainer_options, model, hparams)
+
+
+def test_cpu_restore_training():
+    """
+    Verify continue training session on CPU
+    :return:
+    """
+    hparams = get_hparams()
+    model = LightningTestModel(hparams)
+
+    save_dir = init_save_dir()
+
+    # exp file to get meta
+    test_exp_version = 10
+    exp = get_exp(False, version=test_exp_version)
+    exp.argparse(hparams)
+    exp.save()
+
+    trainer_options = dict(
+        max_nb_epochs=2,
+        val_check_interval=0.50,
+        val_percent_check=0.2,
+        train_percent_check=0.2,
+        experiment=exp,
+        checkpoint_callback=ModelCheckpoint(save_dir)
+    )
+
+    # fit model
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+    real_global_epoch = trainer.current_epoch
+
+    # traning complete
+    assert result == 1, 'amp + ddp model failed to complete'
+
+    # wipe-out trainer and model
+    # retrain with not much data... this simulates picking training back up after slurm
+    # we want to see if the weights come back correctly
+    new_exp = get_exp(False, version=test_exp_version)
+    trainer_options = dict(
+        max_nb_epochs=2,
+        val_check_interval=0.50,
+        val_percent_check=0.2,
+        train_percent_check=0.2,
+        experiment=new_exp,
+        checkpoint_callback=ModelCheckpoint(save_dir),
+    )
+    trainer = Trainer(**trainer_options)
+    model = LightningTestModel(hparams)
+
+    # set the epoch start hook so we can predict before the model does the full training
+    def assert_good_acc():
+        assert trainer.current_epoch == real_global_epoch and trainer.current_epoch > 0
+
+        # if model and state loaded correctly, predictions will be good even though we
+        # haven't trained with the new loaded model
+        trainer.model.eval()
+        _ = [run_prediction(dataloader, trainer.model) for dataloader in trainer.val_dataloader]
+
+    model.on_sanity_check_start = assert_good_acc
+
+    # by calling fit again, we trigger training, loading weights from the cluster
+    # and our hook to predict using current model before any more weight updates
+    trainer.fit(model)
+
+    clear_save_dir()
+
+
 def test_amp_gpu_ddp():
     """
     Make sure DDP + AMP work
     :return:
     """
     if not torch.cuda.is_available():
-        warnings.warn('test_amp_gpu_ddp cannot run. Rerun on a GPU node to run this test')
+        warnings.warn('test_amp_gpu_ddp cannot run.'
+                      'Rerun on a GPU node to run this test')
         return
     if not torch.cuda.device_count() > 1:
-        warnings.warn('test_amp_gpu_ddp cannot run. Rerun on a node with 2+ GPUs to run this test')
+        warnings.warn('test_amp_gpu_ddp cannot run.'
+                      'Rerun on a node with 2+ GPUs to run this test')
         return
 
     os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0])
@@ -105,7 +412,8 @@ def test_cpu_slurm_save_load():
     # wipe-out trainer and model
     # retrain with not much data... this simulates picking training back up after slurm
     # we want to see if the weights come back correctly
-    continue_tng_hparams = get_hparams(continue_training=True, hpc_exp_number=cluster_a.hpc_exp_number)
+    continue_tng_hparams = get_hparams(continue_training=True,
+                                       hpc_exp_number=cluster_a.hpc_exp_number)
     trainer_options = dict(
         max_nb_epochs=1,
         cluster=SlurmCluster(continue_tng_hparams),
@@ -136,11 +444,9 @@ def test_cpu_slurm_save_load():
 def test_loading_meta_tags():
     hparams = get_hparams()
 
-    save_dir = init_save_dir()
-
     # save tags
     exp = get_exp(False)
-    exp.tag({'some_str':'a_str', 'an_int': 1, 'a_float': 2.0})
+    exp.tag({'some_str': 'a_str', 'an_int': 1, 'a_float': 2.0})
     exp.argparse(hparams)
     exp.save()
 
@@ -221,7 +527,8 @@ def test_model_saving_loading():
     # load new model
     tags_path = exp.get_data_path(exp.name, exp.version)
     tags_path = os.path.join(tags_path, 'meta_tags.csv')
-    model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path, tags_csv=tags_path, on_gpu=False)
+    model_2 = LightningTestModel.load_from_metrics(weights_path=new_weights_path,
+                                                   tags_csv=tags_path, on_gpu=False)
     model_2.eval()
 
     # make prediction
@@ -232,8 +539,6 @@ def test_model_saving_loading():
     clear_save_dir()
 
 
-
-
 def test_model_freeze_unfreeze():
     hparams = get_hparams()
     model = LightningTestModel(hparams)
@@ -248,10 +553,12 @@ def test_amp_gpu_ddp_slurm_managed():
     :return:
     """
     if not torch.cuda.is_available():
-        warnings.warn('test_amp_gpu_ddp cannot run. Rerun on a GPU node to run this test')
+        warnings.warn('test_amp_gpu_ddp cannot run.'
+                      ' Rerun on a GPU node to run this test')
         return
     if not torch.cuda.device_count() > 1:
-        warnings.warn('test_amp_gpu_ddp cannot run. Rerun on a node with 2+ GPUs to run this test')
+        warnings.warn('test_amp_gpu_ddp cannot run.'
+                      ' Rerun on a node with 2+ GPUs to run this test')
         return
 
     # simulate setting slurm flags
@@ -320,33 +627,6 @@ def test_amp_gpu_ddp_slurm_managed():
     clear_save_dir()
 
 
-def test_early_stopping_cpu_model():
-    """
-    Test each of the trainer options
-    :return:
-    """
-
-    stopping = EarlyStopping()
-    trainer_options = dict(
-        early_stop_callback=stopping,
-        gradient_clip=1.0,
-        overfit_pct=0.20,
-        track_grad_norm=2,
-        print_nan_grads=True,
-        progress_bar=False,
-        experiment=get_exp(),
-        train_percent_check=0.1,
-        val_percent_check=0.1
-    )
-
-    model, hparams = get_model()
-    run_gpu_model_test(trainer_options, model, hparams, on_gpu=False)
-
-    # test freeze on cpu
-    model.freeze()
-    model.unfreeze()
-
-
 def test_cpu_model_with_amp():
     """
     Make sure model trains on CPU
@@ -400,6 +680,7 @@ def test_all_features_cpu_model():
         print_nan_grads=True,
         progress_bar=False,
         experiment=get_exp(),
+        accumulate_grad_batches=2,
         max_nb_epochs=1,
         train_percent_check=0.4,
         val_percent_check=0.4
@@ -415,7 +696,8 @@ def test_single_gpu_model():
     :return:
     """
     if not torch.cuda.is_available():
-        warnings.warn('test_single_gpu_model cannot run. Rerun on a GPU node to run this test')
+        warnings.warn('test_single_gpu_model cannot run.'
+                      ' Rerun on a GPU node to run this test')
         return
     model, hparams = get_model()
 
@@ -436,10 +718,12 @@ def test_multi_gpu_model_dp():
     :return:
     """
     if not torch.cuda.is_available():
-        warnings.warn('test_multi_gpu_model_dp cannot run. Rerun on a GPU node to run this test')
+        warnings.warn('test_multi_gpu_model_dp cannot run.'
+                      ' Rerun on a GPU node to run this test')
         return
     if not torch.cuda.device_count() > 1:
-        warnings.warn('test_multi_gpu_model_dp cannot run. Rerun on a node with 2+ GPUs to run this test')
+        warnings.warn('test_multi_gpu_model_dp cannot run.'
+                      ' Rerun on a node with 2+ GPUs to run this test')
         return
     model, hparams = get_model()
     trainer_options = dict(
@@ -462,10 +746,12 @@ def test_amp_gpu_dp():
     :return:
     """
     if not torch.cuda.is_available():
-        warnings.warn('test_amp_gpu_dp cannot run. Rerun on a GPU node to run this test')
+        warnings.warn('test_amp_gpu_dp cannot run.'
+                      ' Rerun on a GPU node to run this test')
         return
     if not torch.cuda.device_count() > 1:
-        warnings.warn('test_amp_gpu_dp cannot run. Rerun on a node with 2+ GPUs to run this test')
+        warnings.warn('test_amp_gpu_dp cannot run.'
+                      ' Rerun on a node with 2+ GPUs to run this test')
         return
     model, hparams = get_model()
     trainer_options = dict(
@@ -484,10 +770,12 @@ def test_multi_gpu_model_ddp():
     :return:
     """
     if not torch.cuda.is_available():
-        warnings.warn('test_multi_gpu_model_ddp cannot run. Rerun on a GPU node to run this test')
+        warnings.warn('test_multi_gpu_model_ddp cannot run.'
+                      ' Rerun on a GPU node to run this test')
         return
     if not torch.cuda.device_count() > 1:
-        warnings.warn('test_multi_gpu_model_ddp cannot run. Rerun on a node with 2+ GPUs to run this test')
+        warnings.warn('test_multi_gpu_model_ddp cannot run.'
+                      ' Rerun on a node with 2+ GPUs to run this test')
         return
 
     os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0])
@@ -504,7 +792,6 @@ def test_multi_gpu_model_ddp():
     run_gpu_model_test(trainer_options, model, hparams)
 
 
-
 def test_ddp_sampler_error():
     """
     Make sure DDP + AMP work
@@ -534,12 +821,43 @@ def test_ddp_sampler_error():
         use_amp=True
     )
 
-    with pytest.raises(MisconfigurationException):
+    with pytest.warns(UserWarning):
         trainer.get_dataloaders(model)
 
     clear_save_dir()
 
 
+def test_multiple_val_dataloader():
+    """
+    Verify multiple val_dataloader
+    :return:
+    """
+    hparams = get_hparams()
+    model = LightningTestModel(hparams)
+
+    save_dir = init_save_dir()
+
+    # exp file to get meta
+    trainer_options = dict(
+        max_nb_epochs=1,
+        val_percent_check=0.1,
+        train_percent_check=0.1,
+    )
+
+    # fit model
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model)
+
+    # verify tng completed
+    assert result == 1
+
+    # verify there are 2 val loaders
+    assert len(trainer.val_dataloader) == 2, 'Multiple val_dataloaders not initiated properly'
+
+    # make sure predictions are good for each val set
+    [run_prediction(dataloader, trainer.model) for dataloader in trainer.val_dataloader]
+
+
 # ------------------------------------------------------------------------
 # UTILS
 # ------------------------------------------------------------------------
@@ -589,8 +907,8 @@ def get_hparams(continue_training=False, hpc_exp_number=0):
     args = {
         'drop_prob': 0.2,
         'batch_size': 32,
-        'in_features': 28*28,
+        'in_features': 28 * 28,
-        'learning_rate': 0.001*8,
+        'learning_rate': 0.001 * 8,
         'optimizer_name': 'adam',
         'data_root': os.path.join(root_dir, 'mnist'),
         'out_features': 10,
@@ -604,18 +922,22 @@ def get_hparams(continue_training=False, hpc_exp_number=0):
     return hparams
 
 
-def get_model():
+def get_model(use_test_model=False):
     # set up model with these hyperparams
     hparams = get_hparams()
-    model = LightningTemplateModel(hparams)
+
+    if use_test_model:
+        model = LightningTestModel(hparams)
+    else:
+        model = LightningTemplateModel(hparams)
 
     return model, hparams
 
 
-def get_exp(debug=True):
+def get_exp(debug=True, version=None):
     # set up exp object without actually saving logs
     root_dir = os.path.dirname(os.path.realpath(__file__))
-    exp = Experiment(debug=debug, save_dir=root_dir, name='tests_tt_dir')
+    exp = Experiment(debug=debug, save_dir=root_dir, name='tests_tt_dir', version=version)
     return exp
 
 
@@ -675,13 +997,13 @@ def run_prediction(dataloader, trained_model):
 
     print(val_acc)
 
-    assert val_acc > 0.50, f'this model is expected to get > 0.50 in test set (it got {val_acc})'
+    assert val_acc > 0.50, 'this model is expected to get > 0.50 in test set (it got %f)' % val_acc
 
 
 def assert_ok_acc(trainer):
     # this model should get 0.80+ acc
     acc = trainer.tng_tqdm_dic['val_acc']
-    assert acc > 0.50, f'model failed to get expected 0.50 validation accuracy. Got: {acc}'
+    assert acc > 0.50, 'model failed to get expected 0.50 validation accuracy. Got: %f' % acc
 
 
 if __name__ == '__main__':

tox.ini

@@ -0,0 +1,47 @@
+# this file is *not* meant to cover or endorse the use of tox or pytest or testing in general,
+#
+#  It's meant to show the use of:
+#
+#  - check-manifest
+#     confirm items checked into vcs are in your segdist
+#  - python setup.py check 
+#     confirm required package meta-data in setup.py
+#  - readme_renderer (when using a ReStructuredText README)
+#     confirms your long_description will render correctly on PyPI.
+#
+#  and also to help confirm pull requests to this project.
+
+[tox]
+envlist = py{35,36,37}
+
+[pytest]
+log_cli = 0
+log_cli_level = CRITICAL
+log_cli_format = %(message)s
+log_file = pytest.log
+log_file_level = DEBUG
+log_file_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)
+log_file_date_format=%Y-%m-%d %H:%M:%S
+
+[testenv]
+basepython =
+    py35: python3.5
+    py36: python3.6
+    py37: python3.7
+deps =
+    -r requirements.txt
+    -r ./tests/requirements.txt
+commands =
+    check-manifest --ignore tox.ini
+    python setup.py check -m -s
+    coverage run --source pytorch_lightning -m py.test pytorch_lightning tests examples -v --doctest-modules
+    flake8 .
+
+[flake8]
+exclude = .tox,*.egg,build,temp,examples/templates
+select = E,W,F
+doctests = True
+verbose = 2
+# https://pep8.readthedocs.io/en/latest/intro.html#error-codes
+format = pylint
+max-line-length = 100

update.sh

@@ -11,10 +11,7 @@ rm -rf ./dist/*
 python3 setup.py sdist
 twine upload dist/*
 
-
-
 # to update docs
 # cd to root dir
 # mkdocs gh-deploy
 
-