indents

This will import the ipywidgets version of tqdm if available. This works nicely in notebooks by not filling up the log.

In the terminal it will use the same old tqdm.

We might also want to consider passing in the tqdm we want as an argument since there may be some edge cases where ipywidgets is available but the interface doesn't support it (e.g. vscode?) or isn't working. In which case people will get a warning message, but may want to configure it themselves.

.circleci/config.yml

@@ -0,0 +1,116 @@
+# Python CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-python/ for more details
+#
+version: 2.0
+
+references:
+
+  install_deps: &install_deps
+    run:
+      name: Install Dependences
+      command: |
+        pip install "$TORCH_VERSION" --user
+        # this is temporal fix til test-tube is not merged and released
+        pip install -r requirements.txt --user
+        sudo pip install pytest pytest-cov pytest-flake8
+        pip install -r ./tests/requirements.txt --user
+
+  tests: &tests
+   run:
+     name: Testing
+     command: |
+       python --version ; pip --version ; pip list
+       py.test pytorch_lightning tests pl_examples -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
+     no_output_timeout: 15m
+
+  format: &format
+   run:
+     name: Formatting
+     command: |
+       python --version ; pip --version ; pip list
+       flake8
+
+  make_docs: &make_docs
+   run:
+     name: Make Documentation
+     command: |
+       # sudo apt-get install pandoc
+       pip install -r requirements.txt --user
+       sudo pip install -r docs/requirements.txt
+       # sphinx-apidoc -o ./docs/source ./pytorch_lightning **/test_* --force --follow-links
+       cd docs; make clean ; make html
+
+jobs:
+
+  Build-Docs:
+    docker:
+      - image: circleci/python:3.7
+    steps:
+      - checkout
+      - *make_docs
+
+  Formatting:
+    docker:
+      - image: circleci/python:3.7
+    environment:
+        - TORCH_VERSION: "torch"
+    steps:
+      - checkout
+      - *install_deps
+      - *format
+
+  PyTorch:
+    docker:
+      - image: circleci/python:3.7
+    environment:
+      - TORCH_VERSION: "torch"
+    steps: &steps
+      - checkout
+
+      - *install_deps
+      - *tests
+
+      - store_test_results:
+          path: test-reports
+      - store_artifacts:
+          path: test-reports
+
+  PyTorch-v1.1:
+    docker:
+      - image: circleci/python:3.6
+    environment:
+      - TORCH_VERSION: "torch>=1.1, <1.2"
+    steps: *steps
+
+  PyTorch-v1.2:
+    docker:
+      - image: circleci/python:3.6
+    environment:
+      - TORCH_VERSION: "torch>=1.2, <1.3"
+    steps: *steps
+
+  PyTorch-v1.3:
+    docker:
+      - image: circleci/python:3.6
+    environment:
+      - TORCH_VERSION: "torch>=1.3, <1.4"
+    steps: *steps
+
+  PyTorch-v1.4:
+    docker:
+      - image: circleci/python:3.6
+    environment:
+      - TORCH_VERSION: "torch>=1.4, <1.5"
+    steps: *steps
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - Formatting
+      - Build-Docs
+      - PyTorch-v1.1
+      - PyTorch-v1.2
+      - PyTorch-v1.3
+      - PyTorch-v1.4

.codecov.yml

@@ -0,0 +1,46 @@
+# see https://docs.codecov.io/docs/codecov-yaml
+# Validation check:
+# $ curl --data-binary @.codecov.yml https://codecov.io/validate
+
+codecov:
+  notify:
+    require_ci_to_pass: yes
+
+coverage:
+  precision: 0  # 2 = xx.xx%, 0 = xx%
+  round: nearest # how coverage is rounded: down/up/nearest
+  range: 40...100 # custom range of coverage colors from red -> yellow -> green
+  status:
+    # https://codecov.readme.io/v1.0/docs/commit-status
+    project:
+      default:
+        against: auto
+        target: 99% # specify the target coverage for each commit status
+        threshold: 20% # allow this little decrease on project
+        # https://github.com/codecov/support/wiki/Filtering-Branches
+        # branches: master
+        if_ci_failed: error
+    # https://github.com/codecov/support/wiki/Patch-Status
+    patch:
+      default:
+        against: auto
+        target: 40% # specify the target "X%" coverage to hit
+        # threshold: 50% # allow this much decrease on patch
+    changes: false
+
+parsers:
+  gcov:
+    branch_detection:
+      conditional: true
+      loop: true
+      macro: false
+      method: false
+  javascript:
+    enable_partials: false
+
+comment:
+  layout: header, diff
+  require_changes: false
+  behavior: default  # update if exists else create new
+  # branches: *
+

.github/BECOMING_A_CORE_CONTRIBUTOR.md

@@ -0,0 +1,59 @@
+# How to become a core contributor
+
+Thanks for your interest in joining the Lightning team! We’re a rapidly growing project which is poised to become the go-to framework for DL researchers! 
+We're currently recruiting for a team of 5 core maintainers. 
+
+As a core maintainer you will have a strong say in the direction of the project. Big changes will require a majority of maintainers to agree.
+
+### Code of conduct  
+First and foremost, you'll be evaluated against [these core values](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md). Any code we commit or feature we add needs to align with those core values.
+
+### The bar for joining the team   
+Lightning is being used to solve really hard problems at the top AI labs in the world. As such, the bar for adding team members is extremely high. Candidates must have solid engineering skills, have a good eye for user experience, and must be a power user of Lightning and PyTorch.
+
+With that said, the Lightning team will be diverse and a reflection of an inclusive AI community. You don't have to be an engineer to conntribute! Scientists with great usability intuition and PyTorch ninja skills are welcomed!
+
+### Responsibilities:
+The responsibilities mainly revolve around 3 things.
+
+#### Github issues
+- Here we want to help users have an amazing experience. These range from questions from new people getting into DL to questions from researchers about doing something esoteric with Lightning
+Often, these issues require some sort of bug fix, document clarification or new functionality to be scoped out.
+
+- To become a core member you must resolve at least 10 Github issues which align with the API design goals for Lightning. By the end of these 10 issues I should feel comfortable in the way you answer user questions
+Pleasant/helpful tone.
+
+- Can abstract from that issue or bug into functionality that might solve other related issues or makes the platform more flexible.
+
+- Don’t make users feel like they don’t know what they’re doing. We’re here to help and to make everyone’s experience delightful.
+
+#### Pull requests
+
+- Here we need to ensure the code that enters Lightning is high quality. For each PR we need to:
+- Make sure code coverage does not decrease
+- Documents are updated
+- Code is elegant and simple
+- Code is NOT overly engineered or hard to read
+- Ask yourself, could a non-engineer understand what’s happening here?
+- Make sure new tests are written
+- Is this NECESSARY for Lightning? There are some PRs which are just purely about adding engineering complexity which have no place in Lightning. 
+Guidance
+- Some other PRs are for people who are wanting to get involved and add something unnecessary. We do want their help though! So don’t approve the PR, but direct them to a Github issue that they might be interested in helping with instead!
+- To be considered for core contributor, please review 10 PRs and help the authors land it on master. Once you've finished the review, ping me 
+for a sanity check. At the end of 10 PRs if your PR reviews are inline with expectations described above, then you can merge PRs on your own going forward,
+otherwise we'll do a few more until we're both comfortable :)
+
+#### Project directions
+There are some big decisions which the project must make. For these I expect core contributors to have something meaningful to add if it’s their area of expertise.
+
+#### Diversity
+Lightning should reflect the broader community it serves. As such we should have scientists/researchers from 
+different fields contributing!   
+
+The first 5 core contributors will fit this profile. Thus if you overlap strongly with experiences and expertise as someone else on the team, you might have to wait until the next set of contributors are added.
+
+#### Summary: Requirements to apply
+- Solve 10 Github issues. The goal is to be inline with expectations for solving issues by the last one so you can do them on your own. If not, I might ask you to solve a few more specific ones.
+- Do 10 PR reviews. The goal is to be inline with expectations for solving issues by the last one so you can do them on your own. If not, I might ask you to solve a few more specific ones.
+
+If you want to be considered, ping me on gitter and start [tracking your progress here](https://docs.google.com/spreadsheets/d/15D58gp8DvI0Z6qbbYVRuaWioiwzafcP58-UlbuO_CMU/edit?usp=sharing).

.github/CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at waf2107@columbia.edu. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

.github/CONTRIBUTING.md

@@ -0,0 +1,53 @@
+# Contributing    
+Welcome to the PyTorch Lightning community! We're building the most advanced research platform on the planet to implement the latest, best practices that the amazing PyTorch team rolls out!   
+
+## Main Core Value: One less thing to remember    
+Simplify the API as much as possible from the user perspective. Any additions or improvements should minimize things the user needs to remember.   
+
+For example: One benefit of the validation_step is that the user doesn't have to remember to set the model to .eval(). This avoids all sorts of subtle errors the user could make.  
+
+## Lightning Design Principles   
+We encourage all sorts of contributions you're interested in adding! When coding for lightning, please follow these principles.    
+#### No PyTorch Interference   
+We don't want to add any abstractions on top of pure PyTorch. This gives researchers all the control they need without having to learn yet another framework.    
+
+#### Simple Internal Code    
+It's useful for users to look at the code and understand very quickly what's happening. Many users won't be engineers. Thus we need to value clear, simple code over condensed ninja moves. While that's super cool, this isn't the project for that :)      
+
+#### Force User Decisions To Best Practices    
+There are 1,000 ways to do something. However, something eventually becomes standard practice that everyone does. Thus we pick one way of doing it and force everyone to do it this way. A good example is accumulated gradients. There are many ways to implement, we just pick one and force users to use that one. A bad forced decision would be to make users use a specific library to do something.    
+
+When something becomes a best practice, we add it to the framework. This likely looks like code in utils or in the model file that everyone keeps adding over and over again across projects. When this happens, bring that code inside the trainer and add a flag for it.
+
+#### Simple External API    
+What makes sense to you may not make sense to others. Create an issue with an API change suggestion and validate that it makes sense for others. Treat code changes how you treat a startup: validate that it's a needed feature, then add if it makes sense for many people. 
+
+#### Backward-compatible API   
+We all hate updating our deep learning packages because we don't want to refactor a bunch of stuff. In Lightning, we make sure every change we make which could break an API is backwards compatible with good deprecation warnings.
+
+You shouldn't be afraid to upgrade Lightning :)
+
+#### Gain User Trust    
+As a researcher you can't have any part of your code going wrong. So, make thorough tests that ensure an implementation of a new trick or subbtle change is correct.    
+
+#### Interoperability  
+Have a favorite feature from other libraries like fast.ai or transformers? Those should just work with lightning as well. Grab your favorite model or learning rate scheduler from your favorite library and run it in Lightning.
+
+## Contribution Types    
+Currently looking for help implementing new features or adding bug fixes. 
+
+A lot of good work has already been done in project mechanics (requirements.txt, setup.py, pep8, badges, ci, etc...) we're in a good state there thanks to all the early contributors (even pre-beta release)!   
+
+## Bug Fixes:  
+1. Submit a github issue.   
+2. Fix it.  
+3. Submit a PR! 
+
+## New Features:  
+1. Submit a github issue.   
+2. We'll agree on the feature scope.     
+3. Submit a PR! (with updated docs and tests 🙃).   
+
+## Coding Styleguide
+1. Test the code with flake8.
+2. Use f-strings.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,62 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: bug
+assignees: ''
+
+---
+
+### Common bugs:
+1. Tensorboard not showing in Jupyter-notebook see [issue 79](https://github.com/PyTorchLightning/pytorch-lightning/issues/79).    
+2. PyTorch 1.1.0 vs 1.2.0 support [see FAQ](https://github.com/PyTorchLightning/pytorch-lightning#faq)    
+
+## 🐛 Bug
+
+<!-- A clear and concise description of what the bug is. -->
+
+### To Reproduce
+
+Steps to reproduce the behavior:
+
+1. Go to '...'
+2. Run '....'
+3. Scroll down to '....'
+4. See error
+
+<!-- If you have a code sample, error messages, stack traces, please provide it here as well -->
+
+
+#### Code sample
+<!-- Ideally attach a minimal code sample to reproduce the decried issue. 
+Minimal means having the shortest code but still preserving the bug. -->
+
+### Expected behavior
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+### Environment
+
+Please copy and paste the output from our
+[environment collection script](https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py)
+(or fill out the checklist below manually).
+
+You can get the script and run it with:
+```
+wget https://raw.githubusercontent.com/pytorch/pytorch/master/torch/utils/collect_env.py
+# For security purposes, please check the contents of collect_env.py before running it.
+python collect_env.py
+```
+
+ - PyTorch Version (e.g., 1.0):
+ - OS (e.g., Linux):
+ - How you installed PyTorch (`conda`, `pip`, source):
+ - Build command you used (if compiling from source):
+ - Python version:
+ - CUDA/cuDNN version:
+ - GPU models and configuration:
+ - Any other relevant information:
+
+### Additional context
+
+<!-- Add any other context about the problem here. -->

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,18 @@
+---
+name: Typos and doc fixes
+about: Typos and doc fixes
+title: ''
+labels: typo
+assignees: ''
+
+---
+
+## 📚 Documentation
+
+For typos and doc fixes, please go ahead and:
+
+1. Create an issue.
+2. Fix the typo.   
+3. Submit a PR.
+
+Thanks!

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,27 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: enhancement, help wanted
+assignees: ''
+
+---
+
+## 🚀 Feature
+<!-- A clear and concise description of the feature proposal -->
+
+### Motivation
+
+<!-- Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too -->
+
+### Pitch
+
+<!-- A clear and concise description of what you want to happen. -->
+
+### Alternatives
+
+<!-- A clear and concise description of any alternative solutions or features you've considered, if any. -->
+
+### Additional context
+
+<!-- Add any other context or screenshots about the feature request here. -->

.github/ISSUE_TEMPLATE/how-to-question.md

@@ -0,0 +1,30 @@
+---
+name: How to question
+about: Asking how-to questions
+title: ''
+labels: question
+assignees: ''
+
+---
+
+## ❓ Questions and Help
+
+### Before asking:   
+1. search the issues.   
+2. search the docs.    
+
+<!-- If you still can't find what you need: -->
+
+#### What is your question?
+
+#### Code
+
+<!-- Please paste a code snippet if your question requires it! -->   
+
+#### What have you tried?
+
+#### What's your environment?
+
+ - OS: [e.g. iOS, Linux, Win]
+ - Packaging [e.g. pip, conda]
+ - Version [e.g. 0.5.2.1]

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+# Before submitting
+
+- [ ] Was this discussed/approved via a Github issue? (no need for typos, doc improvements)
+- [ ] Did you read the [contributor guideline](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md)?
+- [ ] Did you make sure to update the docs?   
+- [ ] Did you write any new necessary tests?  
+
+## What does this PR do?
+Fixes # (issue).
+
+## PR review    
+Anyone in the community is free to review the PR once the tests have passed.     
+If we didn't discuss your PR in Github issues there's a high chance it will not be merged.
+
+## Did you have fun?
+Make sure you had fun coding 🙃

.gitignore

@@ -1,23 +1,28 @@
 # project
 .DS_Store
-.data/
 run_configs/
-test_tube_logs/
-test_tube_data/
-datasets/
 model_weights/
 app/models/
 pip-wheel-metadata/
+lightning_logs/
+
+
+# Test-tube
+test_tube_logs/
+test_tube_data/
 test_tube_exp/
-tests/tests_tt_dir/
+
+# Documentations
+docs/source/pl_examples*.rst
+docs/source/pytorch_lightning*.rst
+tests/tests/
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
-example.py
 timit_data/
-LJSpeech-1.1/
+
 
 # C extensions
 *.so
@@ -26,7 +31,6 @@ LJSpeech-1.1/
 
 # Distribution / packaging
 .Python
-env/
 ide_layouts/
 build/
 develop-eggs/
@@ -38,7 +42,6 @@ lib/
 lib64/
 parts/
 sdist/
-var/
 wheels/
 *.egg-info/
 .installed.cfg
@@ -64,6 +67,9 @@ nosetests.xml
 coverage.xml
 *.cover
 .hypothesis/
+tests/tests_tt_dir/
+tests/save_dir
+tests/tests/
 
 # Translations
 *.mo
@@ -81,7 +87,7 @@ instance/
 .scrapy
 
 # Sphinx documentation
-docs/_build/
+docs/build/
 
 # PyBuilder
 target/
@@ -103,6 +109,7 @@ celerybeat-schedule
 
 # virtualenv
 .venv
+env/
 venv/
 ENV/
 
@@ -120,4 +127,6 @@ ENV/
 .mypy_cache/
 
 # data
+.data/
+datasets/
 mnist/

.readthedocs.yml

@@ -5,9 +5,13 @@
 # Required
 version: 2
 
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+  configuration: docs/source/conf.py
+
 # Build documentation with MkDocs
-mkdocs:
-  configuration: mkdocs.yml
+#mkdocs:
+#  configuration: mkdocs.yml
 
 # Optionally build your docs in additional formats such as PDF and ePub
 formats: all
@@ -16,4 +20,5 @@ formats: all
 python:
   version: 3.7
   install:
-    - requirements: docs/doc_requirements.txt
+    - requirements: docs/requirements.txt
+    #- requirements: requirements.txt

.run_local_tests.sh

@@ -0,0 +1,10 @@
+# use this to run tests
+rm -rf _ckpt_*
+rm -rf tests/save_dir*
+rm -rf tests/mlruns_*
+rm -rf tests/cometruns*
+rm -rf tests/wandb*
+rm -rf tests/tests/*
+rm -rf lightning_logs
+coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules
+coverage report -m

.travis.yml

@@ -1,16 +1,90 @@
+# vim ft=yaml
+
+# After changing this file, check it on:
+#   http://yaml-online-parser.appspot.com/
+
+# See doc/travis_notes.txt for some guidelines
+
+# this file is *not* meant to cover or endorse the use of travis, but rather to
+# help confirm pull requests to this project.
+
+env:
+  global:
+    - DISPLAY=""
+
 language: python
-python:
-  - "3.7"
-# command to install dependencies
+
+matrix:
+  include:
+    - dist: xenial  # Ubuntu 16.04
+      python: 3.6
+      env:
+        - TOXENV=py36
+        - MIN_REQUIREMENTS=1
+    - dist: xenial  # Ubuntu 16.04
+      python: 3.7
+      env:
+        - TOXENV=py37
+        - MIN_REQUIREMENTS=1
+    - dist: bionic  # Ubuntu 18.04
+      python: 3.6
+      env: TOXENV=py36
+    - dist: bionic  # Ubuntu 18.04
+      python: 3.7
+      env: TOXENV=py37
+    - os: osx
+      # https://blog.travis-ci.com/2019-08-07-extensive-python-testing-on-travis-ci
+      osx_image: xcode10.3
+      language: generic
+      env: TOXENV=py37
+      #addons:
+      #  homebrew:
+      #    # update: true
+      #    packages: python3.7
+      before_install:
+        - pip3 install virtualenv
+        - virtualenv -p python3 ~/venv
+        - source ~/venv/bin/activate
+#    - os: windows
+#      language: minimal
+#      before_install:
+#        - choco install python3
+#        - export PATH="/c/Python37:/c/Python37/Scripts:$PATH"
+#      env: TOXENV=py37
+
+# See http://docs.travis-ci.com/user/caching/#pip-cache
 cache: pip
+
 install:
-  - pip install -e .
-  - pip install -r requirements.txt
-  - pip install -U numpy
+  - pip install future  # needed for `builtins`
+  - sudo pip install tox
 
-# keep build from timing out
-dist: xenial
+before_script:
+  # rewrite all minimal requirements as strict
+  - if [[ "${MIN_REQUIREMENTS}" == "1" ]]; then
+      python -c "req = open('requirements.txt').read().replace('>', '=') ; open('requirements-ci.txt', 'w').write(req)" ;
+    else
+      cp requirements.txt requirements-ci.txt ;
+    fi
+  - pip install -r requirements-ci.txt -U
 
-# command to run tests
 script:
-  - py.test # or py.test for Python versions 3.5 and below
+  # integration
+  - tox --sitepackages
+
+  #- python setup.py install --dry-run --user
+  - virtualenv vEnv ;
+    source vEnv/bin/activate
+  - pip install --editable . ;
+    cd .. & python -c "import pytorch_lightning ; print(pytorch_lightning.__version__)"
+  - deactivate ;
+    rm -rf vEnv
+
+after_success:
+  - coverage report
+  # disable auto coverage bc it isn't accurate since it misses gpu code.
+  # to get coverage, run local and push results
+  # - codecov
+
+notifications:
+  email: false

COPYING

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2019 William Falcon
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MANIFEST.in

@@ -1,9 +1,42 @@
-graft docs
+# Manifest syntax https://docs.python.org/2/distutils/sourcedist.html
+graft wheelhouse
 
-include COPYING
-include AUTHORS
+recursive-exclude __pycache__  *.py[cod] *.orig
 
-recursive-include src/einsteinpy/tests *.py *.html
+# Include the README
+include *.md
 
-prune docs/source/examples/.ipynb_checkpoints
-global-exclude *.py[cod] __pycache__ *.so *.dylib
+# Include the license file
+include LICENSE
+
+exclude *.sh
+exclude *.toml
+exclude *.svg    
+recursive-include pytorch_lightning *.py
+
+# include examples
+recursive-include pl_examples *.py
+recursive-include pl_examples *.md
+recursive-include pl_examples *.sh
+
+# exclude tests from package
+recursive-exclude tests *
+recursive-exclude site *
+exclude tests
+
+# Exclude the documentation files
+recursive-exclude docs *
+exclude docs
+
+# Include the Requirements
+include requirements.txt
+
+# Exclude build configs
+exclude *.yml
+
+prune .git
+prune .github
+prune .circleci
+prune notebook*
+prune temp*
+prune test*

README.md

@@ -1,143 +1,222 @@
-<p align="center">
-  <a href="https://williamfalcon.github.io/pytorch-lightning/">
-    <img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/lightning_logo.png" width="50">
-  </a>
-</p>
-<h3 align="center">
-  Pytorch Lightning
-</h3>
-<p align="center">
-  The Keras for ML researchers using PyTorch. More control. Less boilerplate.    
-</p>
-<p align="center">
-  <a href="https://badge.fury.io/py/pytorch-lightning"><img src="https://badge.fury.io/py/pytorch-lightning.svg" alt="PyPI version" height="18"></a>
-  <a href="https://pepy.tech/project/pytorch-lightning"><img src="https://pepy.tech/badge/pytorch-lightning" alt="PyPI version" height="18"></a>
-</p>   
+<div align="center">
 
-<p align="center">
-  <a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/tests"><img src="https://github.com/williamFalcon/pytorch-lightning/blob/master/coverage.svg"></a>
-  <a href="https://travis-ci.org/williamFalcon/pytorch-lightning"><img src="https://travis-ci.org/williamFalcon/pytorch-lightning.svg?branch=master"></a>
-  <a href="https://williamfalcon.github.io/pytorch-lightning/"><img src="https://readthedocs.org/projects/pytorch-lightning/badge/?version=latest"></a>
-  <a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/COPYING"><img src="https://img.shields.io/badge/License-MIT-yellow.svg"></a>
-</p>   
+<img src="docs/source/_static/images/lightning_logo.png" width="50" height="50">
 
+# PyTorch Lightning
+
+**The lightweight PyTorch wrapper for ML researchers. Scale your models. Write less boilerplate.**
+
+
+[![PyPI Status](https://badge.fury.io/py/pytorch-lightning.svg)](https://badge.fury.io/py/pytorch-lightning)
+[![PyPI Status](https://pepy.tech/badge/pytorch-lightning)](https://pepy.tech/project/pytorch-lightning)
+[![Build Status](https://travis-ci.org/PytorchLightning/pytorch-lightning.svg?branch=master)](https://travis-ci.org/PytorchLightning/pytorch-lightning)
+[![Build status](https://ci.appveyor.com/api/projects/status/NEW-PROJECT-ID?svg=true)](https://ci.appveyor.com/project/PytorchLightning/pytorch-lightning)
+[![Coverage](docs/source/_static/images/coverage.svg)](https://github.com/PytorchLightning/pytorch-lightning/tree/master/tests#running-coverage)
+[![CodeFactor](https://www.codefactor.io/repository/github/borda/pytorch-lightning/badge)](https://www.codefactor.io/repository/github/borda/pytorch-lightning)    
+
+[![ReadTheDocs](https://readthedocs.org/projects/pytorch-lightning/badge/?version=0.6.0)](https://pytorch-lightning.readthedocs.io/en/0.6.0/)
+[![Slack](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://join.slack.com/t/pytorch-lightning/shared_invite/enQtODU5ODIyNTUzODQwLTFkMDg5Mzc1MDBmNjEzMDgxOTVmYTdhYjA1MDdmODUyOTg2OGQ1ZWZkYTQzODhhNzdhZDA3YmNhMDhlMDY4YzQ)
+[![license](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://github.com/PytorchLightning/pytorch-lightning/blob/master/LICENSE)
+[![Next Release](https://img.shields.io/badge/Next%20Release-Mar%2021-<COLOR>.svg)](https://shields.io/)
+
+<!-- 
+removed until codecov badge isn't empy. likely a config error showing nothing on master.
+[![codecov](https://codecov.io/gh/Borda/pytorch-lightning/branch/master/graph/badge.svg)](https://codecov.io/gh/Borda/pytorch-lightning)
+-->
+
+</div>
+
+Simple installation from PyPI
 ```bash
 pip install pytorch-lightning  
 ```
 
 ## Docs   
-**[View the docs here](https://williamfalcon.github.io/pytorch-lightning/)**
+- [master](https://pytorch-lightning.readthedocs.io/en/latest)   
+- [0.6.0](https://pytorch-lightning.readthedocs.io/en/0.6.0/)
+- [0.5.3.2](https://pytorch-lightning.readthedocs.io/en/0.5.3.2/)
+
+
+## Demo  
+[Copy and run this COLAB!](https://colab.research.google.com/drive/1F_RNcHzTfFuQf-LeKvSlud6x7jXYkG31#scrollTo=HOk9c4_35FKg)
 
 ## What is it?  
-Lightning defers training and validation loop logic to you. It guarantees correct, modern best practices for the core training logic.
+Lightning is a very lightweight wrapper on PyTorch that decouples the science code from the engineering code. It's more of a style-guide than a framework. By refactoring your code, we can automate most of the non-research code.  
 
+To use Lightning, simply refactor your research code into the [LightningModule](https://github.com/PytorchLightning/pytorch-lightning#how-do-i-do-use-it) format (the science) and Lightning will automate the rest (the engineering). Lightning guarantees tested, correct, modern best practices for the automated parts.
+
+- If you are a researcher, Lightning is infinitely flexible, you can modify everything down to the way .backward is called or distributed is set up. 
+- If you are a scientist or production team, lightning is very simple to use with best practice defaults.
+
+## What does lightning control for me?   
+
+Everything in Blue!   
+This is how lightning separates the science (red) from the engineering (blue).
+
+![Overview](docs/source/_static/images/pl.gif)
+
+## How much effort is it to convert?
+You're probably tired of switching frameworks at this point. But it is a very quick process to refactor into the Lightning format (ie: hours). [Check out this tutorial](https://towardsdatascience.com/how-to-refactor-your-pytorch-code-to-get-these-42-benefits-of-pytorch-lighting-6fdd0dc97538)
+
+## Starting a new project?   
+[Use our seed-project aimed at reproducibility!](https://github.com/PytorchLightning/pytorch-lightning-conference-seed)     
 
 ## Why do I want to use lightning?
-When starting a new project the last thing you want to do is recode a training loop, model loading/saving, distributed training, when to validate, etc... You're likely to spend a long time ironing out all the bugs without even getting to the core of your research.
+Every research project starts the same, a model, a training loop, validation loop, etc. As your research advances, you're likely to need distributed training, 16-bit precision, checkpointing, gradient accumulation, etc.   
 
-With lightning, you guarantee those parts of your code work so you can focus on what the meat of the research: Data and training, validation loop logic. Don't worry about multiple gpus or speeding up your code, lightning will do that for you!
+Lightning sets up all the boilerplate state-of-the-art training for you so you can focus on the research.   
+
+---
+ 
+## README Table of Contents        
+- [How do I use it](https://github.com/PytorchLightning/pytorch-lightning#how-do-i-do-use-it)     
+- [What lightning automates](https://github.com/PytorchLightning/pytorch-lightning#what-does-lightning-control-for-me)    
+- [Tensorboard integration](https://github.com/PytorchLightning/pytorch-lightning#tensorboard)    
+- [Lightning features](https://github.com/PytorchLightning/pytorch-lightning#lightning-automates-all-of-the-following-each-is-also-configurable)    
+- [Examples](https://github.com/PytorchLightning/pytorch-lightning#examples)    
+- [Tutorials](https://github.com/PytorchLightning/pytorch-lightning#tutorials)
+- [Contributing](https://github.com/PytorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md)
+- [Bleeding edge install](https://github.com/PytorchLightning/pytorch-lightning#bleeding-edge)   
+- [Lightning Design Principles](https://github.com/PytorchLightning/pytorch-lightning#lightning-design-principles)   
+- [Asking for help](https://github.com/PytorchLightning/pytorch-lightning#asking-for-help)
+- [FAQ](https://github.com/PytorchLightning/pytorch-lightning#faq)    
+
+---
+
+## How do I do use it?   
+Think about Lightning as refactoring your research code instead of using a new framework. The research code goes into a [LightningModule](https://pytorch-lightning.rtfd.io/en/latest/LightningModule/RequiredTrainerInterface/) which you fit using a Trainer.
+
+The LightningModule defines a *system* such as seq-2-seq, GAN, etc... It can ALSO define a simple classifier such as the example below.     
 
 To use lightning do 2 things:  
-1. [Define a LightningModel](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/)         
-```python
-import pytorch_lightning as ptl
-import torch
-from torch.nn import functional as F
-from torch.utils.data import DataLoader
-from torchvision.datasets import MNIST
+1. [Define a LightningModule](https://pytorch-lightning.rtfd.io/en/latest/LightningModule/RequiredTrainerInterface/)
+**WARNING:** This syntax is for version 0.5.0+ where abbreviations were removed.
+    ```python
+    import os
+    
+    import torch
+    from torch.nn import functional as F
+    from torch.utils.data import DataLoader
+    from torchvision.datasets import MNIST
+    from torchvision import transforms
+    
+    import pytorch_lightning as pl
+    
+    class CoolSystem(pl.LightningModule):
+    
+        def __init__(self):
+            super(CoolSystem, self).__init__()
+            # not the best model...
+            self.l1 = torch.nn.Linear(28 * 28, 10)
+    
+        def forward(self, x):
+            return torch.relu(self.l1(x.view(x.size(0), -1)))
+    
+        def training_step(self, batch, batch_idx):
+            # REQUIRED
+            x, y = batch
+            y_hat = self.forward(x)
+            loss = F.cross_entropy(y_hat, y)
+            tensorboard_logs = {'train_loss': loss}
+            return {'loss': loss, 'log': tensorboard_logs}
+    
+        def validation_step(self, batch, batch_idx):
+            # OPTIONAL
+            x, y = batch
+            y_hat = self.forward(x)
+            return {'val_loss': F.cross_entropy(y_hat, y)}
+    
+        def validation_end(self, outputs):
+            # OPTIONAL
+            avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()
+            tensorboard_logs = {'val_loss': avg_loss}
+            return {'avg_val_loss': avg_loss, 'log': tensorboard_logs}
+            
+        def test_step(self, batch, batch_idx):
+            # OPTIONAL
+            x, y = batch
+            y_hat = self.forward(x)
+            return {'test_loss': F.cross_entropy(y_hat, y)}
+    
+        def test_end(self, outputs):
+            # OPTIONAL
+            avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()
+            tensorboard_logs = {'test_loss': avg_loss}
+            return {'avg_test_loss': avg_loss, 'log': tensorboard_logs}
+    
+        def configure_optimizers(self):
+            # REQUIRED
+            # can return multiple optimizers and learning_rate schedulers
+            # (LBFGS it is automatically supported, no need for closure function)
+            return torch.optim.Adam(self.parameters(), lr=0.02)
+    
+        @pl.data_loader
+        def train_dataloader(self):
+            # REQUIRED
+            return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
+    
+        @pl.data_loader
+        def val_dataloader(self):
+            # OPTIONAL
+            return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)
+    
+        @pl.data_loader
+        def test_dataloader(self):
+            # OPTIONAL
+            return DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=transforms.ToTensor()), batch_size=32)
+    ```
+2. Fit with a [trainer](https://pytorch-lightning.rtfd.io/en/latest/Trainer/)    
+    ```python
+    from pytorch_lightning import Trainer
+    
+    model = CoolSystem()
+    
+    # most basic trainer, uses good defaults
+    trainer = Trainer()    
+    trainer.fit(model)   
+    ```
 
-class CoolModel(ptl.LightningModule):
+Trainer sets up a tensorboard logger, early stopping and checkpointing by default (you can modify all of them or
+use something other than tensorboard).   
 
-    def __init(self):
-        super(CoolModel, self).__init__()
-        # not the best model...
-        self.l1 = torch.nn.Linear(28 * 28, 10)
+Here are more advanced examples
+```python   
+# train on cpu using only 10% of the data (for demo purposes)
+trainer = Trainer(max_epochs=1, train_percent_check=0.1)
 
-    def forward(self, x):
-        return torch.relu(self.l1(x))
+# train on 4 gpus (lightning chooses GPUs for you)
+# trainer = Trainer(max_epochs=1, gpus=4, distributed_backend='ddp')  
 
-    def my_loss(self, y_hat, y):
-        return F.cross_entropy(y_hat, y)
+# train on 4 gpus (you choose GPUs)
+# trainer = Trainer(max_epochs=1, gpus=[0, 1, 3, 7], distributed_backend='ddp')   
 
-    def training_step(self, batch, batch_nb):
-        x, y = batch
-        y_hat = self.forward(x)
-        return {'tng_loss': self.my_loss(y_hat, y)}
-
-    def validation_step(self, batch, batch_nb):
-        x, y = batch
-        y_hat = self.forward(x)
-        return {'val_loss': self.my_loss(y_hat, y)}
-
-    def validation_end(self, outputs):
-        avg_loss = torch.stack([x for x in outputs['val_loss']]).mean()
-        return avg_loss
-
-    def configure_optimizers(self):
-        return [torch.optim.Adam(self.parameters(), lr=0.02)]
-
-    @ptl.data_loader
-    def tng_dataloader(self):
-        return DataLoader(MNIST('path/to/save', train=True), batch_size=32)
-
-    @ptl.data_loader
-    def val_dataloader(self):
-        return DataLoader(MNIST('path/to/save', train=False), batch_size=32)
-
-    @ptl.data_loader
-    def test_dataloader(self):
-        return DataLoader(MNIST('path/to/save', train=False), batch_size=32)
-```
-
-2. Fit with a [trainer](https://williamfalcon.github.io/pytorch-lightning/Trainer/)    
-```python
-from pytorch_lightning import Trainer
-from test_tube import Experiment
-
-model = CoolModel()
-
-# fit on 32 gpus across 4 nodes
-exp = Experiment(save_dir='some/dir')
-trainer = Trainer(experiment=exp, nb_gpu_nodes=4, gpus=[0,1,2,3,4,5,6,7])
+# train on 32 gpus across 4 nodes (make sure to submit appropriate SLURM job)
+# trainer = Trainer(max_epochs=1, gpus=8, num_gpu_nodes=4, distributed_backend='ddp')
 
+# train (1 epoch only here for demo)
 trainer.fit(model)
 
-# see all experiment metrics here
-# tensorboard --log_dir some/dir   
+# view tensorboard logs 
+logging.info(f'View tensorboard logs by running\ntensorboard --logdir {os.getcwd()}')
+logging.info('and going to http://localhost:6006 on your browser')
 ```
 
-
-## What does lightning control for me?
-Everything!    
-Except for these 6 core functions which you define:    
-
-```{.python}
-# what to do in the training loop
-def training_step(self, data_batch, batch_nb):
-
-# what to do in the validation loop
-def validation_step(self, data_batch, batch_nb):
-
-# how to aggregate validation_step outputs
-def validation_end(self, outputs):
-
-# and your dataloaders
-def tng_dataloader():
-def val_dataloader():
-def test_dataloader():
+When you're all done you can even run the test set separately.   
+```python
+trainer.test()
 ```
 
 **Could be as complex as seq-2-seq + attention**    
 
 ```python
 # define what happens for training here
-def training_step(self, data_batch, batch_nb):
-    x, y = data_batch
+def training_step(self, batch, batch_idx):
+    x, y = batch
     
     # define your own forward and loss calculation
     hidden_states = self.encoder(x)
      
-    # even as complex as a seq-2seq + attn model
+    # even as complex as a seq-2-seq + attn model
     # (this is just a toy, non-working example to illustrate)
     start_token = '<SOS>'
     last_hidden = torch.zeros(...)
@@ -158,8 +237,8 @@ def training_step(self, data_batch, batch_nb):
 
 ```python
 # define what happens for validation here
-def validation_step(self, data_batch, batch_nb):    
-    x, y = data_batch
+def validation_step(self, batch, batch_idx):    
+    x, y = batch
     
     # or as basic as a CNN classification
     out = self.forward(x)
@@ -184,122 +263,127 @@ def validation_end(self, outputs):
 
     val_loss_mean /= len(outputs)
     val_acc_mean /= len(outputs)
-    tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-    return tqdm_dic
+    logs = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
+    result = {'log': logs}
+    return result
 ```
    
 ## Tensorboard    
-Lightning is fully integrated with tensorboard.   
+Lightning is fully integrated with tensorboard, MLFlow and supports any logging module.   
 
-<p align="center">
-  <a href="https://williamfalcon.github.io/pytorch-lightning/">
-    <img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/tf_loss.png" width="900px">
-  </a>
-</p>
+![tensorboard-support](docs/source/_static/images/tf_loss.png)
 
 Lightning also adds a text column with all the hyperparameters for this experiment.      
 
-<p align="center">
-  <a href="https://williamfalcon.github.io/pytorch-lightning/">
-        <img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/tf_tags.png" width="900px">
-  </a>
-</p>
+![tensorboard-support](docs/source/_static/images/tf_tags.png)
 
-Simply note the path you set for the Experiment    
-``` {.python}   
-from test_tube import Experiment
-from pytorch-lightning import  Trainer
+## Lightning automates all of the following ([each is also configurable](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.html)):
 
-exp = Experiment(save_dir='/some/path')
-trainer = Trainer(experiment=exp)
-...
-```   
 
-And run tensorboard from that dir   
+- [Running grid search on a cluster](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.distrib_data_parallel.html)  
+- [Fast dev run](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.utilities.debugging.html)
+- [Logging](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.logging.html)
+- [Implement Your Own Distributed (DDP) training](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.core.lightning.html#pytorch_lightning.core.lightning.LightningModule.configure_ddp)
+- [Multi-GPU & Multi-node](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.distrib_parts.html)
+- [Training loop](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.training_loop.html)
+- [Hooks](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.core.hooks.html)
+- [Configure optimizers](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.core.lightning.html#pytorch_lightning.core.lightning.LightningModule.configure_optimizers)
+- [Validations](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.evaluation_loop.html)
+- [Model saving & Restoring training session](https://pytorch-lightning.rtfd.io/en/latest/pytorch_lightning.trainer.training_io.html)
+  
+
+## Examples   
+- [GAN](https://github.com/PytorchLightning/pytorch-lightning/tree/master/pl_examples/domain_templates/gan.py)    
+- [MNIST](https://github.com/PytorchLightning/pytorch-lightning/tree/master/pl_examples/basic_examples)      
+- [Other projects using Lightning](https://github.com/PytorchLightning/pytorch-lightning/network/dependents?package_id=UGFja2FnZS0zNzE3NDU4OTM%3D)    
+- [Multi-node](https://github.com/PytorchLightning/pytorch-lightning/tree/master/pl_examples/multi_node_examples)   
+
+## Tutorials   
+- [Basic Lightning use](https://towardsdatascience.com/supercharge-your-ai-research-with-pytorch-lightning-337948a99eec)    
+- [9 key speed features in Pytorch-Lightning](https://towardsdatascience.com/9-tips-for-training-lightning-fast-neural-networks-in-pytorch-8e63a502f565)    
+- [SLURM, multi-node training with Lightning](https://towardsdatascience.com/trivial-multi-node-training-with-pytorch-lightning-ff75dfb809bd)     
+
+---
+
+## Asking for help    
+Welcome to the Lightning community!   
+
+If you have any questions, feel free to:   
+1. [read the docs](https://pytorch-lightning.rtfd.io/en/latest/).     
+2. [Search through the issues](https://github.com/PytorchLightning/pytorch-lightning/issues?utf8=%E2%9C%93&q=my++question).      
+3. [Ask on stackoverflow](https://stackoverflow.com/questions/ask?guided=false) with the tag pytorch-lightning.   
+
+If no one replies to you quickly enough, feel free to post the stackoverflow link to our Gitter chat!   
+
+To chat with the rest of us visit our [gitter channel](https://gitter.im/PyTorch-Lightning/community)!     
+
+---   
+## FAQ    
+**How do I use Lightning for rapid research?**   
+[Here's a walk-through](https://pytorch-lightning.rtfd.io/en/latest/)  
+
+**Why was Lightning created?**     
+Lightning has 3 goals in mind:
+1. Maximal flexibility while abstracting out the common boilerplate across research projects.   
+2. Reproducibility. If all projects use the LightningModule template, it will be much much easier to understand what's going on and where to look! It will also mean every implementation follows a standard format.   
+3. Democratizing PyTorch power user features. Distributed training? 16-bit? know you need them but don't want to take the time to implement? All good... these come built into Lightning.    
+
+**How does Lightning compare with Ignite and fast.ai?**     
+[Here's a thorough comparison](https://medium.com/@_willfalcon/pytorch-lightning-vs-pytorch-ignite-vs-fast-ai-61dc7480ad8a).    
+
+**Is this another library I have to learn?**    
+Nope! We use pure Pytorch everywhere and don't add unecessary abstractions!   
+
+**Are there plans to support Python 2?**          
+Nope.     
+
+**Are there plans to support virtualenv?**    
+Nope. Please use anaconda or miniconda.    
+
+**Which PyTorch versions do you support?**    
+- **PyTorch 1.1.0**       
+    ```bash    
+    # install pytorch 1.1.0 using the official instructions   
+    
+    # install test-tube 0.6.7.6 which supports 1.1.0   
+    pip install test-tube==0.6.7.6   
+    
+    # install latest Lightning version without upgrading deps    
+    pip install -U --no-deps pytorch-lightning
+    ```     
+- **PyTorch 1.2.0, 1.3.0,**
+    Install via pip as normal   
+
+## Custom installation
+
+### Bleeding edge
+
+If you can't wait for the next release, install the most up to date code with:
+* using GIT (locally clone whole repo with full history)
+    ```bash
+    pip install git+https://github.com/PytorchLightning/pytorch-lightning.git@master --upgrade
+    ```
+* using instant zip (last state of the repo without git history)
+    ```bash
+    pip install https://github.com/PytorchLightning/pytorch-lightning/archive/master.zip --upgrade
+    ```
+
+### Any release installation
+
+You can also install any past release `0.X.Y` from this repository:
 ```bash
-tensorboard --logdir /some/path     
-```    
-
-## Lightning automatically automates all of the following ([each is also configurable](https://williamfalcon.github.io/pytorch-lightning/Trainer/)):
-
-###### Checkpointing    
-
-- [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
-- [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics) 
-
-###### Computing cluster (SLURM)    
-
-- [Running grid search on a cluster](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#running-grid-search-on-a-cluster) 
-- [Walltime auto-resubmit](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#walltime-auto-resubmit)   
-
-###### Debugging  
-
-- [Fast dev run](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#fast-dev-run)
-- [Inspect gradient norms](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#inspect-gradient-norms)
-- [Log GPU usage](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage)
-- [Make model overfit on subset of data](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data)
-- [Print the parameter count by layer](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer)
-- [Pring which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
-
-
-###### Distributed training    
-
-- [16-bit mixed precision](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision)
-- [Multi-GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU)
-- [Multi-node](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node)
-- [Single GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#single-gpu)
-- [Self-balancing architecture](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#self-balancing-architecture)
-
-
-###### Experiment Logging   
-
-- [Display metrics in progress bar](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#display-metrics-in-progress-bar)
-- Log arbitrary metrics
-- [Log metric row every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#log-metric-row-every-k-batches)
-- [Process position](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#process-position)
-- [Save a snapshot of all hyperparameters](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#save-a-snapshot-of-all-hyperparameters) 
-- [Snapshot code for a training run](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#snapshot-code-for-a-training-run) 
-- [Write logs file to csv every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#write-logs-file-to-csv-every-k-batches)
-
-###### Training loop    
-
-- [Accumulate gradients](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients)
-- [Anneal Learning rate](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#anneal-learning-rate)
-- [Force training for min or max epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs)
-- [Force disable early stop](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop)
-- [Gradient Clipping](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping)
-- [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/Pytorch-Lightning/LightningModule/#configure_optimizers)
-- [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check)
-
-###### Validation loop    
-
-- [Check validation every n epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#check-validation-every-n-epochs)
-- [Set how much of the validation set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-validation-set-to-check)
-- [Set how much of the test set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-test-set-to-check)
-- [Set validation check frequency within 1 training epoch](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-validation-check-frequency-within-1-training-epoch)
-- [Set the number of validation sanity steps](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-the-number-of-validation-sanity-steps)
-
-
-## Demo
-```bash
-# install lightning
-pip install pytorch-lightning
-
-# clone lightning for the demo
-git clone https://github.com/williamFalcon/pytorch-lightning.git
-cd examples/new_project_templates/
-
-# run demo (on cpu)
-python trainer_gpu_cluster_template.py
+pip install https://github.com/PytorchLightning/pytorch-lightning/archive/0.X.Y.zip --upgrade
 ```
 
-Without changing the model AT ALL, you can run the model on a single gpu, over multiple gpus, or over multiple nodes.
-```bash
-# run a grid search on two gpus
-python fully_featured_trainer.py --gpus "0;1"
-
-# run single model on multiple gpus
-python fully_featured_trainer.py --gpus "0;1" --interactive
+## Bibtex
+If you want to cite the framework feel free to use this (but only if you loved it 😊):
+```
+@misc{Falcon2019,
+  author = {Falcon, W.A. et al.},
+  title = {PyTorch Lightning},
+  year = {2019},
+  publisher = {GitHub},
+  journal = {GitHub repository},
+  howpublished = {\url{https://github.com/PytorchLightning/pytorch-lightning}}
+}
 ```
-
-

appveyor.yml

@@ -0,0 +1,68 @@
+# https://www.appveyor.com/docs/appveyor-yml/
+environment:
+
+  # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
+  # /E:ON and /V:ON options are not enabled in the batch script interpreter
+  # See: http://stackoverflow.com/a/13751649/163740
+  CMD_IN_ENV: "cmd /E:ON /V:ON /C obvci_appveyor_python_build_env.cmd"
+
+  matrix:
+    # Pre-installed Python versions, which Appveyor may upgrade to
+    # a later point release.
+    # See: http://www.appveyor.com/docs/installed-software#python
+
+
+    # - PYTHON: "C:\\Python35-x64"
+    #   PYTHON_VERSION: "3.5.x"
+    #   PYTHON_ARCH: "64"
+    #  TOXENV: "py35"
+
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "64"
+      TOXENV: "py36"
+      PIP_PYVER: "36"
+
+    - PYTHON: "C:\\Python37-x64"
+      PYTHON_VERSION: "3.7.x"
+      PYTHON_ARCH: "64"
+      TOXENV: "py37"
+      PIP_PYVER: "37"
+
+build: off
+
+# https://www.appveyor.com/docs/build-cache/
+cache:
+  - C:\ProgramData\chocolatey\bin -> appveyor.yml
+  - C:\ProgramData\chocolatey\lib -> appveyor.yml
+  - '%LOCALAPPDATA%\pip\Cache -> appveyor.yml'
+
+# scripts that run after cloning repository
+install:
+  # If there is a newer build queued for the same PR, cancel this one.
+  # The AppVeyor 'rollout builds' option is supposed to serve the same
+  # purpose but it is problematic because it tends to cancel builds pushed
+  # directly to master instead of just PR builds (or the converse).
+  - SET PATH=%PYTHON%;%PYTHON%\\Scripts;%path%
+  #- pip install -U --user "pip<19.3"
+  - python -m pip install -r requirements.txt -f https://download.pytorch.org/whl/torch_stable.html
+  - python -m pip install -r ./tests/requirements.txt
+  - python -m pip install pytest-flake8
+
+# scripts to run before tests (working directory and environment changes
+# are persisted from the previous steps such as "before_build")
+before_test:
+  - python --version
+  - pip --version
+  - pip list
+  - dir
+
+# to run your custom scripts instead of automatic tests
+test_script:
+  - coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules --flake8
+  #- python setup.py sdist
+  #- twine check dist/*
+
+on_success:
+  - coverage report
+  # - codecov

docs/LightningModule/RequiredTrainerInterface.md

@@ -1,430 +0,0 @@
-# Lightning Module interface
-[[Github Code](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/root_module/root_module.py)]
-
-A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model.
-
-The easiest thing to do is copy [this template](../../pytorch_lightning/examples/new_project_templates/lightning_module_template.py) and modify accordingly. 
-
-Otherwise, to Define a Lightning Module, implement the following methods:
-
-**Required**:  
-
-- [training_step](RequiredTrainerInterface.md#training_step)   
-- [validation_step](RequiredTrainerInterface.md#validation_step)
-- [validation_end](RequiredTrainerInterface.md#validation_end)
-    
-- [configure_optimizers](RequiredTrainerInterface.md#configure_optimizers)
-- [get_save_dict](RequiredTrainerInterface.md#get_save_dict)
-- [load_model_specific](RequiredTrainerInterface.md#load_model_specific)
-
-- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)
-- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)
-- [test_dataloader](RequiredTrainerInterface.md#test_dataloader)
-
-**Optional**:   
-
-- [update_tng_log_metrics](RequiredTrainerInterface.md#update_tng_log_metrics)
-- [add_model_specific_args](RequiredTrainerInterface.md#add_model_specific_args)
-
----
-**Minimal example**
-```python
-import pytorch_lightning as ptl
-import torch
-from torch.nn import functional as F
-from torch.utils.data import DataLoader
-from torchvision.datasets import MNIST
-
-class CoolModel(ptl.LightningModule):
-
-    def __init(self):
-        super(CoolModel, self).__init__()
-        # not the best model...
-        self.l1 = torch.nn.Linear(28 * 28, 10)
-
-    def forward(self, x):
-        return torch.relu(self.l1(x))
-
-    def my_loss(self, y_hat, y):
-        return F.cross_entropy(y_hat, y)
-
-    def training_step(self, batch, batch_nb):
-        x, y = batch
-        y_hat = self.forward(x)
-        return {'tng_loss': self.my_loss(y_hat, y)}
-
-    def validation_step(self, batch, batch_nb):
-        x, y = batch
-        y_hat = self.forward(x)
-        return {'val_loss': self.my_loss(y_hat, y)}
-
-    def validation_end(self, outputs):
-        avg_loss = torch.stack([x for x in outputs['val_loss']]).mean()
-        return avg_loss
-
-    def configure_optimizers(self):
-        return [torch.optim.Adam(self.parameters(), lr=0.02)]
-
-    @ptl.data_loader
-    def tng_dataloader(self):
-        return DataLoader(MNIST('path/to/save', train=True), batch_size=32)
-
-    @ptl.data_loader
-    def val_dataloader(self):
-        return DataLoader(MNIST('path/to/save', train=False), batch_size=32)
-
-    @ptl.data_loader
-    def test_dataloader(self):
-        return DataLoader(MNIST('path/to/save', train=False), batch_size=32)
-```
-
----
-
-### training_step
-
-``` {.python}
-def training_step(self, data_batch, batch_nb)
-```
-
-In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model.
-
-**Params**    
-
-| Param  | description  |
-|---|---|
-|  data_batch | The output of your dataloader. A tensor, tuple or list  |
-|  batch_nb | Integer displaying which batch this is  |
-
-**Return**   
-
-Dictionary or OrderedDict   
-
-| key  | value  | is required |
-|---|---|---|
-|  loss | tensor scalar  | Y |
-|  prog | Dict for progress bar display. Must have only tensors  | N |
-
-
-**Example**
-
-``` {.python}
-def training_step(self, data_batch, batch_nb):
-    x, y, z = data_batch
-    
-    # implement your own
-    out = self.forward(x)
-    loss = self.loss(out, x)
-    
-    output = {
-        'loss': loss, # required
-        'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional
-    }
-    
-    # return a dict
-    return output
-```
-
----
-
-### validation_step
-
-``` {.python}
-def validation_step(self, data_batch, batch_nb)
-```
-
-In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model.
-This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation.
-
-**Params**   
-
-| Param  | description  |
-|---|---|
-|  data_batch | The output of your dataloader. A tensor, tuple or list  |
-|  batch_nb | Integer displaying which batch this is  |
-
-**Return**   
-
-| Return  | description  | optional |
-|---|---|---|   
-|  dict | Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. | Y |
-
-**Example**
-
-``` {.python}
-def validation_step(self, data_batch, batch_nb):
-    x, y, z = data_batch
-    
-    # implement your own
-    out = self.forward(x)
-    loss = self.loss(out, x)
-    
-    # calculate acc
-    labels_hat = torch.argmax(out, dim=1)
-    val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
-    
-    # all optional...
-    # return whatever you need for the collation function validation_end
-    output = OrderedDict({
-        'val_loss': loss_val,
-        'val_acc': torch.tensor(val_acc), # everything must be a tensor
-    })
-    
-    # return an optional dict
-    return output
-```
-
---- 
-### validation_end
-
-``` {.python}
-def validation_end(self, outputs)
-```
-
-Called at the end of the validation loop with the output of each validation_step.
-
-**Params**    
-
-| Param  | description  |
-|---|---|
-|  outputs | List of outputs you defined in validation_step |
-
-**Return**   
-
-| Return  | description  | optional |
-|---|---|---|   
-|  dict | Dict of OrderedDict with metrics to display in progress bar | Y |
-
-**Example**
-
-``` {.python}
-def validation_end(self, outputs):
-    """
-    Called at the end of validation to aggregate outputs
-    :param outputs: list of individual outputs of each validation step
-    :return:
-    """
-    val_loss_mean = 0
-    val_acc_mean = 0
-    for output in outputs:
-        val_loss_mean += output['val_loss']
-        val_acc_mean += output['val_acc']
-
-    val_loss_mean /= len(outputs)
-    val_acc_mean /= len(outputs)
-    tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-    return tqdm_dic
-```
-
---- 
-### configure_optimizers 
-
-``` {.python}
-def configure_optimizers(self)
-```
-
-Set up as many optimizers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. 
-Lightning will call .backward() and .step() on each one.  If you use 16 bit precision it will also handle that.
-
-
-##### Return
-List - List of optimizers
-
-**Example**
-
-``` {.python}
-# most cases
-def configure_optimizers(self):
-    opt = Adam(lr=0.01)
-    return [opt]
-    
-# gan example
-def configure_optimizers(self):
-    generator_opt = Adam(lr=0.01)
-    disriminator_opt = Adam(lr=0.02)
-    return [generator_opt, disriminator_opt] 
-```
-
---- 
-### get_save_dict 
-
-``` {.python}
-def get_save_dict(self)
-```
-Called by lightning to checkpoint your model. Lightning saves current epoch, current batch nb, etc...
-All you have to return is what specifically about your lightning model you want to checkpoint.
-
-##### Return
-Dictionary - No required keys. Most of the time as described in this example.   
-
-**Example**
-
-``` {.python}
-def get_save_dict(self):
-    # 99% of use cases this is all you need to return
-    checkpoint = {'state_dict': self.state_dict()}
-    return checkpoint
-```
-
---- 
-### load_model_specific 
-
-``` {.python}
-def load_model_specific(self, checkpoint)
-```
-Called by lightning to restore your model. This is your chance to restore your model using the keys you added in get_save_dict.
-Lightning will automatically restore current epoch, batch nb, etc. 
-
-##### Return
-Nothing  
-
-**Example**
-
-``` {.python}
-def load_model_specific(self, checkpoint):
-    # you defined 'state_dict' in get_save_dict()
-    self.load_state_dict(checkpoint['state_dict'])
-```
-
---- 
-### tng_dataloader 
-
-``` {.python}
-@ptl.data_loader
-def tng_dataloader(self)
-```
-Called by lightning during training loop. Make sure to use the @ptl.data_loader decorator, this ensures not calling this function until the data are needed.
-
-##### Return
-Pytorch DataLoader
-
-**Example**
-
-``` {.python}
-@ptl.data_loader
-def tng_dataloader(self):
-    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
-    dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True)
-    loader = torch.utils.data.DataLoader(
-        dataset=dataset,
-        batch_size=self.hparams.batch_size,
-        shuffle=True
-    )
-    return loader
-```
-
---- 
-### val_dataloader 
-
-``` {.python}
-@ptl.data_loader
-def tng_dataloader(self)
-```
-Called by lightning during validation loop. Make sure to use the @ptl.data_loader decorator, this ensures not calling this function until the data are needed.
-
-##### Return
-Pytorch DataLoader
-
-**Example**
-
-``` {.python}
-@ptl.data_loader
-def val_dataloader(self):
-    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
-    dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True)
-    loader = torch.utils.data.DataLoader(
-        dataset=dataset,
-        batch_size=self.hparams.batch_size,
-        shuffle=True
-    )
-    
-    return loader
-```
-
---- 
-### test_dataloader 
-
-``` {.python}
-@ptl.data_loader
-def test_dataloader(self)
-```
-Called by lightning during test loop. Make sure to use the @ptl.data_loader decorator, this ensures not calling this function until the data are needed.
-
-##### Return
-Pytorch DataLoader
-
-**Example**
-
-``` {.python}
-@ptl.data_loader
-def test_dataloader(self):
-    transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
-    dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True)
-    loader = torch.utils.data.DataLoader(
-        dataset=dataset,
-        batch_size=self.hparams.batch_size,
-        shuffle=True
-    )
-    
-    return loader
-```
-
---- 
-### update_tng_log_metrics 
-
-``` {.python}
-def update_tng_log_metrics(self, logs)
-```
-Called by lightning right before it logs metrics for this batch.
-This is a chance to ammend or add to the metrics about to be logged.
-
-##### Return
-Dict 
-
-**Example**
-
-``` {.python}
-def update_tng_log_metrics(self, logs):
-    # modify or add to logs
-    return logs
-```
-
---- 
-### add_model_specific_args 
-
-``` {.python}
-@staticmethod
-def add_model_specific_args(parent_parser, root_dir)
-```
-Lightning has a list of default argparse commands.
-This method is your chance to add or modify commands specific to your model.
-The [hyperparameter argument parser](https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/) is available anywhere in your model by calling self.hparams.
-
-##### Return
-An argument parser
-
-**Example**
-
-``` {.python}
-@staticmethod
-def add_model_specific_args(parent_parser, root_dir):
-    parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
-
-    # param overwrites
-    # parser.set_defaults(gradient_clip=5.0)
-
-    # network params
-    parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
-    parser.add_argument('--in_features', default=28*28)
-    parser.add_argument('--out_features', default=10)
-    parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference
-
-    # data
-    parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
-
-    # training params (opt)
-    parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
-                    tunable=False)
-    parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False)
-    parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
-    return parser
-```

docs/LightningModule/methods.md

@@ -1,49 +0,0 @@
-Lightning modules are strict superclasses of torch.nn.Module. A LightningModule offers the following in addition to that API.
-
----    
-### freeze 
-Freeze all params for inference
-```{.python}
-model = MyLightningModule(...)
-model.freeze()
-```
-
----    
-### load_from_metrics
-This is the easiest/fastest way which uses the meta_tags.csv file from test-tube to rebuild the model.
-The meta_tags.csv file can be found in the test-tube experiment save_dir.       
-
-```{.python}
-pretrained_model = MyLightningModule.load_from_metrics(
-    weights_path='/path/to/pytorch_checkpoint.ckpt',
-    tags_csv='/path/to/test_tube/experiment/version/meta_tags.csv',
-    on_gpu=True,
-    map_location=None
-)
-    
-# predict 
-pretrained_model.freeze()
-y_hat = pretrained_model(x)
-```
-
-**Params**    
-
-| Param  | description  |
-|---|---|
-|  weights_path | Path to a pytorch checkpoint  |
-|  tags_csv | Path to meta_tags.csv file generated by the test-tube Experiment  |
-|  on_gpu | if True, puts model on GPU. Make sure to use transforms option if model devices have changed  |
-|  map_location | A dictionary mapping saved weight GPU devices to new GPU devices |
-
-**Returns**    
-
-LightningModule - The pretrained LightningModule
-
----    
-### unfreeze 
-Unfreeze all params for inference
-```{.python}
-model = MyLightningModule(...)
-model.unfreeze()
-```
-

docs/LightningModule/properties.md

@@ -1,40 +0,0 @@
-A LightningModule has the following properties which you can access at any time
-
---- 
-#### current_epoch
-The current epoch   
-
----
-#### dtype    
-Current dtype    
-
----
-#### experiment    
-An instance of test-tube Experiment which you can use to log anything for tensorboarX.   
-```{.python}
-self.experiment.add_embedding(...)   
-self.experiment.log({'val_loss': 0.9})   
-self.experiment.add_scalars(...)   
-```
-
---- 
-#### global_step   
-Total training batches seen across all epochs   
-
---- 
-#### gradient_clip   
-The current gradient clip value    
-
----
-#### on_gpu    
-True if your model is currently running on GPUs. Useful to set flags around the LightningModule for different CPU vs GPU behavior.    
-
----
-#### trainer    
-Last resort access to any state the trainer has. Changing certain properties here could affect your training run.
-```{.python}   
-self.trainer.optimizers
-self.trainer.current_epoch
-...   
-```   
-

docs/Makefile

@@ -0,0 +1,19 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/Trainer/Checkpointing.md

@@ -1,22 +0,0 @@
-Lightning can automate saving and loading checkpoints.
-
----
-### Model saving
-To enable checkpointing, define the checkpoint callback and give it to the trainer.
-
-``` {.python}
-from pytorch_lightning.utils.pt_callbacks import ModelCheckpoint
-
-checkpoint_callback = ModelCheckpoint(
-    filepath='/path/to/store/weights.ckpt',
-    save_best_only=True,
-    verbose=True,
-    monitor='val_loss',
-    mode='min'
-)
-
-trainer = Trainer(checkpoint_callback=checkpoint_callback)
-```
-
-
-

docs/Trainer/Distributed training.md

@@ -1,128 +0,0 @@
-Lightning makes multi-gpu training and 16 bit training trivial.
-
-*Note:*   
-None of the flags below require changing anything about your lightningModel definition. 
-
----
-#### Choosing a backend  
-Lightning supports two backends. DataParallel and DistributedDataParallel. Both can be used for single-node multi-GPU training.
-For multi-node training you must use DistributedDataParallel.   
-
-You can toggle between each mode by setting this flag.
-``` {.python}
-# DEFAULT uses DataParallel
-trainer = Trainer(distributed_backend='dp')
-
-# change to distributed data parallel
-trainer = Trainer(distributed_backend='ddp')
-```
-
-If you request multiple nodes, the back-end will auto-switch to ddp.
-We recommend you use DistributedDataparallel even for single-node multi-GPU training. It is MUCH faster than DP but *may*
-have configuration issues depending on your cluster.
-
-For a deeper understanding of what lightning is doing, feel free to read [this guide](https://medium.com/@_willfalcon/9-tips-for-training-lightning-fast-neural-networks-in-pytorch-8e63a502f565).   
-
----
-#### 16-bit mixed precision
-16 bit precision can cut your memory footprint by half. If using volta architecture GPUs it can give a dramatic training speed-up as well.    
-First, install apex (if install fails, look [here](https://github.com/NVIDIA/apex)):
-```bash
-$ git clone https://github.com/NVIDIA/apex
-$ cd apex
-$ pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./
-```
-
-then set this use_amp to True.
-``` {.python}
-# DEFAULT
-trainer = Trainer(amp_level='O2', use_amp=False)
-```
-
----
-#### Single-gpu
-Make sure you're on a GPU machine. 
-```python
-# set these flags
-os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-os.environ["CUDA_VISIBLE_DEVICES"] = "0"
-
-# DEFAULT
-trainer = Trainer(gpus=[0])
-```
-
----
-#### multi-gpu 
-Make sure you're on a GPU machine. You can set as many GPUs as you want.
-In this setting, the model will run on all 8 GPUs at once using DataParallel under the hood.
-```python
-# set these flags
-# lightning sets these flags for you automatically
-# no need to set yourself
-# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-# os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3,4,5,6,7"
-
-
-# to use DataParallel (default)
-trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='dp')
-
-# RECOMMENDED use DistributedDataParallel
-trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='ddp')
-```
-
----
-#### Multi-node
-Multi-node training is easily done by specifying these flags.
-```python
-# train on 12*8 GPUs
-trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], nb_gpu_nodes=12)
-```
-
-In addition, make sure to set up your SLURM job correctly via the [SlurmClusterObject](https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/). In particular, specify the number of tasks per node correctly.
-
-```python
-cluster = SlurmCluster(
-    hyperparam_optimizer=test_tube.HyperOptArgumentParser(),
-    log_path='/some/path/to/save',
-)
-
-# OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT
-# which interface your nodes use for communication
-cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo')
-
-# see output of the NCCL connection process
-# NCCL is how the nodes talk to each other
-cluster.add_command('export NCCL_DEBUG=INFO')
-
-# setting a master port here is a good idea.
-cluster.add_command(f'export MASTER_PORT={PORT}')
-
-# good to load the latest NCCL version
-cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0'])
-
-# configure cluster
-cluster.per_experiment_nb_nodes = 12 
-cluster.per_experiment_nb_gpus = 8
-
-cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu')
-```
-
-Finally, make sure to add a distributed sampler to your dataset. The distributed sampler copies a 
-portion of your dataset onto each GPU. (World_size = gpus_per_node * nb_nodes).   
-
-```python
-# ie: this:
-dataset = myDataset()
-dataloader = Dataloader(dataset)
-
-# becomes:
-dataset = myDataset()
-dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
-dataloader = Dataloader(dataset, sampler=dist_sampler)
-```
-
----
-#### Self-balancing architecture
-Here lightning distributes parts of your module across available GPUs to optimize for speed and memory.   
-
-COMING SOON.

docs/Trainer/Logging.md

@@ -1,61 +0,0 @@
-Lighting offers a few options for logging information about model, gpu usage, etc (via test-tube). It also offers printing options for training monitoring.
-
-
----
-#### Display metrics in progress bar 
-``` {.python}
-# DEFAULT
-trainer = Trainer(progress_bar=True)
-```
-
----
-#### Log metric row every k batches 
-Every k batches lightning will make an entry in the metrics log
-``` {.python}
-# DEFAULT (ie: save a .csv log file every 10 batches)
-trainer = Trainer(add_log_row_interval=10)
-```
-
----
-#### Process position
-When running multiple models on the same machine we want to decide which progress bar to use.
-Lightning will stack progress bars according to this value. 
-``` {.python}
-# DEFAULT
-trainer = Trainer(process_position=0)
-
-# if this is the second model on the node, show the second progress bar below
-trainer = Trainer(process_position=1)
-```
-
----
-#### Save a snapshot of all hyperparameters 
-Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use.
-Give lightning a test-tube Experiment object to automate this for you.
-``` {.python}
-from test-tube import Experiment
-
-exp = Experiment(...)
-Trainer(experiment=exp)
-```
-
----
-#### Snapshot code for a training run
-Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag.
-Give lightning a test-tube Experiment object to automate this for you.
-``` {.python}
-from test-tube import Experiment
-
-exp = Experiment(create_git_tag=True)
-Trainer(experiment=exp)
-```
-
-
----
-#### Write logs file to csv every k batches 
-Every k batches, lightning will write the new logs to disk
-``` {.python}
-# DEFAULT (ie: save a .csv log file every 100 batches)
-trainer = Trainer(log_save_interval=100)
-```
-

docs/Trainer/SLURM Managed Cluster.md

@@ -1,104 +0,0 @@
-Lightning supports model training on a cluster managed by SLURM in the following cases:    
-
-1. Training on single or multi-cpus only.
-2. Training on single or multi-gpus on the same node.
-3. Coming SOON: Training across multiple nodes.
-
----
-#### Running grid search on a cluster
-To use lightning to run a hyperparameter search (grid-search or random-search) on a cluster do 4 things:   
-
-(1). Define the parameters for the grid search    
-    
-```{.python}
-from test_tube import HyperOptArgumentParser
-
-# subclass of argparse
-parser = HyperOptArgumentParser(strategy='random_search')
-parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate')
-
-# let's enable optimizing over the number of layers in the network
-parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8])
-
-hparams = parser.parse_args()    
-```    
-    
-     
-(2). Define the cluster options in the [SlurmCluster object](https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/) (over 5 nodes and 8 gpus)    
-
-```{.python}
-from test_tube.hpc import SlurmCluster
-
-# hyperparameters is a test-tube hyper params object
-# see https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/
-hyperparams = args.parse()
-
-# init cluster
-cluster = SlurmCluster(
-    hyperparam_optimizer=hyperparams,
-    log_path='/path/to/log/results/to',
-    python_cmd='python3'
-)
-
-# let the cluster know where to email for a change in job status (ie: complete, fail, etc...)
-cluster.notify_job_status(email='some@email.com', on_done=True, on_fail=True)
-
-# set the job options. In this instance, we'll run 20 different models
-# each with its own set of hyperparameters giving each one 1 GPU (ie: taking up 20 GPUs)
-cluster.per_experiment_nb_gpus = 8
-cluster.per_experiment_nb_nodes = 5
-
-# we'll request 10GB of memory per node
-cluster.memory_mb_per_node = 10000
-
-# set a walltime of 10 minues
-cluster.job_time = '10:00'
-```
-
-(3). Give trainer the cluster_manager in your main function:    
-
-```{.python}
-from pytorch_lightning import Trainer
-
-def train_fx(trial_hparams, cluster_manager, _):
-    # hparams has a specific set of hyperparams
-    
-    my_model = MyLightningModel()
-    
-    # give the trainer the cluster object
-    trainer = Trainer(cluster=cluster_manager)
-    trainer.fit(my_model)
-
-```
-
-(4). Start the grid search     
-```{.python}
-# run the models on the cluster
-cluster.optimize_parallel_cluster_gpu(
-    train_fx, 
-    nb_trials=20, 
-    job_name='my_grid_search_exp_name', 
-    job_display_name='my_exp')
-```
-
-That's it! The SlurmCluster object will automatically checkpoint the lightning model and resubmit if it runs into the walltime!
-
-
----
-#### Walltime auto-resubmit
-Lightning automatically resubmits jobs when they reach the walltime. You get this behavior for free if you give lightning
-a slurm cluster object.
-
-```{.python}
-def my_main_fx(hparams, slurm_manager, _):
-    trainer = Trainer(cluster=slurm_manager)
-``` 
-
-(See the grid search example above for cluster configuration).
-With this feature lightning will:    
-
-1. automatically checkpoint the model
-2. checkpoint the trainer session
-3. resubmit a continuation job.
-4. load the checkpoint and trainer session in the new model
-

docs/Trainer/Training Loop.md

@@ -1,72 +0,0 @@
-The lightning training loop handles everything except the actual computations of your model. To decide what will happen in your training loop, define the [training_step function](../../Pytorch-lightning/LightningModule/#training_step).
-
-Below are all the things lightning automates for you in the training loop.
-
----
-#### Accumulated gradients  
-Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. 
-
-``` {.python}
-# DEFAULT (ie: no accumulated grads)
-trainer = Trainer(accumulate_grad_batches=1)
-```
-
----
-#### Anneal Learning rate
-Cut the learning rate by 10 at every epoch listed in this list.
-``` {.python}
-# DEFAULT (don't anneal)
-trainer = Trainer(lr_scheduler_milestones=None)
-
-# cut LR by 10 at 100, 200, and 300 epochs 
-trainer = Trainer(lr_scheduler_milestones='100, 200, 300')
-```
-
----
-#### Force training for min or max epochs
-It can be useful to force training for a minimum number of epochs or limit to a max number
-``` {.python}
-# DEFAULT
-trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)
-```
-
----
-#### Force disable early stop 
-Use this to turn off early stopping and run training to the [max_epoch](#force-training-for-min-or-max-epochs)
-``` {.python}
-# DEFAULT
-trainer = Trainer(enable_early_stop=True)
-```
-
----
-#### Gradient Clipping 
-Use this to turn off early stopping and run training to the [max_epoch](#force-training-for-min-or-max-epochs)
-``` {.python}
-# DEFAULT (ie: don't clip)
-trainer = Trainer(gradient_clip=0)
-```
-
-
-
----
-#### Inspect gradient norms
-Looking at grad norms can help you figure out where training might be going wrong.
-``` {.python}
-# DEFAULT (-1 doesn't track norms)
-trainer = Trainer(track_grad_norm=-1)
-
-# track the LP norm (P=2 here)
-trainer = Trainer(track_grad_norm=2)
-```
-
-
----
-#### Set how much of the training set to check
-If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag
-``` {.python}
-# DEFAULT
-trainer = Trainer(train_percent_check=1.0)
-
-# check 10% only
-trainer = Trainer(train_percent_check=0.1)
-```

docs/Trainer/Validation loop.md

@@ -1,57 +0,0 @@
-The lightning validation loop handles everything except the actual computations of your model. To decide what will happen in your validation loop, define the [validation_step function](../../Pytorch-lightning/LightningModule/#validation_step).
-Below are all the things lightning automates for you in the validation loop.
-
-**Note**   
-Lightning will run 5 steps of validation in the beginning of training as a sanity check so you don't have to wait until a full epoch to catch possible validation issues.
-
-
-
-
----
-#### Check validation every n epochs
-If you have a small dataset you might want to check validation every n epochs
-``` {.python}
-# DEFAULT
-trainer = Trainer(check_val_every_n_epoch=1)
-```
-
----
-#### Set how much of the validation set to check 
-If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag
-``` {.python}
-# DEFAULT
-trainer = Trainer(val_percent_check=1.0)
-
-# check 10% only
-trainer = Trainer(val_percent_check=0.1)
-```
-
----
-#### Set how much of the test set to check 
-If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag
-``` {.python}
-# DEFAULT
-trainer = Trainer(test_percent_check=1.0)
-
-# check 10% only
-trainer = Trainer(test_percent_check=0.1)
-```
-
----
-####  Set validation check frequency within 1 training epoch
-For large datasets it's often desirable to check validation multiple times within a training loop
-``` {.python}
-# DEFAULT
-trainer = Trainer(val_check_interval=0.95)
-
-# check every .25 of an epoch 
-trainer = Trainer(val_check_interval=0.25)
-```
-
----
-####  Set the number of validation sanity steps
-Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop.
-``` {.python}
-# DEFAULT
-trainer = Trainer(nb_sanity_val_steps=5)
-```

docs/Trainer/debugging.md

@@ -1,48 +0,0 @@
-These flags are useful to help debug a model.
-
----
-#### Fast dev run 
-This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch.
-Use this to debug a full run of your program quickly
-``` {.python}
-# DEFAULT
-trainer = Trainer(fast_dev_run=False)
-```
-
----
-#### Inspect gradient norms
-Looking at grad norms can help you figure out where training might be going wrong.
-``` {.python}
-# DEFAULT (-1 doesn't track norms)
-trainer = Trainer(track_grad_norm=-1)
-
-# track the LP norm (P=2 here)
-trainer = Trainer(track_grad_norm=2)
-```
-
----
-#### Make model overfit on subset of data
-A useful debugging trick is to make your model overfit a tiny fraction of the data.
-``` {.python}
-# DEFAULT don't overfit (ie: normal training)
-trainer = Trainer(overfit_pct=0.0)
-
-# overfit on 1% of data 
-trainer = Trainer(overfit_pct=0.01)
-```
-
----
-#### Print the parameter count by layer
-By default lightning prints a list of parameters *and submodules* when it starts training.
-
----
-#### Print which gradients are nan 
-This option prints a list of tensors with nan gradients.
-``` {.python}
-# DEFAULT
-trainer = Trainer(print_nan_grads=False)
-```
-
----
-#### Log GPU usage
-Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.

docs/Trainer/index.md

@@ -1,74 +0,0 @@
-# Trainer
-[[Github Code](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/models/trainer.py)]
-
-The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research.
-
-This is the basic use of the trainer:
-
-``` {.python}
-from pytorch_lightning import Trainer
-
-model = LightningTemplate()
-
-trainer = Trainer()
-trainer.fit(model)
-```
-
-But of course the fun is in all the advanced things it can do:
-
-
-**Checkpointing**    
-
-- Model saving
-- Model loading 
-
-**Computing cluster (SLURM)**    
-
-- [Running grid search on a cluster](SLURM%20Managed%20Cluster/#running-grid-search-on-a-cluster)
-- [Walltime auto-resubmit](SLURM%20Managed%20Cluster/#walltime-auto-resubmit)   
-
-**Debugging**  
-
-- [Fast dev run](Debugging/#fast-dev-run)
-- [Inspect gradient norms](Debugging/#inspect-gradient-norms)
-- [Log GPU usage](Debugging/#Log-gpu-usage)
-- [Make model overfit on subset of data](Debugging/#make-model-overfit-on-subset-of-data)
-- [Print the parameter count by layer](Debugging/#print-the-parameter-count-by-layer)
-- [Pring which gradients are nan](Debugging/#print-which-gradients-are-nan)
-
-
-**Distributed training**    
-
-- [16-bit mixed precision](Distributed%20training/#16-bit-mixed-precision)
-- [Multi-GPU](Distributed%20training/#Multi-GPU)
-- [Multi-node](Distributed%20training/#Multi-node)
-- [Single GPU](Distributed%20training/#single-gpu)
-- [Self-balancing architecture](Distributed%20training/#self-balancing-architecture)
-
-
-**Experiment Logging**   
-
-- [Display metrics in progress bar](Logging/#display-metrics-in-progress-bar)
-- Log arbitrary metrics
-- [Log metric row every k batches](Logging/#log-metric-row-every-k-batches)
-- [Process position](Logging/#process-position)
-- [Save a snapshot of all hyperparameters](Logging/#save-a-snapshot-of-all-hyperparameters) 
-- [Snapshot code for a training run](Logging/#snapshot-code-for-a-training-run) 
-- [Write logs file to csv every k batches](Logging/#write-logs-file-to-csv-every-k-batches)
-
-**Training loop**    
-
-- [Accumulate gradients](Training%20Loop/#accumulated-gradients)
-- [Anneal Learning rate](Training%20Loop/#anneal-learning-rate)
-- [Force training for min or max epochs](Training%20Loop/#force-training-for-min-or-max-epochs)
-- [Force disable early stop](Training%20Loop/#force-disable-early-stop)
-- [Use multiple optimizers (like GANs)](../Pytorch-lightning/LightningModule/#configure_optimizers)
-- [Set how much of the training set to check (1-100%)](Training%20Loop/#set-how-much-of-the-training-set-to-check)
-
-**Validation loop**    
-
-- [Check validation every n epochs](Validation%20Loop/#check-validation-every-n-epochs)
-- [Set how much of the validation set to check](Validation%20Loop/#set-how-much-of-the-validation-set-to-check)
-- [Set how much of the test set to check](Validation%20Loop/#set-how-much-of-the-test-set-to-check)
-- [Set validation check frequency within 1 training epoch](Validation%20Loop/#set-validation-check-frequency-within-1-training-epoch)
-- [Set the number of validation sanity steps](Validation%20Loop/#set-the-number-of-validation-sanity-steps)

docs/doc_requirements.txt

@@ -1 +0,0 @@
-mkdocs-material==4.4.0

docs/examples/Examples.md

@@ -1,171 +0,0 @@
-### Template model definition
-In 99% of cases you want to just copy [this template](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py) to start a new lightningModule and change the core of what your model is actually trying to do.
-
-```bash
-# get a copy of the module template
-wget https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py
-```
-
----    
-### Trainer Example 
-
-** \_\_main__ function**    
-
-Normally, we want to let the \_\_main__ function start the training.
-Inside the main we parse training arguments with whatever hyperparameters we want. Your LightningModule will have a 
-chance to add hyperparameters.   
-
-```{.python}
-from test_tube import HyperOptArgumentParser
-
-if __name__ == '__main__':
-
-    # use default args given by lightning
-    root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
-    parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False)
-    add_default_args(parent_parser, root_dir)
-
-    # allow model to overwrite or extend args
-    parser = ExampleModel.add_model_specific_args(parent_parser)
-    hyperparams = parser.parse_args()
-
-    # train model
-    main(hyperparams)
-```
-**Main Function**      
-
-The main function is your entry into the program. This is where you init your model, checkpoint directory, and launch the training.
-The main function should have 3 arguments:   
-- hparams: a configuration of hyperparameters.    
-- slurm_manager: Slurm cluster manager object (can be None)
-- dict: for you to return any values you want (useful in meta-learning, otherwise set to _)    
-
-```{}
-def main(hparams, cluster, results_dict):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # init experiment
-    log_dir = os.path.dirname(os.path.realpath(__file__))
-    exp = Experiment(
-        name='test_tube_exp',
-        debug=True,
-        save_dir=log_dir,
-        version=0,
-        autosave=False,
-        description='test demo'
-    )
-    
-    # set the hparams for the experiment
-    exp.argparse(hparams)
-    exp.save()
-
-    # build model
-    model = MyLightningModule(hparams)
-
-    # callbacks
-    early_stop = EarlyStopping(
-        monitor=hparams.early_stop_metric,
-        patience=hparams.early_stop_patience,
-        verbose=True,
-        mode=hparams.early_stop_mode
-    )
-
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_function=None,
-        save_best_only=True,
-        verbose=True,
-        monitor=hparams.model_save_monitor_value,
-        mode=hparams.model_save_monitor_mode
-    )
-
-    # configure trainer
-    trainer = Trainer(
-        experiment=exp,
-        cluster=cluster,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-    )
-
-    # train model
-    trainer.fit(model)
-```
-
-
-
-
-The __main__ function will start training on your **main** function. If you use the HyperParameterOptimizer
-in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple
-argument parser you get the default arguments in the argument parser.
-
-So, calling main(hyperparams) runs the model with the default argparse arguments.       
-```{.python}
-main(hyperparams)
-```
-
----
-#### CPU hyperparameter search      
-
-```{.python}
-# run a grid search over 20 hyperparameter combinations.
-hyperparams.optimize_parallel_cpu(
-    main_local,
-    nb_trials=20,
-    nb_workers=1
-)
-```
-
----
-#### Hyperparameter search on a single or multiple GPUs       
-```{.python}
-# run a grid search over 20 hyperparameter combinations.
-hyperparams.optimize_parallel_gpu(
-    main_local,
-    nb_trials=20,
-    nb_workers=1,
-    gpus=[0,1,2,3]
-)
-```
-
----
-#### Hyperparameter search on a SLURM HPC cluster   
-```{.python}    
-def optimize_on_cluster(hyperparams):
-    # enable cluster training
-    cluster = SlurmCluster(
-        hyperparam_optimizer=hyperparams,
-        log_path=hyperparams.tt_save_path,
-        test_tube_exp_name=hyperparams.tt_name
-    )
-
-    # email for cluster coms
-    cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True)
-
-    # configure cluster
-    cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus
-    cluster.job_time = '48:00:00'
-    cluster.gpu_type = '1080ti'
-    cluster.memory_mb_per_node = 48000
-
-    # any modules for code to run in env
-    cluster.add_command('source activate pytorch_lightning')
-
-    # name of exp
-    job_display_name = hyperparams.tt_name.split('_')[0]
-    job_display_name = job_display_name[0:3]
-
-    # run hopt
-    print('submitting jobs...')
-    cluster.optimize_parallel_cluster_gpu(
-        main,
-        nb_trials=hyperparams.nb_hopt_trials,
-        job_name=job_display_name
-    )
-
-# run cluster hyperparameter search    
-optimize_on_cluster(hyperparams)
-```

docs/index.md

@@ -1,76 +0,0 @@
-###### New project Quick Start    
-To start a new project define these two files.   
-
-1. [Define a LightningModule](/LightningModule/RequiredTrainerInterface/) 
-2. Pick a trainer      
-    - [Basic CPU Trainer](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/examples/new_project_templates/single_cpu_template.py) 
-    - [GPU cluster Trainer](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/examples/new_project_templates/multi_node_cluster_template.py)
-
-###### Docs shortcuts
-- [LightningModule](LightningModule/RequiredTrainerInterface/)  
-- [Trainer](Trainer/)  
-
-###### Quick start examples 
-- [CPU example](examples/Examples/#cpu-hyperparameter-search)   
-- [Hyperparameter search on single GPU](examples/Examples/#hyperparameter-search-on-a-single-or-multiple-gpus)    
-- [Hyperparameter search on multiple GPUs on same node](examples/Examples/#hyperparameter-search-on-a-single-or-multiple-gpus)  
-- [Hyperparameter search on a SLURM HPC cluster](examples/Examples/#Hyperparameter search on a SLURM HPC cluster)      
-
-
-###### Checkpointing    
-
-- [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
-- [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics) 
-
-###### Computing cluster (SLURM)    
-
-- [Running grid search on a cluster](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#running-grid-search-on-a-cluster) 
-- [Walltime auto-resubmit](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#walltime-auto-resubmit)   
-
-###### Debugging  
-
-- [Fast dev run](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#fast-dev-run)
-- [Inspect gradient norms](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#inspect-gradient-norms)
-- [Log GPU usage](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage)
-- [Make model overfit on subset of data](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data)
-- [Print the parameter count by layer](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer)
-- [Pring which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
-
-
-###### Distributed training    
-
-- [16-bit mixed precision](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision)
-- [Multi-GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU)
-- [Multi-node](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node)
-- [Single GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#single-gpu)
-- [Self-balancing architecture](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#self-balancing-architecture)
-
-
-###### Experiment Logging   
-
-- [Display metrics in progress bar](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#display-metrics-in-progress-bar)
-- Log arbitrary metrics
-- [Log metric row every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#log-metric-row-every-k-batches)
-- [Process position](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#process-position)
-- [Save a snapshot of all hyperparameters](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#save-a-snapshot-of-all-hyperparameters) 
-- [Snapshot code for a training run](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#snapshot-code-for-a-training-run) 
-- [Write logs file to csv every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#write-logs-file-to-csv-every-k-batches)
-
-###### Training loop    
-
-- [Accumulate gradients](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients)
-- [Anneal Learning rate](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#anneal-learning-rate)
-- [Force training for min or max epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs)
-- [Force disable early stop](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop)
-- [Gradient Clipping](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping)
-- [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/Pytorch-Lightning/LightningModule/#configure_optimizers)
-- [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check)
-
-######Validation loop    
-
-- [Check validation every n epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#check-validation-every-n-epochs)
-- [Set how much of the validation set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-validation-set-to-check)
-- [Set how much of the test set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-test-set-to-check)
-- [Set validation check frequency within 1 training epoch](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-validation-check-frequency-within-1-training-epoch)
-- [Set the number of validation sanity steps](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-the-number-of-validation-sanity-steps)
-

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/requirements.txt

@@ -0,0 +1,10 @@
+sphinx>=1.8.3
+recommonmark  # fails with badges
+m2r  # fails with multi-line text
+nbsphinx
+pandoc
+docutils
+git+https://github.com/PytorchLightning/lightning_sphinx_theme.git
+sphinxcontrib-fulltoc
+sphinxcontrib-mockautodoc
+pip_shims

docs/source/BECOMING_A_CORE_CONTRIBUTOR.md

@@ -0,0 +1,59 @@
+# How to become a core contributor
+
+Thanks for your interest in joining the Lightning team! We’re a rapidly growing project which is poised to become the go-to framework for DL researchers! 
+We're currently recruiting for a team of 5 core maintainers. 
+
+As a core maintainer you will have a strong say in the direction of the project. Big changes will require a majority of maintainers to agree.
+
+### Code of conduct  
+First and foremost, you'll be evaluated against [these core values](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md). Any code we commit or feature we add needs to align with those core values.
+
+### The bar for joining the team   
+Lightning is being used to solve really hard problems at the top AI labs in the world. As such, the bar for adding team members is extremely high. Candidates must have solid engineering skills, have a good eye for user experience, and must be a power user of Lightning and PyTorch.
+
+With that said, the Lightning team will be diverse and a reflection of an inclusive AI community. You don't have to be an engineer to conntribute! Scientists with great usability intuition and PyTorch ninja skills are welcomed!
+
+### Responsibilities:
+The responsibilities mainly revolve around 3 things.
+
+#### Github issues
+- Here we want to help users have an amazing experience. These range from questions from new people getting into DL to questions from researchers about doing something esoteric with Lightning
+Often, these issues require some sort of bug fix, document clarification or new functionality to be scoped out.
+
+- To become a core member you must resolve at least 10 Github issues which align with the API design goals for Lightning. By the end of these 10 issues I should feel comfortable in the way you answer user questions
+Pleasant/helpful tone.
+
+- Can abstract from that issue or bug into functionality that might solve other related issues or makes the platform more flexible.
+
+- Don’t make users feel like they don’t know what they’re doing. We’re here to help and to make everyone’s experience delightful.
+
+#### Pull requests
+
+- Here we need to ensure the code that enters Lightning is high quality. For each PR we need to:
+- Make sure code coverage does not decrease
+- Documents are updated
+- Code is elegant and simple
+- Code is NOT overly engineered or hard to read
+- Ask yourself, could a non-engineer understand what’s happening here?
+- Make sure new tests are written
+- Is this NECESSARY for Lightning? There are some PRs which are just purely about adding engineering complexity which have no place in Lightning. 
+Guidance
+- Some other PRs are for people who are wanting to get involved and add something unnecessary. We do want their help though! So don’t approve the PR, but direct them to a Github issue that they might be interested in helping with instead!
+- To be considered for core contributor, please review 10 PRs and help the authors land it on master. Once you've finished the review, ping me 
+for a sanity check. At the end of 10 PRs if your PR reviews are inline with expectations described above, then you can merge PRs on your own going forward,
+otherwise we'll do a few more until we're both comfortable :)
+
+#### Project directions
+There are some big decisions which the project must make. For these I expect core contributors to have something meaningful to add if it’s their area of expertise.
+
+#### Diversity
+Lightning should reflect the broader community it serves. As such we should have scientists/researchers from 
+different fields contributing!   
+
+The first 5 core contributors will fit this profile. Thus if you overlap strongly with experiences and expertise as someone else on the team, you might have to wait until the next set of contributors are added.
+
+#### Summary: Requirements to apply
+- Solve 10 Github issues. The goal is to be inline with expectations for solving issues by the last one so you can do them on your own. If not, I might ask you to solve a few more specific ones.
+- Do 10 PR reviews. The goal is to be inline with expectations for solving issues by the last one so you can do them on your own. If not, I might ask you to solve a few more specific ones.
+
+If you want to be considered, ping me on gitter and start [tracking your progress here](https://docs.google.com/spreadsheets/d/15D58gp8DvI0Z6qbbYVRuaWioiwzafcP58-UlbuO_CMU/edit?usp=sharing).

docs/source/CODE_OF_CONDUCT.md

@@ -0,0 +1,76 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, sex characteristics, gender identity and expression,
+level of experience, education, socio-economic status, nationality, personal
+appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+ advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+ address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+ professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at waf2107@columbia.edu. All
+complaints will be reviewed and investigated and will result in a response that
+is deemed necessary and appropriate to the circumstances. The project team is
+obligated to maintain confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see
+https://www.contributor-covenant.org/faq

docs/source/CONTRIBUTING.md

@@ -0,0 +1,53 @@
+# Contributing    
+Welcome to the PyTorch Lightning community! We're building the most advanced research platform on the planet to implement the latest, best practices that the amazing PyTorch team rolls out!   
+
+## Main Core Value: One less thing to remember    
+Simplify the API as much as possible from the user perspective. Any additions or improvements should minimize things the user needs to remember.   
+
+For example: One benefit of the validation_step is that the user doesn't have to remember to set the model to .eval(). This avoids all sorts of subtle errors the user could make.  
+
+## Lightning Design Principles   
+We encourage all sorts of contributions you're interested in adding! When coding for lightning, please follow these principles.    
+#### No PyTorch Interference   
+We don't want to add any abstractions on top of pure PyTorch. This gives researchers all the control they need without having to learn yet another framework.    
+
+#### Simple Internal Code    
+It's useful for users to look at the code and understand very quickly what's happening. Many users won't be engineers. Thus we need to value clear, simple code over condensed ninja moves. While that's super cool, this isn't the project for that :)      
+
+#### Force User Decisions To Best Practices    
+There are 1,000 ways to do something. However, something eventually becomes standard practice that everyone does. Thus we pick one way of doing it and force everyone to do it this way. A good example is accumulated gradients. There are many ways to implement, we just pick one and force users to use that one. A bad forced decision would be to make users use a specific library to do something.    
+
+When something becomes a best practice, we add it to the framework. This likely looks like code in utils or in the model file that everyone keeps adding over and over again across projects. When this happens, bring that code inside the trainer and add a flag for it.
+
+#### Simple External API    
+What makes sense to you may not make sense to others. Create an issue with an API change suggestion and validate that it makes sense for others. Treat code changes how you treat a startup: validate that it's a needed feature, then add if it makes sense for many people. 
+
+#### Backward-compatible API   
+We all hate updating our deep learning packages because we don't want to refactor a bunch of stuff. In Lightning, we make sure every change we make which could break an API is backwards compatible with good deprecation warnings.
+
+You shouldn't be afraid to upgrade Lightning :)
+
+#### Gain User Trust    
+As a researcher you can't have any part of your code going wrong. So, make thorough tests that ensure an implementation of a new trick or subbtle change is correct.    
+
+#### Interoperability  
+Have a favorite feature from other libraries like fast.ai or transformers? Those should just work with lightning as well. Grab your favorite model or learning rate scheduler from your favorite library and run it in Lightning.
+
+## Contribution Types    
+Currently looking for help implementing new features or adding bug fixes. 
+
+A lot of good work has already been done in project mechanics (requirements.txt, setup.py, pep8, badges, ci, etc...) we're in a good state there thanks to all the early contributors (even pre-beta release)!   
+
+## Bug Fixes:  
+1. Submit a github issue.   
+2. Fix it.  
+3. Submit a PR! 
+
+## New Features:  
+1. Submit a github issue.   
+2. We'll agree on the feature scope.     
+3. Submit a PR! (with updated docs and tests 🙃).   
+
+## Coding Styleguide
+1. Test the code with flake8.
+2. Use f-strings.

docs/source/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,16 @@
+# Before submitting
+
+- [ ] Was this discussed/approved via a Github issue? (no need for typos, doc improvements)
+- [ ] Did you read the [contributor guideline](https://github.com/PyTorchLightning/pytorch-lightning/blob/master/.github/CONTRIBUTING.md)?
+- [ ] Did you make sure to update the docs?   
+- [ ] Did you write any new necessary tests?  
+
+## What does this PR do?
+Fixes # (issue).
+
+## PR review    
+Anyone in the community is free to review the PR once the tests have passed.     
+If we didn't discuss your PR in Github issues there's a high chance it will not be merged.
+
+## Did you have fun?
+Make sure you had fun coding 🙃

docs/source/_static/images/lightning_icon.svg

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   id="svg"
+   version="1.1"
+   width="16.000004"
+   height="15.999986"
+   viewBox="0 0 16.000004 15.999986"
+   sodipodi:docname="lightning_icon.svg"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)">
+  <metadata
+     id="metadata13">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title></dc:title>
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs11" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1920"
+     inkscape:window-height="1028"
+     id="namedview9"
+     showgrid="false"
+     inkscape:zoom="0.59"
+     inkscape:cx="-669.05062"
+     inkscape:cy="373.84245"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg" />
+  <path
+     style="fill:#fbfbfb;fill-rule:evenodd;stroke:none;stroke-width:0.04002798"
+     inkscape:connector-curvature="0"
+     d="m 8.987101,1.723485 c -0.05588,0.03422 -4.121881,4.096544 -4.184645,4.180924 -0.02317,0.0311 -0.04587,0.06016 -0.05044,0.06456 -0.0087,0.0084 -0.07477,0.145063 -0.09679,0.20014 -0.05848,0.146583 -0.05804,0.44387 0.001,0.592413 0.08426,0.21243 0.08826,0.216754 1.576864,1.706274 0.779463,0.779947 1.41719,1.426877 1.41719,1.437604 0,0.0232 -0.253177,0.79848 -0.273873,0.838707 -0.0079,0.0153 -0.01433,0.04087 -0.01433,0.05684 0,0.01597 -0.0059,0.03587 -0.01313,0.04423 -0.0072,0.0084 -0.03678,0.09086 -0.06568,0.18333 -0.02893,0.09246 -0.05904,0.180647 -0.06693,0.195937 -0.0079,0.0153 -0.01437,0.04087 -0.01437,0.05684 0,0.01597 -0.0059,0.03586 -0.01313,0.04423 -0.0072,0.0084 -0.03679,0.09086 -0.06569,0.18333 -0.02893,0.09246 -0.05904,0.180643 -0.06693,0.195937 -0.0079,0.0153 -0.01437,0.04187 -0.01437,0.05908 0,0.0172 -0.0072,0.03574 -0.016,0.04119 -0.0088,0.0054 -0.016,0.02607 -0.016,0.04579 0,0.01973 -0.006,0.04271 -0.0134,0.05108 -0.0074,0.0084 -0.04439,0.112477 -0.08222,0.23136 -0.03787,0.118884 -0.151103,0.461124 -0.251693,0.760534 -0.489984,1.45874 -0.462444,1.36155 -0.413611,1.45938 0.06917,0.138657 0.23128,0.199741 0.358251,0.134974 0.07057,-0.03602 4.143298,-4.099985 4.245368,-4.236242 0.03382,-0.04515 0.09094,-0.165796 0.109916,-0.232123 0.0088,-0.03083 0.0243,-0.08498 0.03442,-0.120363 0.03346,-0.11668 0.0068,-0.361134 -0.0566,-0.520084 C 10.880518,9.229614 10.738898,9.079187 9.372744,7.714673 8.601524,6.944416 7.970523,6.302806 7.970523,6.288916 c 0,-0.01393 0.02817,-0.107833 0.0626,-0.208663 0.03442,-0.100834 0.07881,-0.237367 0.09859,-0.303414 0.0198,-0.06605 0.04207,-0.12693 0.04947,-0.135293 0.0074,-0.0084 0.0135,-0.03133 0.0135,-0.05108 0,-0.01973 0.0072,-0.04035 0.016,-0.04579 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04804 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04804 0.0088,-0.0054 0.016,-0.02707 0.016,-0.04803 0,-0.02097 0.0072,-0.04259 0.016,-0.04803 0.0088,-0.0054 0.016,-0.02397 0.016,-0.04119 0,-0.0172 0.0065,-0.04379 0.0144,-0.05908 0.0079,-0.0153 0.119204,-0.34484 0.247334,-0.73231 C 9.064507,2.979766 9.220177,2.513319 9.28226,2.330632 9.408267,1.960092 9.41367,1.921146 9.35255,1.826839 9.27225,1.703032 9.099973,1.654399 8.986893,1.723566"
+     id="path0" />
+  <path
+     style="fill:#540c8c;fill-rule:evenodd;stroke:none;stroke-width:0.04002798"
+     inkscape:connector-curvature="0"
+     d="m 0.07719102,0.01733399 c -0.02187,0.0111 -0.04875,0.03799 -0.05984,0.05984 -0.0161,0.03173 -0.01937,1.62421701 -0.01633,7.94479601 l 0.0038,7.905086 0.03647,0.03646 0.03646,0.03647 H 8.00241 15.927073 l 0.03646,-0.03647 0.03647,-0.03646 V 8.002393 0.07773399 l -0.03647,-0.03646 -0.03646,-0.03647 -7.905086,-0.0038 c -6.320579,-0.003 -7.91305298,2.4e-4 -7.94479598,0.01633 M 9.193764,1.668208 c 0.259903,0.09046 0.275193,0.212427 0.09363,0.74628 C 8.845834,3.776859 8.388843,5.102846 7.991127,6.302606 L 9.415644,7.72492 c 1.24415,1.242111 1.51682,1.523547 1.51682,1.565414 0,0.0051 0.0133,0.03987 0.02953,0.07718 0.12913,0.296607 0.0877,0.664983 -0.103314,0.91872 -0.141456,0.187933 -4.207341,4.228478 -4.273468,4.246848 -0.139417,0.03871 -0.248653,-0.006 -0.34324,-0.140417 -0.07665,-0.108996 -0.06985,-0.137256 0.287004,-1.194633 0.34663,-1.101761 0.75901,-2.243218 1.08916,-3.290661 0,-0.0078 -0.636164,-0.650377 -1.413707,-1.427921 C 4.877658,7.152643 4.728155,6.995813 4.673718,6.87361 4.661948,6.84718 4.645988,6.81305 4.638168,6.79776 4.630368,6.78246 4.624038,6.75689 4.624038,6.74092 c 0,-0.01597 -0.0076,-0.03659 -0.01687,-0.04587 -0.02253,-0.02253 -0.02253,-0.436904 0,-0.45944 0.0093,-0.0093 0.01687,-0.0327 0.01687,-0.05204 0,-0.0363 0.06917,-0.178363 0.130414,-0.267907 0.07965,-0.1164 4.221831,-4.237681 4.259458,-4.237921 0.02047,-1.2e-4 0.04803,-0.0072 0.06124,-0.01577 0.03147,-0.02033 0.04415,-0.01967 0.118603,0.0062"
+     id="path1"
+     sodipodi:nodetypes="ccscccccccccccscccccscccccccccsssscccc" />
+</svg>

docs/source/_static/images/lightning_logo-large.svg

@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   id="svg"
+   version="1.1"
+   width="400"
+   height="400"
+   viewBox="0, 0, 400,400"
+   sodipodi:docname="lightning_logo.svg"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)">
+  <metadata
+     id="metadata13">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs11" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1920"
+     inkscape:window-height="1028"
+     id="namedview9"
+     showgrid="false"
+     inkscape:zoom="9.44"
+     inkscape:cx="203.07907"
+     inkscape:cy="335.32491"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg" />
+  <path
+     style="fill:#fbfbfb;fill-rule:evenodd;stroke:none"
+     inkscape:connector-curvature="0"
+     d="m 224.6,43.137 c -1.396,0.855 -102.975,102.342 -104.543,104.45 -0.579,0.777 -1.146,1.503 -1.26,1.613 -0.218,0.21 -1.868,3.624 -2.418,5 -1.461,3.662 -1.45,11.089 0.022,14.8 2.105,5.307 2.205,5.415 39.394,42.627 19.473,19.485 35.405,35.647 35.405,35.915 0,0.58 -6.325,19.948 -6.842,20.953 -0.197,0.382 -0.358,1.021 -0.358,1.42 0,0.399 -0.147,0.896 -0.328,1.105 -0.18,0.209 -0.919,2.27 -1.641,4.58 -0.723,2.31 -1.475,4.513 -1.672,4.895 -0.198,0.382 -0.359,1.021 -0.359,1.42 0,0.399 -0.147,0.896 -0.328,1.105 -0.18,0.209 -0.919,2.27 -1.641,4.58 -0.723,2.31 -1.475,4.513 -1.672,4.895 -0.198,0.382 -0.359,1.046 -0.359,1.476 0,0.43 -0.18,0.893 -0.4,1.029 -0.22,0.136 -0.4,0.651 -0.4,1.144 0,0.493 -0.151,1.067 -0.335,1.276 -0.184,0.209 -1.109,2.81 -2.054,5.78 -0.946,2.97 -3.775,11.52 -6.288,19 -12.241,36.443 -11.553,34.015 -10.333,36.459 1.728,3.464 5.778,4.99 8.95,3.372 1.763,-0.9 103.51,-102.428 106.06,-105.832 0.845,-1.128 2.272,-4.142 2.746,-5.799 0.22,-0.77 0.607,-2.123 0.86,-3.007 0.836,-2.915 0.171,-9.022 -1.414,-12.993 -1.493,-3.741 -5.031,-7.499 -39.161,-41.588 C 214.964,173.569 199.2,157.54 199.2,157.193 c 0,-0.348 0.704,-2.694 1.564,-5.213 0.86,-2.519 1.969,-5.93 2.463,-7.58 0.495,-1.65 1.051,-3.171 1.236,-3.38 0.186,-0.209 0.337,-0.783 0.337,-1.276 0,-0.493 0.18,-1.008 0.4,-1.144 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.676 0.4,-1.2 0,-0.524 0.18,-1.064 0.4,-1.2 0.22,-0.136 0.4,-0.599 0.4,-1.029 0,-0.43 0.162,-1.094 0.36,-1.476 0.197,-0.382 2.978,-8.615 6.179,-18.295 3.2,-9.68 7.089,-21.333 8.64,-25.897 3.148,-9.257 3.283,-10.23 1.756,-12.586 -2.006,-3.093 -6.31,-4.308 -9.135,-2.58"
+     id="path0" />
+  <path
+     style="fill:#540c8c;fill-rule:evenodd;stroke:none"
+     inkscape:connector-curvature="0"
+     d="M 2.008,0.513 C 1.462,0.79 0.79,1.462 0.513,2.008 0.111,2.801 0.029,42.585 0.105,200.489 L 0.2,397.978 1.111,398.889 2.022,399.8 H 200 397.978 l 0.911,-0.911 0.911,-0.911 V 200 2.022 L 398.889,1.111 397.978,0.2 200.489,0.105 C 42.585,0.029 2.801,0.111 2.008,0.513 m 227.755,41.243 c 6.493,2.26 6.875,5.307 2.339,18.644 -11.0313,34.035452 -22.44803,67.16196 -32.384,97.135 l 35.588,35.533 c 31.082,31.031 37.894,38.062 37.894,39.108 0,0.128 0.332,0.996 0.738,1.928 3.226,7.41 2.191,16.613 -2.581,22.952 -3.534,4.695 -105.11,105.638 -106.762,106.097 -3.483,0.967 -6.212,-0.15 -8.575,-3.508 -1.915,-2.723 -1.745,-3.429 7.17,-29.845 8.65971,-27.52475 18.96205,-56.04122 27.21,-82.209 0,-0.195 -15.893,-16.248 -35.318,-35.673 -33.146,-33.147 -36.881,-37.065 -38.241,-40.118 -0.294,-0.66 -0.693,-1.513 -0.888,-1.895 -0.194,-0.382 -0.353,-1.021 -0.353,-1.42 0,-0.399 -0.189,-0.914 -0.421,-1.146 -0.563,-0.563 -0.563,-10.915 0,-11.478 0.232,-0.232 0.421,-0.817 0.421,-1.3 0,-0.907 1.728,-4.456 3.258,-6.693 C 120.848,144.96 224.33,42 225.27,41.994 c 0.511,-0.003 1.2,-0.181 1.53,-0.394 0.786,-0.508 1.103,-0.491 2.963,0.156"
+     id="path1"
+     sodipodi:nodetypes="ccscccccccccccscccccscccccccccsssscccc" />
+</svg>

docs/source/_static/images/lightning_logo.svg

@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://creativecommons.org/ns#"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   id="svg"
+   version="1.1"
+   width="47.999985"
+   height="47.999943"
+   viewBox="0 0 47.999985 47.999943"
+   sodipodi:docname="lightning_logo.svg"
+   inkscape:version="0.92.3 (2405546, 2018-03-11)">
+  <metadata
+     id="metadata13">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+        <dc:title />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <defs
+     id="defs11" />
+  <sodipodi:namedview
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1"
+     objecttolerance="10"
+     gridtolerance="10"
+     guidetolerance="10"
+     inkscape:pageopacity="0"
+     inkscape:pageshadow="2"
+     inkscape:window-width="1920"
+     inkscape:window-height="1028"
+     id="namedview9"
+     showgrid="false"
+     inkscape:zoom="0.59"
+     inkscape:cx="-347.96588"
+     inkscape:cy="389.84243"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg" />
+  <path
+     style="fill:#fbfbfb;fill-rule:evenodd;stroke:none;stroke-width:0.12008391"
+     inkscape:connector-curvature="0"
+     d="m 26.961294,5.1704519 c -0.16764,0.10267 -12.36564,12.2896301 -12.55393,12.5427701 -0.0695,0.0933 -0.13762,0.18048 -0.15131,0.19369 -0.0262,0.0252 -0.22432,0.43519 -0.29036,0.60042 -0.17544,0.43975 -0.17412,1.33161 0.003,1.77724 0.25278,0.63729 0.26479,0.65026 4.73059,5.11882 2.33839,2.33984 4.25157,4.28063 4.25157,4.31281 0,0.0696 -0.75953,2.39544 -0.82162,2.51612 -0.0237,0.0459 -0.043,0.12261 -0.043,0.17052 0,0.0479 -0.0177,0.1076 -0.0394,0.13269 -0.0216,0.0251 -0.11035,0.27259 -0.19705,0.54999 -0.0868,0.27739 -0.17713,0.54194 -0.20078,0.58781 -0.0238,0.0459 -0.0431,0.1226 -0.0431,0.17052 0,0.0479 -0.0177,0.10759 -0.0394,0.13269 -0.0216,0.0251 -0.11036,0.27259 -0.19706,0.54999 -0.0868,0.27739 -0.17712,0.54193 -0.20078,0.58781 -0.0238,0.0459 -0.0431,0.1256 -0.0431,0.17724 0,0.0516 -0.0216,0.10723 -0.048,0.12357 -0.0264,0.0163 -0.048,0.0782 -0.048,0.13737 0,0.0592 -0.0181,0.12813 -0.0402,0.15323 -0.0221,0.0251 -0.13318,0.33743 -0.24666,0.69408 -0.1136,0.35665 -0.45331,1.38337 -0.75508,2.2816 -1.46995,4.37622 -1.38733,4.08465 -1.24083,4.37814 0.2075,0.41597 0.69384,0.59922 1.07475,0.40492 0.21171,-0.10807 12.42989,-12.29995 12.7361,-12.70872 0.10147,-0.13545 0.27283,-0.49739 0.32975,-0.69637 0.0264,-0.0925 0.0729,-0.25493 0.10327,-0.36109 0.10039,-0.35004 0.0205,-1.0834 -0.1698,-1.56025 -0.17928,-0.44923 -0.60414,-0.90051 -4.7026,-4.99405 -2.31366,-2.31077 -4.20666,-4.2356 -4.20666,-4.27727 0,-0.0418 0.0845,-0.3235 0.18781,-0.62599 0.10327,-0.3025 0.23644,-0.7121 0.29577,-0.91024 0.0594,-0.19814 0.1262,-0.38079 0.14842,-0.40588 0.0223,-0.0251 0.0405,-0.094 0.0405,-0.15323 0,-0.0592 0.0216,-0.12105 0.048,-0.13738 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.14411 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.14411 0.0264,-0.0163 0.048,-0.0812 0.048,-0.1441 0,-0.0629 0.0216,-0.12777 0.048,-0.1441 0.0264,-0.0163 0.048,-0.0719 0.048,-0.12356 0,-0.0516 0.0195,-0.13137 0.0432,-0.17725 0.0237,-0.0459 0.35761,-1.03452 0.742,-2.19693 0.38427,-1.1624101 0.85128,-2.5617501 1.03753,-3.1098101 0.37802,-1.11162 0.39423,-1.22846 0.21087,-1.51138 -0.24089,-0.37142 -0.75773,-0.51732 -1.09697,-0.30982"
+     id="path0" />
+  <path
+     style="fill:#540c8c;fill-rule:evenodd;stroke:none;stroke-width:0.12008391"
+     inkscape:connector-curvature="0"
+     d="m 0.2315739,0.05200186 c -0.0656,0.0333 -0.14626,0.11396 -0.17952,0.17952 -0.0483,0.0952 -0.0581,4.87265004 -0.049,23.83438014 l 0.0114,23.71525 0.1094,0.10939 0.10939,0.1094 h 23.7739701 23.77398 l 0.10939,-0.1094 0.1094,-0.10939 V 24.007172 0.23320186 l -0.1094,-0.10939 -0.10939,-0.1094 -23.71525,-0.0114 c -18.9617301,-0.009 -23.7391501,7.2e-4 -23.8343801,0.049 M 27.581274,5.0046319 c 0.77971,0.27139 0.82558,0.63728 0.28088,2.23884 -1.32468,4.0871101 -2.69565,8.0650701 -3.8888,11.6643501 l 4.27355,4.26694 c 3.73245,3.72633 4.55046,4.57064 4.55046,4.69624 0,0.0154 0.0399,0.11961 0.0886,0.23153 0.38739,0.88982 0.2631,1.99495 -0.30994,2.75616 -0.42437,0.5638 -12.62202,12.68543 -12.8204,12.74054 -0.41825,0.11613 -0.74596,-0.018 -1.02972,-0.42125 -0.22996,-0.32699 -0.20954,-0.41177 0.86101,-3.5839 1.03989,-3.30528 2.27703,-6.72965 3.26748,-9.87198 0,-0.0234 -1.90849,-1.95113 -4.24112,-4.28376 -3.98031,-3.98042 -4.42882,-4.45091 -4.59213,-4.81752 -0.0353,-0.0793 -0.0832,-0.18169 -0.10664,-0.22756 -0.0233,-0.0459 -0.0424,-0.12261 -0.0424,-0.17052 0,-0.0479 -0.0227,-0.10976 -0.0506,-0.13762 -0.0676,-0.0676 -0.0676,-1.31071 0,-1.37832 0.0279,-0.0279 0.0506,-0.0981 0.0506,-0.15611 0,-0.10891 0.20751,-0.53509 0.39124,-0.80372 0.23896,-0.3492 12.66549,-12.7130401 12.77837,-12.7137601 0.0614,-3.6e-4 0.1441,-0.0217 0.18372,-0.0473 0.0944,-0.061 0.13246,-0.059 0.35581,0.0187"
+     id="path1"
+     sodipodi:nodetypes="ccscccccccccccscccccscccccccccsssscccc" />
+</svg>

docs/source/_templates/theme_variables.jinja

@@ -0,0 +1,18 @@
+{%- set external_urls = {
+  'github': 'https://github.com/PytorchLightning/pytorch-lightning',
+  'github_issues': 'https://github.com/PytorchLightning/pytorch-lightning/issues',
+  'contributing': 'https://github.com/PytorchLightning/pytorch-lightning/blob/master/CONTRIBUTING.md',
+  'governance': 'https://github.com/PytorchLightning/pytorch-lightning/blob/master/governance.md',
+  'docs': 'https://pytorch-lightning.rtfd.io/en/latest',
+  'twitter': 'https://twitter.com/PyTorchLightnin',
+  'discuss': 'https://discuss.pytorch.org',
+  'tutorials': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'previous_pytorch_versions': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'home': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'get_started': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'features': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'blog': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'resources': 'https://pytorch-lightning.rtfd.io/en/latest/',
+  'support': 'https://pytorch-lightning.rtfd.io/en/latest/',
+}
+-%}

docs/source/callbacks.rst

@@ -0,0 +1,14 @@
+.. role:: hidden
+    :class: hidden-section
+
+Callbacks
+===========
+.. automodule:: pytorch_lightning.callbacks
+   :exclude-members:
+        _del_model,
+        _save_model,
+        on_epoch_end,
+        on_train_end,
+        on_epoch_begin,
+        check_monitor_top_k,
+        on_train_begin,

docs/source/common-cases.rst

@@ -0,0 +1,21 @@
+Multi-gpu (same node) training
+==============================
+
+Multi-node training
+====================
+
+16-bit precision
+=================
+
+gradient clipping
+=================
+
+modifying training via hooks
+=============================
+
+
+
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples

docs/source/conf.py

@@ -0,0 +1,350 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+import os
+import sys
+import glob
+import shutil
+import inspect
+
+# import m2r
+import builtins
+import pt_lightning_sphinx_theme
+
+PATH_HERE = os.path.abspath(os.path.dirname(__file__))
+PATH_ROOT = os.path.join(PATH_HERE, '..', '..')
+sys.path.insert(0, os.path.abspath(PATH_ROOT))
+
+builtins.__LIGHTNING_SETUP__ = True
+
+import pytorch_lightning  # noqa: E402
+
+# -- Project documents -------------------------------------------------------
+
+# # export the documentation
+# with open('intro.rst', 'w') as fp:
+#     intro = pytorch_lightning.__doc__.replace(os.linesep + ' ', '')
+#     fp.write(m2r.convert(intro))
+#     # fp.write(pytorch_lightning.__doc__)
+
+# # export the READme
+# with open(os.path.join(PATH_ROOT, 'README.md'), 'r') as fp:
+#     readme = fp.read()
+# # replace all paths to relative
+# for ndir in (os.path.basename(p) for p in glob.glob(os.path.join(PATH_ROOT, '*'))
+#              if os.path.isdir(p)):
+#     readme = readme.replace('](%s/' % ndir, '](%s/%s/' % (PATH_ROOT, ndir))
+# with open('readme.md', 'w') as fp:
+#     fp.write(readme)
+
+for md in glob.glob(os.path.join(PATH_ROOT, '.github', '*.md')):
+    shutil.copy(md, os.path.join(PATH_HERE, os.path.basename(md)))
+
+# -- Project information -----------------------------------------------------
+
+project = 'PyTorch-Lightning'
+copyright = pytorch_lightning.__copyright__
+author = pytorch_lightning.__author__
+
+# The short X.Y version
+version = pytorch_lightning.__version__
+# The full version, including alpha/beta/rc tags
+release = pytorch_lightning.__version__
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+
+needs_sphinx = '1.4'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinxcontrib.mockautodoc',
+    # 'sphinxcontrib.fulltoc',  # breaks pytorch-theme with unexpected kw argument 'titles_only'
+    'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.linkcode',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.napoleon',
+    'recommonmark',
+    'sphinx.ext.autosectionlabel',
+    # 'm2r',
+    'nbsphinx',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# https://berkeley-stat159-f17.github.io/stat159-f17/lectures/14-sphinx..html#conf.py-(cont.)
+# https://stackoverflow.com/questions/38526888/embed-ipython-notebook-in-sphinx-document
+# I execute the notebooks manually in advance. If notebooks test the code,
+# they should be run at build time.
+nbsphinx_execute = 'never'
+nbsphinx_allow_errors = True
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+# source_suffix = ['.rst', '.md', '.ipynb']
+source_suffix = {
+    '.rst': 'restructuredtext',
+    '.txt': 'markdown',
+    '.md': 'markdown',
+    '.ipynb': 'nbsphinx',
+}
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['*.test_*']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+# http://www.sphinx-doc.org/en/master/usage/theming.html#builtin-themes
+# html_theme = 'bizstyle'
+# https://sphinx-themes.org
+html_theme = 'pt_lightning_sphinx_theme'
+html_theme_path = [pt_lightning_sphinx_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+
+html_theme_options = {
+    'pytorch_project': pytorch_lightning.__homepage__,
+    'canonical_url': pytorch_lightning.__homepage__,
+    'collapse_navigation': False,
+    'display_version': True,
+    'logo_only': False,
+}
+
+html_logo = '_static/images/lightning_logo-name.svg'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = project + '-doc'
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, project + '.tex', project + ' Documentation', author, 'manual'),
+]
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, project, project + ' Documentation', [author], 1)
+]
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, project, project + ' Documentation', author, project,
+     'One line description of project.', 'Miscellaneous'),
+]
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+# https://github.com/rtfd/readthedocs.org/issues/1139
+# I use sphinx-apidoc to auto-generate API documentation for my project.
+# Right now I have to commit these auto-generated files to my repository
+# so that RTD can build them into HTML docs. It'd be cool if RTD could run
+# sphinx-apidoc for me, since it's easy to forget to regen API docs
+# and commit them to my repo after making changes to my code.
+
+PACKAGES = [
+    pytorch_lightning.__name__,
+    'pl_examples',
+]
+
+
+def run_apidoc(_):
+    for pkg in PACKAGES:
+        argv = ['-e', '-o', PATH_HERE, os.path.join(PATH_HERE, PATH_ROOT, pkg),
+                '**/test_*', '--force', '--private', '--module-first']
+        try:
+            # Sphinx 1.7+
+            from sphinx.ext import apidoc
+            apidoc.main(argv)
+        except ImportError:
+            # Sphinx 1.6 (and earlier)
+            from sphinx import apidoc
+            argv.insert(0, apidoc.__file__)
+            apidoc.main(argv)
+
+
+def setup(app):
+    app.connect('builder-inited', run_apidoc)
+
+
+# copy all notebooks to local folder
+path_nbs = os.path.join(PATH_HERE, 'notebooks')
+if not os.path.isdir(path_nbs):
+    os.mkdir(path_nbs)
+for path_ipynb in glob.glob(os.path.join(PATH_ROOT, 'notebooks', '*.ipynb')):
+    path_ipynb2 = os.path.join(path_nbs, os.path.basename(path_ipynb))
+    shutil.copy(path_ipynb, path_ipynb2)
+
+# Ignoring Third-party packages
+# https://stackoverflow.com/questions/15889621/sphinx-how-to-exclude-imports-in-automodule
+
+MOCK_REQUIRE_PACKAGES = []
+with open(os.path.join(PATH_ROOT, 'requirements.txt'), 'r') as fp:
+    for ln in fp.readlines():
+        found = [ln.index(ch) for ch in list(',=<>#') if ch in ln]
+        pkg = ln[:min(found)] if found else ln
+        if pkg.rstrip():
+            MOCK_REQUIRE_PACKAGES.append(pkg.rstrip())
+
+# TODO: better parse from package since the import name and package name may differ
+MOCK_MANUAL_PACKAGES = ['torch', 'torchvision', 'sklearn', 'test_tube', 'mlflow', 'comet_ml', 'wandb', 'neptune']
+autodoc_mock_imports = MOCK_REQUIRE_PACKAGES + MOCK_MANUAL_PACKAGES
+# for mod_name in MOCK_REQUIRE_PACKAGES:
+#     sys.modules[mod_name] = mock.Mock()
+
+
+# Options for the linkcode extension
+# ----------------------------------
+github_user = 'PyTorchLightning'
+github_repo = project
+
+
+# Resolve function
+# This function is used to populate the (source) links in the API
+def linkcode_resolve(domain, info):
+    def find_source():
+        # try to find the file and line number, based on code from numpy:
+        # https://github.com/numpy/numpy/blob/master/doc/source/conf.py#L286
+        obj = sys.modules[info['module']]
+        for part in info['fullname'].split('.'):
+            obj = getattr(obj, part)
+        fname = inspect.getsourcefile(obj)
+        # https://github.com/rtfd/readthedocs.org/issues/5735
+        if any([s in fname for s in ('readthedocs', 'checkouts')]):
+            # /home/docs/checkouts/readthedocs.org/user_builds/pytorch_lightning/checkouts/
+            #  devel/pytorch_lightning/utilities/cls_experiment.py#L26-L176
+            path_top = os.path.abspath(os.path.join('..', '..', '..'))
+            fname = os.path.relpath(fname, start=path_top)
+        else:
+            # Local build, imitate master
+            fname = 'master/' + os.path.relpath(fname, start=os.path.abspath('..'))
+        source, lineno = inspect.getsourcelines(obj)
+        return fname, lineno, lineno + len(source) - 1
+
+    if domain != 'py' or not info['module']:
+        return None
+    try:
+        filename = '%s#L%d-L%d' % find_source()
+    except Exception:
+        filename = info['module'].replace('.', '/') + '.py'
+    # import subprocess
+    # tag = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
+    #                        universal_newlines=True).communicate()[0][:-1]
+    return "https://github.com/%s/%s/blob/%s" \
+           % (github_user, github_repo, filename)
+
+
+autodoc_member_order = 'groupwise'
+autoclass_content = 'both'
+autodoc_default_flags = [
+    'members', 'undoc-members', 'show-inheritance', 'private-members',
+    # 'special-members', 'inherited-members'
+]

docs/source/documentation.rst

@@ -0,0 +1,8 @@
+Documentation
+=============
+
+
+.. toctree::
+   :maxdepth: 4
+
+   pytorch_lightning

docs/source/examples.rst

@@ -0,0 +1,34 @@
+GAN
+====
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples.domain_templates.gan
+
+MNIST
+====
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples.basic_examples.lightning_module_template
+
+Multi-node (ddp) MNIST
+====
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples.multi_node_examples.multi_node_ddp_demo
+
+Multi-node (ddp2) MNIST
+====
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples.multi_node_examples.multi_node_ddp2_demo
+
+Imagenet
+====
+.. toctree::
+   :maxdepth: 3
+
+   pl_examples.full_examples.imagenet.imagenet_example

docs/source/governance.md

@@ -0,0 +1,8 @@
+# Pytorch Lightning Governance | Persons of interest
+
+### Maintainers  
+- William Falcon ([williamFalcon](https://github.com/williamFalcon))   
+- Jirka Borovek ([Borda](https://github.com/Borda))   
+- Nick Eggert ([neggert](https://github.com/neggert))   
+- Jeff Ling ([jeffling](https://github.com/jeffling))   
+- Tullie Murrell ([tullie](https://github.com/tullie))

docs/source/index.rst

@@ -0,0 +1,63 @@
+.. PyTorch-Lightning documentation master file, created by
+   sphinx-quickstart on Fri Nov 15 07:48:22 2019.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+PyTorch-Lightning Documentation
+=============================
+
+.. toctree::
+   :maxdepth: 1
+   :name: start
+   :caption: Start Here
+
+   new-project
+
+.. toctree::
+   :maxdepth: 4
+   :name: docs
+   :caption: Python API
+
+   callbacks
+   lightning-module
+   logging
+   trainer
+
+.. toctree::
+   :maxdepth: 1
+   :name: Examples
+   :caption: Examples
+
+   examples
+
+.. toctree::
+   :maxdepth: 1
+   :name: Tutorials
+   :caption: Tutorials
+
+   tutorials
+
+.. toctree::
+   :maxdepth: 1
+   :name: Common Use Cases
+   :caption: Common Use Cases
+
+   common-cases
+
+.. toctree::
+   :maxdepth: 1
+   :name: community
+   :caption: Community
+
+
+   CODE_OF_CONDUCT.md
+   CONTRIBUTING.md
+   BECOMING_A_CORE_CONTRIBUTOR.md
+   governance.md
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/source/lightning-module.rst

@@ -0,0 +1,10 @@
+.. role:: hidden
+    :class: hidden-section
+
+LightningModule
+===========
+.. automodule:: pytorch_lightning.core
+   :exclude-members:
+        _abc_impl,
+        summarize,
+

docs/source/logging.rst

@@ -0,0 +1,12 @@
+.. role:: hidden
+    :class: hidden-section
+
+Logging
+===========
+.. automodule:: pytorch_lightning.logging
+   :exclude-members:
+        _abc_impl,
+        _save_model,
+        on_epoch_end,
+        on_train_end,
+        on_epoch_begin,

docs/source/modules.rst

@@ -0,0 +1,7 @@
+pl_examples
+===========
+
+.. toctree::
+   :maxdepth: 4
+
+   pl_examples

docs/source/new-project.rst

@@ -0,0 +1,72 @@
+Quick Start
+===========
+| To start a new project define two files, a LightningModule and a Trainer file.
+| To illustrate the power of Lightning and its simplicity, here's an example of a typical research flow.
+
+Case 1: BERT
+------------
+
+| Let's say you're working on something like BERT but want to try different ways of training or even different networks.
+| You would define a single LightningModule and use flags to switch between your different ideas.
+
+.. code-block:: python
+
+    class BERT(pl.LightningModule):
+        def __init__(self, model_name, task):
+            self.task = task
+
+            if model_name == 'transformer':
+                self.net = Transformer()
+            elif model_name == 'my_cool_version':
+                self.net = MyCoolVersion()
+
+        def training_step(self, batch, batch_idx):
+            if self.task == 'standard_bert':
+                # do standard bert training with self.net...
+                # return loss
+
+            if self.task == 'my_cool_task':
+                # do my own version with self.net
+                # return loss
+
+
+Case 2: COOLER NOT BERT
+-----------------------
+
+But if you wanted to try something **completely** different, you'd define a new module for that.
+
+
+.. code-block:: python
+
+    class CoolerNotBERT(pl.LightningModule):
+        def __init__(self):
+            self.net = ...
+
+        def training_step(self, batch, batch_idx):
+            # do some other cool task
+            # return loss
+
+
+Rapid research flow
+-------------------
+
+Then you could do rapid research by switching between these two and using the same trainer.
+
+
+.. code-block:: python
+
+    if use_bert:
+        model = BERT()
+    else:
+        model = CoolerNotBERT()
+
+    trainer = Trainer(gpus=4, use_amp=True)
+    trainer.fit(model)
+
+
+**Notice a few things about this flow:**
+
+1. You're writing pure PyTorch... no unnecessary abstractions or new libraries to learn.
+2. You get free GPU and 16-bit support without writing any of that code in your model.
+3. You also get early stopping, multi-gpu training, 16-bit and MUCH more without coding anything!
+

docs/source/trainer.rst

@@ -0,0 +1,21 @@
+.. role:: hidden
+    :class: hidden-section
+
+Trainer
+===========
+.. automodule:: pytorch_lightning.trainer
+   :members: fit, test
+   :exclude-members: 
+        run_pretrain_routine, 
+        _abc_impl, 
+        _Trainer__set_root_gpu, 
+        _Trainer__init_optimizers,
+        _Trainer__parse_gpu_ids,
+        _Trainer__configure_schedulers,
+        data_parallel,
+        num_gpus,
+        slurm_job_id,
+        tng_tqdm_dic,
+        training_tqdm_dict,
+        init_optimizers,
+        configure_schedulers

docs/source/tutorials.rst

@@ -0,0 +1,20 @@
+Refactoring PyTorch into Lightning
+==================================
+`Tutorial <https://towardsdatascience.com/how-to-refactor-your-pytorch-code-to-get-these-42-benefits-of-pytorch-lighting-6fdd0dc97538>`_
+
+Start a research project
+=========================
+`Research seed <https://github.com/PytorchLightning/pytorch-lightning-conference-seed>`_
+
+Basic Lightning use
+====================
+`Tutorial <https://towardsdatascience.com/supercharge-your-ai-research-with-pytorch-lightning-337948a99eec>`_
+
+9 key Lightning  tricks
+========================
+`Tutorial <9 key speed features in Pytorch-Lightning>`_
+
+Multi-node training on SLURM
+=============================
+`Tutorial <https://towardsdatascience.com/trivial-multi-node-training-with-pytorch-lightning-ff75dfb809bd>`_
+

mkdocs.yml

@@ -1,10 +0,0 @@
-site_name: Pytorch lightning Documentation
-theme:
-  name: 'material'
-docs_dir: docs
-repo_url: https://github.com/williamFalcon/pytorch-lightning
-site_dir: 'site'
-site_description: 'Documentation for Pytorch LightningModule, the researcher version of keras.'
-
-dev_addr: '0.0.0.0:8000'
-#google_analytics: ['UA-aasd', 'sitename']

pl_examples/README.md

@@ -0,0 +1,11 @@
+# Examples   
+This folder has 3 sections:   
+
+### Domain templates   
+These are templates to show common approaches such as GANs and RL.   
+
+### Basic examples   
+These show the most common use of Lightning for either CPU or GPU training.   
+
+### Multi-node examples   
+These show how to run jobs on a GPU cluster using lightning.

pl_examples/__init__.py

@@ -0,0 +1,146 @@
+"""
+Template model definition
+-------------------------
+
+In 99% of cases you want to just copy `one of the examples
+ <https://github.com/PyTorchLightning/pytorch-lightning/tree/master/pl_examples>`_
+ to start a new lightningModule and change the core of what your model is actually trying to do.
+
+.. code-block:: bash
+
+    # get a copy of the module template
+    wget https://raw.githubusercontent.com/PyTorchLightning/pytorch-lightning/master/pl_examples/new_project_templates/lightning_module_template.py  # noqa: E501
+
+
+Trainer Example
+---------------
+
+**`__main__` function**
+
+Normally, we want to let the `__main__` function start the training.
+ Inside the main we parse training arguments with whatever hyperparameters we want.
+ Your LightningModule will have a chance to add hyperparameters.
+
+.. code-block:: python
+
+    from test_tube import HyperOptArgumentParser
+
+    if __name__ == '__main__':
+
+        # use default args given by lightning
+        root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
+        parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False)
+        add_default_args(parent_parser, root_dir)
+
+        # allow model to overwrite or extend args
+        parser = ExampleModel.add_model_specific_args(parent_parser)
+        hyperparams = parser.parse_args()
+
+        # train model
+        main(hyperparams)
+
+**Main Function**
+
+The main function is your entry into the program. This is where you init your model, checkpoint directory,
+ and launch the training. The main function should have 3 arguments:
+- hparams: a configuration of hyperparameters.
+- slurm_manager: Slurm cluster manager object (can be None)
+- dict: for you to return any values you want (useful in meta-learning, otherwise set to)
+
+.. code-block:: python
+
+    def main(hparams, cluster, results_dict):
+        # build model
+        model = MyLightningModule(hparams)
+
+        # configure trainer
+        trainer = Trainer()
+
+        # train model
+        trainer.fit(model)
+
+
+The `__main__` function will start training on your **main** function.
+ If you use the HyperParameterOptimizer in hyper parameter optimization mode,
+ this main function will get one set of hyperparameters. If you use it as a simple
+ argument parser you get the default arguments in the argument parser.
+
+So, calling main(hyperparams) runs the model with the default argparse arguments.::
+
+    main(hyperparams)
+
+
+CPU hyperparameter search
+-------------------------
+
+.. code-block:: python
+
+    # run a grid search over 20 hyperparameter combinations.
+    hyperparams.optimize_parallel_cpu(
+        main_local,
+        nb_trials=20,
+        nb_workers=1
+    )
+
+
+Hyperparameter search on a single or multiple GPUs
+--------------------------------------------------
+
+.. code-block:: python
+
+    # run a grid search over 20 hyperparameter combinations.
+    hyperparams.optimize_parallel_gpu(
+        main_local,
+        nb_trials=20,
+        nb_workers=1,
+        gpus=[0,1,2,3]
+    )
+
+
+Hyperparameter search on a SLURM HPC cluster
+--------------------------------------------
+
+.. code-block:: python
+
+    def optimize_on_cluster(hyperparams):
+        # enable cluster training
+        cluster = SlurmCluster(
+            hyperparam_optimizer=hyperparams,
+            log_path=hyperparams.tt_save_path,
+            test_tube_exp_name=hyperparams.tt_name
+        )
+
+        # email for cluster coms
+        cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True)
+
+        # configure cluster
+        cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus
+        cluster.job_time = '48:00:00'
+        cluster.gpu_type = '1080ti'
+        cluster.memory_mb_per_node = 48000
+
+        # any modules for code to run in env
+        cluster.add_command('source activate pytorch_lightning')
+
+        # name of exp
+        job_display_name = hyperparams.tt_name.split('_')[0]
+        job_display_name = job_display_name[0:3]
+
+        # run hopt
+        logging.info('submitting jobs...')
+        cluster.optimize_parallel_cluster_gpu(
+            main,
+            nb_trials=hyperparams.nb_hopt_trials,
+            job_name=job_display_name
+        )
+
+    # run cluster hyperparameter search
+    optimize_on_cluster(hyperparams)
+
+"""
+
+from .basic_examples.lightning_module_template import LightningTemplateModel
+
+__all__ = [
+    'LightningTemplateModel'
+]

pl_examples/basic_examples/README.md

@@ -0,0 +1,39 @@
+# Basic Examples   
+Use these examples to test how lightning works.   
+
+#### Test on CPU  
+```bash
+python cpu_template.py
+```
+
+---   
+#### Train on a single GPU
+```bash
+python gpu_template.py --gpus 1
+```   
+
+---    
+#### DataParallel (dp)   
+Train on multiple GPUs using DataParallel.
+
+```bash
+python gpu_template.py --gpus 2 --distributed_backend dp
+```   
+
+---
+#### DistributedDataParallel (ddp)    
+
+Train on multiple GPUs using DistributedDataParallel   
+```bash
+python gpu_template.py --gpus 2 --distributed_backend ddp
+```
+
+---
+#### DistributedDataParallel+DP (ddp2)    
+
+Train on multiple GPUs using DistributedDataParallel + dataparallel.
+On a single node, uses all GPUs for 1 model. Then shares gradient information
+across nodes.   
+```bash
+python gpu_template.py --gpus 2 --distributed_backend ddp2
+```

pl_examples/basic_examples/cpu_template.py

@@ -0,0 +1,54 @@
+"""
+Runs a model on a single node across N-gpus.
+"""
+import os
+from argparse import ArgumentParser
+
+import numpy as np
+import torch
+
+from pl_examples.basic_examples.lightning_module_template import LightningTemplateModel
+from pytorch_lightning import Trainer
+
+SEED = 2334
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+
+def main(hparams):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    """
+    # ------------------------
+    # 1 INIT LIGHTNING MODEL
+    # ------------------------
+    model = LightningTemplateModel(hparams)
+
+    # ------------------------
+    # 2 INIT TRAINER
+    # ------------------------
+    trainer = Trainer()
+
+    # ------------------------
+    # 3 START TRAINING
+    # ------------------------
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+    # ------------------------
+    # TRAINING ARGUMENTS
+    # ------------------------
+    # these are project-wide arguments
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    parent_parser = ArgumentParser(add_help=False)
+
+    # each LightningModule defines arguments relevant to it
+    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
+    hyperparams = parser.parse_args()
+
+    # ---------------------
+    # RUN TRAINING
+    # ---------------------
+    main(hyperparams)

pl_examples/basic_examples/gpu_template.py

@@ -0,0 +1,79 @@
+"""
+Runs a model on a single node across N-gpus.
+"""
+import os
+from argparse import ArgumentParser
+
+import numpy as np
+import torch
+
+from pl_examples.basic_examples.lightning_module_template import LightningTemplateModel
+from pytorch_lightning import Trainer
+
+SEED = 2334
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+
+def main(hparams):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    """
+    # ------------------------
+    # 1 INIT LIGHTNING MODEL
+    # ------------------------
+    model = LightningTemplateModel(hparams)
+
+    # ------------------------
+    # 2 INIT TRAINER
+    # ------------------------
+    trainer = Trainer(
+        gpus=hparams.gpus,
+        distributed_backend=hparams.distributed_backend,
+        use_amp=hparams.use_16bit
+    )
+
+    # ------------------------
+    # 3 START TRAINING
+    # ------------------------
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+    # ------------------------
+    # TRAINING ARGUMENTS
+    # ------------------------
+    # these are project-wide arguments
+
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    parent_parser = ArgumentParser(add_help=False)
+
+    # gpu args
+    parent_parser.add_argument(
+        '--gpus',
+        type=int,
+        default=2,
+        help='how many gpus'
+    )
+    parent_parser.add_argument(
+        '--distributed_backend',
+        type=str,
+        default='dp',
+        help='supports three options dp, ddp, ddp2'
+    )
+    parent_parser.add_argument(
+        '--use_16bit',
+        dest='use_16bit',
+        action='store_true',
+        help='if true uses 16 bit precision'
+    )
+
+    # each LightningModule defines arguments relevant to it
+    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
+    hyperparams = parser.parse_args()
+
+    # ---------------------
+    # RUN TRAINING
+    # ---------------------
+    main(hyperparams)

pl_examples/basic_examples/lightning_module_template.py

@@ -1,17 +1,22 @@
+"""
+Example template for defining a system
+"""
+import logging
 import os
+from argparse import ArgumentParser
 from collections import OrderedDict
-import torch.nn as nn
-from torchvision.datasets import MNIST
-import torchvision.transforms as transforms
+
 import torch
+import torch.nn as nn
 import torch.nn.functional as F
-from test_tube import HyperOptArgumentParser
+import torchvision.transforms as transforms
 from torch import optim
 from torch.utils.data import DataLoader
 from torch.utils.data.distributed import DistributedSampler
+from torchvision.datasets import MNIST
 
-import pytorch_lightning as ptl
-from pytorch_lightning.root_module.root_module import LightningModule
+import pytorch_lightning as pl
+from pytorch_lightning.core.lightning import LightningModule
 
 
 class LightningTemplateModel(LightningModule):
@@ -44,11 +49,13 @@ class LightningTemplateModel(LightningModule):
         Layout model
         :return:
         """
-        self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
+        self.c_d1 = nn.Linear(in_features=self.hparams.in_features,
+                              out_features=self.hparams.hidden_dim)
         self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
         self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
 
-        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)
+        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim,
+                              out_features=self.hparams.out_features)
 
     # ---------------------
     # TRAINING
@@ -74,14 +81,14 @@ class LightningTemplateModel(LightningModule):
         nll = F.nll_loss(logits, labels)
         return nll
 
-    def training_step(self, data_batch, batch_i):
+    def training_step(self, batch, batch_idx):
         """
         Lightning calls this inside the training loop
-        :param data_batch:
+        :param batch:
         :return:
         """
         # forward pass
-        x, y = data_batch
+        x, y = batch
         x = x.view(x.size(0), -1)
 
         y_hat = self.forward(x)
@@ -90,23 +97,26 @@ class LightningTemplateModel(LightningModule):
         loss_val = self.loss(y, y_hat)
 
         # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
-        if self.trainer.use_dp:
+        if self.trainer.use_dp or self.trainer.use_ddp2:
             loss_val = loss_val.unsqueeze(0)
 
+        tqdm_dict = {'train_loss': loss_val}
         output = OrderedDict({
-            'loss': loss_val
+            'loss': loss_val,
+            'progress_bar': tqdm_dict,
+            'log': tqdm_dict
         })
 
         # can also return just a scalar instead of a dict (return loss_val)
         return output
 
-    def validation_step(self, data_batch, batch_i):
+    def validation_step(self, batch, batch_idx):
         """
         Lightning calls this inside the validation loop
-        :param data_batch:
+        :param batch:
         :return:
         """
-        x, y = data_batch
+        x, y = batch
         x = x.view(x.size(0), -1)
         y_hat = self.forward(x)
 
@@ -121,7 +131,7 @@ class LightningTemplateModel(LightningModule):
             val_acc = val_acc.cuda(loss_val.device.index)
 
         # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
-        if self.trainer.use_dp:
+        if self.trainer.use_dp or self.trainer.use_ddp2:
             loss_val = loss_val.unsqueeze(0)
             val_acc = val_acc.unsqueeze(0)
 
@@ -146,24 +156,25 @@ class LightningTemplateModel(LightningModule):
         val_loss_mean = 0
         val_acc_mean = 0
         for output in outputs:
-            val_loss_mean += output['val_loss']
-            val_acc_mean += output['val_acc']
+            val_loss = output['val_loss']
+
+            # reduce manually when using dp
+            if self.trainer.use_dp or self.trainer.use_ddp2:
+                val_loss = torch.mean(val_loss)
+            val_loss_mean += val_loss
+
+            # reduce manually when using dp
+            val_acc = output['val_acc']
+            if self.trainer.use_dp or self.trainer.use_ddp2:
+                val_acc = torch.mean(val_acc)
+
+            val_acc_mean += val_acc
 
         val_loss_mean /= len(outputs)
         val_acc_mean /= len(outputs)
-        tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-        return tqdm_dic
-
-    # ---------------------
-    # MODEL SAVING
-    # ---------------------
-    def get_save_dict(self):
-        checkpoint = {'state_dict': self.state_dict()}
-        return checkpoint
-
-    def load_model_specific(self, checkpoint):
-        self.load_state_dict(checkpoint['state_dict'])
-        pass
+        tqdm_dict = {'val_loss': val_loss_mean, 'val_acc': val_acc_mean}
+        result = {'progress_bar': tqdm_dict, 'log': tqdm_dict, 'val_loss': val_loss_mean}
+        return result
 
     # ---------------------
     # TRAINING SETUP
@@ -174,77 +185,74 @@ class LightningTemplateModel(LightningModule):
         :return: list of optimizers
         """
         optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
-        return [optimizer]
+        scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)
+        return [optimizer], [scheduler]
 
     def __dataloader(self, train):
         # init data generators
-        transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
-        dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)
+        transform = transforms.Compose([transforms.ToTensor(),
+                                        transforms.Normalize((0.5,), (1.0,))])
+        dataset = MNIST(root=self.hparams.data_root, train=train,
+                        transform=transform, download=True)
 
-        # when using multi-node we need to add the datasampler
+        # when using multi-node (ddp) we need to add the  datasampler
         train_sampler = None
         batch_size = self.hparams.batch_size
 
-        try:
-            if self.on_gpu:
-                train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
-                batch_size = batch_size // self.trainer.world_size  # scale batch size
-        except Exception as e:
-            pass
+        if self.use_ddp:
+            train_sampler = DistributedSampler(dataset)
 
         should_shuffle = train_sampler is None
         loader = DataLoader(
             dataset=dataset,
             batch_size=batch_size,
             shuffle=should_shuffle,
-            sampler=train_sampler
+            sampler=train_sampler,
+            num_workers=0
         )
 
         return loader
 
-    @ptl.data_loader
-    def tng_dataloader(self):
-        print('tng data loader called')
+    @pl.data_loader
+    def train_dataloader(self):
+        logging.info('training data loader called')
         return self.__dataloader(train=True)
 
-    @ptl.data_loader
+    @pl.data_loader
     def val_dataloader(self):
-        print('val data loader called')
+        logging.info('val data loader called')
         return self.__dataloader(train=False)
 
-    @ptl.data_loader
+    @pl.data_loader
     def test_dataloader(self):
-        print('test data loader called')
+        logging.info('test data loader called')
         return self.__dataloader(train=False)
 
     @staticmethod
-    def add_model_specific_args(parent_parser, root_dir): # pragma: no cover
+    def add_model_specific_args(parent_parser, root_dir):  # pragma: no cover
         """
         Parameters you define here will be available to your model through self.hparams
         :param parent_parser:
         :param root_dir:
         :return:
         """
-        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+        parser = ArgumentParser(parents=[parent_parser])
 
         # param overwrites
-        # parser.set_defaults(gradient_clip=5.0)
+        # parser.set_defaults(gradient_clip_val=5.0)
 
         # network params
-        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
-        parser.add_argument('--in_features', default=28*28, type=int)
+        parser.add_argument('--in_features', default=28 * 28, type=int)
         parser.add_argument('--out_features', default=10, type=int)
-        parser.add_argument('--hidden_dim', default=50000, type=int) # use 500 for CPU, 50000 for GPU to see speed difference
+        # use 500 for CPU, 50000 for GPU to see speed difference
+        parser.add_argument('--hidden_dim', default=50000, type=int)
+        parser.add_argument('--drop_prob', default=0.2, type=float)
+        parser.add_argument('--learning_rate', default=0.001, type=float)
 
         # data
         parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
 
         # training params (opt)
-        parser.opt_list('--learning_rate', default=0.001*8, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
-                        tunable=False)
-        parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
-
-        # if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu
-        parser.opt_list('--batch_size', default=256*8, type=int, options=[32, 64, 128, 256], tunable=False,
-                        help='batch size will be divided over all the gpus being used across all nodes')
+        parser.add_argument('--optimizer_name', default='adam', type=str)
+        parser.add_argument('--batch_size', default=64, type=int)
         return parser

pl_examples/domain_templates/gan.py

@@ -0,0 +1,217 @@
+"""
+To run this template just do:
+python gan.py
+
+After a few epochs, launch tensorboard to see the images being generated at every batch.
+
+tensorboard --logdir default
+"""
+import os
+from argparse import ArgumentParser
+from collections import OrderedDict
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision
+import torchvision.transforms as transforms
+from torch.utils.data import DataLoader
+from torchvision.datasets import MNIST
+
+import pytorch_lightning as pl
+
+
+class Generator(nn.Module):
+    def __init__(self, latent_dim, img_shape):
+        super(Generator, self).__init__()
+        self.img_shape = img_shape
+
+        def block(in_feat, out_feat, normalize=True):
+            layers = [nn.Linear(in_feat, out_feat)]
+            if normalize:
+                layers.append(nn.BatchNorm1d(out_feat, 0.8))
+            layers.append(nn.LeakyReLU(0.2, inplace=True))
+            return layers
+
+        self.model = nn.Sequential(
+            *block(latent_dim, 128, normalize=False),
+            *block(128, 256),
+            *block(256, 512),
+            *block(512, 1024),
+            nn.Linear(1024, int(np.prod(img_shape))),
+            nn.Tanh()
+        )
+
+    def forward(self, z):
+        img = self.model(z)
+        img = img.view(img.size(0), *self.img_shape)
+        return img
+
+
+class Discriminator(nn.Module):
+    def __init__(self, img_shape):
+        super(Discriminator, self).__init__()
+
+        self.model = nn.Sequential(
+            nn.Linear(int(np.prod(img_shape)), 512),
+            nn.LeakyReLU(0.2, inplace=True),
+            nn.Linear(512, 256),
+            nn.LeakyReLU(0.2, inplace=True),
+            nn.Linear(256, 1),
+            nn.Sigmoid(),
+        )
+
+    def forward(self, img):
+        img_flat = img.view(img.size(0), -1)
+        validity = self.model(img_flat)
+
+        return validity
+
+
+class GAN(pl.LightningModule):
+
+    def __init__(self, hparams):
+        super(GAN, self).__init__()
+        self.hparams = hparams
+
+        # networks
+        mnist_shape = (1, 28, 28)
+        self.generator = Generator(latent_dim=hparams.latent_dim, img_shape=mnist_shape)
+        self.discriminator = Discriminator(img_shape=mnist_shape)
+
+        # cache for generated images
+        self.generated_imgs = None
+        self.last_imgs = None
+
+    def forward(self, z):
+        return self.generator(z)
+
+    def adversarial_loss(self, y_hat, y):
+        return F.binary_cross_entropy(y_hat, y)
+
+    def training_step(self, batch, batch_idx, optimizer_idx):
+        imgs, _ = batch
+        self.last_imgs = imgs
+
+        # train generator
+        if optimizer_idx == 0:
+            # sample noise
+            z = torch.randn(imgs.shape[0], self.hparams.latent_dim)
+
+            # match gpu device (or keep as cpu)
+            if self.on_gpu:
+                z = z.cuda(imgs.device.index)
+
+            # generate images
+            self.generated_imgs = self.forward(z)
+
+            # log sampled images
+            # sample_imgs = self.generated_imgs[:6]
+            # grid = torchvision.utils.make_grid(sample_imgs)
+            # self.logger.experiment.add_image('generated_images', grid, 0)
+
+            # ground truth result (ie: all fake)
+            # put on GPU because we created this tensor inside training_loop
+            valid = torch.ones(imgs.size(0), 1)
+            if self.on_gpu:
+                valid = valid.cuda(imgs.device.index)
+
+            # adversarial loss is binary cross-entropy
+            g_loss = self.adversarial_loss(self.discriminator(self.generated_imgs), valid)
+            tqdm_dict = {'g_loss': g_loss}
+            output = OrderedDict({
+                'loss': g_loss,
+                'progress_bar': tqdm_dict,
+                'log': tqdm_dict
+            })
+            return output
+
+        # train discriminator
+        if optimizer_idx == 1:
+            # Measure discriminator's ability to classify real from generated samples
+
+            # how well can it label as real?
+            valid = torch.ones(imgs.size(0), 1)
+            if self.on_gpu:
+                valid = valid.cuda(imgs.device.index)
+
+            real_loss = self.adversarial_loss(self.discriminator(imgs), valid)
+
+            # how well can it label as fake?
+            fake = torch.zeros(imgs.size(0), 1)
+            if self.on_gpu:
+                fake = fake.cuda(imgs.device.index)
+
+            fake_loss = self.adversarial_loss(
+                self.discriminator(self.generated_imgs.detach()), fake)
+
+            # discriminator loss is the average of these
+            d_loss = (real_loss + fake_loss) / 2
+            tqdm_dict = {'d_loss': d_loss}
+            output = OrderedDict({
+                'loss': d_loss,
+                'progress_bar': tqdm_dict,
+                'log': tqdm_dict
+            })
+            return output
+
+    def configure_optimizers(self):
+        lr = self.hparams.lr
+        b1 = self.hparams.b1
+        b2 = self.hparams.b2
+
+        opt_g = torch.optim.Adam(self.generator.parameters(), lr=lr, betas=(b1, b2))
+        opt_d = torch.optim.Adam(self.discriminator.parameters(), lr=lr, betas=(b1, b2))
+        return [opt_g, opt_d], []
+
+    @pl.data_loader
+    def train_dataloader(self):
+        transform = transforms.Compose([transforms.ToTensor(),
+                                        transforms.Normalize([0.5], [0.5])])
+        dataset = MNIST(os.getcwd(), train=True, download=True, transform=transform)
+        return DataLoader(dataset, batch_size=self.hparams.batch_size)
+
+    def on_epoch_end(self):
+        z = torch.randn(8, self.hparams.latent_dim)
+        # match gpu device (or keep as cpu)
+        if self.on_gpu:
+            z = z.cuda(self.last_imgs.device.index)
+
+        # log sampled images
+        sample_imgs = self.forward(z)
+        grid = torchvision.utils.make_grid(sample_imgs)
+        self.logger.experiment.add_image(f'generated_images', grid, self.current_epoch)
+
+
+def main(hparams):
+    # ------------------------
+    # 1 INIT LIGHTNING MODEL
+    # ------------------------
+    model = GAN(hparams)
+
+    # ------------------------
+    # 2 INIT TRAINER
+    # ------------------------
+    trainer = pl.Trainer()
+
+    # ------------------------
+    # 3 START TRAINING
+    # ------------------------
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument("--batch_size", type=int, default=64, help="size of the batches")
+    parser.add_argument("--lr", type=float, default=0.0002, help="adam: learning rate")
+    parser.add_argument("--b1", type=float, default=0.5,
+                        help="adam: decay of first order momentum of gradient")
+    parser.add_argument("--b2", type=float, default=0.999,
+                        help="adam: decay of first order momentum of gradient")
+    parser.add_argument("--latent_dim", type=int, default=100,
+                        help="dimensionality of the latent space")
+
+    hparams = parser.parse_args()
+
+    main(hparams)

pl_examples/full_examples/imagenet/imagenet_example.py

@@ -0,0 +1,249 @@
+"""
+This example is largely adapted from https://github.com/pytorch/examples/blob/master/imagenet/main.py
+"""
+import argparse
+import os
+import random
+from collections import OrderedDict
+
+import torch
+import torch.backends.cudnn as cudnn
+import torch.nn.functional as F
+import torch.nn.parallel
+import torch.optim as optim
+import torch.optim.lr_scheduler as lr_scheduler
+import torch.utils.data
+import torch.utils.data.distributed
+import torchvision.datasets as datasets
+import torchvision.models as models
+import torchvision.transforms as transforms
+
+import pytorch_lightning as pl
+
+# pull out resnet names from torchvision models
+MODEL_NAMES = sorted(
+    name for name in models.__dict__
+    if name.islower() and not name.startswith("__") and callable(models.__dict__[name])
+)
+
+
+class ImageNetLightningModel(pl.LightningModule):
+
+    def __init__(self, hparams):
+        super(ImageNetLightningModel, self).__init__()
+        self.hparams = hparams
+        self.model = models.__dict__[self.hparams.arch](pretrained=self.hparams.pretrained)
+
+    def forward(self, x):
+        return self.model(x)
+
+    def training_step(self, batch, batch_idx):
+        images, target = batch
+        output = self.forward(images)
+        loss_val = F.cross_entropy(output, target)
+        acc1, acc5 = self.__accuracy(output, target, topk=(1, 5))
+
+        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
+        if self.trainer.use_dp or self.trainer.use_ddp2:
+            loss_val = loss_val.unsqueeze(0)
+            acc1 = acc1.unsqueeze(0)
+            acc5 = acc5.unsqueeze(0)
+
+        tqdm_dict = {'train_loss': loss_val}
+        output = OrderedDict({
+            'loss': loss_val,
+            'acc1': acc1,
+            'acc5': acc5,
+            'progress_bar': tqdm_dict,
+            'log': tqdm_dict
+        })
+
+        return output
+
+    def validation_step(self, batch, batch_idx):
+        images, target = batch
+        output = self.forward(images)
+        loss_val = F.cross_entropy(output, target)
+        acc1, acc5 = self.__accuracy(output, target, topk=(1, 5))
+
+        # in DP mode (default) make sure if result is scalar, there's another dim in the beginning
+        if self.trainer.use_dp or self.trainer.use_ddp2:
+            loss_val = loss_val.unsqueeze(0)
+            acc1 = acc1.unsqueeze(0)
+            acc5 = acc5.unsqueeze(0)
+
+        output = OrderedDict({
+            'val_loss': loss_val,
+            'val_acc1': acc1,
+            'val_acc5': acc5,
+        })
+
+        return output
+
+    def validation_end(self, outputs):
+
+        tqdm_dict = {}
+
+        for metric_name in ["val_loss", "val_acc1", "val_acc5"]:
+            metric_total = 0
+
+            for output in outputs:
+                metric_value = output[metric_name]
+
+                # reduce manually when using dp
+                if self.trainer.use_dp or self.trainer.use_ddp2:
+                    metric_value = torch.mean(metric_value)
+
+                metric_total += metric_value
+
+            tqdm_dict[metric_name] = metric_total / len(outputs)
+
+        result = {'progress_bar': tqdm_dict, 'log': tqdm_dict, 'val_loss': tqdm_dict["val_loss"]}
+        return result
+
+    @classmethod
+    def __accuracy(cls, output, target, topk=(1,)):
+        """Computes the accuracy over the k top predictions for the specified values of k"""
+        with torch.no_grad():
+            maxk = max(topk)
+            batch_size = target.size(0)
+
+            _, pred = output.topk(maxk, 1, True, True)
+            pred = pred.t()
+            correct = pred.eq(target.view(1, -1).expand_as(pred))
+
+            res = []
+            for k in topk:
+                correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
+                res.append(correct_k.mul_(100.0 / batch_size))
+            return res
+
+    def configure_optimizers(self):
+        optimizer = optim.SGD(
+            self.parameters(),
+            lr=self.hparams.lr,
+            momentum=self.hparams.momentum,
+            weight_decay=self.hparams.weight_decay
+        )
+        scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.1)
+        return [optimizer], [scheduler]
+
+    @pl.data_loader
+    def train_dataloader(self):
+        normalize = transforms.Normalize(
+            mean=[0.485, 0.456, 0.406],
+            std=[0.229, 0.224, 0.225],
+        )
+
+        train_dir = os.path.join(self.hparams.data_path, 'train')
+        train_dataset = datasets.ImageFolder(
+            train_dir,
+            transforms.Compose([
+                transforms.RandomResizedCrop(224),
+                transforms.RandomHorizontalFlip(),
+                transforms.ToTensor(),
+                normalize,
+            ]))
+
+        if self.use_ddp:
+            train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
+        else:
+            train_sampler = None
+
+        train_loader = torch.utils.data.DataLoader(
+            dataset=train_dataset,
+            batch_size=self.hparams.batch_size,
+            shuffle=(train_sampler is None),
+            num_workers=0,
+            sampler=train_sampler
+        )
+        return train_loader
+
+    @pl.data_loader
+    def val_dataloader(self):
+        normalize = transforms.Normalize(
+            mean=[0.485, 0.456, 0.406],
+            std=[0.229, 0.224, 0.225],
+        )
+        val_dir = os.path.join(self.hparams.data_path, 'val')
+        val_loader = torch.utils.data.DataLoader(
+            datasets.ImageFolder(val_dir, transforms.Compose([
+                transforms.Resize(256),
+                transforms.CenterCrop(224),
+                transforms.ToTensor(),
+                normalize,
+            ])),
+            batch_size=self.hparams.batch_size,
+            shuffle=False,
+            num_workers=0,
+        )
+        return val_loader
+
+    @staticmethod
+    def add_model_specific_args(parent_parser):  # pragma: no cover
+        parser = argparse.ArgumentParser(parents=[parent_parser])
+        parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet18', choices=MODEL_NAMES,
+                            help='model architecture: ' +
+                                 ' | '.join(MODEL_NAMES) +
+                                 ' (default: resnet18)')
+        parser.add_argument('--epochs', default=90, type=int, metavar='N',
+                            help='number of total epochs to run')
+        parser.add_argument('--seed', type=int, default=42,
+                            help='seed for initializing training. ')
+        parser.add_argument('-b', '--batch-size', default=256, type=int,
+                            metavar='N',
+                            help='mini-batch size (default: 256), this is the total '
+                                 'batch size of all GPUs on the current node when '
+                                 'using Data Parallel or Distributed Data Parallel')
+        parser.add_argument('--lr', '--learning-rate', default=0.1, type=float,
+                            metavar='LR', help='initial learning rate', dest='lr')
+        parser.add_argument('--momentum', default=0.9, type=float, metavar='M',
+                            help='momentum')
+        parser.add_argument('--wd', '--weight-decay', default=1e-4, type=float,
+                            metavar='W', help='weight decay (default: 1e-4)',
+                            dest='weight_decay')
+        parser.add_argument('--pretrained', dest='pretrained', action='store_true',
+                            help='use pre-trained model')
+        return parser
+
+
+def get_args():
+    parent_parser = argparse.ArgumentParser(add_help=False)
+    parent_parser.add_argument('--data-path', metavar='DIR', type=str,
+                               help='path to dataset')
+    parent_parser.add_argument('--save-path', metavar='DIR', default=".", type=str,
+                               help='path to save output')
+    parent_parser.add_argument('--gpus', type=int, default=1,
+                               help='how many gpus')
+    parent_parser.add_argument('--distributed-backend', type=str, default='dp', choices=('dp', 'ddp', 'ddp2'),
+                               help='supports three options dp, ddp, ddp2')
+    parent_parser.add_argument('--use-16bit', dest='use_16bit', action='store_true',
+                               help='if true uses 16 bit precision')
+    parent_parser.add_argument('-e', '--evaluate', dest='evaluate', action='store_true',
+                               help='evaluate model on validation set')
+
+    parser = ImageNetLightningModel.add_model_specific_args(parent_parser)
+    return parser.parse_args()
+
+
+def main(hparams):
+    model = ImageNetLightningModel(hparams)
+    if hparams.seed is not None:
+        random.seed(hparams.seed)
+        torch.manual_seed(hparams.seed)
+        cudnn.deterministic = True
+    trainer = pl.Trainer(
+        default_save_path=hparams.save_path,
+        gpus=hparams.gpus,
+        max_epochs=hparams.epochs,
+        distributed_backend=hparams.distributed_backend,
+        use_amp=hparams.use_16bit
+    )
+    if hparams.evaluate:
+        trainer.run_evaluation()
+    else:
+        trainer.fit(model)
+
+
+if __name__ == '__main__':
+    main(get_args())

pl_examples/multi_node_examples/README.md

@@ -0,0 +1,21 @@
+# Multi-node example   
+
+This demo launches a job using 2 GPUs on 2 different nodes (4 GPUs total).
+To run this demo do the following:
+
+1. Log into the jumphost node of your SLURM-managed cluster.  
+2. Create a conda environment with Lightning and a GPU PyTorch version.   
+3. Choose a script to submit    
+
+#### DDP  
+Submit this job to run with distributedDataParallel (2 nodes, 2 gpus each)
+```bash
+sbatch ddp_job_submit.sh YourEnv
+```
+
+#### DDP2  
+Submit this job to run with a different implementation of distributedDataParallel.
+In this version, each node acts like DataParallel but syncs across nodes like DDP.
+```bash
+sbatch ddp2_job_submit.sh YourEnv
+```

pl_examples/multi_node_examples/ddp2_job_submit.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -l
+
+# SLURM SUBMIT SCRIPT
+#SBATCH --nodes=2
+#SBATCH --gres=gpu:2
+#SBATCH --ntasks-per-node=1
+#SBATCH --mem=0
+#SBATCH --time=0-02:00:00
+
+# activate conda env
+source activate $1
+
+# -------------------------
+# debugging flags (optional)
+ export NCCL_DEBUG=INFO
+ export PYTHONFAULTHANDLER=1
+
+# on your cluster you might need these:
+# set the network interface
+# export NCCL_SOCKET_IFNAME=^docker0,lo
+
+# might need the latest cuda
+# module load NCCL/2.4.7-1-cuda.10.0
+# -------------------------
+
+# run script from above
+srun python3 multi_node_ddp2_demo.py

pl_examples/multi_node_examples/ddp_job_submit.sh

@@ -0,0 +1,27 @@
+#!/bin/bash -l
+
+# SLURM SUBMIT SCRIPT
+#SBATCH --nodes=2
+#SBATCH --gres=gpu:2
+#SBATCH --ntasks-per-node=2
+#SBATCH --mem=0
+#SBATCH --time=0-02:00:00
+
+# activate conda env
+source activate $1
+
+# -------------------------
+# debugging flags (optional)
+ export NCCL_DEBUG=INFO
+ export PYTHONFAULTHANDLER=1
+
+# on your cluster you might need these:
+# set the network interface
+# export NCCL_SOCKET_IFNAME=^docker0,lo
+
+# might need the latest cuda
+# module load NCCL/2.4.7-1-cuda.10.0
+# -------------------------
+
+# run script from above
+srun python3 multi_node_ddp_demo.py

pl_examples/multi_node_examples/multi_node_ddp2_demo.py

@@ -0,0 +1,55 @@
+"""
+Multi-node example (GPU)
+"""
+import os
+from argparse import ArgumentParser
+
+import numpy as np
+import torch
+
+from pl_examples.basic_examples.lightning_module_template import LightningTemplateModel
+from pytorch_lightning import Trainer
+
+SEED = 2334
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+
+def main(hparams):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    :return:
+    """
+    # ------------------------
+    # 1 INIT LIGHTNING MODEL
+    # ------------------------
+    model = LightningTemplateModel(hparams)
+
+    # ------------------------
+    # 2 INIT TRAINER
+    # ------------------------
+    trainer = Trainer(
+        gpus=2,
+        num_nodes=2,
+        distributed_backend='ddp2'
+    )
+
+    # ------------------------
+    # 3 START TRAINING
+    # ------------------------
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    parent_parser = ArgumentParser(add_help=False)
+
+    # each LightningModule defines arguments relevant to it
+    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
+    hyperparams = parser.parse_args()
+
+    # ---------------------
+    # RUN TRAINING
+    # ---------------------
+    main(hyperparams)

pl_examples/multi_node_examples/multi_node_ddp_demo.py

@@ -0,0 +1,55 @@
+"""
+Multi-node example (GPU)
+"""
+import os
+from argparse import ArgumentParser
+
+import numpy as np
+import torch
+
+from pl_examples.basic_examples.lightning_module_template import LightningTemplateModel
+from pytorch_lightning import Trainer
+
+SEED = 2334
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+
+def main(hparams):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    :return:
+    """
+    # ------------------------
+    # 1 INIT LIGHTNING MODEL
+    # ------------------------
+    model = LightningTemplateModel(hparams)
+
+    # ------------------------
+    # 2 INIT TRAINER
+    # ------------------------
+    trainer = Trainer(
+        gpus=2,
+        num_nodes=2,
+        distributed_backend='ddp'
+    )
+
+    # ------------------------
+    # 3 START TRAINING
+    # ------------------------
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+    root_dir = os.path.dirname(os.path.realpath(__file__))
+    parent_parser = ArgumentParser(add_help=False)
+
+    # each LightningModule defines arguments relevant to it
+    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
+    hyperparams = parser.parse_args()
+
+    # ---------------------
+    # RUN TRAINING
+    # ---------------------
+    main(hyperparams)

pytorch_lightning/__init__.py

@@ -1,3 +1,38 @@
-from .models import Trainer
-from .root_module.root_module import LightningModule
-from .root_module.decorators import data_loader
+"""Package info"""
+
+__version__ = '0.6.0'
+__author__ = 'William Falcon et al.'
+__author_email__ = 'waf2107@columbia.edu'
+__license__ = 'Apache-2.0'
+__copyright__ = 'Copyright (c) 2018-2019, %s.' % __author__
+__homepage__ = 'https://github.com/PyTorchLightning/pytorch-lightning'
+# this has to be simple string, see: https://github.com/pypa/twine/issues/522
+__docs__ = "PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers." \
+           " Scale your models. Write less boilerplate."
+
+
+try:
+    # This variable is injected in the __builtins__ by the build
+    # process. It used to enable importing subpackages of skimage when
+    # the binaries are not built
+    __LIGHTNING_SETUP__
+except NameError:
+    __LIGHTNING_SETUP__ = False
+
+if __LIGHTNING_SETUP__:
+    import sys
+    sys.stderr.write('Partial import of torchlightning during the build process.\n')
+    # We are not importing the rest of the scikit during the build
+    # process, as it may not be compiled yet
+else:
+    from .trainer.trainer import Trainer
+    from .core.lightning import LightningModule
+    from .core.decorators import data_loader
+    import logging
+
+    __all__ = [
+        'Trainer',
+        'LightningModule',
+        'data_loader',
+    ]
+    logging.basicConfig(level=logging.INFO)

pytorch_lightning/callbacks/__init__.py

@@ -1 +1,7 @@
-from .pt_callbacks import EarlyStopping, ModelCheckpoint
+from .pt_callbacks import EarlyStopping, ModelCheckpoint, GradientAccumulationScheduler
+
+__all__ = [
+    'EarlyStopping',
+    'ModelCheckpoint',
+    'GradientAccumulationScheduler',
+]

pytorch_lightning/callbacks/pt_callbacks.py

@@ -1,29 +1,20 @@
+"""
+Callbacks
+====================================
+Callbacks supported by Lightning
+"""
+
+import os
+import shutil
+import logging
+import warnings
 import numpy as np
-import os, shutil
-from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel
+
+from pytorch_lightning.overrides.data_parallel import LightningDistributedDataParallel
 
 
 class Callback(object):
-    """Abstract base class used to build new callbacks.
-    # Properties
-        params: dict. Training parameters
-            (eg. verbosity, batch size, number of epochs...).
-        model: instance of `keras.models.Model`.
-            Reference of the model being trained.
-    The `logs` dictionary that callback methods
-    take as argument will contain keys for quantities relevant to
-    the current batch or epoch.
-    Currently, the `.fit()` method of the `Sequential` model class
-    will include the following quantities in the `logs` that
-    it passes to its callbacks:
-        on_epoch_end: logs include `acc` and `loss`, and
-            optionally include `val_loss`
-            (if validation is enabled in `fit`), and `val_acc`
-            (if validation and accuracy monitoring are enabled).
-        on_batch_begin: logs include `size`,
-            the number of samples in the current batch.
-        on_batch_end: logs include `loss`, and optionally `acc`
-            (if accuracy monitoring is enabled).
+    r"""Abstract base class used to build new callbacks.
     """
 
     def __init__(self):
@@ -39,12 +30,30 @@ class Callback(object):
         self.model = model
 
     def on_epoch_begin(self, epoch, logs=None):
+        """
+        called when the epoch begins
+
+        Args:
+            epoch (int): current epoch
+            logs (dict): key-value pairs of quantities to monitor
+
+            Example:
+
+                on_epoch_begin(epoch=2, logs={'val_loss': 0.2})
+        """
         pass
 
     def on_epoch_end(self, epoch, logs=None):
         pass
 
     def on_batch_begin(self, batch, logs=None):
+        """
+        called when the batch starts.
+
+        Args:
+            batch (Tensor): current batch tensor
+            logs (dict): key-value pairs of quantities to monitor
+        """
         pass
 
     def on_batch_end(self, batch, logs=None):
@@ -58,38 +67,52 @@ class Callback(object):
 
 
 class EarlyStopping(Callback):
-    """Stop training when a monitored quantity has stopped improving.
-    # Arguments
-        monitor: quantity to be monitored.
-        min_delta: minimum change in the monitored quantity
+    r"""
+    Stop training when a monitored quantity has stopped improving.
+
+    Args:
+        monitor (str): quantity to be monitored. Default: ``'val_loss'``.
+        min_delta (float): minimum change in the monitored quantity
             to qualify as an improvement, i.e. an absolute
-            change of less than min_delta, will count as no
-            improvement.
-        patience: number of epochs with no improvement
-            after which training will be stopped.
-        verbose: verbosity mode.
-        mode: one of {auto, min, max}. In `min` mode,
+            change of less than `min_delta`, will count as no
+            improvement. Default: ``0``.
+        patience (int): number of epochs with no improvement
+            after which training will be stopped. Default: ``0``.
+        verbose (bool): verbosity mode. Default: ``0``.
+        mode (str): one of {auto, min, max}. In `min` mode,
             training will stop when the quantity
             monitored has stopped decreasing; in `max`
             mode it will stop when the quantity
             monitored has stopped increasing; in `auto`
             mode, the direction is automatically inferred
-            from the name of the monitored quantity.
+            from the name of the monitored quantity. Default: ``'auto'``.
+        strict (bool): whether to crash the training if `monitor` is
+            not found in the metrics. Default: ``True``.
+
+    Example::
+
+        from pytorch_lightning import Trainer
+        from pytorch_lightning.callbacks import EarlyStopping
+
+        early_stopping = EarlyStopping('val_loss')
+        Trainer(early_stop_callback=early_stopping)
     """
 
     def __init__(self, monitor='val_loss',
-                 min_delta=0.0, patience=0, verbose=0, mode='auto'):
+                 min_delta=0.0, patience=0, verbose=0, mode='auto', strict=True):
         super(EarlyStopping, self).__init__()
 
         self.monitor = monitor
         self.patience = patience
         self.verbose = verbose
+        self.strict = strict
         self.min_delta = min_delta
         self.wait = 0
         self.stopped_epoch = 0
 
         if mode not in ['auto', 'min', 'max']:
-            print('EarlyStopping mode %s is unknown, fallback to auto mode.' % mode)
+            if self.verbose > 0:
+                logging.info(f'EarlyStopping mode {mode} is unknown, fallback to auto mode.')
             mode = 'auto'
 
         if mode == 'min':
@@ -109,6 +132,22 @@ class EarlyStopping(Callback):
 
         self.on_train_begin()
 
+    def check_metrics(self, logs):
+        monitor_val = logs.get(self.monitor)
+        error_msg = (f'Early stopping conditioned on metric `{self.monitor}`'
+                     f' which is not available. Available metrics are:'
+                     f' `{"`, `".join(list(logs.keys()))}`')
+
+        if monitor_val is None:
+            if self.strict:
+                raise RuntimeError(error_msg)
+            elif self.verbose > 0:
+                warnings.warn(error_msg, RuntimeWarning)
+
+            return False
+
+        return True
+
     def on_train_begin(self, logs=None):
         # Allow instances to be re-used
         self.wait = 0
@@ -116,14 +155,11 @@ class EarlyStopping(Callback):
         self.best = np.Inf if self.monitor_op == np.less else -np.Inf
 
     def on_epoch_end(self, epoch, logs=None):
-        current = logs.get(self.monitor)
         stop_training = False
-        if current is None:
-            print('Early stopping conditioned on metric `%s` ''which is not available. Available metrics are: %s' %
-                (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning
-            )
-            exit(-1)
+        if not self.check_metrics(logs):
+            return stop_training
 
+        current = logs.get(self.monitor)
         if self.monitor_op(current - self.min_delta, self.best):
             self.best = current
             self.wait = 0
@@ -138,126 +174,248 @@ class EarlyStopping(Callback):
 
     def on_train_end(self, logs=None):
         if self.stopped_epoch > 0 and self.verbose > 0:
-            print('Epoch %05d: early stopping' % (self.stopped_epoch + 1))
+            logging.info(f'Epoch {self.stopped_epoch + 1:05d}: early stopping')
 
 
 class ModelCheckpoint(Callback):
-    """Save the model after every epoch.
-    `filepath` can contain named formatting options,
-    which will be filled the value of `epoch` and
-    keys in `logs` (passed in `on_epoch_end`).
-    For example: if `filepath` is `weights.{epoch:02d}-{val_loss:.2f}.hdf5`,
-    then the model checkpoints will be saved with the epoch number and
-    the validation loss in the filename.
-    # Arguments
-        filepath: string, path to save the model file.
-        monitor: quantity to monitor.
-        verbose: verbosity mode, 0 or 1.
-        save_best_only: if `save_best_only=True`,
-            the latest best model according to
-            the quantity monitored will not be overwritten.
-        mode: one of {auto, min, max}.
-            If `save_best_only=True`, the decision
+    r"""
+
+    Save the model after every epoch.
+
+    Args:
+        filepath (str): path to save the model file.
+            Can contain named formatting options to be auto-filled.
+
+            Example::
+
+                # save epoch and val_loss in name
+                ModelCheckpoint(filepath='{epoch:02d}-{val_loss:.2f}.hdf5')
+                # saves file like: /path/epoch_2-val_loss_0.2.hdf5
+        monitor (str): quantity to monitor.
+        verbose (bool): verbosity mode, 0 or 1.
+        save_top_k (int): if `save_top_k == k`,
+            the best k models according to
+            the quantity monitored will be saved.
+            if `save_top_k == 0`, no models are saved.
+            if `save_top_k == -1`, all models are saved.
+            Please note that the monitors are checked every `period` epochs.
+            if `save_top_k >= 2` and the callback is called multiple
+            times inside an epoch, the name of the saved file will be
+            appended with a version count starting with `v0`.
+        mode (str): one of {auto, min, max}.
+            If `save_top_k != 0`, the decision
             to overwrite the current save file is made
             based on either the maximization or the
             minimization of the monitored quantity. For `val_acc`,
             this should be `max`, for `val_loss` this should
             be `min`, etc. In `auto` mode, the direction is
             automatically inferred from the name of the monitored quantity.
-        save_weights_only: if True, then only the model's weights will be
+        save_weights_only (bool): if True, then only the model's weights will be
             saved (`model.save_weights(filepath)`), else the full model
             is saved (`model.save(filepath)`).
-        period: Interval (number of epochs) between checkpoints.
+        period (int): Interval (number of epochs) between checkpoints.
+
+    Example::
+
+        from pytorch_lightning import Trainer
+        from pytorch_lightning.callbacks import ModelCheckpoint
+
+        checkpoint_callback = ModelCheckpoint(filepath='my_path')
+        Trainer(checkpoint_callback=checkpoint_callback)
+
+        # saves checkpoints to my_path whenever 'val_loss' has a new min
     """
 
     def __init__(self, filepath, monitor='val_loss', verbose=0,
-                 save_best_only=False, save_weights_only=False,
+                 save_top_k=1, save_weights_only=False,
                  mode='auto', period=1, prefix=''):
         super(ModelCheckpoint, self).__init__()
+        if (
+            save_top_k and
+            os.path.isdir(filepath) and
+            len(os.listdir(filepath)) > 0
+        ):
+            warnings.warn(
+                f"Checkpoint directory {filepath} exists and is not empty with save_top_k != 0."
+                "All files in this directory will be deleted when a checkpoint is saved!"
+            )
+
         self.monitor = monitor
         self.verbose = verbose
         self.filepath = filepath
-        self.save_best_only = save_best_only
+        os.makedirs(filepath, exist_ok=True)
+        self.save_top_k = save_top_k
         self.save_weights_only = save_weights_only
         self.period = period
-        self.epochs_since_last_save = 0
+        self.epochs_since_last_check = 0
         self.prefix = prefix
+        self.best_k_models = {}
+        # {filename: monitor}
+        self.kth_best_model = ''
+        self.best = 0
 
         if mode not in ['auto', 'min', 'max']:
-            print('ModelCheckpoint mode %s is unknown, '
-                          'fallback to auto mode.' % (mode),
-                          RuntimeWarning)
+            warnings.warn(
+                f'ModelCheckpoint mode {mode} is unknown, '
+                'fallback to auto mode.', RuntimeWarning)
             mode = 'auto'
 
         if mode == 'min':
             self.monitor_op = np.less
-            self.best = np.Inf
+            self.kth_value = np.Inf
+            self.mode = 'min'
         elif mode == 'max':
             self.monitor_op = np.greater
-            self.best = -np.Inf
+            self.kth_value = -np.Inf
+            self.mode = 'max'
         else:
             if 'acc' in self.monitor or self.monitor.startswith('fmeasure'):
                 self.monitor_op = np.greater
-                self.best = -np.Inf
+                self.kth_value = -np.Inf
+                self.mode = 'max'
             else:
                 self.monitor_op = np.less
-                self.best = np.Inf
+                self.kth_value = np.Inf
+                self.mode = 'min'
 
-    def save_model(self, filepath, overwrite):
-        dirpath = '/'.join(filepath.split('/')[:-1])
+    def _del_model(self, filepath):
+        dirpath = os.path.dirname(filepath)
 
         # make paths
-        os.makedirs(os.path.dirname(filepath), exist_ok=True)
+        os.makedirs(dirpath, exist_ok=True)
 
-        if overwrite:
-            for filename in os.listdir(dirpath):
-                if self.prefix in filename:
-                    path_to_delete = os.path.join(dirpath, filename)
-                    try:
-                        shutil.rmtree(path_to_delete)
-                    except OSError:
-                        os.remove(path_to_delete)
+        try:
+            shutil.rmtree(filepath)
+        except OSError:
+            os.remove(filepath)
+
+    def _save_model(self, filepath):
+        dirpath = os.path.dirname(filepath)
+
+        # make paths
+        os.makedirs(dirpath, exist_ok=True)
 
         # delegate the saving to the model
         self.save_function(filepath)
 
+    def check_monitor_top_k(self, current):
+        less_than_k_models = len(self.best_k_models.keys()) < self.save_top_k
+        if less_than_k_models:
+            return True
+        return self.monitor_op(current, self.best_k_models[self.kth_best_model])
+
     def on_epoch_end(self, epoch, logs=None):
         logs = logs or {}
-        self.epochs_since_last_save += 1
-        if self.epochs_since_last_save >= self.period:
-            self.epochs_since_last_save = 0
-            filepath = '{}/{}_ckpt_epoch_{}.ckpt'.format(self.filepath, self.prefix, epoch + 1)
-            if self.save_best_only:
+        self.epochs_since_last_check += 1
+
+        if self.save_top_k == 0:
+            # no models are saved
+            return
+        if self.epochs_since_last_check >= self.period:
+            self.epochs_since_last_check = 0
+            filepath = f'{self.filepath}/{self.prefix}_ckpt_epoch_{epoch}.ckpt'
+            version_cnt = 0
+            while os.path.isfile(filepath):
+                # this epoch called before
+                filepath = f'{self.filepath}/{self.prefix}_ckpt_epoch_{epoch}_v{version_cnt}.ckpt'
+                version_cnt += 1
+
+            if self.save_top_k != -1:
                 current = logs.get(self.monitor)
+
                 if current is None:
-                    print('Can save best model only with %s available, '
-                                  'skipping.' % (self.monitor), RuntimeWarning)
+                    warnings.warn(
+                        f'Can save best model only with {self.monitor} available,'
+                        ' skipping.', RuntimeWarning)
                 else:
-                    if self.monitor_op(current, self.best):
+                    if self.check_monitor_top_k(current):
+
+                        # remove kth
+                        if len(self.best_k_models.keys()) == self.save_top_k:
+                            delpath = self.kth_best_model
+                            self.best_k_models.pop(self.kth_best_model)
+                            self._del_model(delpath)
+
+                        self.best_k_models[filepath] = current
+                        if len(self.best_k_models.keys()) == self.save_top_k:
+                            # monitor dict has reached k elements
+                            if self.mode == 'min':
+                                self.kth_best_model = max(self.best_k_models, key=self.best_k_models.get)
+                            else:
+                                self.kth_best_model = min(self.best_k_models, key=self.best_k_models.get)
+                            self.kth_value = self.best_k_models[self.kth_best_model]
+
+                        if self.mode == 'min':
+                            self.best = min(self.best_k_models.values())
+                        else:
+                            self.best = max(self.best_k_models.values())
                         if self.verbose > 0:
-                            print('\nEpoch %05d: %s improved from %0.5f to %0.5f,'
-                                  ' saving model to %s'
-                                  % (epoch + 1, self.monitor, self.best,
-                                     current, filepath))
-                        self.best = current
-                        self.save_model(filepath, overwrite=True)
+                            logging.info(
+                                f'\nEpoch {epoch:05d}: {self.monitor} reached'
+                                f' {current:0.5f} (best {self.best:0.5f}), saving model to'
+                                f' {filepath} as top {self.save_top_k}')
+                        self._save_model(filepath)
 
                     else:
                         if self.verbose > 0:
-                            print('\nEpoch %05d: %s did not improve' %
-                                  (epoch + 1, self.monitor))
+                            logging.info(
+                                f'\nEpoch {epoch:05d}: {self.monitor}'
+                                f' was not in top {self.save_top_k}')
+
             else:
                 if self.verbose > 0:
-                    print('\nEpoch %05d: saving model to %s' % (epoch + 1, filepath))
-                self.save_model(filepath, overwrite=False)
+                    logging.info(f'\nEpoch {epoch:05d}: saving model to {filepath}')
+                self._save_model(filepath)
 
 
-if __name__ == '__main__':
-    c = EarlyStopping(min_delta=0.9, patience=2, verbose=True)
-    losses = [10, 9, 8, 8, 6, 4.3, 5, 4.4, 2.8, 2.5]
-    for i, loss in enumerate(losses):
-        should_stop = c.on_epoch_end(i, logs={'val_loss': loss})
-        print(loss)
-        if should_stop:
-            break
+class GradientAccumulationScheduler(Callback):
+    r"""
+    Change gradient accumulation factor according to scheduling.
 
+    Args:
+        scheduling (dict): scheduling in format {epoch: accumulation_factor}
+
+    Example::
+
+        from pytorch_lightning import Trainer
+        from pytorch_lightning.callbacks import GradientAccumulationScheduler
+
+        # at epoch 5 start accumulating every 2 batches
+        accumulator = GradientAccumulationScheduler(scheduling: {5: 2})
+        Trainer(accumulate_grad_batches=accumulator)
+    """
+
+    def __init__(self, scheduling: dict):
+        if scheduling == {}:  # empty dict error
+            raise TypeError("Empty dict cannot be interpreted correct")
+
+        for key in scheduling.keys():
+            if not isinstance(key, int) or not isinstance(scheduling[key], int):
+                raise TypeError("All epoches and accumulation factor must be integers")
+
+        minimal_epoch = min(scheduling.keys())
+        if minimal_epoch < 1:
+            msg = f"Epochs indexing from 1, epoch {minimal_epoch} cannot be interpreted correct"
+            raise IndexError(msg)
+        elif minimal_epoch != 1:  # if user didnt define first epoch accumulation factor
+            scheduling.update({1: 1})
+
+        self.scheduling = scheduling
+        self.epochs = sorted(scheduling.keys())
+
+    def on_epoch_begin(self, epoch, trainer):
+        epoch += 1  # indexing epochs from 1
+        for i in reversed(range(len(self.epochs))):
+            if epoch >= self.epochs[i]:
+                trainer.accumulate_grad_batches = self.scheduling.get(self.epochs[i])
+                break
+
+
+# if __name__ == '__main__':
+#     c = EarlyStopping(min_delta=0.9, patience=2, verbose=True)
+#     losses = [10, 9, 8, 8, 6, 4.3, 5, 4.4, 2.8, 2.5]
+#     for i, loss in enumerate(losses):
+#         should_stop = c.on_epoch_end(i, logs={'val_loss': loss})
+#         logging.info(loss)
+#         if should_stop:
+#             break

pytorch_lightning/core/__init__.py

@@ -0,0 +1,100 @@
+"""
+A LightningModule is a strict superclass of torch.nn.Module but provides an interface to standardize
+the "ingredients" for a research or production system.
+
+- The model/system definition (__init__)
+- The model/system computations (forward)
+- What happens in the training loop (training_step, training_end)
+- What happens in the validation loop (validation_step, validation_end)
+- What happens in the test loop (test_step, test_end)
+- What optimizers to use (configure_optimizers)
+- What data to use (train_dataloader, val_dataloader, test_dataloader)
+
+Most methods are optional. Here's a minimal example.
+
+.. code-block:: python
+
+    import os
+    import torch
+    from torch.nn import functional as F
+    from torch.utils.data import DataLoader
+    from torchvision.datasets import MNIST
+    import torchvision.transforms as transforms
+
+    import pytorch_lightning as pl
+
+    class CoolModel(pl.LightningModule):
+
+        def __init__(self):
+            super(CoolModel, self).__init__()
+            self.l1 = torch.nn.Linear(28 * 28, 10)
+
+        def forward(self, x):
+            return torch.relu(self.l1(x.view(x.size(0), -1)))
+
+        def training_step(self, batch, batch_idx):
+            x, y = batch
+            y_hat = self.forward(x)
+            return {'loss': F.cross_entropy(y_hat, y)}
+
+        def validation_step(self, batch, batch_idx):
+            # OPTIONAL
+            x, y = batch
+            y_hat = self.forward(x)
+            return {'val_loss': F.cross_entropy(y_hat, y)}
+
+        def validation_end(self, outputs):
+            # OPTIONAL
+            val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean()
+            return {'val_loss': val_loss_mean}
+
+        def test_step(self, batch, batch_idx):
+            # OPTIONAL
+            x, y = batch
+            y_hat = self.forward(x)
+            return {'test_loss': F.cross_entropy(y_hat, y)}
+
+        def test_end(self, outputs):
+            # OPTIONAL
+            test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean()
+            return {'test_loss': test_loss_mean}
+
+        def configure_optimizers(self):
+            # REQUIRED
+            return torch.optim.Adam(self.parameters(), lr=0.02)
+
+        @pl.data_loader
+        def train_dataloader(self):
+            return DataLoader(MNIST(os.getcwd(), train=True, download=True,
+                              transform=transforms.ToTensor()), batch_size=32)
+
+        @pl.data_loader
+        def val_dataloader(self):
+            # OPTIONAL
+            # can also return a list of val dataloaders
+            return DataLoader(MNIST(os.getcwd(), train=True, download=True,
+                              transform=transforms.ToTensor()), batch_size=32)
+
+        @pl.data_loader
+        def test_dataloader(self):
+            # OPTIONAL
+            # can also return a list of test dataloaders
+            return DataLoader(MNIST(os.getcwd(), train=False, download=True,
+                              transform=transforms.ToTensor()), batch_size=32)
+
+Once you've defined the LightningModule, fit  it using a trainer.
+
+.. code-block:: python
+   trainer = pl.Trainer()
+   model = CoolModel()
+
+   trainer.fit(model)
+
+Check out this
+`COLAB <https://colab.research.google.com/drive/1F_RNcHzTfFuQf-LeKvSlud6x7jXYkG31#scrollTo=HOk9c4_35FKg>`_
+for a live demo.
+
+"""
+from .lightning import LightningModule
+
+__all__ = ['LightningModule']

pytorch_lightning/core/decorators.py

@@ -0,0 +1,35 @@
+import traceback
+from functools import wraps
+
+
+def data_loader(fn):
+    """
+    Decorator to make any fx with this use the lazy property
+    :param fn:
+    :return:
+    """
+
+    wraps(fn)
+    attr_name = '_lazy_' + fn.__name__
+
+    def _get_data_loader(self):
+        try:
+            value = getattr(self, attr_name)
+        except AttributeError:
+            try:
+                value = fn(self)  # Lazy evaluation, done only once.
+                if (
+                        value is not None and
+                        not isinstance(value, list) and
+                        fn.__name__ in ['test_dataloader', 'val_dataloader']
+                ):
+                    value = [value]
+            except AttributeError as e:
+                # Guard against AttributeError suppression. (Issue #142)
+                traceback.print_exc()
+                error = f'{fn.__name__}: An AttributeError was encountered: ' + str(e)
+                raise RuntimeError(error) from e
+            setattr(self, attr_name, value)  # Memoize evaluation.
+        return value
+
+    return _get_data_loader

pytorch_lightning/core/grads.py

@@ -1,29 +1,29 @@
-import numpy as np
-from torch import nn
-
 """
 Module to describe gradients
 """
 
+from torch import nn
+
 
 class GradInformation(nn.Module):
 
     def grad_norm(self, norm_type):
         results = {}
         total_norm = 0
-        for i, p in enumerate(self.parameters()):
+        for name, p in self.named_parameters():
             if p.requires_grad:
                 try:
                     param_norm = p.grad.data.norm(norm_type)
                     total_norm += param_norm ** norm_type
                     norm = param_norm ** (1 / norm_type)
 
-                    results['grad_{}_norm_{}'.format(norm_type, i)] = round(norm.data.cpu().numpy().flatten()[0], 3)
-                except Exception as e:
+                    grad = round(norm.data.cpu().numpy().flatten()[0], 3)
+                    results['grad_{}_norm_{}'.format(norm_type, name)] = grad
+                except Exception:
                     # this param had no grad
                     pass
 
         total_norm = total_norm ** (1. / norm_type)
-        results['grad_{}_norm_total'.format(norm_type)] = round(total_norm.data.cpu().numpy().flatten()[0], 3)
+        grad = round(total_norm.data.cpu().numpy().flatten()[0], 3)
+        results['grad_{}_norm_total'.format(norm_type)] = grad
         return results
-

pytorch_lightning/core/hooks.py

@@ -0,0 +1,155 @@
+"""
+Hooks
+=====
+
+There are cases when you might want to do something different at different parts of the training/validation loop.
+ To enable a hook, simply override the method in your LightningModule and the trainer will call it at the correct time.
+
+**Contributing** If there's a hook you'd like to add, simply:
+1. Fork PyTorchLightning.
+2. Add the hook :py:mod:`pytorch_lightning.base_module.hooks.py`.
+3. Add the correct place in the :py:mod:`pytorch_lightning.models.trainer` where it should be called.
+
+"""
+
+
+import torch
+
+
+try:
+    from apex import amp
+
+    APEX_AVAILABLE = True
+except ImportError:
+    APEX_AVAILABLE = False
+
+
+class ModelHooks(torch.nn.Module):
+
+    def on_sanity_check_start(self):
+        """
+        Called before starting evaluate
+        .. warning:: will be deprecated.
+        :return:
+        """
+        pass
+
+    def on_train_start(self):
+        """Called at the beginning of training before sanity check
+        :return:
+        """
+        # do something at the start of training
+        pass
+
+    def on_train_end(self):
+        """
+        Called at the end of training before logger experiment is closed
+        :return:
+        """
+        # do something at the end of training
+        pass
+
+    def on_batch_start(self, batch):
+        """Called in the training loop before anything happens for that batch.
+
+        :param batch:
+        :return:
+        """
+        # do something when the batch starts
+        pass
+
+    def on_batch_end(self):
+        """Called in the training loop after the batch."""
+        # do something when the batch ends
+        pass
+
+    def on_epoch_start(self):
+        """Called in the training loop at the very beginning of the epoch."""
+        # do something when the epoch starts
+        pass
+
+    def on_epoch_end(self):
+        """Called in the training loop at the very end of the epoch."""
+        # do something when the epoch ends
+        pass
+
+    def on_pre_performance_check(self):
+        """Called at the very beginning of the validation loop."""
+        # do something before validation starts
+        pass
+
+    def on_post_performance_check(self):
+        """Called at the very end of the validation loop."""
+        # do something before validation end
+        pass
+
+    def on_before_zero_grad(self, optimizer):
+        """Called after optimizer.step() and before optimizer.zero_grad()
+
+        Called in the training loop after taking an optimizer step and before zeroing grads.
+        Good place to inspect weight information with weights updated.
+
+        for optimizer in optimizers::
+
+            optimizer.step()
+            model.on_before_zero_grad(optimizer) # < ---- called here
+            optimizer.zero_grad
+
+        :param optimizer:
+        :return:
+        """
+        # do something with the optimizer or inspect it.
+        pass
+
+    def on_after_backward(self):
+        """Called after loss.backward() and before optimizers do anything.
+
+        :return:
+
+        Called in the training loop after model.backward()
+        This is the ideal place to inspect or log gradient information
+
+        .. code-block:: python
+
+            def on_after_backward(self):
+                # example to inspect gradient information in tensorboard
+                if self.trainer.global_step % 25 == 0:  # don't make the tf file huge
+                    params = self.state_dict()
+                    for k, v in params.items():
+                        grads = v
+                        name = k
+                        self.logger.experiment.add_histogram(tag=name, values=grads,
+                                                             global_step=self.trainer.global_step)
+
+        """
+        pass
+
+    def backward(self, use_amp, loss, optimizer, optimizer_idx):
+        """Override backward with your own implementation if you need to
+
+        :param use_amp: Whether amp was requested or not
+        :param loss: Loss is already scaled by accumulated grads
+        :param optimizer: Current optimizer being used
+        :param optimizer_idx: Index of the current optimizer being used
+        :return:
+
+        Called to perform backward step.
+        Feel free to override as needed.
+
+        The loss passed in has already been scaled for accumulated gradients if requested.
+
+        .. code-block:: python
+
+            def backward(self, use_amp, loss, optimizer):
+                if use_amp:
+                    with amp.scale_loss(loss, optimizer) as scaled_loss:
+                        scaled_loss.backward()
+                else:
+                    loss.backward()
+
+        """
+        if use_amp:
+            with amp.scale_loss(loss, optimizer) as scaled_loss:
+                scaled_loss.backward()
+        else:
+            loss.backward()

pytorch_lightning/core/memory.py

@@ -0,0 +1,270 @@
+'''
+Generates a summary of a model's layers and dimensionality
+'''
+
+import gc
+import logging
+import os
+import subprocess
+from subprocess import PIPE
+
+import numpy as np
+import pandas as pd
+import torch
+
+
+class ModelSummary(object):
+
+    def __init__(self, model, mode='full'):
+        '''
+        Generates summaries of model layers and dimensions.
+        '''
+        self.model = model
+        self.mode = mode
+        self.in_sizes = []
+        self.out_sizes = []
+
+        self.summarize()
+
+    def __str__(self):
+        return self.summary.__str__()
+
+    def __repr__(self):
+        return self.summary.__str__()
+
+    def named_modules(self):
+        if self.mode == 'full':
+            mods = self.model.named_modules()
+            mods = list(mods)[1:]  # do not include root module (LightningModule)
+        elif self.mode == 'top':
+            # the children are the top-level modules
+            mods = self.model.named_children()
+        else:
+            mods = []
+        return list(mods)
+
+    def get_variable_sizes(self):
+        '''Run sample input through each layer to get output sizes'''
+        mods = self.named_modules()
+        in_sizes = []
+        out_sizes = []
+        input_ = self.model.example_input_array
+
+        if self.model.on_gpu:
+            device = next(self.model.parameters()).get_device()
+            # test if input is a list or a tuple
+            if isinstance(input_, (list, tuple)):
+                input_ = [input_i.cuda(device) if torch.is_tensor(input_i) else input_i
+                          for input_i in input_]
+            else:
+                input_ = input_.cuda(device)
+
+        if self.model.trainer.use_amp:
+            # test if it is not a list or a tuple
+            if isinstance(input_, (list, tuple)):
+                input_ = [input_i.half() if torch.is_tensor(input_i) else input_i
+                          for input_i in input_]
+            else:
+                input_ = input_.half()
+
+        with torch.no_grad():
+
+            for _, m in mods:
+                if isinstance(input_, (list, tuple)):  # pragma: no cover
+                    out = m(*input_)
+                else:
+                    out = m(input_)
+
+                if isinstance(input_, (list, tuple)):  # pragma: no cover
+                    in_size = []
+                    for x in input_:
+                        if type(x) is list:
+                            in_size.append(len(x))
+                        else:
+                            in_size.append(x.size())
+                else:
+                    in_size = np.array(input_.size())
+
+                in_sizes.append(in_size)
+
+                if isinstance(out, (list, tuple)):  # pragma: no cover
+                    out_size = np.asarray([x.size() for x in out])
+                else:
+                    out_size = np.array(out.size())
+
+                out_sizes.append(out_size)
+                input_ = out
+
+        self.in_sizes = in_sizes
+        self.out_sizes = out_sizes
+        assert len(in_sizes) == len(out_sizes)
+        return
+
+    def get_layer_names(self):
+        '''Collect Layer Names'''
+        mods = self.named_modules()
+        names = []
+        layers = []
+        for name, m in mods:
+            names += [name]
+            layers += [str(m.__class__)]
+
+        layer_types = [x.split('.')[-1][:-2] for x in layers]
+
+        self.layer_names = names
+        self.layer_types = layer_types
+        return
+
+    def get_parameter_sizes(self):
+        '''Get sizes of all parameters in `model`'''
+        mods = self.named_modules()
+        sizes = []
+        for _, m in mods:
+            p = list(m.parameters())
+            modsz = []
+            for j in range(len(p)):
+                modsz.append(np.array(p[j].size()))
+            sizes.append(modsz)
+
+        self.param_sizes = sizes
+        return
+
+    def get_parameter_nums(self):
+        '''Get number of parameters in each layer'''
+        param_nums = []
+        for mod in self.param_sizes:
+            all_params = 0
+            for p in mod:
+                all_params += np.prod(p)
+            param_nums.append(all_params)
+        self.param_nums = param_nums
+        return
+
+    def make_summary(self):
+        '''
+        Makes a summary listing with:
+
+        Layer Name, Layer Type, Input Size, Output Size, Number of Parameters
+        '''
+
+        cols = ['Name', 'Type', 'Params']
+        if self.model.example_input_array is not None:
+            cols.extend(['In_sizes', 'Out_sizes'])
+
+        df = pd.DataFrame(np.zeros((len(self.layer_names), len(cols))))
+        df.columns = cols
+
+        df['Name'] = self.layer_names
+        df['Type'] = self.layer_types
+        df['Params'] = self.param_nums
+        df['Params'] = df['Params'].map(get_human_readable_count)
+
+        if self.model.example_input_array is not None:
+            df['In_sizes'] = self.in_sizes
+            df['Out_sizes'] = self.out_sizes
+
+        self.summary = df
+        return
+
+    def summarize(self):
+        self.get_layer_names()
+        self.get_parameter_sizes()
+        self.get_parameter_nums()
+
+        if self.model.example_input_array is not None:
+            self.get_variable_sizes()
+        self.make_summary()
+
+
+def print_mem_stack():  # pragma: no cover
+    for obj in gc.get_objects():
+        try:
+            if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
+                logging.info(type(obj), obj.size())
+        except Exception:
+            pass
+
+
+def count_mem_items():  # pragma: no cover
+    num_params = 0
+    num_tensors = 0
+    for obj in gc.get_objects():
+        try:
+            if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
+                obj_type = str(type(obj))
+                if 'parameter' in obj_type:
+                    num_params += 1
+                else:
+                    num_tensors += 1
+        except Exception:
+            pass
+
+    return num_params, num_tensors
+
+
+def get_memory_profile(mode):
+    """
+    'all' means return memory for all gpus
+    'min_max' means return memory for max and min
+    :param mode:
+    :return:
+    """
+    memory_map = get_gpu_memory_map()
+
+    if mode == 'min_max':
+        min_index, min_memory = min(memory_map.items(), key=lambda item: item[1])
+        max_index, max_memory = max(memory_map.items(), key=lambda item: item[1])
+
+        memory_map = {min_index: min_memory, max_index: max_memory}
+
+    return memory_map
+
+
+def get_gpu_memory_map():
+    """Get the current gpu usage.
+
+    Returns
+    -------
+    usage: dict
+        Keys are device ids as integers.
+        Values are memory usage as integers in MB.
+    """
+    result = subprocess.run(
+        [
+            'nvidia-smi',
+            '--query-gpu=memory.used',
+            '--format=csv,nounits,noheader',
+        ],
+        encoding='utf-8',
+        # capture_output=True,          # valid for python version >=3.7
+        stdout=PIPE, stderr=PIPE,       # for backward compatibility with python version 3.6
+        check=True)
+    # Convert lines into a dictionary
+    gpu_memory = [int(x) for x in result.stdout.strip().split(os.linesep)]
+    gpu_memory_map = {f'gpu_{index}': memory for index, memory in enumerate(gpu_memory)}
+    return gpu_memory_map
+
+
+def get_human_readable_count(number):
+    """
+    Abbreviates an integer number with K, M, B, T for thousands, millions,
+    billions and trillions, respectively.
+    Examples:
+        123     -> 123
+        1234    -> 1 K       (one thousand)
+        2e6     -> 2 M       (two million)
+        3e9     -> 3 B       (three billion)
+        4e12    -> 4 T       (four trillion)
+        5e15    -> 5,000 T
+    :param number: a positive integer number
+    :returns a string formatted according to the pattern described above.
+    """
+    assert number >= 0
+    labels = [' ', 'K', 'M', 'B', 'T']
+    num_digits = int(np.floor(np.log10(number)) + 1 if number > 0 else 1)
+    num_groups = int(np.ceil(num_digits / 3))
+    num_groups = min(num_groups, len(labels))  # don't abbreviate beyond trillions
+    shift = -3 * (num_groups - 1)
+    number = number * (10 ** shift)
+    index = num_groups - 1
+    return f'{int(number):,d} {labels[index]}'

pytorch_lightning/core/model_saving.py

@@ -0,0 +1,10 @@
+"""
+.. warning:: `model_saving` module has been renamed to `saving` since v0.6.0 and will be removed in v0.8.0
+"""
+
+import warnings
+
+warnings.warn("`model_saving` module has been renamed to `saving` since v0.6.0"
+              " and will be removed in v0.8.0", DeprecationWarning)
+
+from pytorch_lightning.core.saving import ModelIO  # noqa: E402

pytorch_lightning/core/root_module.py

@@ -0,0 +1,8 @@
+"""
+.. warning:: `root_module` module has been renamed to `lightning` since v0.6.0 and will be removed in v0.8.0
+"""
+
+import warnings
+
+warnings.warn("`root_module` module has been renamed to `lightning` since v0.6.0"
+              " and will be removed in v0.8.0", DeprecationWarning)

pytorch_lightning/core/saving.py

@@ -0,0 +1,34 @@
+class ModelIO(object):
+
+    def on_load_checkpoint(self, checkpoint):
+        """
+        Do something with the checkpoint
+        Gives model a chance to load something before state_dict is restored
+        :param checkpoint:
+        :return:
+        """
+        pass
+
+    def on_save_checkpoint(self, checkpoint):
+        """
+        Give the model a chance to add something to the checkpoint.
+        state_dict is already there
+        """
+        pass
+
+    # -------------------------
+    # OPTIONAL HOOKS
+    # -------------------------
+    def on_hpc_save(self, checkpoint):
+        """
+        Hook to do whatever you need right before Slurm manager saves the model
+        :return:
+        """
+        pass
+
+    def on_hpc_load(self, checkpoint):
+        """
+        Hook to do whatever you need right before Slurm manager loads the model
+        :return:
+        """
+        pass

pytorch_lightning/examples/__init__.py

@@ -1 +0,0 @@
-from .new_project_templates.lightning_module_template import LightningTemplateModel

pytorch_lightning/examples/new_project_templates/multi_node_cluster_template.py

@@ -1,172 +0,0 @@
-import os
-import sys
-import numpy as np
-from time import sleep
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
-from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-# ---------------------
-# DEFINE MODEL HERE
-# ---------------------
-from lightning_module_template import LightningTemplateModel
-# ---------------------
-
-"""
-Allows training by using command line arguments
-Run by:
-# TYPE YOUR RUN COMMAND HERE
-"""
-
-
-def main_local(hparams):
-    main(hparams, None, None)
-
-
-def main(hparams, cluster, results_dict):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    print('loading model...')
-    model = LightningTemplateModel(hparams)
-    print('model built')
-
-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-    # when using grid search, it's possible for all models to start at once
-    # and use the same test tube experiment version
-    relative_node_id = int(os.environ['SLURM_NODEID'])
-    sleep(relative_node_id + 1)
-
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        cluster=cluster,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-        gpus=hparams.gpus,
-        nb_gpu_nodes=hyperparams.nb_gpu_nodes
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-def optimize_on_cluster(hyperparams):
-    # enable cluster training
-    # log all scripts to the test tube folder
-    cluster = SlurmCluster(
-        hyperparam_optimizer=hyperparams,
-        log_path=hyperparams.slurm_log_path,
-    )
-
-    # email for cluster coms
-    cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True)
-
-    # configure cluster
-    cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus
-    cluster.per_experiment_nb_nodes = hyperparams.nb_gpu_nodes
-    cluster.job_time = '2:00:00'
-    cluster.gpu_type = 'volta'
-    cluster.memory_mb_per_node = 0
-
-    # any modules for code to run in env
-    cluster.add_command('source activate lightning')
-
-    # run only on 32GB voltas
-    cluster.add_slurm_cmd(cmd='constraint', value='volta32gb', comment='use 32gb gpus')
-    cluster.add_slurm_cmd(cmd='partition', value=hyperparams.gpu_partition, comment='use 32gb gpus')
-
-    # run hopt
-    # creates and submits jobs to slurm
-    cluster.optimize_parallel_cluster_gpu(
-        main,
-        nb_trials=hyperparams.nb_hopt_trials,
-        job_name=hyperparams.experiment_name
-    )
-
-
-if __name__ == '__main__':
-
-    # use default args
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-    slurm_out_dir = os.path.join(demo_log_dir, 'slurm_scripts')
-
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # cluster args not defined inside the model
-    parent_parser.add_argument('--gpu_partition', type=str, help='consult your cluster manual')
-
-    # TODO: make 1 param
-    parent_parser.add_argument('--per_experiment_nb_gpus', type=int, help='how many gpus to use in a node')
-    parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node')
-
-    parent_parser.add_argument('--nb_gpu_nodes', type=int, default=1, help='how many nodes to use in a cluster')
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
-    parent_parser.add_argument('--slurm_log_path', type=str, default=slurm_out_dir, help='where to save slurm meta')
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
-    parent_parser.add_argument('--nb_hopt_trials', type=int, default=1, help='how many grid search trials to run')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print('RUNNING ON SLURM CLUSTER')
-    optimize_on_cluster(hyperparams)

pytorch_lightning/examples/new_project_templates/single_cpu_template.py

@@ -1,110 +0,0 @@
-"""
-Runs a model on a single node across N-gpus.
-"""
-import os
-import sys
-import numpy as np
-from time import sleep
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
-from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-from lightning_module_template import LightningTemplateModel
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    print('loading model...')
-    model = LightningTemplateModel(hparams)
-    print('model built')
-
-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # dirs
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-
-    # although we user hyperOptParser, we are using it only as argparse right now
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # gpu args
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print(f'RUNNING ON CPU')
-    main(hyperparams)

pytorch_lightning/examples/new_project_templates/single_gpu_node_16bit_template.py

@@ -1,113 +0,0 @@
-"""
-Runs a model on a single node across N-gpus.
-"""
-import os
-import sys
-import numpy as np
-from time import sleep
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
-from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-from lightning_module_template import LightningTemplateModel
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    print('loading model...')
-    model = LightningTemplateModel(hparams)
-    print('model built')
-
-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-        gpus=hparams.gpus,
-        use_amp=True
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # dirs
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-
-    # although we user hyperOptParser, we are using it only as argparse right now
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # gpu args
-    parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node. -1 uses all the gpus on the node')
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
-    main(hyperparams)

pytorch_lightning/examples/new_project_templates/single_gpu_node_dp_template.py

@@ -1,112 +0,0 @@
-"""
-Runs a model on a single node across N-gpus.
-"""
-import os
-import sys
-import numpy as np
-from time import sleep
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
-from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-from lightning_module_template import LightningTemplateModel
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # ------------------------
-    # 1 INIT LIGHTNING MODEL
-    # ------------------------
-    print('loading model...')
-    model = LightningTemplateModel(hparams)
-    print('model built')
-
-    # ------------------------
-    # 2 INIT TEST TUBE EXP
-    # ------------------------
-
-    # init experiment
-    exp = Experiment(
-        name=hyperparams.experiment_name,
-        save_dir=hyperparams.test_tube_save_path,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # ------------------------
-    # 3 DEFINE CALLBACKS
-    # ------------------------
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        verbose=True,
-        mode='max'
-    )
-
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_loss',
-        mode='min'
-    )
-
-    # ------------------------
-    # 4 INIT TRAINER
-    # ------------------------
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-        gpus=hparams.gpus,
-    )
-
-    # ------------------------
-    # 5 START TRAINING
-    # ------------------------
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # dirs
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
-    checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
-    test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
-
-    # although we user hyperOptParser, we are using it only as argparse right now
-    parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
-
-    # gpu args
-    parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node. -1 uses all the gpus on the node')
-    parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
-    parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
-    parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
-
-    # allow model to overwrite or extend args
-    parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-    # run on HPC cluster
-    print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
-    main(hyperparams)