Sphinx generated documentation (#521)

* upgrade req.

* move MkDocs

* create Sphinx

* init Sphinx

* move md from MkDocs to Sphinx

* CI: build docs

* build Sphinx

formatting

move docs from MD to docstring in particular package/modules

formatting

add Sphinx ext.

rename root_module to core

drop implicit name "_logger"

drop duplicate name "overwrite"

fix imports

use pytorch theme

add sample link mapping

try fix RTD build

use forked template

fix some docs warnings

fix paths

add deprecation warnings

fix flake8

fix paths

revert refactor

revert MLFlowLogger

* revert example import

* update link

* Update lightning_module_template.py

docs/source/_templates/theme_variables.jinja

@@ -0,0 +1,17 @@
+{%- set external_urls = {
+  'github': 'https://github.com/williamFalcon/pytorch-lightning',
+  'github_issues': 'https://github.com/williamFalcon/pytorch-lightning/issues',
+  'contributing': 'https://github.com/williamFalcon/pytorch-lightning/blob/master/CONTRIBUTING.md',
+  'docs': 'https://williamfalcon.github.io/pytorch-lightning',
+  'twitter': 'https://twitter.com/PyTorchLightnin',
+  'discuss': 'https://discuss.pytorch.org',
+  'tutorials': 'https://williamfalcon.github.io/pytorch-lightning/',
+  'previous_pytorch_versions': 'https://williamfalcon.github.io/pytorch-lightning/',
+  'home': 'https://williamfalcon.github.io/pytorch-lightning/',
+  'get_started': 'https://williamfalcon.github.io/pytorch-lightning/',
+  'features': 'https://williamfalcon.github.io/pytorch-lightning/',
+  'blog': 'https://williamfalcon.github.io/pytorch-lightning/',
+  'resources': 'https://williamfalcon.github.io/pytorch-lightning/',
+  'support': 'https://williamfalcon.github.io/pytorch-lightning/',
+}
+-%}

docs/source/conf.py

@@ -0,0 +1,354 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+
+import os
+import sys
+import glob
+import shutil
+import inspect
+
+# import m2r
+import builtins
+import pt_lightning_sphinx_theme
+
+PATH_HERE = os.path.abspath(os.path.dirname(__file__))
+PATH_ROOT = os.path.join(PATH_HERE, '..', '..')
+sys.path.insert(0, os.path.abspath(PATH_ROOT))
+
+builtins.__LIGHTNING_SETUP__ = True
+
+import pytorch_lightning  # noqa: E402
+
+# -- Project documents -------------------------------------------------------
+
+# export the documentation
+# with open('intro.rst', 'w') as fp:
+#     intro = pytorch_lightning.__doc__.replace(os.linesep + ' ', '')
+#     fp.write(m2r.convert(intro))
+#     # fp.write(pytorch_lightning.__doc__)
+
+# export the READme
+# with open(os.path.join(PATH_ROOT, 'README.md'), 'r') as fp:
+#     readme = fp.read()
+# # replace all paths to relative
+# for ndir in (os.path.basename(p) for p in glob.glob(os.path.join(PATH_ROOT, '*'))
+#              if os.path.isdir(p)):
+#     readme = readme.replace('](%s/' % ndir, '](%s/%s/' % (PATH_ROOT, ndir))
+# with open('readme.md', 'w') as fp:
+#     fp.write(readme)
+
+# -- Project information -----------------------------------------------------
+
+project = 'PyTorch-Lightning'
+copyright = pytorch_lightning.__copyright__
+author = pytorch_lightning.__author__
+
+# The short X.Y version
+version = pytorch_lightning.__version__
+# The full version, including alpha/beta/rc tags
+release = pytorch_lightning.__version__
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+
+needs_sphinx = '1.4'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinxcontrib.mockautodoc',
+    # 'sphinxcontrib.fulltoc',  # breaks pytorch-theme with unexpected kw argument 'titles_only'
+    'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.linkcode',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.napoleon',
+    'recommonmark',
+    # 'm2r',
+    'nbsphinx',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# https://berkeley-stat159-f17.github.io/stat159-f17/lectures/14-sphinx..html#conf.py-(cont.)
+# https://stackoverflow.com/questions/38526888/embed-ipython-notebook-in-sphinx-document
+# I execute the notebooks manually in advance. If notebooks test the code,
+# they should be run at build time.
+nbsphinx_execute = 'never'
+nbsphinx_allow_errors = True
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+# source_suffix = ['.rst', '.md', '.ipynb']
+source_suffix = {
+    '.rst': 'restructuredtext',
+    '.txt': 'markdown',
+    '.md': 'markdown',
+    '.ipynb': 'nbsphinx',
+}
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['*.test_*']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+# http://www.sphinx-doc.org/en/master/usage/theming.html#builtin-themes
+# html_theme = 'bizstyle'
+# https://sphinx-themes.org
+html_theme = 'pt_lightning_sphinx_theme'
+html_theme_path = [pt_lightning_sphinx_theme.get_html_theme_path()]
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+
+html_theme_options = {
+    'pytorch_project': pytorch_lightning.__homepage__,
+    'canonical_url': pytorch_lightning.__homepage__,
+    'collapse_navigation': False,
+    'display_version': True,
+    'logo_only': False,
+}
+
+html_logo = '_static/images/lightning_logo_small.png'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = project + '-doc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, project + '.tex', project + ' Documentation', author, 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, project, project + ' Documentation', [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, project, project + ' Documentation', author, project,
+     'One line description of project.', 'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# https://github.com/rtfd/readthedocs.org/issues/1139
+# I use sphinx-apidoc to auto-generate API documentation for my project.
+# Right now I have to commit these auto-generated files to my repository
+# so that RTD can build them into HTML docs. It'd be cool if RTD could run
+# sphinx-apidoc for me, since it's easy to forget to regen API docs
+# and commit them to my repo after making changes to my code.
+
+PACKAGES = [
+    pytorch_lightning.__name__,
+    'pl_examples',
+]
+
+
+def run_apidoc(_):
+    for pkg in PACKAGES:
+        argv = ['-e', '-o', PATH_HERE, os.path.join(PATH_HERE, PATH_ROOT, pkg),
+                '**/test_*', '--force', '--private', '--module-first']
+        try:
+            # Sphinx 1.7+
+            from sphinx.ext import apidoc
+            apidoc.main(argv)
+        except ImportError:
+            # Sphinx 1.6 (and earlier)
+            from sphinx import apidoc
+            argv.insert(0, apidoc.__file__)
+            apidoc.main(argv)
+
+
+def setup(app):
+    app.connect('builder-inited', run_apidoc)
+
+
+# copy all notebooks to local folder
+path_nbs = os.path.join(PATH_HERE, 'notebooks')
+if not os.path.isdir(path_nbs):
+    os.mkdir(path_nbs)
+for path_ipynb in glob.glob(os.path.join(PATH_ROOT, 'notebooks', '*.ipynb')):
+    path_ipynb2 = os.path.join(path_nbs, os.path.basename(path_ipynb))
+    shutil.copy(path_ipynb, path_ipynb2)
+
+# Ignoring Third-party packages
+# https://stackoverflow.com/questions/15889621/sphinx-how-to-exclude-imports-in-automodule
+
+MOCK_REQUIRE_PACKAGES = []
+with open(os.path.join(PATH_ROOT, 'requirements.txt'), 'r') as fp:
+    for ln in fp.readlines():
+        found = [ln.index(ch) for ch in list(',=<>#') if ch in ln]
+        pkg = ln[:min(found)] if found else ln
+        if pkg.rstrip():
+            MOCK_REQUIRE_PACKAGES.append(pkg.rstrip())
+
+# TODO: better parse from package since the import name and package name may differ
+MOCK_MANUAL_PACKAGES = ['torch', 'torchvision', 'sklearn', 'test_tube', 'mlflow', 'comet_ml']
+autodoc_mock_imports = MOCK_REQUIRE_PACKAGES + MOCK_MANUAL_PACKAGES
+# for mod_name in MOCK_REQUIRE_PACKAGES:
+#     sys.modules[mod_name] = mock.Mock()
+
+
+# Options for the linkcode extension
+# ----------------------------------
+github_user = 'williamFalcon'
+github_repo = project
+
+
+# Resolve function
+# This function is used to populate the (source) links in the API
+def linkcode_resolve(domain, info):
+    def find_source():
+        # try to find the file and line number, based on code from numpy:
+        # https://github.com/numpy/numpy/blob/master/doc/source/conf.py#L286
+        obj = sys.modules[info['module']]
+        for part in info['fullname'].split('.'):
+            obj = getattr(obj, part)
+        fname = inspect.getsourcefile(obj)
+        # https://github.com/rtfd/readthedocs.org/issues/5735
+        if any([s in fname for s in ('readthedocs', 'checkouts')]):
+            # /home/docs/checkouts/readthedocs.org/user_builds/pytorch_lightning/checkouts/
+            #  devel/pytorch_lightning/utilities/cls_experiment.py#L26-L176
+            path_top = os.path.abspath(os.path.join('..', '..', '..'))
+            fname = os.path.relpath(fname, start=path_top)
+        else:
+            # Local build, imitate master
+            fname = 'master/' + os.path.relpath(fname, start=os.path.abspath('..'))
+        source, lineno = inspect.getsourcelines(obj)
+        return fname, lineno, lineno + len(source) - 1
+
+    if domain != 'py' or not info['module']:
+        return None
+    try:
+        filename = '%s#L%d-L%d' % find_source()
+    except Exception:
+        filename = info['module'].replace('.', '/') + '.py'
+    # import subprocess
+    # tag = subprocess.Popen(['git', 'rev-parse', 'HEAD'], stdout=subprocess.PIPE,
+    #                        universal_newlines=True).communicate()[0][:-1]
+    return "https://github.com/%s/%s/blob/%s" \
+           % (github_user, github_repo, filename)
+
+
+autodoc_member_order = 'groupwise'
+autoclass_content = 'both'
+autodoc_default_flags = [
+    'members', 'undoc-members', 'show-inheritance', 'private-members',
+    # 'special-members', 'inherited-members'
+]

docs/source/index.rst

@@ -0,0 +1,26 @@
+.. PyTorch-Lightning documentation master file, created by
+   sphinx-quickstart on Fri Nov 15 07:48:22 2019.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to PyTorch-Lightning!
+=============================
+
+Table of content
+----------------
+
+.. toctree::
+   :maxdepth: 1
+
+   intro
+   pytorch_lightning
+   pl_examples
+
+
+Indices and tables
+------------------
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`
+

docs/source/intro.md

@@ -0,0 +1,60 @@
+## New project Quick Start    
+To start a new project define two files, a LightningModule and a Trainer file.    
+To illustrate Lightning power and simplicity, here's an example of a typical research flow.    
+
+### Case 1: BERT    
+Let's say you're working on something like BERT but want to try different ways of training or even different networks.  
+You would define a single LightningModule and use flags to switch between your different ideas.
+
+```python
+class BERT(pl.LightningModule):
+    def __init__(self, model_name, task):
+        self.task = task
+    
+        if model_name == 'transformer':
+            self.net = Transformer()
+        elif model_name == 'my_cool_version':
+            self.net = MyCoolVersion()
+            
+    def training_step(self, batch, batch_nb):
+        if self.task == 'standard_bert':
+            # do standard bert training with self.net...
+            # return loss
+            
+        if self.task == 'my_cool_task':
+            # do my own version with self.net
+            # return loss
+```   
+
+### Case 2: COOLER NOT BERT    
+But if you wanted to try something **completely** different, you'd define a new module for that.
+
+```python
+
+class CoolerNotBERT(pl.LightningModule):
+    def __init__(self):
+        self.net = ...
+        
+    def training_step(self, batch, batch_nb):
+        # do some other cool task
+        # return loss   
+```   
+
+### Rapid research flow    
+Then you could do rapid research by switching between these two and using the same trainer.
+
+```python
+
+if use_bert:
+    model = BERT()
+else:
+    model = CoolerNotBERT()
+    
+trainer = Trainer(gpus=4, use_amp=True)
+trainer.fit(model)
+```
+
+Notice a few things about this flow:   
+1. You're writing pure PyTorch... no unnecessary abstractions or new libraries to learn.   
+2. You get free GPU and 16-bit support without writing any of that code in your model.   
+3. You also get all of the capabilities below (without coding or testing yourself).