From ee20b83349fb847ad4a7d1c69d9cbce084ce28be Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 15:12:54 -0500 Subject: [PATCH 01/15] updated gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 654e70b1..43541c9d 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ tests/save_dir default/ lightning_logs/ tests/tests/ +*.rst +/docs/source/*.md # Byte-compiled / optimized / DLL files __pycache__/ From 2916a05f72ccc23a378ccecdb8a916b0b63b1aef Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 15:12:54 -0500 Subject: [PATCH 02/15] updated gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 654e70b1..43541c9d 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,8 @@ tests/save_dir default/ lightning_logs/ tests/tests/ +*.rst +/docs/source/*.md # Byte-compiled / optimized / DLL files __pycache__/ From 8efaba1591c2cba15a801e254a4b9c91f70c6c88 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 18:35:01 -0500 Subject: [PATCH 03/15] updated links in ninja file --- docs/source/_templates/theme_variables.jinja | 24 ++++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/source/_templates/theme_variables.jinja b/docs/source/_templates/theme_variables.jinja index 4982f358..3a67ad64 100644 --- a/docs/source/_templates/theme_variables.jinja +++ b/docs/source/_templates/theme_variables.jinja @@ -1,17 +1,17 @@ {%- set external_urls = { - 'github': 'https://github.com/williamFalcon/pytorch-lightning', - 'github_issues': 'https://github.com/williamFalcon/pytorch-lightning/issues', - 'contributing': 'https://github.com/williamFalcon/pytorch-lightning/blob/master/CONTRIBUTING.md', - 'docs': 'https://williamfalcon.github.io/pytorch-lightning', + 'github': 'https://github.com/PytorchLightning/pytorch-lightning', + 'github_issues': 'https://github.com/PytorchLightning/pytorch-lightning/issues', + 'contributing': 'https://github.com/PytorchLightning/pytorch-lightning/blob/master/CONTRIBUTING.md', + 'docs': 'https://pytorchlightning.github.io/pytorch-lightning', 'twitter': 'https://twitter.com/PyTorchLightnin', 'discuss': 'https://discuss.pytorch.org', - 'tutorials': 'https://williamfalcon.github.io/pytorch-lightning/', - 'previous_pytorch_versions': 'https://williamfalcon.github.io/pytorch-lightning/', - 'home': 'https://williamfalcon.github.io/pytorch-lightning/', - 'get_started': 'https://williamfalcon.github.io/pytorch-lightning/', - 'features': 'https://williamfalcon.github.io/pytorch-lightning/', - 'blog': 'https://williamfalcon.github.io/pytorch-lightning/', - 'resources': 'https://williamfalcon.github.io/pytorch-lightning/', - 'support': 'https://williamfalcon.github.io/pytorch-lightning/', + 'tutorials': 'https://pytorchlightning.github.io/pytorch-lightning/', + 'previous_pytorch_versions': 'https://pytorchlightning.github.io/pytorch-lightning/', + 'home': 'https://pytorchlightning.github.io/pytorch-lightning/', + 'get_started': 'https://pytorchlightning.github.io/pytorch-lightning/', + 'features': 'https://pytorchlightning.github.io/pytorch-lightning/', + 'blog': 'https://pytorchlightning.github.io/pytorch-lightning/', + 'resources': 'https://pytorchlightning.github.io/pytorch-lightning/', + 'support': 'https://pytorchlightning.github.io/pytorch-lightning/', } -%} From f3d517deb5146dbb8d772817397e9fed354edd93 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 19:44:02 -0500 Subject: [PATCH 04/15] updated docs --- docs/source/conf.py | 1 + docs/source/index.rst | 8 ++++---- docs/source/new-project.rst | 18 +++++++++++------- pytorch_lightning/trainer/__init__.py | 6 +++++- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index da0774a1..fa3558ab 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -83,6 +83,7 @@ extensions = [ 'sphinx.ext.autosummary', 'sphinx.ext.napoleon', 'recommonmark', + 'sphinx.ext.autosectionlabel', # 'm2r', 'nbsphinx', ] diff --git a/docs/source/index.rst b/docs/source/index.rst index 0dd7a6af..2fde388c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,13 +3,13 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to PyTorch-Lightning! +PyTorch-Lightning Documentation ============================= .. toctree:: - :maxdepth: 4 + :maxdepth: 1 :name: start - :caption: Quick Start + :caption: Start Here new-project examples @@ -17,7 +17,7 @@ Welcome to PyTorch-Lightning! .. toctree:: :maxdepth: 4 :name: docs - :caption: Docs + :caption: Python API documentation diff --git a/docs/source/new-project.rst b/docs/source/new-project.rst index 448e7e38..3bf8e3d4 100644 --- a/docs/source/new-project.rst +++ b/docs/source/new-project.rst @@ -1,13 +1,13 @@ Quick Start =========== -To start a new project define two files, a LightningModule and a Trainer file. -To illustrate Lightning power and simplicity, here's an example of a typical research flow. +| To start a new project define two files, a LightningModule and a Trainer file. +| To illustrate the power of Lightning and its simplicity, here's an example of a typical research flow. Case 1: BERT ------------ -Let's say you're working on something like BERT but want to try different ways of training or even different networks. -You would define a single LightningModule and use flags to switch between your different ideas. +| Let's say you're working on something like BERT but want to try different ways of training or even different networks. +| You would define a single LightningModule and use flags to switch between your different ideas. .. code-block:: python @@ -66,6 +66,10 @@ Then you could do rapid research by switching between these two and using the sa **Notice a few things about this flow:** -1. You're writing pure PyTorch... no unnecessary abstractions or new libraries to learn. -2. You get free GPU and 16-bit support without writing any of that code in your model. -3. You also get all of the capabilities below (without coding or testing yourself). +1. You're writing pure PyTorch... no unnecessary abstractions or new libraries to learn. +2. You get free GPU and 16-bit support without writing any of that code in your model. +3. You also get all of the capabilities below (without coding or testing yourself). + +- :ref:`Examples & Tutorials` +- :ref:`Examples & Tutorials` +- :ref:`Examples & Tutorials` diff --git a/pytorch_lightning/trainer/__init__.py b/pytorch_lightning/trainer/__init__.py index 88087a24..96254c26 100644 --- a/pytorch_lightning/trainer/__init__.py +++ b/pytorch_lightning/trainer/__init__.py @@ -16,4 +16,8 @@ This is the basic use of the trainer: trainer = Trainer() trainer.fit(model) -""" +The Trainer holds all the engineering code you might need such as distributing over GPUs or early stopping. +The LightningTemplate holds the core computations, train, val, test loop, optimizer and dataloaders. + +This pattern de-couples the engineering from the science which makes your code reusable and free to run on any hardware. +""" \ No newline at end of file From 7003f74751cf9a08bbb1b51ce8fe23a58156e9e0 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 20:36:57 -0500 Subject: [PATCH 05/15] finished callbacks --- docs/source/new-project.rst | 2 +- pytorch_lightning/callbacks/pt_callbacks.py | 110 ++++++++++++-------- 2 files changed, 65 insertions(+), 47 deletions(-) diff --git a/docs/source/new-project.rst b/docs/source/new-project.rst index 3bf8e3d4..62de6879 100644 --- a/docs/source/new-project.rst +++ b/docs/source/new-project.rst @@ -70,6 +70,6 @@ Then you could do rapid research by switching between these two and using the sa 2. You get free GPU and 16-bit support without writing any of that code in your model. 3. You also get all of the capabilities below (without coding or testing yourself). -- :ref:`Examples & Tutorials` +- :ref:`Callbacks` - :ref:`Examples & Tutorials` - :ref:`Examples & Tutorials` diff --git a/pytorch_lightning/callbacks/pt_callbacks.py b/pytorch_lightning/callbacks/pt_callbacks.py index 19517198..dc4158f8 100644 --- a/pytorch_lightning/callbacks/pt_callbacks.py +++ b/pytorch_lightning/callbacks/pt_callbacks.py @@ -1,3 +1,9 @@ +""" +Callbacks +==================================== +Callbacks supported by Lightning +""" + import os import shutil import logging @@ -8,26 +14,7 @@ from pytorch_lightning.overrides.data_parallel import LightningDistributedDataPa class Callback(object): - """Abstract base class used to build new callbacks. - - # Properties - * params: dict. Training parameters - (eg. verbosity, batch size, number of epochs...). - Reference of the model being trained. - - The `logs` dictionary that callback methods take as argument will contain keys - for quantities relevant to the current batch or epoch. - Currently, the `.fit()` method of the `Sequential` model class will include the following - quantities in the `logs` that it passes to its callbacks: - * on_epoch_end: logs include `acc` and `loss`, and - optionally include `val_loss` - (if validation is enabled in `fit`), and `val_acc` - (if validation and accuracy monitoring are enabled). - * on_batch_begin: logs include `size`, - the number of samples in the current batch. - * on_batch_end: logs include `loss`, and optionally `acc` - (if accuracy monitoring is enabled). - + r"""Abstract base class used to build new callbacks. """ def __init__(self): @@ -43,12 +30,29 @@ class Callback(object): self.model = model def on_epoch_begin(self, epoch, logs=None): + r""" + called when the epoch begins + + Args: + epoch (int): current epoch + logs (dict): key-value pairs of quantities to monitor + + Example: + >>> on_epoch_begin(epoch=2, logs={'val_loss': 0.2}) + """ pass def on_epoch_end(self, epoch, logs=None): pass def on_batch_begin(self, batch, logs=None): + r""" + called when the batch starts. + + Args: + batch (Tensor): current batch tensor + logs (dict): key-value pairs of quantities to monitor + """ pass def on_batch_end(self, batch, logs=None): @@ -62,25 +66,28 @@ class Callback(object): class EarlyStopping(Callback): - """Stop training when a monitored quantity has stopped improving. + r""" + Stop training when a monitored quantity has stopped improving. - # Arguments - monitor: quantity to be monitored. - min_delta: minimum change in the monitored quantity + Args: + monitor (str): quantity to be monitored. + min_delta (float): minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement. - patience: number of epochs with no improvement + patience (int): number of epochs with no improvement after which training will be stopped. - verbose: verbosity mode. - mode: one of {auto, min, max}. In `min` mode, + verbose (bool): verbosity mode. + mode (str): one of {auto, min, max}. In `min` mode, training will stop when the quantity monitored has stopped decreasing; in `max` mode it will stop when the quantity monitored has stopped increasing; in `auto` mode, the direction is automatically inferred from the name of the monitored quantity. - + Example: + >>> from pytorch_lightning.callbacks import EarlyStopping + >>> EarlyStopping('val_loss') """ def __init__(self, monitor='val_loss', @@ -150,20 +157,21 @@ class EarlyStopping(Callback): class ModelCheckpoint(Callback): - """Save the model after every epoch. + r""" - The `filepath` can contain named formatting options, - which will be filled the value of `epoch` and - keys in `logs` (passed in `on_epoch_end`). - For example: if `filepath` is `weights.{epoch:02d}-{val_loss:.2f}.hdf5`, - then the model checkpoints will be saved with the epoch number and - the validation loss in the filename. + Save the model after every epoch. - # Arguments - filepath: string, path to save the model file. - monitor: quantity to monitor. - verbose: verbosity mode, 0 or 1. - save_top_k: if `save_top_k == k`, + Args: + filepath (str): path to save the model file. + Can contain named formatting options to be auto-filled. + + Example: + >>> # save epoch and val_loss in name + >>> ModelCheckpoint(filepath='{epoch:02d}-{val_loss:.2f}.hdf5') + >>> # saves file like: /path/epoch_2-val_loss_0.2.hdf5 + monitor (str): quantity to monitor. + verbose (bool): verbosity mode, 0 or 1. + save_top_k (int): if `save_top_k == k`, the best k models according to the quantity monitored will be saved. if `save_top_k == 0`, no models are saved. @@ -172,7 +180,7 @@ class ModelCheckpoint(Callback): if `save_top_k >= 2` and the callback is called multiple times inside an epoch, the name of the saved file will be appended with a version count starting with `v0`. - mode: one of {auto, min, max}. + mode (str): one of {auto, min, max}. If `save_top_k != 0`, the decision to overwrite the current save file is made based on either the maximization or the @@ -180,11 +188,16 @@ class ModelCheckpoint(Callback): this should be `max`, for `val_loss` this should be `min`, etc. In `auto` mode, the direction is automatically inferred from the name of the monitored quantity. - save_weights_only: if True, then only the model's weights will be + save_weights_only (bool): if True, then only the model's weights will be saved (`model.save_weights(filepath)`), else the full model is saved (`model.save(filepath)`). - period: Interval (number of epochs) between checkpoints. + period (int): Interval (number of epochs) between checkpoints. + Example: + >>> from pytorch_lightning.callbacks import ModelCheckpoint + >>> ModelCheckpoint(filepath='my_path') + >>> # saves checkpoints to my_path whenever 'val_loss' has a new min + """ def __init__(self, filepath, monitor='val_loss', verbose=0, @@ -330,11 +343,16 @@ class ModelCheckpoint(Callback): class GradientAccumulationScheduler(Callback): - """Change gradient accumulation factor according to scheduling. + r""" + Change gradient accumulation factor according to scheduling. - # Arguments - scheduling: dict, scheduling in format {epoch: accumulation_factor} + Args: + scheduling (dict): scheduling in format {epoch: accumulation_factor} + Example: + >>> from pytorch_lightning.callbacks import GradientAccumulationScheduler + >>> # at epoch 5 start accumulating every 2 batches + >>> GradientAccumulationScheduler(scheduling: {5: 2}) """ def __init__(self, scheduling: dict): From 592e087df17f0498ee5400d2d0e997fa6735dd21 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 20:50:46 -0500 Subject: [PATCH 06/15] finished callbacks --- pytorch_lightning/trainer/__init__.py | 4 +++- pytorch_lightning/trainer/auto_mix_precision.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/__init__.py b/pytorch_lightning/trainer/__init__.py index 96254c26..e8b9dc05 100644 --- a/pytorch_lightning/trainer/__init__.py +++ b/pytorch_lightning/trainer/__init__.py @@ -1,5 +1,7 @@ """ -# Trainer +Trainer +==================================== + The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes diff --git a/pytorch_lightning/trainer/auto_mix_precision.py b/pytorch_lightning/trainer/auto_mix_precision.py index b28193c0..2915f246 100644 --- a/pytorch_lightning/trainer/auto_mix_precision.py +++ b/pytorch_lightning/trainer/auto_mix_precision.py @@ -1,3 +1,4 @@ + from abc import ABC try: From 519f70edf02a69e0a07520df290545734cf1f288 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 20:57:04 -0500 Subject: [PATCH 07/15] finished callbacks --- pytorch_lightning/trainer/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pytorch_lightning/trainer/__init__.py b/pytorch_lightning/trainer/__init__.py index e8b9dc05..e3ffd3e1 100644 --- a/pytorch_lightning/trainer/__init__.py +++ b/pytorch_lightning/trainer/__init__.py @@ -2,8 +2,7 @@ Trainer ==================================== - -The lightning trainer abstracts best practices for running a training, val, test routine. +The lightning Trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. From da721527a884314c62812af0223db5b3b9fd8821 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 21:17:16 -0500 Subject: [PATCH 08/15] fixed left menu --- docs/source/index.rst | 3 ++- pytorch_lightning/core/lightning.py | 5 +++++ pytorch_lightning/trainer/trainer.py | 5 +++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 2fde388c..db1a051c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,7 +19,8 @@ PyTorch-Lightning Documentation :name: docs :caption: Python API - documentation + trainer + lightning-module .. toctree:: :maxdepth: 1 diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 22fff333..07af367f 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1,3 +1,8 @@ +""" +LightningModule +==================================== + +""" import os import warnings import collections diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index a1133004..8a768abd 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -1,4 +1,7 @@ """ +AAA +==================================== + The trainer handles all the logic for running a val loop, training loop, distributing, etc.. . """ @@ -50,6 +53,8 @@ class Trainer(TrainerIOMixin, TrainerTrainLoopMixin, TrainerCallbackConfigMixin, ): + r"""Abstract base class used to build new callbacks. + """ def __init__( self, From 88c84dccb0c080d79e09bcf6397cf56ac5d69281 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 21:29:30 -0500 Subject: [PATCH 09/15] added callbacks to menu --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index db1a051c..5687013c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -21,6 +21,7 @@ PyTorch-Lightning Documentation trainer lightning-module + callbacks .. toctree:: :maxdepth: 1 From 2f12f21f3401edb387bbbb1467a4c1e37bc931e3 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 21:36:43 -0500 Subject: [PATCH 10/15] added direct links to docs --- docs/source/index.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 5687013c..c219884b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,9 +19,10 @@ PyTorch-Lightning Documentation :name: docs :caption: Python API - trainer - lightning-module callbacks + lightning-module + logging + trainer .. toctree:: :maxdepth: 1 From 9bccb4ccddf7247f4aa2b33c2d68480e44e4003f Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 21:43:04 -0500 Subject: [PATCH 11/15] added direct links to docs --- pytorch_lightning/core/lightning.py | 12 +++++++++--- pytorch_lightning/trainer/__init__.py | 3 --- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 07af367f..53646356 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1,8 +1,14 @@ """ -LightningModule -==================================== +The LightningModule is the "system recipe." It groups the following in one file: + - computational system definition + - computations done on forward + - training loop + - validation loop + - testing loop + - train, val, test dataloaders + - optimizers +""" -""" import os import warnings import collections diff --git a/pytorch_lightning/trainer/__init__.py b/pytorch_lightning/trainer/__init__.py index e3ffd3e1..71138861 100644 --- a/pytorch_lightning/trainer/__init__.py +++ b/pytorch_lightning/trainer/__init__.py @@ -1,7 +1,4 @@ """ -Trainer -==================================== - The lightning Trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. From d71342cb6145eab2a0de1c4f3f22576725b04b14 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 21:49:12 -0500 Subject: [PATCH 12/15] added direct links to docs --- pytorch_lightning/core/lightning.py | 2 -- pytorch_lightning/trainer/trainer.py | 6 ------ 2 files changed, 8 deletions(-) diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 53646356..4a2b1fe7 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -80,8 +80,6 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks): def __init__(self): # put the dimensions of the first input to your system self.example_input_array = torch.rand(5, 28 * 28) - - """ def __init__(self, *args, **kwargs): diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 8a768abd..258b7cff 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -1,9 +1,3 @@ -""" -AAA -==================================== - -The trainer handles all the logic for running a val loop, training loop, distributing, etc.. . -""" import os import sys From 937978f0d352a153b3b1a285bfecd6245249e5b0 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 15 Jan 2020 22:01:11 -0500 Subject: [PATCH 13/15] added direct links to docs --- pytorch_lightning/callbacks/pt_callbacks.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pytorch_lightning/callbacks/pt_callbacks.py b/pytorch_lightning/callbacks/pt_callbacks.py index dc4158f8..55a41e7e 100644 --- a/pytorch_lightning/callbacks/pt_callbacks.py +++ b/pytorch_lightning/callbacks/pt_callbacks.py @@ -86,8 +86,11 @@ class EarlyStopping(Callback): mode, the direction is automatically inferred from the name of the monitored quantity. Example: + >>> from pytorch_lightning import Trainer >>> from pytorch_lightning.callbacks import EarlyStopping - >>> EarlyStopping('val_loss') + >>> + >>> early_stopping = EarlyStopping('val_loss') + >>> Trainer(early_stop_callback=early_stopping) """ def __init__(self, monitor='val_loss', @@ -194,10 +197,13 @@ class ModelCheckpoint(Callback): period (int): Interval (number of epochs) between checkpoints. Example: + >>> from pytorch_lightning import Trainer >>> from pytorch_lightning.callbacks import ModelCheckpoint - >>> ModelCheckpoint(filepath='my_path') + >>> + >>> checkpoint_callback = ModelCheckpoint(filepath='my_path') + >>> Trainer(checkpoint_callback=checkpoint_callback) + >>> # saves checkpoints to my_path whenever 'val_loss' has a new min - """ def __init__(self, filepath, monitor='val_loss', verbose=0, @@ -350,9 +356,12 @@ class GradientAccumulationScheduler(Callback): scheduling (dict): scheduling in format {epoch: accumulation_factor} Example: + >>> from pytorch_lightning import Trainer >>> from pytorch_lightning.callbacks import GradientAccumulationScheduler + >>> >>> # at epoch 5 start accumulating every 2 batches - >>> GradientAccumulationScheduler(scheduling: {5: 2}) + >>> accumulator = GradientAccumulationScheduler(scheduling: {5: 2}) + >>> Trainer(accumulate_grad_batches=accumulator) """ def __init__(self, scheduling: dict): From 610edf8c3e24801e94a6672fcc02f9efe4b46a90 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 16 Jan 2020 06:04:17 -0500 Subject: [PATCH 14/15] added direct links to docs --- pytorch_lightning/trainer/__init__.py | 5 +- pytorch_lightning/trainer/trainer.py | 95 ++++++++++++++------------- 2 files changed, 54 insertions(+), 46 deletions(-) diff --git a/pytorch_lightning/trainer/__init__.py b/pytorch_lightning/trainer/__init__.py index 71138861..318ec82d 100644 --- a/pytorch_lightning/trainer/__init__.py +++ b/pytorch_lightning/trainer/__init__.py @@ -1,4 +1,7 @@ """ +Trainer +==================================== + The lightning Trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. @@ -18,4 +21,4 @@ The Trainer holds all the engineering code you might need such as distributing o The LightningTemplate holds the core computations, train, val, test loop, optimizer and dataloaders. This pattern de-couples the engineering from the science which makes your code reusable and free to run on any hardware. -""" \ No newline at end of file +""" diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 258b7cff..24b6d888 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -1,4 +1,5 @@ + import os import sys import warnings @@ -47,7 +48,9 @@ class Trainer(TrainerIOMixin, TrainerTrainLoopMixin, TrainerCallbackConfigMixin, ): - r"""Abstract base class used to build new callbacks. + r""" + Main trainer class + """ def __init__( @@ -91,52 +94,54 @@ class Trainer(TrainerIOMixin, truncated_bptt_steps=None, resume_from_checkpoint=None, ): + r""" + + Customize every aspect of training via flags """ + # Args: + # logger (ref:`Logger`): Logger for experiment tracking + # checkpoint_callback (ref:`Callback`): Callback for checkpointing + # :param early_stop_callback: Callback for early stopping + # :param str default_save_path: Default path for logs+weights if no logger/ckpt_callback passed + # :param int gradient_clip_val: 0 means don't clip. + # :param int gradient_clip: 0 means don't clip. Deprecated. + # :param process_position: shown in the tqdm bar + # :param int num_nodes: number of GPU nodes + # :param list|str|int gpus: int. (ie: 2 gpus) OR list to specify which GPUs [0, 1] OR '0,1' + # OR '-1' / -1 to use all available gpus + # :param str log_gpu_memory: None, 'min_max', 'all' + # :param bool show_progress_bar: If true shows tqdm bar + # :param float overfit_pct: uses this much of all datasets + # :param int track_grad_norm: -1 no tracking. Otherwise tracks that norm + # :param int check_val_every_n_epoch: check val every n train epochs + # :param bool fast_dev_run: runs full iteration over everything to find bugs + # :param int accumulate_grad_batches: Accumulates grads every k batches + # :param int max_epochs: + # :param int min_epochs: + # :param int train_percent_check: How much of train set to check + # :param int val_percent_check: How much of val set to check + # :param int test_percent_check: How much of test set to check + # :param float|int val_check_interval: If float, % of tng epoch. If int, check every n batch + # :param int log_save_interval: Writes logs to disk this often + # :param int row_log_interval: How often to add logging rows + # :param int add_row_log_interval: How often to add logging rows. Deprecated. + # :param str distributed_backend: Options: 'dp', 'ddp', 'ddp2'. + # :param bool use_amp: If true uses apex for 16bit precision + # :param bool print_nan_grads: Prints nan gradients + # :param str weights_summary: Options: 'full', 'top', None to not print. + # :param bool weights_save_path: Where to save weights if on cluster + # :param str amp_level: Check nvidia docs for level + # :param int num_sanity_val_steps: How many val steps before a full train loop. + # :param int truncated_bptt_steps: Enables multiple backward passes for each batch. + # + # .. warning:: Following arguments become deprecated and they will be removed in v0.8.0: + # - `gradient_clip`, + # - `nb_gpu_nodes`, + # - `max_nb_epochs`, + # - `min_nb_epochs`, + # - `add_row_log_interval`, + # - `nb_sanity_val_steps` - :param logger: Logger for experiment tracking - :param checkpoint_callback: Callback for checkpointing - :param early_stop_callback: Callback for early stopping - :param str default_save_path: Default path for logs+weights if no logger/ckpt_callback passed - :param int gradient_clip_val: 0 means don't clip. - :param int gradient_clip: 0 means don't clip. Deprecated. - :param process_position: shown in the tqdm bar - :param int num_nodes: number of GPU nodes - :param list|str|int gpus: int. (ie: 2 gpus) OR list to specify which GPUs [0, 1] OR '0,1' - OR '-1' / -1 to use all available gpus - :param str log_gpu_memory: None, 'min_max', 'all' - :param bool show_progress_bar: If true shows tqdm bar - :param float overfit_pct: uses this much of all datasets - :param int track_grad_norm: -1 no tracking. Otherwise tracks that norm - :param int check_val_every_n_epoch: check val every n train epochs - :param bool fast_dev_run: runs full iteration over everything to find bugs - :param int accumulate_grad_batches: Accumulates grads every k batches - :param int max_epochs: - :param int min_epochs: - :param int train_percent_check: How much of train set to check - :param int val_percent_check: How much of val set to check - :param int test_percent_check: How much of test set to check - :param float|int val_check_interval: If float, % of tng epoch. If int, check every n batch - :param int log_save_interval: Writes logs to disk this often - :param int row_log_interval: How often to add logging rows - :param int add_row_log_interval: How often to add logging rows. Deprecated. - :param str distributed_backend: Options: 'dp', 'ddp', 'ddp2'. - :param bool use_amp: If true uses apex for 16bit precision - :param bool print_nan_grads: Prints nan gradients - :param str weights_summary: Options: 'full', 'top', None to not print. - :param bool weights_save_path: Where to save weights if on cluster - :param str amp_level: Check nvidia docs for level - :param int num_sanity_val_steps: How many val steps before a full train loop. - :param int truncated_bptt_steps: Enables multiple backward passes for each batch. - - .. warning:: Following arguments become deprecated and they will be removed in v0.8.0: - - `gradient_clip`, - - `nb_gpu_nodes`, - - `max_nb_epochs`, - - `min_nb_epochs`, - - `add_row_log_interval`, - - `nb_sanity_val_steps` - - """ # Transfer params # Backward compatibility if nb_gpu_nodes is not None: From 9f9bf65edeab5e4442446e216f0806f5f1dd1931 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 16 Jan 2020 07:25:16 -0500 Subject: [PATCH 15/15] added direct links to docs --- pytorch_lightning/trainer/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pytorch_lightning/trainer/__init__.py b/pytorch_lightning/trainer/__init__.py index 318ec82d..4993c20c 100644 --- a/pytorch_lightning/trainer/__init__.py +++ b/pytorch_lightning/trainer/__init__.py @@ -22,3 +22,7 @@ The LightningTemplate holds the core computations, train, val, test loop, optimi This pattern de-couples the engineering from the science which makes your code reusable and free to run on any hardware. """ + +from .trainer import Trainer + +__all__ = ['Trainer']