mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-08-24 12:19:51 +08:00
Docs (#1024)
* added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * added community examples * added community examples
This commit is contained in:
William Falcon
@@ -0,0 +1,62 @@
|
||||
Child Modules
|
||||
-------------
|
||||
Research projects tend to test different approaches to the same dataset.
|
||||
This is very easy to do in Lightning with inheritance.
|
||||
|
||||
For example, imaging we now want to train an Autoencoder to use as a feature extractor for MNIST images.
|
||||
Recall that `CoolMNIST` already defines all the dataloading etc... The only things
|
||||
that change in the `Autoencoder` model are the init, forward, training, validation and test step.
|
||||
|
||||
.. code-block::
|
||||
|
||||
class Encoder(torch.nn.Module):
|
||||
...
|
||||
|
||||
class AutoEncoder(CoolMNIST):
|
||||
def __init__(self):
|
||||
self.encoder = Encoder()
|
||||
self.decoder = Decoder()
|
||||
|
||||
def forward(self, x):
|
||||
generated = self.decoder(x)
|
||||
|
||||
def training_step(self, batch, batch_idx):
|
||||
x, _ = batch
|
||||
|
||||
representation = self.encoder(x)
|
||||
x_hat = self.forward(representation)
|
||||
|
||||
loss = MSE(x, x_hat)
|
||||
return loss
|
||||
|
||||
def validation_step(self, batch, batch_idx):
|
||||
return self._shared_eval(batch, batch_idx, 'val'):
|
||||
|
||||
def test_step(self, batch, batch_idx):
|
||||
return self._shared_eval(batch, batch_idx, 'test'):
|
||||
|
||||
def _shared_eval(self, batch, batch_idx, prefix):
|
||||
x, y = batch
|
||||
representation = self.encoder(x)
|
||||
x_hat = self.forward(representation)
|
||||
|
||||
loss = F.nll_loss(logits, y)
|
||||
return {f'{prefix}_loss': loss}
|
||||
|
||||
and we can train this using the same trainer
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
autoencoder = AutoEncoder()
|
||||
trainer = Trainer()
|
||||
trainer.fit(autoencoder)
|
||||
|
||||
And remember that the forward method is to define the practical use of a LightningModule.
|
||||
In this case, we want to use the `AutoEncoder` to extract image representations
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
some_images = torch.Tensor(32, 1, 28, 28)
|
||||
representations = autoencoder(some_images)
|
||||
|
||||
..
|
||||
@@ -1,6 +1,16 @@
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:name: Examples
|
||||
:caption: Examples
|
||||
:name: Community Examples
|
||||
:caption: Community Examples
|
||||
|
||||
MNIST on TPU <https://colab.research.google.com/drive/1-_LKx4HwAxl5M6xPJmqAAu444LTDQoa3#scrollTo=BHBz1_AnamN_>
|
||||
|
||||
Fast Aging GAN <https://github.com/PyTorchLightning/Fast-AgingGAN>
|
||||
Generative Adversarial Network <https://colab.research.google.com/drive/1F_RNcHzTfFuQf-LeKvSlud6x7jXYkG31#scrollTo=TyYOdg8g77P0>
|
||||
Hyperparameter optimization with Optuna <https://github.com/optuna/optuna/blob/master/examples/pytorch_lightning_simple.py>
|
||||
Recurrent Attentive Neural Process <https://github.com/PyTorchLightning/attentive-neural-processes>
|
||||
Siamese Nets for One-shot Image Recognition <https://github.com/PyTorchLightning/Siamese-Neural-Networks>
|
||||
Speech Transformers <https://github.com/PyTorchLightning/speech-transformer-pytorch_lightning>
|
||||
Transformers transfer learning (Huggingface) <https://colab.research.google.com/drive/1F_RNcHzTfFuQf-LeKvSlud6x7jXYkG31#scrollTo=yr7eaxkF-djf>
|
||||
Transformers text classification <https://github.com/ricardorei/lightning-text-classification>
|
||||
MNIST on TPU <https://colab.research.google.com/drive/1-_LKx4HwAxl5M6xPJmqAAu444LTDQoa3#scrollTo=BHBz1_AnamN_>
|
||||
NER (transformers, TPU) <https://colab.research.google.com/drive/1dBN-wwYUngLYVt985wGs_OKPlK_ANB9D>
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
Hyperparameters
|
||||
---------------
|
||||
Lightning has utilities to interact seamlessly with the command line ArgumentParser
|
||||
and plays well with the hyperparameter optimization framework of your choice.
|
||||
|
||||
LightiningModule hparams
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Normally, we don't hard-code the values to a model. We usually use the command line to
|
||||
modify the network. The `Trainer` can add all the available options to an ArgumentParser.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from argparse import ArgumentParser
|
||||
|
||||
parser = ArgumentParser()
|
||||
|
||||
# parametrize the network
|
||||
parser.add_argument('--layer_1_dim', type=int, default=128)
|
||||
parser.add_argument('--layer_1_dim', type=int, default=256)
|
||||
parser.add_argument('--batch_size', type=int, default=64)
|
||||
args = parser.parse_args()
|
||||
|
||||
Now we can parametrize the LightningModule.
|
||||
|
||||
.. code-block:: python
|
||||
:emphasize-lines: 5,6,7,12,14
|
||||
|
||||
class CoolMNIST(pl.LightningModule):
|
||||
def __init__(self, hparams):
|
||||
super(CoolMNIST, self).__init__()
|
||||
self.hparams = hparams
|
||||
|
||||
self.layer_1 = torch.nn.Linear(28 * 28, hparams.layer_1_dim)
|
||||
self.layer_2 = torch.nn.Linear(hparams.layer_1_dim, hparams.layer_2_dim)
|
||||
self.layer_3 = torch.nn.Linear(hparams.layer_2_dim, 10)
|
||||
|
||||
def forward(self, x):
|
||||
...
|
||||
|
||||
def train_dataloader(self):
|
||||
...
|
||||
return DataLoader(mnist_train, batch_size=self.hparams.batch_size)
|
||||
|
||||
def configure_optimizers(self):
|
||||
return Adam(self.parameters(), lr=self.hparams.learning_rate)
|
||||
|
||||
hparams = parse_args()
|
||||
model = CoolMNIST(hparams)
|
||||
|
||||
.. note:: Bonus! if (hparams) is in your module, Lightning will save it into the checkpoint and restore your
|
||||
model using those hparams exactly.
|
||||
|
||||
Trainer args
|
||||
^^^^^^^^^^^^
|
||||
|
||||
It also gets annoying to map each argument into the Argparser. Luckily we have
|
||||
a default parser
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
parser = ArgumentParser()
|
||||
|
||||
# add all options available in the trainer such as (max_epochs, etc...)
|
||||
parser = Trainer.add_argparse_args(parser)
|
||||
|
||||
We set up the main training entry point file like this:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def main(args):
|
||||
model = CoolMNIST(hparams=args)
|
||||
trainer = Trainer(max_epochs=args.max_epochs)
|
||||
trainer.fit(model)
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = ArgumentParser()
|
||||
|
||||
# adds all the trainer options as default arguments (like max_epochs)
|
||||
parser = Trainer.add_argparse_args(parser)
|
||||
|
||||
# parametrize the network
|
||||
parser.add_argument('--layer_1_dim', type=int, default=128)
|
||||
parser.add_argument('--layer_1_dim', type=int, default=256)
|
||||
parser.add_argument('--batch_size', type=int, default=64)
|
||||
args = parser.parse_args()
|
||||
|
||||
# train
|
||||
main(args)
|
||||
|
||||
And now we can train like this:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ python main.py --layer_1_dim 128 --layer_2_dim 256 --batch_size 64 --max_epochs 64
|
||||
|
||||
But it would also be nice to pass in any arbitrary argument to the trainer.
|
||||
We can do it by changing how we init the trainer.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def main(args):
|
||||
model = CoolMNIST(hparams=args)
|
||||
|
||||
# makes all trainer options available from the command line
|
||||
trainer = Trainer.from_argparse_args(args)
|
||||
|
||||
and now we can do this:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ python main.py --gpus 1 --min_epochs 12 --max_epochs 64 --arbitrary_trainer_arg some_value
|
||||
|
||||
Multiple Lightning Modules
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
We often have multiple Lightning Modules where each one has different arguments. Instead of
|
||||
polluting the main.py file, the LightningModule lets you define arguments for each one.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class CoolMNIST(pl.LightningModule):
|
||||
def __init__(self, hparams):
|
||||
super(CoolMNIST, self).__init__()
|
||||
self.layer_1 = torch.nn.Linear(28 * 28, hparams.layer_1_dim)
|
||||
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser):
|
||||
parser = ArgumentParser(parents=[parent_parser])
|
||||
parser.add_argument('--layer_1_dim', type=int, default=128)
|
||||
return parser
|
||||
|
||||
class GoodGAN(pl.LightningModule):
|
||||
def __init__(self, hparams):
|
||||
super(GoodGAN, self).__init__()
|
||||
self.encoder = Encoder(layers=hparams.encoder_layers)
|
||||
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser):
|
||||
parser = ArgumentParser(parents=[parent_parser])
|
||||
parser.add_argument('--encoder_layers', type=int, default=12)
|
||||
return parser
|
||||
|
||||
Now we can allow each model to inject the arguments it needs in the main.py
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def main(args):
|
||||
|
||||
# pick model
|
||||
if args.model_name == 'gan':
|
||||
model = GoodGAN(hparams=args)
|
||||
elif args.model_name == 'mnist':
|
||||
model = CoolMNIST(hparams=args)
|
||||
|
||||
model = CoolMNIST(hparams=args)
|
||||
trainer = Trainer(max_epochs=args.max_epochs)
|
||||
trainer.fit(model)
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = ArgumentParser()
|
||||
parser = Trainer.add_argparse_args(parser)
|
||||
|
||||
# figure out which model to use
|
||||
parser.add_argument('--model_name', type=str, default='gan', help='gan or mnist')
|
||||
temp_args = parser.parse_known_args()
|
||||
|
||||
# let the model add what it wants
|
||||
if temp_args.model_name == 'gan':
|
||||
parser = GoodGAN.add_model_specific_args(parser)
|
||||
elif temp_args.model_name == 'mnist':
|
||||
parser = CoolMNIST.add_model_specific_args(parser)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# train
|
||||
main(args)
|
||||
|
||||
and now we can train MNIST or the gan using the command line interface!
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
$ python main.py --model_name gan --encoder_layers 24
|
||||
$ python main.py --model_name mnist --layer_1_dim 128
|
||||
|
||||
Hyperparameter Optimization
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
Lightning is fully compatible with the hyperparameter optimization libraries!
|
||||
Here are some useful ones:
|
||||
|
||||
- `Hydra <https://medium.com/pytorch/hydra-a-fresh-look-at-configuration-for-machine-learning-projects-50583186b710>`_
|
||||
- `Optuna <https://github.com/optuna/optuna/blob/master/examples/pytorch_lightning_simple.py>`_
|
||||
@@ -27,8 +27,8 @@ PyTorch-Lightning Documentation
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 1
|
||||
:name: Examples
|
||||
:caption: Examples
|
||||
:name: Community Examples
|
||||
:caption: Community Examples
|
||||
|
||||
examples
|
||||
|
||||
@@ -46,17 +46,20 @@ PyTorch-Lightning Documentation
|
||||
|
||||
apex
|
||||
slurm
|
||||
child_modules
|
||||
debugging
|
||||
experiment_logging
|
||||
experiment_reporting
|
||||
early_stopping
|
||||
fast_training
|
||||
hooks
|
||||
hyperparameters
|
||||
multi_gpu
|
||||
weights_loading
|
||||
single_gpu
|
||||
sequences
|
||||
training_tricks
|
||||
transfer_learning
|
||||
tpu
|
||||
test_set
|
||||
optimizers
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
Transfer Learning
|
||||
-----------------
|
||||
|
||||
Using Pretrained Models
|
||||
^^^^^^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Sometimes we want to use a LightningModule as a pretrained model. This is fine because
|
||||
a LightningModule is just a `torch.nn.Module`!
|
||||
|
||||
Let's use the `AutoEncoder` as a feature extractor in a separate model.
|
||||
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class Encoder(torch.nn.Module):
|
||||
...
|
||||
|
||||
class AutoEncoder(pl.LightningModule):
|
||||
def __init__(self):
|
||||
self.encoder = Encoder()
|
||||
self.decoder = Decoder()
|
||||
|
||||
class CIFAR10Classifier(pl.LightingModule):
|
||||
def __init__(self):
|
||||
# init the pretrained LightningModule
|
||||
self.feature_extractor = AutoEncoder.load_from_checkpoint(PATH)
|
||||
self.feature_extractor.freeze()
|
||||
|
||||
# the autoencoder outputs a 100-dim representation and CIFAR-10 has 10 classes
|
||||
self.classifier = nn.Liner(100, 10)
|
||||
|
||||
def forward(self, x):
|
||||
representations = self.feature_extractor(x)
|
||||
x = self.classifier(representations)
|
||||
...
|
||||
|
||||
We used our pretrained Autoencoder (a LightningModule) for transfer learning!
|
||||
|
||||
Example: BERT (transformers)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
Lightning is completely agnostic to what's used for tranfer learning so long
|
||||
as it is a `torch.nn.Module` subclass.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from transformers import BertModel
|
||||
|
||||
class BertMNLIFinetuner(pl.LightningModule):
|
||||
|
||||
def __init__(self):
|
||||
super(BertMNLIFinetuner, self).__init__()
|
||||
|
||||
self.bert = BertModel.from_pretrained('bert-base-cased', output_attentions=True)
|
||||
self.W = nn.Linear(bert.config.hidden_size, 3)
|
||||
self.num_classes = 3
|
||||
|
||||
|
||||
def forward(self, input_ids, attention_mask, token_type_ids):
|
||||
|
||||
h, _, attn = self.bert(input_ids=input_ids,
|
||||
attention_mask=attention_mask,
|
||||
token_type_ids=token_type_ids)
|
||||
|
||||
h_cls = h[:, 0]
|
||||
logits = self.W(h_cls)
|
||||
return logits, attn
|
||||
@@ -448,8 +448,59 @@ Notice the epoch is MUCH faster!
|
||||
.. figure:: /_images/mnist_imgs/tpu_fast.png
|
||||
:alt: TPU speed
|
||||
|
||||
Validation loop
|
||||
Hyperparameters
|
||||
---------------
|
||||
Normally, we don't hard-code the values to a model. We usually use the command line to
|
||||
modify the network. The `Trainer` can add all the available options to an ArgumentParser.
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
from argparse import ArgumentParser
|
||||
|
||||
parser = ArgumentParser()
|
||||
|
||||
# parametrize the network
|
||||
parser.add_argument('--layer_1_dim', type=int, default=128)
|
||||
parser.add_argument('--layer_1_dim', type=int, default=256)
|
||||
parser.add_argument('--batch_size', type=int, default=64)
|
||||
args = parser.parse_args()
|
||||
|
||||
Now we can parametrize the LightningModule.
|
||||
|
||||
.. code-block:: python
|
||||
:emphasize-lines: 5,6,7,12,14
|
||||
|
||||
class CoolMNIST(pl.LightningModule):
|
||||
def __init__(self, hparams):
|
||||
super(CoolMNIST, self).__init__()
|
||||
self.hparams = hparams
|
||||
|
||||
self.layer_1 = torch.nn.Linear(28 * 28, hparams.layer_1_dim)
|
||||
self.layer_2 = torch.nn.Linear(hparams.layer_1_dim, hparams.layer_2_dim)
|
||||
self.layer_3 = torch.nn.Linear(hparams.layer_2_dim, 10)
|
||||
|
||||
def forward(self, x):
|
||||
...
|
||||
|
||||
def train_dataloader(self):
|
||||
...
|
||||
return DataLoader(mnist_train, batch_size=self.hparams.batch_size)
|
||||
|
||||
def configure_optimizers(self):
|
||||
return Adam(self.parameters(), lr=self.hparams.learning_rate)
|
||||
|
||||
hparams = parse_args()
|
||||
model = CoolMNIST(hparams)
|
||||
|
||||
.. note:: Bonus! if (hparams) is in your module, Lightning will save it into the checkpoint and restore your
|
||||
model using those hparams exactly.
|
||||
|
||||
For a full guide on using hyperparameters, `check out the hyperparameters docs <hyperparameters.rst>`_.
|
||||
|
||||
|
||||
|
||||
Validating
|
||||
----------
|
||||
|
||||
For most cases, we stop training the model when the performance on a validation
|
||||
split of the data reaches a minimum.
|
||||
@@ -523,8 +574,8 @@ in the validation loop, you won't need to potentially wait a full epoch to find
|
||||
|
||||
.. note:: Lightning disables gradients, puts model in eval mode and does everything needed for validation.
|
||||
|
||||
Testing loop
|
||||
------------
|
||||
Testing
|
||||
-------
|
||||
Once our research is done and we're about to publish or deploy a model, we normally want to figure out
|
||||
how it will generalize in the "real world." For this, we use a held-out split of the data for testing.
|
||||
|
||||
@@ -579,6 +630,8 @@ You can also run the test from a saved lightning model
|
||||
|
||||
.. note:: Lightning disables gradients, puts model in eval mode and does everything needed for testing.
|
||||
|
||||
.. warning:: .test() is not stable yet on TPUs. We're working on getting around the multiprocessing challenges.
|
||||
|
||||
Predicting
|
||||
----------
|
||||
Again, a LightningModule is exactly the same as a PyTorch module. This means you can load it
|
||||
@@ -642,3 +695,95 @@ In this case, we've set this LightningModel to predict logits. But we could also
|
||||
How you split up what goes in `forward` vs `training_step` depends on how you want to use this model for
|
||||
prediction.
|
||||
|
||||
Extensibility
|
||||
-------------
|
||||
Although lightning makes everything super simple, it doesn't sacrifice any flexibility or control.
|
||||
Lightning offers multiple ways of managing the training state.
|
||||
|
||||
Training overrides
|
||||
^^^^^^^^^^^^^^^^^^
|
||||
|
||||
Any part of the training, validation and testing loop can be modified.
|
||||
For instance, if you wanted to do your own backward pass, you would override the
|
||||
default implementation
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def backward(self, use_amp, loss, optimizer):
|
||||
if use_amp:
|
||||
with amp.scale_loss(loss, optimizer) as scaled_loss:
|
||||
scaled_loss.backward()
|
||||
else:
|
||||
loss.backward()
|
||||
|
||||
With your own
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class CoolMNIST(pl.LightningModule):
|
||||
|
||||
def backward(self, use_amp, loss, optimizer):
|
||||
# do a custom way of backward
|
||||
loss.backward(retain_graph=True)
|
||||
|
||||
Or if you wanted to initialize ddp in a different way than the default one
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
def configure_ddp(self, model, device_ids):
|
||||
# Lightning DDP simply routes to test_step, val_step, etc...
|
||||
model = LightningDistributedDataParallel(
|
||||
model,
|
||||
device_ids=device_ids,
|
||||
find_unused_parameters=True
|
||||
)
|
||||
return model
|
||||
|
||||
you could do your own:
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
class CoolMNIST(pl.LightningModule):
|
||||
|
||||
def configure_ddp(self, model, device_ids):
|
||||
|
||||
model = Horovod(model)
|
||||
# model = Ray(model)
|
||||
return model
|
||||
|
||||
Every single part of training is configurable this way.
|
||||
For a full list look at `lightningModule <lightning-module.rst>`_.
|
||||
|
||||
|
||||
Callbacks
|
||||
---------
|
||||
Another way to add arbitrary functionality is to add a custom callback
|
||||
for hooks that you might care about
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
import pytorch_lightning as pl
|
||||
|
||||
class MyPrintingCallback(pl.Callback):
|
||||
|
||||
def on_init_start(self, trainer):
|
||||
print('Starting to init trainer!')
|
||||
|
||||
def on_init_end(self, trainer):
|
||||
print('trainer is init now')
|
||||
|
||||
def on_train_end(self, trainer, pl_module):
|
||||
print('do something when training ends')
|
||||
|
||||
And pass the callbacks into the trainer
|
||||
|
||||
.. code-block:: python
|
||||
|
||||
Trainer(callbacks=[MyPrintingCallback()])
|
||||
|
||||
.. note:: See full list of 12+ hooks in the `Callback docs <callbacks.rst#callback-class>`_
|
||||
|
||||
.. include:: child_modules.rst
|
||||
|
||||
.. include:: transfer_learning.rst
|
||||
|
||||
|
||||
Reference in New Issue
Block a user