From 4aaa7d28ea283fce058647bc11215db27c29fa0d Mon Sep 17 00:00:00 2001 From: William Falcon Date: Tue, 3 Mar 2020 10:52:16 -0500 Subject: [PATCH] Docs (#1024) * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * added checkpoint defaults * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * docs * added community examples * added community examples --- docs/source/child_modules.rst | 62 ++++++++++ docs/source/examples.rst | 16 ++- docs/source/hyperparameters.rst | 192 ++++++++++++++++++++++++++++++ docs/source/index.rst | 7 +- docs/source/transfer_learning.rst | 66 ++++++++++ docs/source/tutorial_mnist.rst | 151 ++++++++++++++++++++++- 6 files changed, 486 insertions(+), 8 deletions(-) create mode 100644 docs/source/child_modules.rst create mode 100644 docs/source/hyperparameters.rst create mode 100644 docs/source/transfer_learning.rst diff --git a/docs/source/child_modules.rst b/docs/source/child_modules.rst new file mode 100644 index 00000000..33a25cb3 --- /dev/null +++ b/docs/source/child_modules.rst @@ -0,0 +1,62 @@ +Child Modules +------------- +Research projects tend to test different approaches to the same dataset. +This is very easy to do in Lightning with inheritance. + +For example, imaging we now want to train an Autoencoder to use as a feature extractor for MNIST images. +Recall that `CoolMNIST` already defines all the dataloading etc... The only things +that change in the `Autoencoder` model are the init, forward, training, validation and test step. + +.. code-block:: + + class Encoder(torch.nn.Module): + ... + + class AutoEncoder(CoolMNIST): + def __init__(self): + self.encoder = Encoder() + self.decoder = Decoder() + + def forward(self, x): + generated = self.decoder(x) + + def training_step(self, batch, batch_idx): + x, _ = batch + + representation = self.encoder(x) + x_hat = self.forward(representation) + + loss = MSE(x, x_hat) + return loss + + def validation_step(self, batch, batch_idx): + return self._shared_eval(batch, batch_idx, 'val'): + + def test_step(self, batch, batch_idx): + return self._shared_eval(batch, batch_idx, 'test'): + + def _shared_eval(self, batch, batch_idx, prefix): + x, y = batch + representation = self.encoder(x) + x_hat = self.forward(representation) + + loss = F.nll_loss(logits, y) + return {f'{prefix}_loss': loss} + +and we can train this using the same trainer + +.. code-block:: python + + autoencoder = AutoEncoder() + trainer = Trainer() + trainer.fit(autoencoder) + +And remember that the forward method is to define the practical use of a LightningModule. +In this case, we want to use the `AutoEncoder` to extract image representations + +.. code-block:: python + + some_images = torch.Tensor(32, 1, 28, 28) + representations = autoencoder(some_images) + +.. \ No newline at end of file diff --git a/docs/source/examples.rst b/docs/source/examples.rst index dbf4b71e..9bc77a21 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -1,6 +1,16 @@ .. toctree:: :maxdepth: 1 - :name: Examples - :caption: Examples + :name: Community Examples + :caption: Community Examples - MNIST on TPU \ No newline at end of file + + Fast Aging GAN + Generative Adversarial Network + Hyperparameter optimization with Optuna + Recurrent Attentive Neural Process + Siamese Nets for One-shot Image Recognition + Speech Transformers + Transformers transfer learning (Huggingface) + Transformers text classification + MNIST on TPU + NER (transformers, TPU) diff --git a/docs/source/hyperparameters.rst b/docs/source/hyperparameters.rst new file mode 100644 index 00000000..3f559298 --- /dev/null +++ b/docs/source/hyperparameters.rst @@ -0,0 +1,192 @@ +Hyperparameters +--------------- +Lightning has utilities to interact seamlessly with the command line ArgumentParser +and plays well with the hyperparameter optimization framework of your choice. + +LightiningModule hparams +^^^^^^^^^^^^^^^^^^^^^^^^ + +Normally, we don't hard-code the values to a model. We usually use the command line to +modify the network. The `Trainer` can add all the available options to an ArgumentParser. + +.. code-block:: python + + from argparse import ArgumentParser + + parser = ArgumentParser() + + # parametrize the network + parser.add_argument('--layer_1_dim', type=int, default=128) + parser.add_argument('--layer_1_dim', type=int, default=256) + parser.add_argument('--batch_size', type=int, default=64) + args = parser.parse_args() + +Now we can parametrize the LightningModule. + +.. code-block:: python + :emphasize-lines: 5,6,7,12,14 + + class CoolMNIST(pl.LightningModule): + def __init__(self, hparams): + super(CoolMNIST, self).__init__() + self.hparams = hparams + + self.layer_1 = torch.nn.Linear(28 * 28, hparams.layer_1_dim) + self.layer_2 = torch.nn.Linear(hparams.layer_1_dim, hparams.layer_2_dim) + self.layer_3 = torch.nn.Linear(hparams.layer_2_dim, 10) + + def forward(self, x): + ... + + def train_dataloader(self): + ... + return DataLoader(mnist_train, batch_size=self.hparams.batch_size) + + def configure_optimizers(self): + return Adam(self.parameters(), lr=self.hparams.learning_rate) + + hparams = parse_args() + model = CoolMNIST(hparams) + +.. note:: Bonus! if (hparams) is in your module, Lightning will save it into the checkpoint and restore your + model using those hparams exactly. + +Trainer args +^^^^^^^^^^^^ + +It also gets annoying to map each argument into the Argparser. Luckily we have +a default parser + +.. code-block:: python + + parser = ArgumentParser() + + # add all options available in the trainer such as (max_epochs, etc...) + parser = Trainer.add_argparse_args(parser) + +We set up the main training entry point file like this: + +.. code-block:: python + + def main(args): + model = CoolMNIST(hparams=args) + trainer = Trainer(max_epochs=args.max_epochs) + trainer.fit(model) + + if __name__ == '__main__': + parser = ArgumentParser() + + # adds all the trainer options as default arguments (like max_epochs) + parser = Trainer.add_argparse_args(parser) + + # parametrize the network + parser.add_argument('--layer_1_dim', type=int, default=128) + parser.add_argument('--layer_1_dim', type=int, default=256) + parser.add_argument('--batch_size', type=int, default=64) + args = parser.parse_args() + + # train + main(args) + +And now we can train like this: + +.. code-block:: bash + + $ python main.py --layer_1_dim 128 --layer_2_dim 256 --batch_size 64 --max_epochs 64 + +But it would also be nice to pass in any arbitrary argument to the trainer. +We can do it by changing how we init the trainer. + +.. code-block:: python + + def main(args): + model = CoolMNIST(hparams=args) + + # makes all trainer options available from the command line + trainer = Trainer.from_argparse_args(args) + +and now we can do this: + +.. code-block:: bash + + $ python main.py --gpus 1 --min_epochs 12 --max_epochs 64 --arbitrary_trainer_arg some_value + +Multiple Lightning Modules +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +We often have multiple Lightning Modules where each one has different arguments. Instead of +polluting the main.py file, the LightningModule lets you define arguments for each one. + +.. code-block:: python + + class CoolMNIST(pl.LightningModule): + def __init__(self, hparams): + super(CoolMNIST, self).__init__() + self.layer_1 = torch.nn.Linear(28 * 28, hparams.layer_1_dim) + + @staticmethod + def add_model_specific_args(parent_parser): + parser = ArgumentParser(parents=[parent_parser]) + parser.add_argument('--layer_1_dim', type=int, default=128) + return parser + + class GoodGAN(pl.LightningModule): + def __init__(self, hparams): + super(GoodGAN, self).__init__() + self.encoder = Encoder(layers=hparams.encoder_layers) + + @staticmethod + def add_model_specific_args(parent_parser): + parser = ArgumentParser(parents=[parent_parser]) + parser.add_argument('--encoder_layers', type=int, default=12) + return parser + +Now we can allow each model to inject the arguments it needs in the main.py + +.. code-block:: python + + def main(args): + + # pick model + if args.model_name == 'gan': + model = GoodGAN(hparams=args) + elif args.model_name == 'mnist': + model = CoolMNIST(hparams=args) + + model = CoolMNIST(hparams=args) + trainer = Trainer(max_epochs=args.max_epochs) + trainer.fit(model) + + if __name__ == '__main__': + parser = ArgumentParser() + parser = Trainer.add_argparse_args(parser) + + # figure out which model to use + parser.add_argument('--model_name', type=str, default='gan', help='gan or mnist') + temp_args = parser.parse_known_args() + + # let the model add what it wants + if temp_args.model_name == 'gan': + parser = GoodGAN.add_model_specific_args(parser) + elif temp_args.model_name == 'mnist': + parser = CoolMNIST.add_model_specific_args(parser) + + args = parser.parse_args() + + # train + main(args) + +and now we can train MNIST or the gan using the command line interface! + +.. code-block:: bash + + $ python main.py --model_name gan --encoder_layers 24 + $ python main.py --model_name mnist --layer_1_dim 128 + +Hyperparameter Optimization +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Lightning is fully compatible with the hyperparameter optimization libraries! +Here are some useful ones: + +- `Hydra `_ +- `Optuna `_ diff --git a/docs/source/index.rst b/docs/source/index.rst index d44c80ef..91d4e191 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -27,8 +27,8 @@ PyTorch-Lightning Documentation .. toctree:: :maxdepth: 1 - :name: Examples - :caption: Examples + :name: Community Examples + :caption: Community Examples examples @@ -46,17 +46,20 @@ PyTorch-Lightning Documentation apex slurm + child_modules debugging experiment_logging experiment_reporting early_stopping fast_training hooks + hyperparameters multi_gpu weights_loading single_gpu sequences training_tricks + transfer_learning tpu test_set optimizers diff --git a/docs/source/transfer_learning.rst b/docs/source/transfer_learning.rst new file mode 100644 index 00000000..add502db --- /dev/null +++ b/docs/source/transfer_learning.rst @@ -0,0 +1,66 @@ +Transfer Learning +----------------- + +Using Pretrained Models +^^^^^^^^^^^^^^^^^^^^^^^ + +Sometimes we want to use a LightningModule as a pretrained model. This is fine because +a LightningModule is just a `torch.nn.Module`! + +Let's use the `AutoEncoder` as a feature extractor in a separate model. + + +.. code-block:: python + + class Encoder(torch.nn.Module): + ... + + class AutoEncoder(pl.LightningModule): + def __init__(self): + self.encoder = Encoder() + self.decoder = Decoder() + + class CIFAR10Classifier(pl.LightingModule): + def __init__(self): + # init the pretrained LightningModule + self.feature_extractor = AutoEncoder.load_from_checkpoint(PATH) + self.feature_extractor.freeze() + + # the autoencoder outputs a 100-dim representation and CIFAR-10 has 10 classes + self.classifier = nn.Liner(100, 10) + + def forward(self, x): + representations = self.feature_extractor(x) + x = self.classifier(representations) + ... + +We used our pretrained Autoencoder (a LightningModule) for transfer learning! + +Example: BERT (transformers) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Lightning is completely agnostic to what's used for tranfer learning so long +as it is a `torch.nn.Module` subclass. + +.. code-block:: python + + from transformers import BertModel + + class BertMNLIFinetuner(pl.LightningModule): + + def __init__(self): + super(BertMNLIFinetuner, self).__init__() + + self.bert = BertModel.from_pretrained('bert-base-cased', output_attentions=True) + self.W = nn.Linear(bert.config.hidden_size, 3) + self.num_classes = 3 + + + def forward(self, input_ids, attention_mask, token_type_ids): + + h, _, attn = self.bert(input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids) + + h_cls = h[:, 0] + logits = self.W(h_cls) + return logits, attn \ No newline at end of file diff --git a/docs/source/tutorial_mnist.rst b/docs/source/tutorial_mnist.rst index d5a084c5..1dc59178 100644 --- a/docs/source/tutorial_mnist.rst +++ b/docs/source/tutorial_mnist.rst @@ -448,8 +448,59 @@ Notice the epoch is MUCH faster! .. figure:: /_images/mnist_imgs/tpu_fast.png :alt: TPU speed -Validation loop +Hyperparameters --------------- +Normally, we don't hard-code the values to a model. We usually use the command line to +modify the network. The `Trainer` can add all the available options to an ArgumentParser. + +.. code-block:: python + + from argparse import ArgumentParser + + parser = ArgumentParser() + + # parametrize the network + parser.add_argument('--layer_1_dim', type=int, default=128) + parser.add_argument('--layer_1_dim', type=int, default=256) + parser.add_argument('--batch_size', type=int, default=64) + args = parser.parse_args() + +Now we can parametrize the LightningModule. + +.. code-block:: python + :emphasize-lines: 5,6,7,12,14 + + class CoolMNIST(pl.LightningModule): + def __init__(self, hparams): + super(CoolMNIST, self).__init__() + self.hparams = hparams + + self.layer_1 = torch.nn.Linear(28 * 28, hparams.layer_1_dim) + self.layer_2 = torch.nn.Linear(hparams.layer_1_dim, hparams.layer_2_dim) + self.layer_3 = torch.nn.Linear(hparams.layer_2_dim, 10) + + def forward(self, x): + ... + + def train_dataloader(self): + ... + return DataLoader(mnist_train, batch_size=self.hparams.batch_size) + + def configure_optimizers(self): + return Adam(self.parameters(), lr=self.hparams.learning_rate) + + hparams = parse_args() + model = CoolMNIST(hparams) + +.. note:: Bonus! if (hparams) is in your module, Lightning will save it into the checkpoint and restore your + model using those hparams exactly. + +For a full guide on using hyperparameters, `check out the hyperparameters docs `_. + + + +Validating +---------- For most cases, we stop training the model when the performance on a validation split of the data reaches a minimum. @@ -523,8 +574,8 @@ in the validation loop, you won't need to potentially wait a full epoch to find .. note:: Lightning disables gradients, puts model in eval mode and does everything needed for validation. -Testing loop ------------- +Testing +------- Once our research is done and we're about to publish or deploy a model, we normally want to figure out how it will generalize in the "real world." For this, we use a held-out split of the data for testing. @@ -579,6 +630,8 @@ You can also run the test from a saved lightning model .. note:: Lightning disables gradients, puts model in eval mode and does everything needed for testing. +.. warning:: .test() is not stable yet on TPUs. We're working on getting around the multiprocessing challenges. + Predicting ---------- Again, a LightningModule is exactly the same as a PyTorch module. This means you can load it @@ -642,3 +695,95 @@ In this case, we've set this LightningModel to predict logits. But we could also How you split up what goes in `forward` vs `training_step` depends on how you want to use this model for prediction. +Extensibility +------------- +Although lightning makes everything super simple, it doesn't sacrifice any flexibility or control. +Lightning offers multiple ways of managing the training state. + +Training overrides +^^^^^^^^^^^^^^^^^^ + +Any part of the training, validation and testing loop can be modified. +For instance, if you wanted to do your own backward pass, you would override the +default implementation + +.. code-block:: python + + def backward(self, use_amp, loss, optimizer): + if use_amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + +With your own + +.. code-block:: python + + class CoolMNIST(pl.LightningModule): + + def backward(self, use_amp, loss, optimizer): + # do a custom way of backward + loss.backward(retain_graph=True) + +Or if you wanted to initialize ddp in a different way than the default one + +.. code-block:: python + + def configure_ddp(self, model, device_ids): + # Lightning DDP simply routes to test_step, val_step, etc... + model = LightningDistributedDataParallel( + model, + device_ids=device_ids, + find_unused_parameters=True + ) + return model + +you could do your own: + +.. code-block:: python + + class CoolMNIST(pl.LightningModule): + + def configure_ddp(self, model, device_ids): + + model = Horovod(model) + # model = Ray(model) + return model + +Every single part of training is configurable this way. +For a full list look at `lightningModule `_. + + +Callbacks +--------- +Another way to add arbitrary functionality is to add a custom callback +for hooks that you might care about + +.. code-block:: python + + import pytorch_lightning as pl + + class MyPrintingCallback(pl.Callback): + + def on_init_start(self, trainer): + print('Starting to init trainer!') + + def on_init_end(self, trainer): + print('trainer is init now') + + def on_train_end(self, trainer, pl_module): + print('do something when training ends') + +And pass the callbacks into the trainer + +.. code-block:: python + + Trainer(callbacks=[MyPrintingCallback()]) + +.. note:: See full list of 12+ hooks in the `Callback docs `_ + +.. include:: child_modules.rst + +.. include:: transfer_learning.rst +