From f6934e5f14589ef3d35cfdae7e3c74eb3020637b Mon Sep 17 00:00:00 2001 From: William Falcon Date: Tue, 3 Mar 2020 16:42:49 -0500 Subject: [PATCH] Docs5 (#1033) * changed path * changed path * changed path * changed path * changed path * changed path * changed path * changed path * changed path * changed path * changed path * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv * added cv --- docs/source/callbacks.rst | 8 ++ docs/source/child_modules.rst | 2 +- docs/source/examples.rst | 2 - docs/source/hooks.rst | 13 +-- docs/source/index.rst | 4 +- docs/source/introduction_guide.rst | 129 ++++++++++++++++++++++++--- docs/source/trainer.rst | 9 +- docs/source/transfer_learning.rst | 57 +++++++++++- pytorch_lightning/core/__init__.py | 83 +++++++++-------- pytorch_lightning/core/lightning.py | 6 +- pytorch_lightning/trainer/trainer.py | 16 ++-- 11 files changed, 250 insertions(+), 79 deletions(-) diff --git a/docs/source/callbacks.rst b/docs/source/callbacks.rst index fe2bf2ec..63012121 100644 --- a/docs/source/callbacks.rst +++ b/docs/source/callbacks.rst @@ -35,6 +35,8 @@ Example We successfully extended functionality without polluting our super clean LightningModule research code +--------- + .. automodule:: pytorch_lightning.callbacks.base :noindex: :exclude-members: @@ -43,6 +45,8 @@ We successfully extended functionality without polluting our super clean Lightni _abc_impl, check_monitor_top_k, +--------- + .. automodule:: pytorch_lightning.callbacks.early_stopping :noindex: :exclude-members: @@ -51,6 +55,8 @@ We successfully extended functionality without polluting our super clean Lightni _abc_impl, check_monitor_top_k, +--------- + .. automodule:: pytorch_lightning.callbacks.model_checkpoint :noindex: :exclude-members: @@ -59,6 +65,8 @@ We successfully extended functionality without polluting our super clean Lightni _abc_impl, check_monitor_top_k, +--------- + .. automodule:: pytorch_lightning.callbacks.gradient_accumulation_scheduler :noindex: :exclude-members: diff --git a/docs/source/child_modules.rst b/docs/source/child_modules.rst index 33a25cb3..6360171c 100644 --- a/docs/source/child_modules.rst +++ b/docs/source/child_modules.rst @@ -7,7 +7,7 @@ For example, imaging we now want to train an Autoencoder to use as a feature ext Recall that `CoolMNIST` already defines all the dataloading etc... The only things that change in the `Autoencoder` model are the init, forward, training, validation and test step. -.. code-block:: +.. code-block:: python class Encoder(torch.nn.Module): ... diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 9bc77a21..54d6a550 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -3,8 +3,6 @@ :name: Community Examples :caption: Community Examples - - Fast Aging GAN Generative Adversarial Network Hyperparameter optimization with Optuna Recurrent Attentive Neural Process diff --git a/docs/source/hooks.rst b/docs/source/hooks.rst index eb6d9f6c..50468c70 100644 --- a/docs/source/hooks.rst +++ b/docs/source/hooks.rst @@ -3,11 +3,12 @@ Hooks .. automodule:: pytorch_lightning.core.hooks -Full list of hooks - +Hooks lifecycle +--------------- Training set-up -================ +^^^^^^^^^^^^^^^ + - init_ddp_connection - init_optimizers - configure_apex @@ -19,7 +20,7 @@ Training set-up - restore_weights Training loop -============= +^^^^^^^^^^^^^ - on_epoch_start - on_batch_start @@ -33,7 +34,7 @@ Training loop - on_epoch_end Validation loop -=============== +^^^^^^^^^^^^^^^ - model.zero_grad() - model.eval() @@ -45,7 +46,7 @@ Validation loop - on_post_performance_check Test loop -========= +^^^^^^^^^ - model.zero_grad() - model.eval() diff --git a/docs/source/index.rst b/docs/source/index.rst index 54b47bf4..7f2689b0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -56,14 +56,14 @@ PyTorch-Lightning Documentation hyperparameters multi_gpu weights_loading + optimizers + profiler single_gpu sequences training_tricks transfer_learning tpu test_set - optimizers - profiler .. toctree:: :maxdepth: 1 diff --git a/docs/source/introduction_guide.rst b/docs/source/introduction_guide.rst index 1dc59178..271a08d5 100644 --- a/docs/source/introduction_guide.rst +++ b/docs/source/introduction_guide.rst @@ -11,6 +11,8 @@ To illustrate, here's the typical PyTorch project structure organized in a Light As your project grows in complexity with things like 16-bit precision, distributed training, etc... the part in blue quickly becomes onerous and starts distracting from the core research code. +--------- + Goal of this guide ------------------ This guide walks through the major parts of the library to help you understand @@ -18,26 +20,39 @@ what each parts does. But at the end of the day, you write the same PyTorch code into the LightningModule template which means you keep ALL the flexibility without having to deal with any of the boilerplate code -To show how Lightning works, we'll start with an MNIST classifier and move into -a Variational Autoencoder and a Generative Adversarial Network (GAN). +To show how Lightning works, we'll start with an MNIST classifier. We'll end showing how +to use inheritance to very quickly create an AutoEncoder. .. note:: Any DL/ML PyTorch project fits into the Lightning structure. Here we just focus on 3 types of research to illustrate. +--------- + Lightning Philosophy -------------------- Lightning factors DL/ML code into three types: -1. Core research code. -2. Engineering code. -3. Non-essential research code. +- Research code +- Engineerng code +- Non-essential code Research code ^^^^^^^^^^^^^ In the MNIST generation example, the research code would be the particular system and how it's trained (ie: A GAN or VAE). - In Lightning, this code is abstracted out by the `LightningModule`. +.. code-block:: python + + l1 = nn.Linear(...) + l2 = nn.Linear(...) + decoder = Decoder() + + x1 = l1(x) + x2 = l2(x2) + out = decoder(features, x) + + loss = perceptual_loss(x1, x2, x) + CE(out, x) + Engineering code ^^^^^^^^^^^^^^^^ @@ -46,6 +61,18 @@ over GPUs, 16-bit precision, etc. This is normally code that is THE SAME across In Lightning, this code is abstracted out by the `Trainer`. +.. code-block:: python + + model.cuda(0) + x = x.cuda(0) + + distributed = DistributedParallel(model) + + with gpu_zero: + download_data() + + dist.barrier() + Non-essential code ^^^^^^^^^^^^^^^^^^ This is code that helps the research but isn't relevant to the research code. Some examples might be: @@ -54,6 +81,15 @@ This is code that helps the research but isn't relevant to the research code. So In Lightning this code is abstracted out by `Callbacks`. +.. code-block:: python + + # log samples + z = Q.rsample() + generated = decoder(z) + self.experiment.log('images', generated) + +--------- + Elements of a research project ------------------------------ Every research project requires the same core ingredients: @@ -244,6 +280,8 @@ in the LightningModule Again, this is the same PyTorch code except that it has been organized by the LightningModule. This code is not restricted which means it can be as complicated as a full seq-2-seq, RL loop, GAN, etc... +--------- + Training -------- So far we defined 4 key ingredients in pure PyTorch but organized the code inside the LightningModule. @@ -295,6 +333,9 @@ For clarity, we'll recall that the full LightningModule now looks like this. Again, this is the same PyTorch code, except that it's organized by the LightningModule. This organization now lets us train this model +Train on CPU +^^^^^^^^^^^^ + .. code-block:: python from pytorch_lightning import Trainer @@ -308,6 +349,9 @@ You should see the following weights summary and progress bar .. figure:: /_images/mnist_imgs/mnist_cpu_bar.png :alt: mnist CPU bar +Logging +^^^^^^^ + When we added the `log` key in the return dictionary it went into the built in tensorboard logger. But you could have also logged by calling: @@ -323,6 +367,10 @@ Which will generate automatic tensorboard logs. .. figure:: /_images/mnist_imgs/mnist_tb.png :alt: mnist CPU bar +But you can also use any of the `number of other loggers `_ we support. + +GPU training +^^^^^^^^^^^^ But the beauty is all the magic you can do with the trainer flags. For instance, to run this model on a GPU: @@ -336,7 +384,10 @@ But the beauty is all the magic you can do with the trainer flags. For instance, .. figure:: /_images/mnist_imgs/mnist_gpu.png :alt: mnist GPU bar -Or you can also train on multiple GPUs (not on colab though) +Multi-GPU training +^^^^^^^^^^^^^^^^^^ + +Or you can also train on multiple GPUs. .. code-block:: python @@ -353,7 +404,15 @@ Or multiple nodes trainer = Trainer(gpus=8, num_nodes=4, distributed_backend='ddp') trainer.fit(model) -And even TPUs. Let's do it on the colab! +Refer to the `distributed computing guide for more details `_. + +TPUs +^^^^ +Did you know you can use PyTorch on TPUs? It's very hard to do, but we've +worked with the xla team to use their awesome library to get this to work +out of the box! + +Let's train on Colab (`full demo available here `_) First, change the runtime to TPU (and reinstall lightning). @@ -448,6 +507,8 @@ Notice the epoch is MUCH faster! .. figure:: /_images/mnist_imgs/tpu_fast.png :alt: TPU speed +--------- + Hyperparameters --------------- Normally, we don't hard-code the values to a model. We usually use the command line to @@ -497,7 +558,7 @@ Now we can parametrize the LightningModule. For a full guide on using hyperparameters, `check out the hyperparameters docs `_. - +--------- Validating ---------- @@ -574,6 +635,8 @@ in the validation loop, you won't need to potentially wait a full epoch to find .. note:: Lightning disables gradients, puts model in eval mode and does everything needed for validation. +--------- + Testing ------- Once our research is done and we're about to publish or deploy a model, we normally want to figure out @@ -632,6 +695,8 @@ You can also run the test from a saved lightning model .. warning:: .test() is not stable yet on TPUs. We're working on getting around the multiprocessing challenges. +--------- + Predicting ---------- Again, a LightningModule is exactly the same as a PyTorch module. This means you can load it @@ -649,7 +714,7 @@ within it. .. code-block:: python - class CoolMNIST(pl.LightningModule): + class MNISTClassifier(pl.LightningModule): def forward(self, x): batch_size, channels, width, height = x.size() @@ -668,11 +733,17 @@ within it. loss = F.nll_loss(logits, y) return loss +.. code-block:: python + + model = MNISTClassifier() + x = mnist_image() + logits = model(x) + In this case, we've set this LightningModel to predict logits. But we could also have it predict feature maps: .. code-block:: python - class CoolMNIST(pl.LightningModule): + class MNISTRepresentator(pl.LightningModule): def forward(self, x): batch_size, channels, width, height = x.size() @@ -692,9 +763,41 @@ In this case, we've set this LightningModel to predict logits. But we could also loss = perceptual_loss(l1_feats, l2_feats, l3_feats) + ce_loss return loss +.. code-block:: python + + model = MNISTRepresentator.load_from_checkpoint(PATH) + x = mnist_image() + feature_maps = model(x) + +Or maybe we have a model that we use to do generation + +.. code-block:: python + + class CoolMNISTDreamer(pl.LightningModule): + + def forward(self, z): + imgs = self.decoder(z) + return imgs + + def training_step(self, batch, batch_idx): + x, y = batch + representation = self.encoder(x) + imgs = self.forward(representation) + + loss = perceptual_loss(imgs, x) + return loss + +.. code-block:: python + + model = CoolMNISTDreamer.load_from_checkpoint(PATH) + z = sample_noise() + generated_imgs = model(z) + How you split up what goes in `forward` vs `training_step` depends on how you want to use this model for prediction. +--------- + Extensibility ------------- Although lightning makes everything super simple, it doesn't sacrifice any flexibility or control. @@ -783,7 +886,11 @@ And pass the callbacks into the trainer .. note:: See full list of 12+ hooks in the `Callback docs `_ +--------- + .. include:: child_modules.rst +--------- + .. include:: transfer_learning.rst diff --git a/docs/source/trainer.rst b/docs/source/trainer.rst index a9f65ba9..e83bd94e 100644 --- a/docs/source/trainer.rst +++ b/docs/source/trainer.rst @@ -7,10 +7,11 @@ Trainer .. automodule:: pytorch_lightning.trainer :members: fit, test :noindex: - :exclude-members: - run_pretrain_routine, - _abc_impl, - _Trainer__set_root_gpu, + :exclude-members: + run_pretrain_routine, + _abc_impl, + _Trainer__set_random_port, + _Trainer__set_root_gpu, _Trainer__init_optimizers, _Trainer__parse_gpu_ids, _Trainer__configure_schedulers, diff --git a/docs/source/transfer_learning.rst b/docs/source/transfer_learning.rst index add502db..9737d7d8 100644 --- a/docs/source/transfer_learning.rst +++ b/docs/source/transfer_learning.rst @@ -7,6 +7,8 @@ Using Pretrained Models Sometimes we want to use a LightningModule as a pretrained model. This is fine because a LightningModule is just a `torch.nn.Module`! +.. note:: Remember that a pl.LightningModule is EXACTLY a torch.nn.Module but with more capabilities. + Let's use the `AutoEncoder` as a feature extractor in a separate model. @@ -27,7 +29,7 @@ Let's use the `AutoEncoder` as a feature extractor in a separate model. self.feature_extractor.freeze() # the autoencoder outputs a 100-dim representation and CIFAR-10 has 10 classes - self.classifier = nn.Liner(100, 10) + self.classifier = nn.Linear(100, 10) def forward(self, x): representations = self.feature_extractor(x) @@ -36,11 +38,58 @@ Let's use the `AutoEncoder` as a feature extractor in a separate model. We used our pretrained Autoencoder (a LightningModule) for transfer learning! -Example: BERT (transformers) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Lightning is completely agnostic to what's used for tranfer learning so long +Example: Imagenet (computer Vision) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: python + + import torchvision.models as models + + class ImagenetTranferLearning(pl.LightingModule): + def __init__(self): + # init a pretrained resnet + num_target_classes = 10 + self.feature_extractor = model.resnet50( + pretrained=True, + num_classes=num_target_classes) + self.feature_extractor.eval() + + # use the pretrained model to classify cifar-10 (10 image classes) + self.classifier = nn.Linear(2048, num_target_classes) + + def forward(self, x): + representations = self.feature_extractor(x) + x = self.classifier(representations) + ... + +Finetune + +.. code-block:: python + + model = ImagenetTranferLearning() + trainer = Trainer() + trainer.fit(model) + +And use it to predict your data of interest + +.. code-block:: python + + model = ImagenetTranferLearning.load_from_checkpoint(PATH) + model.freeze() + + x = some_images_from_cifar10() + predictions = model(x) + +We used a pretrained model on imagenet, finetuned on CIFAR-10 to predict on CIFAR-10. +In the non-academic world we would finetune on a tiny dataset you have and predict on your dataset. + +Example: BERT (NLP) +^^^^^^^^^^^^^^^^^^^ +Lightning is completely agnostic to what's used for transfer learning so long as it is a `torch.nn.Module` subclass. +Here's a model that uses `Huggingface transformers `_. + .. code-block:: python from transformers import BertModel diff --git a/pytorch_lightning/core/__init__.py b/pytorch_lightning/core/__init__.py index a2fb7110..40eef1cd 100644 --- a/pytorch_lightning/core/__init__.py +++ b/pytorch_lightning/core/__init__.py @@ -37,52 +37,14 @@ Most methods are optional. Here's a minimal example. y_hat = self.forward(x) return {'loss': F.cross_entropy(y_hat, y)} - def validation_step(self, batch, batch_idx): - # OPTIONAL - x, y = batch - y_hat = self.forward(x) - return {'val_loss': F.cross_entropy(y_hat, y)} - - def validation_end(self, outputs): - # OPTIONAL - val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() - return {'val_loss': val_loss_mean} - - def test_step(self, batch, batch_idx): - # OPTIONAL - x, y = batch - y_hat = self.forward(x) - return {'test_loss': F.cross_entropy(y_hat, y)} - - def test_end(self, outputs): - # OPTIONAL - test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean() - return {'test_loss': test_loss_mean} - def configure_optimizers(self): - # REQUIRED return torch.optim.Adam(self.parameters(), lr=0.02) - @pl.data_loader def train_dataloader(self): return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) - @pl.data_loader - def val_dataloader(self): - # OPTIONAL - # can also return a list of val dataloaders - return DataLoader(MNIST(os.getcwd(), train=True, download=True, - transform=transforms.ToTensor()), batch_size=32) - - @pl.data_loader - def test_dataloader(self): - # OPTIONAL - # can also return a list of test dataloaders - return DataLoader(MNIST(os.getcwd(), train=False, download=True, - transform=transforms.ToTensor()), batch_size=32) - -Once you've defined the LightningModule, fit it using a trainer. +Which you can train by doing: .. code-block:: python @@ -91,10 +53,53 @@ Once you've defined the LightningModule, fit it using a trainer. trainer.fit(model) +If you wanted to add a validation loop + +.. code-block:: python + + class CoolModel(pl.LightningModule): + def validation_step(self, batch, batch_idx): + x, y = batch + y_hat = self.forward(x) + return {'val_loss': F.cross_entropy(y_hat, y)} + + def validation_end(self, outputs): + val_loss_mean = torch.stack([x['val_loss'] for x in outputs]).mean() + return {'val_loss': val_loss_mean} + + def val_dataloader(self): + # can also return a list of val dataloaders + return DataLoader(MNIST(os.getcwd(), train=True, download=True, + transform=transforms.ToTensor()), batch_size=32) + +Or add a test loop + +.. code_block:: python + + class CoolModel(pl.LightningModule): + + def test_step(self, batch, batch_idx): + x, y = batch + y_hat = self.forward(x) + return {'test_loss': F.cross_entropy(y_hat, y)} + + def test_end(self, outputs): + test_loss_mean = torch.stack([x['test_loss'] for x in outputs]).mean() + return {'test_loss': test_loss_mean} + + def test_dataloader(self): + # OPTIONAL + # can also return a list of test dataloaders + return DataLoader(MNIST(os.getcwd(), train=False, download=True, + transform=transforms.ToTensor()), batch_size=32) + Check out this `COLAB `_ for a live demo. +.. note:: Remove all .cuda() or .to() calls from LightningModules. See: + `the multi-gpu training guide for details `_. + """ from .decorators import data_loader diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index 97a10319..8529a404 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -268,6 +268,8 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks): loss = nce_loss(loss) return {'loss': loss} + .. note:: see the `multi-gpu guide for more details `_. + If you define multiple optimizers, this step will also be called with an additional `optimizer_idx` param. .. code-block:: python @@ -280,7 +282,7 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks): # do training_step with decoder If you add truncated back propagation through time you will also get an additional argument - with the hidden states of the previous step. + with the hidden states of the previous step. .. code-block:: python @@ -1248,7 +1250,7 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks): self.eval() def unfreeze(self): - """Unfreeze all params for inference. + """Unfreeze all params for training. .. code-block:: python diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 33daf8de..04e9010c 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -276,16 +276,16 @@ class Trainer(TrainerIOMixin, # -1: train on all available TPUs trainer = Trainer(num_tpu_cores=-1) - To train on more than 8 cores (ie: a POD), - submit this script using the xla_dist script. + To train on more than 8 cores (ie: a POD), + submit this script using the xla_dist script. - Example:: + Example:: - $ python -m torch_xla.distributed.xla_dist - --tpu=$TPU_POD_NAME - --conda-env=torch-xla-nightly - --env=XLA_USE_BF16=1 - -- python your_trainer_file.py + $ python -m torch_xla.distributed.xla_dist + --tpu=$TPU_POD_NAME + --conda-env=torch-xla-nightly + --env=XLA_USE_BF16=1 + -- python your_trainer_file.py log_gpu_memory: None, 'min_max', 'all'. Might slow performance because it uses the output of nvidia-smi.