mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-08-21 11:20:03 +08:00
Adrian WälchliandJirka Borovec
a6de1b8d75
* add doctest to circleci * Revert "add doctest to circleci" This reverts commit c45b34ea911a81f87989f6c3a832b1e8d8c471c6. * Revert "Revert "add doctest to circleci"" This reverts commit 41fca97fdcfe1cf4f6bdb3bbba75d25fa3b11f70. * doctest docs rst files * Revert "doctest docs rst files" This reverts commit b4a2e83e3da5ed1909de500ec14b6b614527c07f. * doctest only rst * doctest debugging.rst * doctest apex * doctest callbacks * doctest early stopping * doctest for child modules * doctest experiment reporting * indentation * doctest fast training * doctest for hyperparams * doctests for lr_finder * doctests multi-gpu * more doctest * make doctest drone * fix label build error * update fast training * update invalid imports * fix problem with int device count * rebase stuff * wip * wip * wip * intro guide * add missing code block * circleci * logger import for doctest * test if doctest runs on drone * fix mnist download * also run install deps for building docs * install cmake * try sudo * hide output * try pip stuff * try to mock horovod * Tranfer -> Transfer * add torchvision to extras * revert pip stuff * mlflow file location * do not mock torch * torchvision * drone extra req. * try higher sphinx version * Revert "try higher sphinx version" This reverts commit 490ac28e46d6fd52352640dfdf0d765befa56988. * try coverage command * try coverage command * try undoc flag * newline * undo drone * report coverage * review Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com> * remove torchvision from extras * skip tests only if torchvision not available * fix testoutput torchvision Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com>
118 lines
3.5 KiB
ReStructuredText
118 lines
3.5 KiB
ReStructuredText
.. testsetup:: *
|
|
|
|
from pytorch_lightning.core.lightning import LightningModule
|
|
|
|
Transfer Learning
|
|
-----------------
|
|
|
|
Using Pretrained Models
|
|
^^^^^^^^^^^^^^^^^^^^^^^
|
|
|
|
Sometimes we want to use a LightningModule as a pretrained model. This is fine because
|
|
a LightningModule is just a `torch.nn.Module`!
|
|
|
|
.. note:: Remember that a LightningModule is EXACTLY a torch.nn.Module but with more capabilities.
|
|
|
|
Let's use the `AutoEncoder` as a feature extractor in a separate model.
|
|
|
|
|
|
.. testcode::
|
|
|
|
class Encoder(torch.nn.Module):
|
|
...
|
|
|
|
class AutoEncoder(LightningModule):
|
|
def __init__(self):
|
|
self.encoder = Encoder()
|
|
self.decoder = Decoder()
|
|
|
|
class CIFAR10Classifier(LightningModule):
|
|
def __init__(self):
|
|
# init the pretrained LightningModule
|
|
self.feature_extractor = AutoEncoder.load_from_checkpoint(PATH)
|
|
self.feature_extractor.freeze()
|
|
|
|
# the autoencoder outputs a 100-dim representation and CIFAR-10 has 10 classes
|
|
self.classifier = nn.Linear(100, 10)
|
|
|
|
def forward(self, x):
|
|
representations = self.feature_extractor(x)
|
|
x = self.classifier(representations)
|
|
...
|
|
|
|
We used our pretrained Autoencoder (a LightningModule) for transfer learning!
|
|
|
|
Example: Imagenet (computer Vision)
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
|
|
.. testcode::
|
|
:skipif: not TORCHVISION_AVAILABLE
|
|
|
|
import torchvision.models as models
|
|
|
|
class ImagenetTransferLearning(LightningModule):
|
|
def __init__(self):
|
|
# init a pretrained resnet
|
|
num_target_classes = 10
|
|
self.feature_extractor = models.resnet50(
|
|
pretrained=True,
|
|
num_classes=num_target_classes)
|
|
self.feature_extractor.eval()
|
|
|
|
# use the pretrained model to classify cifar-10 (10 image classes)
|
|
self.classifier = nn.Linear(2048, num_target_classes)
|
|
|
|
def forward(self, x):
|
|
representations = self.feature_extractor(x)
|
|
x = self.classifier(representations)
|
|
...
|
|
|
|
Finetune
|
|
|
|
.. code-block:: python
|
|
|
|
model = ImagenetTransferLearning()
|
|
trainer = Trainer()
|
|
trainer.fit(model)
|
|
|
|
And use it to predict your data of interest
|
|
|
|
.. code-block:: python
|
|
|
|
model = ImagenetTransferLearning.load_from_checkpoint(PATH)
|
|
model.freeze()
|
|
|
|
x = some_images_from_cifar10()
|
|
predictions = model(x)
|
|
|
|
We used a pretrained model on imagenet, finetuned on CIFAR-10 to predict on CIFAR-10.
|
|
In the non-academic world we would finetune on a tiny dataset you have and predict on your dataset.
|
|
|
|
Example: BERT (NLP)
|
|
^^^^^^^^^^^^^^^^^^^
|
|
Lightning is completely agnostic to what's used for transfer learning so long
|
|
as it is a `torch.nn.Module` subclass.
|
|
|
|
Here's a model that uses `Huggingface transformers <https://github.com/huggingface/transformers>`_.
|
|
|
|
.. testcode::
|
|
|
|
class BertMNLIFinetuner(LightningModule):
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
|
|
self.bert = BertModel.from_pretrained('bert-base-cased', output_attentions=True)
|
|
self.W = nn.Linear(bert.config.hidden_size, 3)
|
|
self.num_classes = 3
|
|
|
|
|
|
def forward(self, input_ids, attention_mask, token_type_ids):
|
|
|
|
h, _, attn = self.bert(input_ids=input_ids,
|
|
attention_mask=attention_mask,
|
|
token_type_ids=token_type_ids)
|
|
|
|
h_cls = h[:, 0]
|
|
logits = self.W(h_cls)
|
|
return logits, attn |