mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-08-21 11:20:03 +08:00
Adrian WälchliandJirka Borovec
a6de1b8d75
* add doctest to circleci * Revert "add doctest to circleci" This reverts commit c45b34ea911a81f87989f6c3a832b1e8d8c471c6. * Revert "Revert "add doctest to circleci"" This reverts commit 41fca97fdcfe1cf4f6bdb3bbba75d25fa3b11f70. * doctest docs rst files * Revert "doctest docs rst files" This reverts commit b4a2e83e3da5ed1909de500ec14b6b614527c07f. * doctest only rst * doctest debugging.rst * doctest apex * doctest callbacks * doctest early stopping * doctest for child modules * doctest experiment reporting * indentation * doctest fast training * doctest for hyperparams * doctests for lr_finder * doctests multi-gpu * more doctest * make doctest drone * fix label build error * update fast training * update invalid imports * fix problem with int device count * rebase stuff * wip * wip * wip * intro guide * add missing code block * circleci * logger import for doctest * test if doctest runs on drone * fix mnist download * also run install deps for building docs * install cmake * try sudo * hide output * try pip stuff * try to mock horovod * Tranfer -> Transfer * add torchvision to extras * revert pip stuff * mlflow file location * do not mock torch * torchvision * drone extra req. * try higher sphinx version * Revert "try higher sphinx version" This reverts commit 490ac28e46d6fd52352640dfdf0d765befa56988. * try coverage command * try coverage command * try undoc flag * newline * undo drone * report coverage * review Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com> * remove torchvision from extras * skip tests only if torchvision not available * fix testoutput torchvision Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com>
120 lines
3.6 KiB
ReStructuredText
120 lines
3.6 KiB
ReStructuredText
Optimization
|
|
===============
|
|
|
|
Learning rate scheduling
|
|
-------------------------------------
|
|
Every optimizer you use can be paired with any `LearningRateScheduler <https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate>`_.
|
|
|
|
.. testcode::
|
|
|
|
# no LR scheduler
|
|
def configure_optimizers(self):
|
|
return Adam(...)
|
|
|
|
# Adam + LR scheduler
|
|
def configure_optimizers(self):
|
|
optimizer = Adam(...)
|
|
scheduler = ReduceLROnPlateau(optimizer, ...)
|
|
return [optimizer], [scheduler]
|
|
|
|
# Two optimziers each with a scheduler
|
|
def configure_optimizers(self):
|
|
optimizer1 = Adam(...)
|
|
optimizer2 = SGD(...)
|
|
scheduler1 = ReduceLROnPlateau(optimizer1, ...)
|
|
scheduler2 = LambdaLR(optimizer2, ...)
|
|
return [optimizer1, optimizer2], [scheduler1, scheduler2]
|
|
|
|
# Same as above with additional params passed to the first scheduler
|
|
def configure_optimizers(self):
|
|
optimizers = [Adam(...), SGD(...)]
|
|
schedulers = [
|
|
{
|
|
'scheduler': ReduceLROnPlateau(optimizers[0], ...),
|
|
'monitor': 'val_recall', # Default: val_loss
|
|
'interval': 'epoch',
|
|
'frequency': 1
|
|
},
|
|
LambdaLR(optimizers[1], ...)
|
|
]
|
|
return optimizers, schedulers
|
|
|
|
|
|
Use multiple optimizers (like GANs)
|
|
-------------------------------------
|
|
To use multiple optimizers return > 1 optimizers from :meth:`pytorch_lightning.core.LightningModule.configure_optimizers`
|
|
|
|
.. testcode::
|
|
|
|
# one optimizer
|
|
def configure_optimizers(self):
|
|
return Adam(...)
|
|
|
|
# two optimizers, no schedulers
|
|
def configure_optimizers(self):
|
|
return Adam(...), SGD(...)
|
|
|
|
# Two optimizers, one scheduler for adam only
|
|
def configure_optimizers(self):
|
|
return [Adam(...), SGD(...)], [ReduceLROnPlateau()]
|
|
|
|
Lightning will call each optimizer sequentially:
|
|
|
|
.. code-block:: python
|
|
|
|
for epoch in epochs:
|
|
for batch in data:
|
|
for opt in optimizers:
|
|
train_step(opt)
|
|
opt.step()
|
|
|
|
for scheduler in scheduler:
|
|
scheduler.step()
|
|
|
|
|
|
Step optimizers at arbitrary intervals
|
|
----------------------------------------
|
|
To do more interesting things with your optimizers such as learning rate warm-up or odd scheduling,
|
|
override the :meth:`optimizer_step` function.
|
|
|
|
For example, here step optimizer A every 2 batches and optimizer B every 4 batches
|
|
|
|
.. testcode::
|
|
|
|
def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None):
|
|
optimizer.step()
|
|
optimizer.zero_grad()
|
|
|
|
# Alternating schedule for optimizer steps (ie: GANs)
|
|
def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None):
|
|
# update generator opt every 2 steps
|
|
if optimizer_i == 0:
|
|
if batch_nb % 2 == 0 :
|
|
optimizer.step()
|
|
optimizer.zero_grad()
|
|
|
|
# update discriminator opt every 4 steps
|
|
if optimizer_i == 1:
|
|
if batch_nb % 4 == 0 :
|
|
optimizer.step()
|
|
optimizer.zero_grad()
|
|
|
|
# ...
|
|
# add as many optimizers as you want
|
|
|
|
Here we add a learning-rate warm up
|
|
|
|
.. testcode::
|
|
|
|
# learning rate warm-up
|
|
def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i, second_order_closure=None):
|
|
# warm up lr
|
|
if self.trainer.global_step < 500:
|
|
lr_scale = min(1., float(self.trainer.global_step + 1) / 500.)
|
|
for pg in optimizer.param_groups:
|
|
pg['lr'] = lr_scale * self.hparams.learning_rate
|
|
|
|
# update params
|
|
optimizer.step()
|
|
optimizer.zero_grad()
|