wassname/pytorch-lightning
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-lightning.git synced 2026-09-12 12:40:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Checkpointing interval (#1272)

Browse Source 
* formatting

* formatting

* fix interval

* fix train loop

* fix test

* parametrize test

* Apply suggestions from code review

Co-Authored-By: Adrian Wälchli <adrian.waelchli@students.unibe.ch>

* fix calling

* flake8

* add types

Co-authored-by: Adrian Wälchli <adrian.waelchli@students.unibe.ch>
Co-authored-by: William Falcon <waf2107@columbia.edu>
This commit is contained in:
Jirka Borovec
2020-03-30 18:37:02 -04:00
committed by GitHub
co-authored by Adrian Wälchli William Falcon
parent 3476d2f279
commit 09167efdb5
15 changed files with 162 additions and 294 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pytorch_lightning/trainer/training_loop.py
+20 -13
View File
@@ -461,8 +461,7 @@ class TrainerTrainLoopMixin(ABC):
# CHECKPOINTING, EARLY STOPPING
# ---------------
# save checkpoint even when no test or val step are defined
train_step_only = not self.is_overriden('validation_step')
if self.fast_dev_run or should_check_val or train_step_only:
if self.fast_dev_run or should_check_val:
self.call_checkpoint_callback()
if self.enable_early_stop:
@@ -483,6 +482,13 @@ class TrainerTrainLoopMixin(ABC):
if early_stop_epoch or self.fast_dev_run:
break
# in case validation step is missing and you are not running fast-dev to duplicate last batch
if not self.is_overriden('validation_step') and not (self.fast_dev_run or should_check_val):
self.call_checkpoint_callback()
if self.enable_early_stop:
self.early_stop_callback.check_metrics(self.callback_metrics)
# Epoch end events
with self.profiler.profile('on_epoch_end'):
# callbacks
@@ -709,20 +715,20 @@ class TrainerTrainLoopMixin(ABC):
with self.profiler.profile('training_end'):
output = model_ref.training_end(output)
m = 'training_end was deprecated in 0.7.0 and will be removed 1.0.0. ' \
'Use training_epoch_end instead'
warnings.warn(m, DeprecationWarning)
warnings.warn('`training_end` was deprecated in 0.7.0 and will be removed 1.0.0.'
' Use training_epoch_end instead', DeprecationWarning)
# format and reduce outputs accordingly
output = self.process_output(output, train=True)
return output
def update_learning_rates(self, interval):
''' Update learning rates
def update_learning_rates(self, interval: str):
"""Update learning rates.
Args:
interval (str): either 'epoch' or 'step'.
'''
interval: either 'epoch' or 'step'.
"""
if not self.lr_schedulers:
return
@@ -738,10 +744,11 @@ class TrainerTrainLoopMixin(ABC):
monitor_val = self.callback_metrics.get(monitor_key)
if monitor_val is None:
avail_metrics = ','.join(list(self.callback_metrics.keys()))
m = f'ReduceLROnPlateau conditioned on metric {monitor_key} ' \
f'which is not available. Available metrics are: {avail_metrics}. ' \
'Condition can be set using `monitor` key in lr scheduler dict'
raise MisconfigurationException(m)
raise MisconfigurationException(
f'ReduceLROnPlateau conditioned on metric {monitor_key}'
f' which is not available. Available metrics are: {avail_metrics}.'
' Condition can be set using `monitor` key in lr scheduler dict'
)
lr_scheduler['scheduler'].step(monitor_val)
else:
lr_scheduler['scheduler'].step()