mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-22 13:30:11 +08:00
3e8f2d99a9
* squash and rebase sanity check hooks sanity check callback hook finish moved core progress bar functionality into callback wip remove duplicate merge clean up imports docs sanity check progress bar main sanity move callback calls init progrss bar callback configuration and docs changelog rate decorator pass process_position disable on rank > 0 position index is_enabled remove decorator refactor init tqdm bars callback method ordering cannot reset when disabled sequence -> list default values fix has no attr _time() move on_val_end to proper place fix the pickle issue update warning properties check for None remove old comment switch order pull out non-tqdm functionality into base class documentation for the base class docs fix refresh rate issue in validation restrict type hint of trainer arg more docs update trainer docs rst docs fix lines too long fix test add missing type hints fix typo move docstring to __init__ solves doctest failures remove doctest :(( can't fix the pickle error fix example simplify by saving trainer reference fix docs errors move docstring initial value multiple val checks per epoch simpler handling of inf dataset sizes update inf docs renamed training_tqdm_dict rename get_tqdm_dict rename occurences of tqdm update changelog fix doctest fix formatting errors added callback tests progress bar on off test more tests for progress bar weird test fix? add ignored property disable default progress bar in LR finder change enable/disable behavior trying doctest in CI again undo doctest pickle error undo doctest pickle error :(( remove progress_bar_callback Trainer arg and fix tests restore progress bar after auto lr find update docs fix rebase fix wrong negation * fix fast dev run total * more thorough testing * remove old args * fix merge * fix merge * separate tests * type hint total batches * reduce if Co-Authored-By: Jirka Borovec <Borda@users.noreply.github.com> * is_disabled Co-Authored-By: Jirka Borovec <Borda@users.noreply.github.com> * is_enabled Co-Authored-By: Jirka Borovec <Borda@users.noreply.github.com> * rename enabled/disabled * move deprecated api * remove duplicated test from merge * fix rename is_disabled * newline * test also testprogress for fast dev run Co-authored-by: J. Borovec <jirka.borovec@seznam.cz> Co-authored-by: Jirka Borovec <Borda@users.noreply.github.com>
265 lines
9.7 KiB
Python
265 lines
9.7 KiB
Python
import tests.base.utils as tutils
|
|
from pytorch_lightning import Callback
|
|
from pytorch_lightning import Trainer, LightningModule
|
|
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
|
|
from tests.base import (
|
|
LightTrainDataloader,
|
|
LightTestMixin,
|
|
LightValidationMixin,
|
|
TestModelBase
|
|
)
|
|
|
|
|
|
def test_trainer_callback_system(tmpdir):
|
|
"""Test the callback system."""
|
|
|
|
class CurrentTestModel(
|
|
LightTrainDataloader,
|
|
LightTestMixin,
|
|
LightValidationMixin,
|
|
TestModelBase,
|
|
):
|
|
pass
|
|
|
|
hparams = tutils.get_default_hparams()
|
|
model = CurrentTestModel(hparams)
|
|
|
|
def _check_args(trainer, pl_module):
|
|
assert isinstance(trainer, Trainer)
|
|
assert isinstance(pl_module, LightningModule)
|
|
|
|
class TestCallback(Callback):
|
|
def __init__(self):
|
|
super().__init__()
|
|
self.on_init_start_called = False
|
|
self.on_init_end_called = False
|
|
self.on_sanity_check_start_called = False
|
|
self.on_sanity_check_end_called = False
|
|
self.on_epoch_start_called = False
|
|
self.on_epoch_end_called = False
|
|
self.on_batch_start_called = False
|
|
self.on_batch_end_called = False
|
|
self.on_validation_batch_start_called = False
|
|
self.on_validation_batch_end_called = False
|
|
self.on_test_batch_start_called = False
|
|
self.on_test_batch_end_called = False
|
|
self.on_train_start_called = False
|
|
self.on_train_end_called = False
|
|
self.on_validation_start_called = False
|
|
self.on_validation_end_called = False
|
|
self.on_test_start_called = False
|
|
self.on_test_end_called = False
|
|
|
|
def on_init_start(self, trainer):
|
|
assert isinstance(trainer, Trainer)
|
|
self.on_init_start_called = True
|
|
|
|
def on_init_end(self, trainer):
|
|
assert isinstance(trainer, Trainer)
|
|
self.on_init_end_called = True
|
|
|
|
def on_sanity_check_start(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_sanity_check_start_called = True
|
|
|
|
def on_sanity_check_end(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_sanity_check_end_called = True
|
|
|
|
def on_epoch_start(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_epoch_start_called = True
|
|
|
|
def on_epoch_end(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_epoch_end_called = True
|
|
|
|
def on_batch_start(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_batch_start_called = True
|
|
|
|
def on_batch_end(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_batch_end_called = True
|
|
|
|
def on_validation_batch_start(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_validation_batch_start_called = True
|
|
|
|
def on_validation_batch_end(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_validation_batch_end_called = True
|
|
|
|
def on_test_batch_start(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_test_batch_start_called = True
|
|
|
|
def on_test_batch_end(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_test_batch_end_called = True
|
|
|
|
def on_train_start(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_train_start_called = True
|
|
|
|
def on_train_end(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_train_end_called = True
|
|
|
|
def on_validation_start(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_validation_start_called = True
|
|
|
|
def on_validation_end(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_validation_end_called = True
|
|
|
|
def on_test_start(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_test_start_called = True
|
|
|
|
def on_test_end(self, trainer, pl_module):
|
|
_check_args(trainer, pl_module)
|
|
self.on_test_end_called = True
|
|
|
|
test_callback = TestCallback()
|
|
|
|
trainer_options = {
|
|
'callbacks': [test_callback],
|
|
'max_epochs': 1,
|
|
'val_percent_check': 0.1,
|
|
'train_percent_check': 0.2,
|
|
'progress_bar_refresh_rate': 0
|
|
}
|
|
|
|
assert not test_callback.on_init_start_called
|
|
assert not test_callback.on_init_end_called
|
|
assert not test_callback.on_sanity_check_start_called
|
|
assert not test_callback.on_sanity_check_end_called
|
|
assert not test_callback.on_epoch_start_called
|
|
assert not test_callback.on_epoch_start_called
|
|
assert not test_callback.on_batch_start_called
|
|
assert not test_callback.on_batch_end_called
|
|
assert not test_callback.on_validation_batch_start_called
|
|
assert not test_callback.on_validation_batch_end_called
|
|
assert not test_callback.on_test_batch_start_called
|
|
assert not test_callback.on_test_batch_end_called
|
|
assert not test_callback.on_train_start_called
|
|
assert not test_callback.on_train_end_called
|
|
assert not test_callback.on_validation_start_called
|
|
assert not test_callback.on_validation_end_called
|
|
assert not test_callback.on_test_start_called
|
|
assert not test_callback.on_test_end_called
|
|
|
|
# fit model
|
|
trainer = Trainer(**trainer_options)
|
|
|
|
assert trainer.callbacks[0] == test_callback
|
|
assert test_callback.on_init_start_called
|
|
assert test_callback.on_init_end_called
|
|
assert not test_callback.on_sanity_check_start_called
|
|
assert not test_callback.on_sanity_check_end_called
|
|
assert not test_callback.on_epoch_start_called
|
|
assert not test_callback.on_epoch_start_called
|
|
assert not test_callback.on_batch_start_called
|
|
assert not test_callback.on_batch_end_called
|
|
assert not test_callback.on_validation_batch_start_called
|
|
assert not test_callback.on_validation_batch_end_called
|
|
assert not test_callback.on_test_batch_start_called
|
|
assert not test_callback.on_test_batch_end_called
|
|
assert not test_callback.on_train_start_called
|
|
assert not test_callback.on_train_end_called
|
|
assert not test_callback.on_validation_start_called
|
|
assert not test_callback.on_validation_end_called
|
|
assert not test_callback.on_test_start_called
|
|
assert not test_callback.on_test_end_called
|
|
|
|
trainer.fit(model)
|
|
|
|
assert test_callback.on_init_start_called
|
|
assert test_callback.on_init_end_called
|
|
assert test_callback.on_sanity_check_start_called
|
|
assert test_callback.on_sanity_check_end_called
|
|
assert test_callback.on_epoch_start_called
|
|
assert test_callback.on_epoch_start_called
|
|
assert test_callback.on_batch_start_called
|
|
assert test_callback.on_batch_end_called
|
|
assert test_callback.on_validation_batch_start_called
|
|
assert test_callback.on_validation_batch_end_called
|
|
assert test_callback.on_train_start_called
|
|
assert test_callback.on_train_end_called
|
|
assert test_callback.on_validation_start_called
|
|
assert test_callback.on_validation_end_called
|
|
assert not test_callback.on_test_batch_start_called
|
|
assert not test_callback.on_test_batch_end_called
|
|
assert not test_callback.on_test_start_called
|
|
assert not test_callback.on_test_end_called
|
|
|
|
test_callback = TestCallback()
|
|
trainer_options['callbacks'] = [test_callback]
|
|
trainer = Trainer(**trainer_options)
|
|
trainer.test(model)
|
|
|
|
assert test_callback.on_test_batch_start_called
|
|
assert test_callback.on_test_batch_end_called
|
|
assert test_callback.on_test_start_called
|
|
assert test_callback.on_test_end_called
|
|
assert not test_callback.on_validation_start_called
|
|
assert not test_callback.on_validation_end_called
|
|
assert not test_callback.on_validation_batch_end_called
|
|
assert not test_callback.on_validation_batch_start_called
|
|
|
|
|
|
def test_early_stopping_no_val_step(tmpdir):
|
|
"""Test that early stopping callback falls back to training metrics when no validation defined."""
|
|
tutils.reset_seed()
|
|
|
|
class ModelWithoutValStep(LightTrainDataloader, TestModelBase):
|
|
|
|
def training_step(self, *args, **kwargs):
|
|
output = super().training_step(*args, **kwargs)
|
|
loss = output['loss'] # could be anything else
|
|
output.update({'my_train_metric': loss})
|
|
return output
|
|
|
|
hparams = tutils.get_default_hparams()
|
|
model = ModelWithoutValStep(hparams)
|
|
|
|
stopping = EarlyStopping(monitor='my_train_metric', min_delta=0.1)
|
|
trainer_options = dict(
|
|
default_root_dir=tmpdir,
|
|
early_stop_callback=stopping,
|
|
overfit_pct=0.20,
|
|
max_epochs=5,
|
|
)
|
|
|
|
trainer = Trainer(**trainer_options)
|
|
result = trainer.fit(model)
|
|
|
|
assert result == 1, 'training failed to complete'
|
|
assert trainer.current_epoch < trainer.max_epochs
|
|
|
|
|
|
def test_model_checkpoint_with_non_string_input(tmpdir):
|
|
""" Test that None in checkpoint callback is valid and that chkp_path is
|
|
set correctly """
|
|
tutils.reset_seed()
|
|
|
|
class CurrentTestModel(LightTrainDataloader, TestModelBase):
|
|
pass
|
|
|
|
hparams = tutils.get_default_hparams()
|
|
model = CurrentTestModel(hparams)
|
|
|
|
checkpoint = ModelCheckpoint(filepath=None, save_top_k=-1)
|
|
|
|
trainer = Trainer(default_root_dir=tmpdir,
|
|
checkpoint_callback=checkpoint,
|
|
overfit_pct=0.20,
|
|
max_epochs=5
|
|
)
|
|
result = trainer.fit(model)
|
|
|
|
# These should be different if the dirpath has be overridden
|
|
assert trainer.ckpt_path != trainer.default_root_dir
|