From 043ae697c2bea15e193e9731a982aae6fcc0fcb6 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 4 May 2020 22:52:22 +0200 Subject: [PATCH] Tests: refactor callbacks (#1688) * refactor default model * drop redundant seeds * path * refactor callback tests * update * fix sch * wip * fix return * review --- tests/callbacks/test_callbacks.py | 53 ++++++++-------------------- tests/callbacks/test_progress_bar.py | 40 +++------------------ tests/trainer/test_dataloaders.py | 1 - 3 files changed, 18 insertions(+), 76 deletions(-) diff --git a/tests/callbacks/test_callbacks.py b/tests/callbacks/test_callbacks.py index b1fb71dc..32fdcdae 100644 --- a/tests/callbacks/test_callbacks.py +++ b/tests/callbacks/test_callbacks.py @@ -3,28 +3,14 @@ import tests.base.utils as tutils from pytorch_lightning import Callback from pytorch_lightning import Trainer, LightningModule from pytorch_lightning.callbacks import EarlyStopping, LearningRateLogger, ModelCheckpoint -from tests.base import ( - LightTrainDataloader, - LightTestMixin, - LightValidationMixin, - LightTestOptimizersWithMixedSchedulingMixin, - TestModelBase -) +from tests.base import EvalModelTemplate def test_trainer_callback_system(tmpdir): """Test the callback system.""" - class CurrentTestModel( - LightTrainDataloader, - LightTestMixin, - LightValidationMixin, - TestModelBase, - ): - pass - hparams = tutils.get_default_hparams() - model = CurrentTestModel(hparams) + model = EvalModelTemplate(hparams) def _check_args(trainer, pl_module): assert isinstance(trainer, Trainer) @@ -214,18 +200,18 @@ def test_trainer_callback_system(tmpdir): def test_early_stopping_no_val_step(tmpdir): """Test that early stopping callback falls back to training metrics when no validation defined.""" - class ModelWithoutValStep(LightTrainDataloader, TestModelBase): + class CurrentModel(EvalModelTemplate): def training_step(self, *args, **kwargs): output = super().training_step(*args, **kwargs) - loss = output['loss'] # could be anything else - output.update({'my_train_metric': loss}) + output.update({'my_train_metric': output['loss']}) # could be anything else return output - model = ModelWithoutValStep(tutils.get_default_hparams()) + model = CurrentModel(tutils.get_default_hparams()) + model.validation_step = None + model.val_dataloader = None stopping = EarlyStopping(monitor='my_train_metric', min_delta=0.1) - trainer = Trainer( default_root_dir=tmpdir, early_stop_callback=stopping, @@ -251,12 +237,7 @@ def test_pickling(tmpdir): def test_model_checkpoint_with_non_string_input(tmpdir, save_top_k): """ Test that None in checkpoint callback is valid and that chkp_path is set correctly """ tutils.reset_seed() - - class CurrentTestModel(LightTrainDataloader, TestModelBase): - pass - - hparams = tutils.get_default_hparams() - model = CurrentTestModel(hparams) + model = EvalModelTemplate(tutils.get_default_hparams()) checkpoint = ModelCheckpoint(filepath=None, save_top_k=save_top_k) @@ -275,11 +256,8 @@ def test_lr_logger_single_lr(tmpdir): """ Test that learning rates are extracted and logged for single lr scheduler""" tutils.reset_seed() - class CurrentTestModel(LightTrainDataloader, TestModelBase): - pass - - hparams = tutils.get_default_hparams() - model = CurrentTestModel(hparams) + model = EvalModelTemplate(tutils.get_default_hparams()) + model.configure_optimizers = model.configure_optimizers__single_scheduler lr_logger = LearningRateLogger() trainer = Trainer( @@ -291,6 +269,7 @@ def test_lr_logger_single_lr(tmpdir): ) results = trainer.fit(model) + assert results == 1 assert lr_logger.lrs, 'No learning rates logged' assert len(lr_logger.lrs) == len(trainer.lr_schedulers), \ 'Number of learning rates logged does not match number of lr schedulers' @@ -302,13 +281,8 @@ def test_lr_logger_multi_lrs(tmpdir): """ Test that learning rates are extracted and logged for multi lr schedulers """ tutils.reset_seed() - class CurrentTestModel(LightTestOptimizersWithMixedSchedulingMixin, - LightTrainDataloader, - TestModelBase): - pass - - hparams = tutils.get_default_hparams() - model = CurrentTestModel(hparams) + model = EvalModelTemplate(tutils.get_default_hparams()) + model.configure_optimizers = model.configure_optimizers__multiple_schedulers lr_logger = LearningRateLogger() trainer = Trainer( @@ -320,6 +294,7 @@ def test_lr_logger_multi_lrs(tmpdir): ) results = trainer.fit(model) + assert results == 1 assert lr_logger.lrs, 'No learning rates logged' assert len(lr_logger.lrs) == len(trainer.lr_schedulers), \ 'Number of learning rates logged does not match number of lr schedulers' diff --git a/tests/callbacks/test_progress_bar.py b/tests/callbacks/test_progress_bar.py index 7cd5d543..ebd35fed 100644 --- a/tests/callbacks/test_progress_bar.py +++ b/tests/callbacks/test_progress_bar.py @@ -4,12 +4,7 @@ import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ProgressBarBase, ProgressBar, ModelCheckpoint from pytorch_lightning.utilities.exceptions import MisconfigurationException -from tests.base import ( - LightTrainDataloader, - LightTestMixin, - LightValidationMixin, - TestModelBase -) +from tests.base import EvalModelTemplate @pytest.mark.parametrize('callbacks,refresh_rate', [ @@ -63,16 +58,7 @@ def test_progress_bar_misconfiguration(): def test_progress_bar_totals(): """Test that the progress finishes with the correct total steps processed.""" - class CurrentTestModel( - LightTrainDataloader, - LightTestMixin, - LightValidationMixin, - TestModelBase, - ): - pass - - hparams = tutils.get_default_hparams() - model = CurrentTestModel(hparams) + model = EvalModelTemplate(tutils.get_default_hparams()) trainer = Trainer( progress_bar_refresh_rate=1, @@ -121,16 +107,7 @@ def test_progress_bar_totals(): def test_progress_bar_fast_dev_run(): - class CurrentTestModel( - LightTrainDataloader, - LightTestMixin, - LightValidationMixin, - TestModelBase, - ): - pass - - hparams = tutils.get_default_hparams() - model = CurrentTestModel(hparams) + model = EvalModelTemplate(tutils.get_default_hparams()) trainer = Trainer( fast_dev_run=True, @@ -163,16 +140,7 @@ def test_progress_bar_fast_dev_run(): def test_progress_bar_progress_refresh(refresh_rate): """Test that the three progress bars get correctly updated when using different refresh rates.""" - class CurrentTestModel( - LightTrainDataloader, - LightTestMixin, - LightValidationMixin, - TestModelBase, - ): - pass - - hparams = tutils.get_default_hparams() - model = CurrentTestModel(hparams) + model = EvalModelTemplate(tutils.get_default_hparams()) class CurrentProgressBar(ProgressBar): diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py index d847b6c8..d0a6dd86 100644 --- a/tests/trainer/test_dataloaders.py +++ b/tests/trainer/test_dataloaders.py @@ -33,7 +33,6 @@ from tests.base import ( dict(val_check_interval=10000), ]) def test_dataloader_config_errors(tmpdir, dataloader_options): - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader,