Tests: refactor callbacks (#1688)

* refactor default model

* drop redundant seeds

* path

* refactor callback tests

* update

* fix sch

* wip

* fix return

* review

tests/callbacks/test_callbacks.py

@@ -3,28 +3,14 @@ import tests.base.utils as tutils
 from pytorch_lightning import Callback
 from pytorch_lightning import Trainer, LightningModule
 from pytorch_lightning.callbacks import EarlyStopping, LearningRateLogger, ModelCheckpoint
-from tests.base import (
-    LightTrainDataloader,
-    LightTestMixin,
-    LightValidationMixin,
-    LightTestOptimizersWithMixedSchedulingMixin,
-    TestModelBase
-)
+from tests.base import EvalModelTemplate
 
 
 def test_trainer_callback_system(tmpdir):
     """Test the callback system."""
 
-    class CurrentTestModel(
-        LightTrainDataloader,
-        LightTestMixin,
-        LightValidationMixin,
-        TestModelBase,
-    ):
-        pass
-
     hparams = tutils.get_default_hparams()
-    model = CurrentTestModel(hparams)
+    model = EvalModelTemplate(hparams)
 
     def _check_args(trainer, pl_module):
         assert isinstance(trainer, Trainer)
@@ -214,18 +200,18 @@ def test_trainer_callback_system(tmpdir):
 
 def test_early_stopping_no_val_step(tmpdir):
     """Test that early stopping callback falls back to training metrics when no validation defined."""
-    class ModelWithoutValStep(LightTrainDataloader, TestModelBase):
 
+    class CurrentModel(EvalModelTemplate):
         def training_step(self, *args, **kwargs):
             output = super().training_step(*args, **kwargs)
-            loss = output['loss']  # could be anything else
-            output.update({'my_train_metric': loss})
+            output.update({'my_train_metric': output['loss']})  # could be anything else
             return output
 
-    model = ModelWithoutValStep(tutils.get_default_hparams())
+    model = CurrentModel(tutils.get_default_hparams())
+    model.validation_step = None
+    model.val_dataloader = None
 
     stopping = EarlyStopping(monitor='my_train_metric', min_delta=0.1)
-
     trainer = Trainer(
         default_root_dir=tmpdir,
         early_stop_callback=stopping,
@@ -251,12 +237,7 @@ def test_pickling(tmpdir):
 def test_model_checkpoint_with_non_string_input(tmpdir, save_top_k):
     """ Test that None in checkpoint callback is valid and that chkp_path is set correctly """
     tutils.reset_seed()
-
-    class CurrentTestModel(LightTrainDataloader, TestModelBase):
-        pass
-
-    hparams = tutils.get_default_hparams()
-    model = CurrentTestModel(hparams)
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     checkpoint = ModelCheckpoint(filepath=None, save_top_k=save_top_k)
 
@@ -275,11 +256,8 @@ def test_lr_logger_single_lr(tmpdir):
     """ Test that learning rates are extracted and logged for single lr scheduler"""
     tutils.reset_seed()
 
-    class CurrentTestModel(LightTrainDataloader, TestModelBase):
-        pass
-
-    hparams = tutils.get_default_hparams()
-    model = CurrentTestModel(hparams)
+    model = EvalModelTemplate(tutils.get_default_hparams())
+    model.configure_optimizers = model.configure_optimizers__single_scheduler
 
     lr_logger = LearningRateLogger()
     trainer = Trainer(
@@ -291,6 +269,7 @@ def test_lr_logger_single_lr(tmpdir):
     )
     results = trainer.fit(model)
 
+    assert results == 1
     assert lr_logger.lrs, 'No learning rates logged'
     assert len(lr_logger.lrs) == len(trainer.lr_schedulers), \
         'Number of learning rates logged does not match number of lr schedulers'
@@ -302,13 +281,8 @@ def test_lr_logger_multi_lrs(tmpdir):
     """ Test that learning rates are extracted and logged for multi lr schedulers """
     tutils.reset_seed()
 
-    class CurrentTestModel(LightTestOptimizersWithMixedSchedulingMixin,
-                           LightTrainDataloader,
-                           TestModelBase):
-        pass
-
-    hparams = tutils.get_default_hparams()
-    model = CurrentTestModel(hparams)
+    model = EvalModelTemplate(tutils.get_default_hparams())
+    model.configure_optimizers = model.configure_optimizers__multiple_schedulers
 
     lr_logger = LearningRateLogger()
     trainer = Trainer(
@@ -320,6 +294,7 @@ def test_lr_logger_multi_lrs(tmpdir):
     )
     results = trainer.fit(model)
 
+    assert results == 1
     assert lr_logger.lrs, 'No learning rates logged'
     assert len(lr_logger.lrs) == len(trainer.lr_schedulers), \
         'Number of learning rates logged does not match number of lr schedulers'

tests/callbacks/test_progress_bar.py

@@ -4,12 +4,7 @@ import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks import ProgressBarBase, ProgressBar, ModelCheckpoint
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
-from tests.base import (
-    LightTrainDataloader,
-    LightTestMixin,
-    LightValidationMixin,
-    TestModelBase
-)
+from tests.base import EvalModelTemplate
 
 
 @pytest.mark.parametrize('callbacks,refresh_rate', [
@@ -63,16 +58,7 @@ def test_progress_bar_misconfiguration():
 def test_progress_bar_totals():
     """Test that the progress finishes with the correct total steps processed."""
 
-    class CurrentTestModel(
-        LightTrainDataloader,
-        LightTestMixin,
-        LightValidationMixin,
-        TestModelBase,
-    ):
-        pass
-
-    hparams = tutils.get_default_hparams()
-    model = CurrentTestModel(hparams)
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     trainer = Trainer(
         progress_bar_refresh_rate=1,
@@ -121,16 +107,7 @@ def test_progress_bar_totals():
 
 
 def test_progress_bar_fast_dev_run():
-    class CurrentTestModel(
-        LightTrainDataloader,
-        LightTestMixin,
-        LightValidationMixin,
-        TestModelBase,
-    ):
-        pass
-
-    hparams = tutils.get_default_hparams()
-    model = CurrentTestModel(hparams)
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     trainer = Trainer(
         fast_dev_run=True,
@@ -163,16 +140,7 @@ def test_progress_bar_fast_dev_run():
 def test_progress_bar_progress_refresh(refresh_rate):
     """Test that the three progress bars get correctly updated when using different refresh rates."""
 
-    class CurrentTestModel(
-        LightTrainDataloader,
-        LightTestMixin,
-        LightValidationMixin,
-        TestModelBase,
-    ):
-        pass
-
-    hparams = tutils.get_default_hparams()
-    model = CurrentTestModel(hparams)
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     class CurrentProgressBar(ProgressBar):
 

tests/trainer/test_dataloaders.py

@@ -33,7 +33,6 @@ from tests.base import (
     dict(val_check_interval=10000),
 ])
 def test_dataloader_config_errors(tmpdir, dataloader_options):
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,