Tests: refactor cleanup (#1744)

* wip

* cleaning

* optim imports

* -

* default hparams

* fix restore

* fix imports

tests/models/data/horovod/train_default_model.py

@@ -30,7 +30,7 @@ sys.path.insert(0, os.path.abspath(PATH_ROOT))
 from pytorch_lightning import Trainer  # noqa: E402
 from pytorch_lightning.callbacks import ModelCheckpoint  # noqa: E402
 from tests.base import EvalModelTemplate  # noqa: E402
-from tests.base.utils import set_random_master_port, get_default_hparams, run_model_test  # noqa: E402
+from tests.base.utils import set_random_master_port, run_model_test  # noqa: E402
 
 
 parser = argparse.ArgumentParser()
@@ -45,7 +45,7 @@ def run_test_from_config(trainer_options):
     ckpt_path = trainer_options['default_root_dir']
     trainer_options.update(checkpoint_callback=ModelCheckpoint(ckpt_path))
 
-    model = EvalModelTemplate(get_default_hparams())
+    model = EvalModelTemplate(EvalModelTemplate.get_default_hparams())
     run_model_test(trainer_options, model, on_gpu=args.on_gpu, version=0, with_hpc=False)
 
     # Horovod should be initialized following training. If not, this will raise an exception.

tests/models/test_amp.py

@@ -23,7 +23,7 @@ def test_amp_single_gpu(tmpdir, backend):
         precision=16
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
     # tutils.run_model_test(trainer_options, model)
     result = trainer.fit(model)
 
@@ -37,7 +37,7 @@ def test_amp_multi_gpu(tmpdir, backend):
     """Make sure DP/DDP + AMP work."""
     tutils.set_random_master_port()
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     trainer_options = dict(
         default_root_dir=tmpdir,
@@ -62,7 +62,7 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir):
     tutils.set_random_master_port()
     os.environ['SLURM_LOCALID'] = str(0)
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     # exp file to get meta
     logger = tutils.get_default_logger(tmpdir)
@@ -103,7 +103,7 @@ def test_cpu_model_with_amp(tmpdir):
         precision=16
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     with pytest.raises((MisconfigurationException, ModuleNotFoundError)):
         tutils.run_model_test(trainer_options, model, on_gpu=False)

tests/models/test_cpu.py

@@ -1,5 +1,5 @@
-from collections import namedtuple
 import platform
+from collections import namedtuple
 
 import pytest
 import torch
@@ -24,7 +24,7 @@ def test_early_stopping_cpu_model(tmpdir):
         val_percent_check=0.1,
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
     # test freeze on cpu
@@ -53,7 +53,7 @@ def test_multi_cpu_model_ddp(tmpdir):
         distributed_backend='ddp_cpu'
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
 
@@ -68,7 +68,7 @@ def test_lbfgs_cpu_model(tmpdir):
         val_percent_check=0.2,
     )
 
-    hparams = tutils.get_default_hparams()
+    hparams = EvalModelTemplate.get_default_hparams()
     setattr(hparams, 'optimizer_name', 'lbfgs')
     setattr(hparams, 'learning_rate', 0.002)
     model = EvalModelTemplate(hparams)
@@ -88,7 +88,7 @@ def test_default_logger_callbacks_cpu_model(tmpdir):
         val_percent_check=0.01,
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
     tutils.run_model_test_without_loggers(trainer_options, model)
 
     # test freeze on cpu
@@ -98,7 +98,7 @@ def test_default_logger_callbacks_cpu_model(tmpdir):
 
 def test_running_test_after_fitting(tmpdir):
     """Verify test() on fitted model."""
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     # logger file to get meta
     logger = tutils.get_default_logger(tmpdir)
@@ -129,7 +129,7 @@ def test_running_test_after_fitting(tmpdir):
 
 def test_running_test_no_val(tmpdir):
     """Verify `test()` works on a model with no `val_loader`."""
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     # logger file to get meta
     logger = tutils.get_default_logger(tmpdir)
@@ -207,7 +207,7 @@ def test_single_gpu_batch_parse():
 
 def test_simple_cpu(tmpdir):
     """Verify continue training session on CPU."""
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     # fit model
     trainer = Trainer(
@@ -232,7 +232,7 @@ def test_cpu_model(tmpdir):
         val_percent_check=0.4
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
@@ -251,7 +251,7 @@ def test_all_features_cpu_model(tmpdir):
         val_percent_check=0.4
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
 
@@ -302,7 +302,7 @@ def test_tbptt_cpu_model(tmpdir):
                 sampler=None,
             )
 
-    hparams = tutils.get_default_hparams()
+    hparams = EvalModelTemplate.get_default_hparams()
     hparams.batch_size = batch_size
     hparams.in_features = truncated_bptt_steps
     hparams.hidden_dim = truncated_bptt_steps
@@ -336,5 +336,5 @@ def test_single_gpu_model(tmpdir):
         gpus=1
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
     tutils.run_model_test(trainer_options, model)

tests/models/test_gpu.py

@@ -30,7 +30,7 @@ def test_multi_gpu_model(tmpdir, backend):
         distributed_backend=backend,
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
     # tutils.run_model_test(trainer_options, model)
     trainer = Trainer(**trainer_options)
     result = trainer.fit(model)
@@ -53,7 +53,7 @@ def test_ddp_all_dataloaders_passed_to_fit(tmpdir):
                            gpus=[0, 1],
                            distributed_backend='ddp')
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
     fit_options = dict(train_dataloader=model.train_dataloader(),
                        val_dataloaders=model.val_dataloader())
 
@@ -64,7 +64,7 @@ def test_ddp_all_dataloaders_passed_to_fit(tmpdir):
 
 def test_cpu_slurm_save_load(tmpdir):
     """Verify model save/load/checkpoint on CPU."""
-    hparams = tutils.get_default_hparams()
+    hparams = EvalModelTemplate.get_default_hparams()
     model = EvalModelTemplate(hparams)
 
     # logger file to get meta
@@ -142,7 +142,7 @@ def test_multi_gpu_none_backend(tmpdir):
         gpus='-1'
     )
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
     with pytest.warns(UserWarning):
         tutils.run_model_test(trainer_options, model)
 

tests/models/test_hooks.py

@@ -14,7 +14,7 @@ def test_on_before_zero_grad_called(max_steps):
         def on_before_zero_grad(self, optimizer):
             self.on_before_zero_grad_called += 1
 
-    model = CurrentTestModel(tutils.get_default_hparams())
+    model = CurrentTestModel()
 
     trainer = Trainer(
         max_steps=max_steps,

tests/models/test_horovod.py

@@ -8,9 +8,8 @@ import sys
 import pytest
 import torch
 
-from pytorch_lightning import Trainer
-
 import tests.base.utils as tutils
+from pytorch_lightning import Trainer
 from tests.base import EvalModelTemplate
 from tests.base.models import TestGAN
 
@@ -121,7 +120,7 @@ def test_horovod_transfer_batch_to_gpu(tmpdir):
             assert str(y.device) != 'cpu'
             return super(TestTrainingStepModel, self).validation_step(batch, *args, **kwargs)
 
-    hparams = tutils.get_default_hparams()
+    hparams = EvalModelTemplate.get_default_hparams()
     model = TestTrainingStepModel(hparams)
 
     trainer_options = dict(
@@ -139,7 +138,7 @@ def test_horovod_transfer_batch_to_gpu(tmpdir):
 @pytest.mark.skipif(sys.version_info >= (3, 8), reason="Horovod not yet supported in Python 3.8")
 @pytest.mark.skipif(platform.system() == "Windows", reason="Horovod is not supported on Windows")
 def test_horovod_multi_optimizer(tmpdir):
-    hparams = tutils.get_default_hparams()
+    hparams = EvalModelTemplate.get_default_hparams()
     model = TestGAN(hparams)
 
     trainer_options = dict(

tests/models/test_module_hooks.py

@@ -27,7 +27,7 @@ def test_training_epoch_end_metrics_collection(tmpdir):
                 }
             }
 
-    model = CurrentModel(tutils.get_default_hparams())
+    model = CurrentModel()
     trainer = Trainer(
         max_epochs=num_epochs,
         default_root_dir=tmpdir,

tests/models/test_restore.py

@@ -19,7 +19,7 @@ def test_running_test_pretrained_model_distrib(tmpdir, backend):
     """Verify `test()` on pretrained model."""
     tutils.set_random_master_port()
 
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     # exp file to get meta
     logger = tutils.get_default_logger(tmpdir)
@@ -67,7 +67,7 @@ def test_running_test_pretrained_model_distrib(tmpdir, backend):
 
 def test_running_test_pretrained_model_cpu(tmpdir):
     """Verify test() on pretrained model."""
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     # logger file to get meta
     logger = tutils.get_default_logger(tmpdir)
@@ -103,7 +103,7 @@ def test_running_test_pretrained_model_cpu(tmpdir):
 
 def test_load_model_from_checkpoint(tmpdir):
     """Verify test() on pretrained model."""
-    hparams = tutils.get_default_hparams()
+    hparams = EvalModelTemplate.get_default_hparams()
     model = EvalModelTemplate(hparams)
 
     trainer_options = dict(
@@ -145,7 +145,7 @@ def test_load_model_from_checkpoint(tmpdir):
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_dp_resume(tmpdir):
     """Make sure DP continues training correctly."""
-    hparams = tutils.get_default_hparams()
+    hparams = EvalModelTemplate.get_default_hparams()
     model = EvalModelTemplate(hparams)
 
     trainer_options = dict(
@@ -217,7 +217,7 @@ def test_dp_resume(tmpdir):
 
 def test_model_saving_loading(tmpdir):
     """Tests use case where trainer saves the model, and user loads it from tags independently."""
-    model = EvalModelTemplate(tutils.get_default_hparams())
+    model = EvalModelTemplate()
 
     # logger file to get meta
     logger = tutils.get_default_logger(tmpdir)
@@ -284,13 +284,11 @@ def test_load_model_with_missing_hparams(tmpdir):
 
     class CurrentModelWithoutHparams(EvalModelTemplate):
         def __init__(self):
-            hparams = tutils.get_default_hparams()
-            super().__init__(hparams)
+            super().__init__()
 
     class CurrentModelUnusedHparams(EvalModelTemplate):
         def __init__(self, hparams):
-            hparams = tutils.get_default_hparams()
-            super().__init__(hparams)
+            super().__init__()
 
     model = CurrentModelWithoutHparams()
     trainer.fit(model)