refactor default model (#1652)

* refactor default model

* drop redundant seeds

* formatting

* path

* formatting

* rename

tests/base/eval_model_optimizers.py

@@ -15,7 +15,7 @@ class ConfigureOptimizersPool(ABC):
     def configure_optimizers_empty(self):
         return None
 
-    def configure_optimizers_lbfgs(self):
+    def configure_optimizers__lbfgs(self):
         """
         return whatever optimizers we want here.
         :return: list of optimizers
@@ -23,7 +23,7 @@ class ConfigureOptimizersPool(ABC):
         optimizer = optim.LBFGS(self.parameters(), lr=self.hparams.learning_rate)
         return optimizer
 
-    def configure_optimizers_multiple_optimizers(self):
+    def configure_optimizers__multiple_optimizers(self):
         """
         return whatever optimizers we want here.
         :return: list of optimizers
@@ -33,12 +33,12 @@ class ConfigureOptimizersPool(ABC):
         optimizer2 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
         return optimizer1, optimizer2
 
-    def configure_optimizers_single_scheduler(self):
+    def configure_optimizers__single_scheduler(self):
         optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
         lr_scheduler = optim.lr_scheduler.StepLR(optimizer, 1, gamma=0.1)
         return [optimizer], [lr_scheduler]
 
-    def configure_optimizers_multiple_schedulers(self):
+    def configure_optimizers__multiple_schedulers(self):
         optimizer1 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
         optimizer2 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
         lr_scheduler1 = optim.lr_scheduler.StepLR(optimizer1, 1, gamma=0.1)
@@ -46,7 +46,7 @@ class ConfigureOptimizersPool(ABC):
 
         return [optimizer1, optimizer2], [lr_scheduler1, lr_scheduler2]
 
-    def configure_optimizers_mixed_scheduling(self):
+    def configure_optimizers__mixed_scheduling(self):
         optimizer1 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
         optimizer2 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
         lr_scheduler1 = optim.lr_scheduler.StepLR(optimizer1, 4, gamma=0.1)
@@ -55,7 +55,7 @@ class ConfigureOptimizersPool(ABC):
         return [optimizer1, optimizer2], \
             [{'scheduler': lr_scheduler1, 'interval': 'step'}, lr_scheduler2]
 
-    def configure_optimizers_reduce_lr_on_plateau(self):
+    def configure_optimizers__reduce_lr_on_plateau(self):
         optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
         lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer)
         return [optimizer], [lr_scheduler]

tests/base/eval_model_template.py

@@ -33,7 +33,7 @@ class EvalModelTemplate(
     """
     This template houses all  combinations of model  configurations  we want to test
     """
-    def __init__(self, hparams):
+    def __init__(self, hparams: object) -> object:
         """Pass in parsed HyperOptArgumentParser to the model."""
         # init superclass
         super().__init__()

tests/base/eval_model_test_steps.py

@@ -45,7 +45,7 @@ class TestStepVariations(ABC):
             })
             return output
 
-    def test_step_multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs):
+    def test_step__multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs):
         """
         Default, baseline test_step
         :param batch:

tests/base/eval_model_valid_steps.py

@@ -51,7 +51,7 @@ class ValidationStepVariations(ABC):
             })
             return output
 
-    def validation_step_multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs):
+    def validation_step__multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs):
         """
         Lightning calls this inside the validation loop
         :param batch:

tests/base/utils.py

@@ -9,7 +9,7 @@ from pytorch_lightning import Trainer
 from pytorch_lightning.callbacks import ModelCheckpoint
 from pytorch_lightning.loggers import TensorBoardLogger
 from tests import TEMP_PATH, RANDOM_PORTS, RANDOM_SEEDS
-from tests.base import LightningTestModel
+from tests.base import LightningTestModel, EvalModelTemplate
 from tests.base.datasets import PATH_DATASETS
 
 
@@ -27,6 +27,8 @@ def assert_speed_parity(pl_times, pt_times, num_epochs):
 
 
 def run_model_test_without_loggers(trainer_options, model, min_acc=0.50):
+    reset_seed()
+
     # fit model
     trainer = Trainer(**trainer_options)
     result = trainer.fit(model)
@@ -54,6 +56,7 @@ def run_model_test_without_loggers(trainer_options, model, min_acc=0.50):
 
 
 def run_model_test(trainer_options, model, on_gpu=True, version=None, with_hpc=True):
+    reset_seed()
     save_dir = trainer_options['default_root_dir']
 
     # logger file to get meta
@@ -95,8 +98,6 @@ def run_model_test(trainer_options, model, on_gpu=True, version=None, with_hpc=T
 
 
 def get_default_hparams(continue_training=False, hpc_exp_number=0):
-    _ = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-
     args = {
         'drop_prob': 0.2,
         'batch_size': 32,
@@ -120,18 +121,6 @@ def get_default_hparams(continue_training=False, hpc_exp_number=0):
     return hparams
 
 
-def get_default_model(lbfgs=False):
-    # set up model with these hyperparams
-    hparams = get_default_hparams()
-    if lbfgs:
-        setattr(hparams, 'optimizer_name', 'lbfgs')
-        setattr(hparams, 'learning_rate', 0.005)
-
-    model = LightningTestModel(hparams)
-
-    return model, hparams
-
-
 def get_default_logger(save_dir, version=None):
     # set up logger object without actually saving logs
     logger = TensorBoardLogger(save_dir, name='lightning_logs', version=version)
@@ -229,6 +218,7 @@ def reset_seed():
 
 
 def set_random_master_port():
+    reset_seed()
     port = RANDOM_PORTS.pop()
     os.environ['MASTER_PORT'] = str(port)
 

tests/callbacks/test_callbacks.py

@@ -214,8 +214,6 @@ def test_trainer_callback_system(tmpdir):
 
 def test_early_stopping_no_val_step(tmpdir):
     """Test that early stopping callback falls back to training metrics when no validation defined."""
-    tutils.reset_seed()
-
     class ModelWithoutValStep(LightTrainDataloader, TestModelBase):
 
         def training_step(self, *args, **kwargs):
@@ -224,8 +222,7 @@ def test_early_stopping_no_val_step(tmpdir):
             output.update({'my_train_metric': loss})
             return output
 
-    hparams = tutils.get_default_hparams()
-    model = ModelWithoutValStep(hparams)
+    model = ModelWithoutValStep(tutils.get_default_hparams())
 
     stopping = EarlyStopping(monitor='my_train_metric', min_delta=0.1)
 
@@ -269,7 +266,7 @@ def test_model_checkpoint_with_non_string_input(tmpdir, save_top_k):
                       overfit_pct=0.20,
                       max_epochs=5
                       )
-    result = trainer.fit(model)
+    trainer.fit(model)
 
     # These should be different if the dirpath has be overridden
     assert trainer.ckpt_path != trainer.default_root_dir

tests/loggers/test_all.py

@@ -7,6 +7,7 @@ import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import (
     TensorBoardLogger, MLFlowLogger, NeptuneLogger, TestTubeLogger, CometLogger)
+from tests.base import EvalModelTemplate
 
 
 def _get_logger_args(logger_class, save_dir):
@@ -29,14 +30,12 @@ def _get_logger_args(logger_class, save_dir):
 ])
 def test_loggers_fit_test(tmpdir, monkeypatch, logger_class):
     """Verify that basic functionality of all loggers."""
-    tutils.reset_seed()
-
     # prevent comet logger from trying to print at exit, since
     # pytest's stdout/stderr redirection breaks it
     import atexit
     monkeypatch.setattr(atexit, 'register', lambda _: None)
 
-    model, _ = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     class StoreHistoryLogger(logger_class):
         def __init__(self, *args, **kwargs):
@@ -78,8 +77,6 @@ def test_loggers_fit_test(tmpdir, monkeypatch, logger_class):
 ])
 def test_loggers_pickle(tmpdir, monkeypatch, logger_class):
     """Verify that pickling trainer with logger works."""
-    tutils.reset_seed()
-
     # prevent comet logger from trying to print at exit, since
     # pytest's stdout/stderr redirection breaks it
     import atexit

tests/loggers/test_base.py

@@ -7,7 +7,7 @@ import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.loggers import LightningLoggerBase, LoggerCollection
 from pytorch_lightning.utilities import rank_zero_only
-from tests.base import LightningTestModel
+from tests.base import LightningTestModel, EvalModelTemplate
 
 
 def test_logger_collection():
@@ -139,7 +139,7 @@ def test_adding_step_key(tmpdir):
 
         return decorated
 
-    model, hparams = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
     model.validation_epoch_end = _validation_epoch_end
     model.training_epoch_end = _training_epoch_end
     trainer = Trainer(

tests/loggers/test_neptune.py

@@ -61,10 +61,7 @@ def test_neptune_additional_methods(neptune):
 
 def test_neptune_leave_open_experiment_after_fit(tmpdir):
     """Verify that neptune experiment was closed after training"""
-    tutils.reset_seed()
-
-    hparams = tutils.get_default_hparams()
-    model = LightningTestModel(hparams)
+    model = LightningTestModel(tutils.get_default_hparams())
 
     def _run_training(logger):
         logger._experiment = MagicMock()

tests/loggers/test_trains.py

@@ -8,8 +8,6 @@ from tests.base import LightningTestModel
 
 def test_trains_logger(tmpdir):
     """Verify that basic functionality of TRAINS logger works."""
-    tutils.reset_seed()
-
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
     TrainsLogger.set_bypass_mode(True)
@@ -33,8 +31,6 @@ def test_trains_logger(tmpdir):
 
 def test_trains_pickle(tmpdir):
     """Verify that pickling trainer with TRAINS logger works."""
-    tutils.reset_seed()
-
     # hparams = tutils.get_default_hparams()
     # model = LightningTestModel(hparams)
     TrainsLogger.set_bypass_mode(True)

tests/loggers/test_wandb.py

@@ -11,8 +11,6 @@ from pytorch_lightning.loggers import WandbLogger
 def test_wandb_logger(wandb):
     """Verify that basic functionality of wandb logger works.
     Wandb doesn't work well with pytest so we have to mock it out here."""
-    tutils.reset_seed()
-
     logger = WandbLogger(anonymous=True, offline=True)
 
     logger.log_metrics({'acc': 1.0})
@@ -38,8 +36,6 @@ def test_wandb_pickle(wandb):
 
     Wandb doesn't work well with pytest so we have to mock it out here.
     """
-    tutils.reset_seed()
-
     class Experiment:
         id = 'the_id'
 

tests/models/data/horovod/train_default_model.py

@@ -29,7 +29,8 @@ sys.path.insert(0, os.path.abspath(PATH_ROOT))
 
 from pytorch_lightning import Trainer  # noqa: E402
 from pytorch_lightning.callbacks import ModelCheckpoint  # noqa: E402
-import tests.base.utils as tutils  # noqa: E402
+from tests.base import EvalModelTemplate  # noqa: E402
+from tests.base.utils import set_random_master_port, get_default_hparams, run_model_test  # noqa: E402
 
 
 parser = argparse.ArgumentParser()
@@ -39,14 +40,13 @@ parser.add_argument('--on-gpu', action='store_true', default=False)
 
 def run_test_from_config(trainer_options):
     """Trains the default model with the given config."""
-    tutils.reset_seed()
-    tutils.set_random_master_port()
+    set_random_master_port()
 
     ckpt_path = trainer_options['default_root_dir']
-    trainer_options['checkpoint_callback'] = ModelCheckpoint(ckpt_path)
+    trainer_options.update(checkpoint_callback=ModelCheckpoint(ckpt_path))
 
-    model, hparams = tutils.get_default_model()
-    tutils.run_model_test(trainer_options, model, on_gpu=args.on_gpu, version=0, with_hpc=False)
+    model = EvalModelTemplate(get_default_hparams())
+    run_model_test(trainer_options, model, on_gpu=args.on_gpu, version=0, with_hpc=False)
 
     # Horovod should be initialized following training. If not, this will raise an exception.
     assert hvd.size() == 2

tests/models/test_amp.py

@@ -6,9 +6,7 @@ import torch
 import tests.base.utils as tutils
 from pytorch_lightning import Trainer
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
-from tests.base import (
-    LightningTestModel,
-)
+from tests.base import LightningTestModel, EvalModelTemplate
 
 
 @pytest.mark.spawn
@@ -18,8 +16,6 @@ def test_amp_single_gpu(tmpdir, backend):
     """Make sure DP/DDP + AMP work."""
     tutils.reset_seed()
 
-    model, hparams = tutils.get_default_model()
-
     trainer = Trainer(
         default_root_dir=tmpdir,
         max_epochs=1,
@@ -28,6 +24,7 @@ def test_amp_single_gpu(tmpdir, backend):
         precision=16
     )
 
+    model = EvalModelTemplate(tutils.get_default_hparams())
     # tutils.run_model_test(trainer_options, model)
     result = trainer.fit(model)
 
@@ -39,10 +36,9 @@ def test_amp_single_gpu(tmpdir, backend):
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_amp_multi_gpu(tmpdir, backend):
     """Make sure DP/DDP + AMP work."""
-    tutils.reset_seed()
     tutils.set_random_master_port()
 
-    model, hparams = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     trainer_options = dict(
         default_root_dir=tmpdir,
@@ -63,8 +59,6 @@ def test_amp_multi_gpu(tmpdir, backend):
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_amp_gpu_ddp_slurm_managed(tmpdir):
     """Make sure DDP + AMP work."""
-    tutils.reset_seed()
-
     # simulate setting slurm flags
     tutils.set_random_master_port()
     os.environ['SLURM_LOCALID'] = str(0)
@@ -102,8 +96,6 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir):
 
 def test_cpu_model_with_amp(tmpdir):
     """Make sure model trains on CPU."""
-    tutils.reset_seed()
-
     trainer_options = dict(
         default_root_dir=tmpdir,
         progress_bar_refresh_rate=0,
@@ -113,7 +105,7 @@ def test_cpu_model_with_amp(tmpdir):
         precision=16
     )
 
-    model, hparams = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     with pytest.raises((MisconfigurationException, ModuleNotFoundError)):
         tutils.run_model_test(trainer_options, model, on_gpu=False)

tests/models/test_cpu.py

@@ -15,13 +15,12 @@ from tests.base import (
     LightTrainDataloader,
     LightningTestModel,
     LightTestMixin,
+    EvalModelTemplate,
 )
 
 
 def test_early_stopping_cpu_model(tmpdir):
     """Test each of the trainer options."""
-    tutils.reset_seed()
-
     stopping = EarlyStopping(monitor='val_loss', min_delta=0.1)
     trainer_options = dict(
         default_root_dir=tmpdir,
@@ -33,7 +32,7 @@ def test_early_stopping_cpu_model(tmpdir):
         val_percent_check=0.1,
     )
 
-    model, hparams = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
     # test freeze on cpu
@@ -49,10 +48,8 @@ def test_early_stopping_cpu_model(tmpdir):
                     reason="Distributed training is not supported on MacOS before Torch 1.3.0")
 def test_multi_cpu_model_ddp(tmpdir):
     """Make sure DDP works."""
-    tutils.reset_seed()
     tutils.set_random_master_port()
 
-    model, hparams = tutils.get_default_model()
     trainer_options = dict(
         default_root_dir=tmpdir,
         progress_bar_refresh_rate=0,
@@ -64,13 +61,12 @@ def test_multi_cpu_model_ddp(tmpdir):
         distributed_backend='ddp_cpu'
     )
 
+    model = EvalModelTemplate(tutils.get_default_hparams())
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
 
 def test_lbfgs_cpu_model(tmpdir):
     """Test each of the trainer options."""
-    tutils.reset_seed()
-
     trainer_options = dict(
         default_root_dir=tmpdir,
         max_epochs=2,
@@ -80,15 +76,16 @@ def test_lbfgs_cpu_model(tmpdir):
         val_percent_check=0.2,
     )
 
-    model, hparams = tutils.get_default_model(lbfgs=True)
-    # the test is there for the closure not the performance
-    tutils.run_model_test_without_loggers(trainer_options, model, min_acc=0.)
+    hparams = tutils.get_default_hparams()
+    setattr(hparams, 'optimizer_name', 'lbfgs')
+    setattr(hparams, 'learning_rate', 0.002)
+    model = EvalModelTemplate(hparams)
+    model.configure_optimizers = model.configure_optimizers__lbfgs
+    tutils.run_model_test_without_loggers(trainer_options, model, min_acc=0.5)
 
 
 def test_default_logger_callbacks_cpu_model(tmpdir):
     """Test each of the trainer options."""
-    tutils.reset_seed()
-
     trainer_options = dict(
         default_root_dir=tmpdir,
         max_epochs=1,
@@ -99,7 +96,7 @@ def test_default_logger_callbacks_cpu_model(tmpdir):
         val_percent_check=0.01,
     )
 
-    model, hparams = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
     tutils.run_model_test_without_loggers(trainer_options, model)
 
     # test freeze on cpu
@@ -109,8 +106,6 @@ def test_default_logger_callbacks_cpu_model(tmpdir):
 
 def test_running_test_after_fitting(tmpdir):
     """Verify test() on fitted model."""
-    tutils.reset_seed()
-
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
@@ -143,8 +138,6 @@ def test_running_test_after_fitting(tmpdir):
 
 def test_running_test_no_val(tmpdir):
     """Verify `test()` works on a model with no `val_loader`."""
-    tutils.reset_seed()
-
     class CurrentTestModel(LightTrainDataloader, LightTestMixin, TestModelBase):
         pass
 
@@ -180,8 +173,6 @@ def test_running_test_no_val(tmpdir):
 
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine")
 def test_single_gpu_batch_parse():
-    tutils.reset_seed()
-
     trainer = Trainer()
 
     # batch is just a tensor
@@ -229,8 +220,6 @@ def test_single_gpu_batch_parse():
 
 def test_simple_cpu(tmpdir):
     """Verify continue training session on CPU."""
-    tutils.reset_seed()
-
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
@@ -249,8 +238,6 @@ def test_simple_cpu(tmpdir):
 
 def test_cpu_model(tmpdir):
     """Make sure model trains on CPU."""
-    tutils.reset_seed()
-
     trainer_options = dict(
         default_root_dir=tmpdir,
         progress_bar_refresh_rate=0,
@@ -259,15 +246,13 @@ def test_cpu_model(tmpdir):
         val_percent_check=0.4
     )
 
-    model, hparams = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
 
 def test_all_features_cpu_model(tmpdir):
     """Test each of the trainer options."""
-    tutils.reset_seed()
-
     trainer_options = dict(
         default_root_dir=tmpdir,
         gradient_clip_val=1.0,
@@ -280,14 +265,12 @@ def test_all_features_cpu_model(tmpdir):
         val_percent_check=0.4
     )
 
-    model, hparams = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
     tutils.run_model_test(trainer_options, model, on_gpu=False)
 
 
 def test_tbptt_cpu_model(tmpdir):
     """Test truncated back propagation through time works."""
-    tutils.reset_seed()
-
     truncated_bptt_steps = 2
     sequence_size = 30
     batch_size = 30
@@ -358,10 +341,6 @@ def test_tbptt_cpu_model(tmpdir):
 @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine")
 def test_single_gpu_model(tmpdir):
     """Make sure single GPU works (DP mode)."""
-    tutils.reset_seed()
-
-    model, hparams = tutils.get_default_model()
-
     trainer_options = dict(
         default_root_dir=tmpdir,
         progress_bar_refresh_rate=0,
@@ -371,8 +350,5 @@ def test_single_gpu_model(tmpdir):
         gpus=1
     )
 
+    model = EvalModelTemplate(tutils.get_default_hparams())
     tutils.run_model_test(trainer_options, model)
-
-
-# if __name__ == '__main__':
-#     pytest.main([__file__])

tests/models/test_gpu.py

@@ -9,7 +9,7 @@ from pytorch_lightning.callbacks import ModelCheckpoint
 from pytorch_lightning.core import memory
 from pytorch_lightning.trainer.distrib_parts import parse_gpu_ids, determine_root_gpu_device
 from pytorch_lightning.utilities.exceptions import MisconfigurationException
-from tests.base import LightningTestModel
+from tests.base import LightningTestModel, EvalModelTemplate
 
 PRETEND_N_OF_GPUS = 16
 
@@ -19,11 +19,8 @@ PRETEND_N_OF_GPUS = 16
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_multi_gpu_model(tmpdir, backend):
     """Make sure DDP works."""
-
-    tutils.reset_seed()
     tutils.set_random_master_port()
 
-    model, hparams = tutils.get_default_model()
     trainer_options = dict(
         default_root_dir=tmpdir,
         max_epochs=1,
@@ -33,6 +30,7 @@ def test_multi_gpu_model(tmpdir, backend):
         distributed_backend=backend,
     )
 
+    model = EvalModelTemplate(tutils.get_default_hparams())
     # tutils.run_model_test(trainer_options, model)
     trainer = Trainer(**trainer_options)
     result = trainer.fit(model)
@@ -45,31 +43,27 @@ def test_multi_gpu_model(tmpdir, backend):
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_ddp_all_dataloaders_passed_to_fit(tmpdir):
     """Make sure DDP works with dataloaders passed to fit()"""
-
-    tutils.reset_seed()
     tutils.set_random_master_port()
 
-    model, hparams = tutils.get_default_model()
+    trainer_options = dict(default_root_dir=tmpdir,
+                           progress_bar_refresh_rate=0,
+                           max_epochs=1,
+                           train_percent_check=0.4,
+                           val_percent_check=0.2,
+                           gpus=[0, 1],
+                           distributed_backend='ddp')
 
-    trainer = Trainer(
-        default_root_dir=tmpdir,
-        progress_bar_refresh_rate=0,
-        max_epochs=1,
-        train_percent_check=0.4,
-        val_percent_check=0.2,
-        gpus=[0, 1],
-        distributed_backend='ddp'
-    )
-    result = trainer.fit(model,
-                         train_dataloader=model.train_dataloader(),
-                         val_dataloaders=model.val_dataloader())
+    model = EvalModelTemplate(tutils.get_default_hparams())
+    fit_options = dict(train_dataloader=model.train_dataloader(),
+                       val_dataloaders=model.val_dataloader())
+
+    trainer = Trainer(**trainer_options)
+    result = trainer.fit(model, **fit_options)
     assert result == 1, "DDP doesn't work with dataloaders passed to fit()."
 
 
 def test_cpu_slurm_save_load(tmpdir):
     """Verify model save/load/checkpoint on CPU."""
-    tutils.reset_seed()
-
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
@@ -139,9 +133,6 @@ def test_cpu_slurm_save_load(tmpdir):
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_multi_gpu_none_backend(tmpdir):
     """Make sure when using multiple GPUs the user can't use `distributed_backend = None`."""
-    tutils.reset_seed()
-
-    model, hparams = tutils.get_default_model()
     trainer_options = dict(
         default_root_dir=tmpdir,
         progress_bar_refresh_rate=0,
@@ -151,6 +142,7 @@ def test_multi_gpu_none_backend(tmpdir):
         gpus='-1'
     )
 
+    model = EvalModelTemplate(tutils.get_default_hparams())
     with pytest.warns(UserWarning):
         tutils.run_model_test(trainer_options, model)
 

tests/models/test_horovod.py

@@ -40,8 +40,12 @@ def _nccl_available():
 
 def _run_horovod(trainer_options, on_gpu=False):
     """Execute the training script across multiple workers in parallel."""
-    cmdline = ['horovodrun', '-np', '2', sys.executable, TEST_SCRIPT,
-               '--trainer-options', shlex.quote(json.dumps(trainer_options))]
+    cmdline = [
+        'horovodrun',
+        '-np', '2',
+        sys.executable, TEST_SCRIPT,
+        '--trainer-options', shlex.quote(json.dumps(trainer_options))
+    ]
     if on_gpu:
         cmdline += ['--on-gpu']
     exit_code = subprocess.call(' '.join(cmdline), shell=True, env=os.environ.copy())

tests/models/test_restore.py

@@ -21,8 +21,6 @@ from tests.base import (
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_running_test_pretrained_model_distrib(tmpdir, backend):
     """Verify `test()` on pretrained model."""
-
-    tutils.reset_seed()
     tutils.set_random_master_port()
 
     hparams = tutils.get_default_hparams()
@@ -74,8 +72,6 @@ def test_running_test_pretrained_model_distrib(tmpdir, backend):
 
 def test_running_test_pretrained_model_cpu(tmpdir):
     """Verify test() on pretrained model."""
-    tutils.reset_seed()
-
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
@@ -113,8 +109,6 @@ def test_running_test_pretrained_model_cpu(tmpdir):
 
 def test_load_model_from_checkpoint(tmpdir):
     """Verify test() on pretrained model."""
-    tutils.reset_seed()
-
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
@@ -157,9 +151,6 @@ def test_load_model_from_checkpoint(tmpdir):
 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine")
 def test_dp_resume(tmpdir):
     """Make sure DP continues training correctly."""
-
-    tutils.reset_seed()
-
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 
@@ -232,8 +223,6 @@ def test_dp_resume(tmpdir):
 
 def test_model_saving_loading(tmpdir):
     """Tests use case where trainer saves the model, and user loads it from tags independently."""
-    tutils.reset_seed()
-
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
 

tests/trainer/test_checks.py

@@ -15,7 +15,6 @@ from tests.base import (
 
 def test_error_on_no_train_step(tmpdir):
     """ Test that an error is thrown when no `training_step()` is defined """
-    tutils.reset_seed()
 
     class CurrentTestModel(LightningModule):
         def forward(self, x):
@@ -30,7 +29,6 @@ def test_error_on_no_train_step(tmpdir):
 
 def test_error_on_no_train_dataloader(tmpdir):
     """ Test that an error is thrown when no `training_dataloader()` is defined """
-    tutils.reset_seed()
     hparams = tutils.get_default_hparams()
 
     class CurrentTestModel(TestModelBase):
@@ -45,7 +43,6 @@ def test_error_on_no_train_dataloader(tmpdir):
 
 def test_error_on_no_configure_optimizers(tmpdir):
     """ Test that an error is thrown when no `configure_optimizers()` is defined """
-    tutils.reset_seed()
 
     class CurrentTestModel(LightTrainDataloader, LightningModule):
         def forward(self, x):
@@ -68,7 +65,6 @@ def test_warning_on_wrong_validation_settings(tmpdir):
             throw warning if `val_epoch_end()` is not defined
         * error if `validation_step()` is overriden but `val_dataloader()` is not
     """
-    tutils.reset_seed()
     hparams = tutils.get_default_hparams()
 
     trainer = Trainer(default_root_dir=tmpdir, max_epochs=1)
@@ -111,7 +107,6 @@ def test_warning_on_wrong_test_settigs(tmpdir):
             throw warning if `test_epoch_end()` is not defined
         * error if `test_step()` is overriden but `test_dataloader()` is not
     """
-    tutils.reset_seed()
     hparams = tutils.get_default_hparams()
     trainer = Trainer(default_root_dir=tmpdir, max_epochs=1)
 

tests/trainer/test_dataloaders.py

@@ -55,7 +55,6 @@ def test_dataloader_config_errors(tmpdir, dataloader_options):
 
 def test_multiple_val_dataloader(tmpdir):
     """Verify multiple val_dataloader."""
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,
@@ -90,7 +89,6 @@ def test_multiple_val_dataloader(tmpdir):
 
 def test_multiple_test_dataloader(tmpdir):
     """Verify multiple test_dataloader."""
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,
@@ -127,7 +125,6 @@ def test_multiple_test_dataloader(tmpdir):
 
 def test_train_dataloaders_passed_to_fit(tmpdir):
     """Verify that train dataloader can be passed to fit """
-    tutils.reset_seed()
 
     class CurrentTestModel(LightTrainDataloader, TestModelBase):
         pass
@@ -149,7 +146,6 @@ def test_train_dataloaders_passed_to_fit(tmpdir):
 
 def test_train_val_dataloaders_passed_to_fit(tmpdir):
     """ Verify that train & val dataloader can be passed to fit """
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,
@@ -178,7 +174,6 @@ def test_train_val_dataloaders_passed_to_fit(tmpdir):
 
 def test_all_dataloaders_passed_to_fit(tmpdir):
     """Verify train, val & test dataloader can be passed to fit """
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,
@@ -215,7 +210,6 @@ def test_all_dataloaders_passed_to_fit(tmpdir):
 
 def test_multiple_dataloaders_passed_to_fit(tmpdir):
     """Verify that multiple val & test dataloaders can be passed to fit."""
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightningTestModel,
@@ -252,7 +246,6 @@ def test_multiple_dataloaders_passed_to_fit(tmpdir):
 
 def test_mixing_of_dataloader_options(tmpdir):
     """Verify that dataloaders can be passed to fit"""
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,
@@ -294,7 +287,6 @@ def test_mixing_of_dataloader_options(tmpdir):
 
 def test_inf_train_dataloader(tmpdir):
     """Test inf train data loader (e.g. IterableDataset)"""
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightInfTrainDataloader,
@@ -336,7 +328,6 @@ def test_inf_train_dataloader(tmpdir):
 
 def test_inf_val_dataloader(tmpdir):
     """Test inf val data loader (e.g. IterableDataset)"""
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightInfValDataloader,
@@ -369,7 +360,6 @@ def test_inf_val_dataloader(tmpdir):
 
 def test_inf_test_dataloader(tmpdir):
     """Test inf test data loader (e.g. IterableDataset)"""
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightInfTestDataloader,
@@ -404,7 +394,6 @@ def test_inf_test_dataloader(tmpdir):
 
 def test_error_on_zero_len_dataloader(tmpdir):
     """ Test that error is raised if a zero-length dataloader is defined """
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightZeroLenDataloader,
@@ -428,7 +417,6 @@ def test_error_on_zero_len_dataloader(tmpdir):
 @pytest.mark.skipif(platform.system() == 'Windows', reason='Does not apply to Windows platform.')
 def test_warning_with_few_workers(tmpdir):
     """ Test that error is raised if dataloader with only a few workers is used """
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,

tests/trainer/test_lr_finder.py

@@ -12,8 +12,7 @@ from tests.base import (
 
 
 def test_error_on_more_than_1_optimizer(tmpdir):
-    ''' Check that error is thrown when more than 1 optimizer is passed '''
-    tutils.reset_seed()
+    """ Check that error is thrown when more than 1 optimizer is passed """
 
     class CurrentTestModel(
         LightTestMultipleOptimizersWithSchedulingMixin,
@@ -36,8 +35,7 @@ def test_error_on_more_than_1_optimizer(tmpdir):
 
 
 def test_model_reset_correctly(tmpdir):
-    ''' Check that model weights are correctly reset after lr_find() '''
-    tutils.reset_seed()
+    """ Check that model weights are correctly reset after lr_find() """
 
     class CurrentTestModel(
         LightTrainDataloader,
@@ -66,8 +64,7 @@ def test_model_reset_correctly(tmpdir):
 
 
 def test_trainer_reset_correctly(tmpdir):
-    ''' Check that all trainer parameters are reset correctly after lr_find() '''
-    tutils.reset_seed()
+    """ Check that all trainer parameters are reset correctly after lr_find() """
 
     class CurrentTestModel(
         LightTrainDataloader,
@@ -104,7 +101,6 @@ def test_trainer_reset_correctly(tmpdir):
 
 
 def test_trainer_arg_bool(tmpdir):
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,
@@ -129,7 +125,6 @@ def test_trainer_arg_bool(tmpdir):
 
 
 def test_trainer_arg_str(tmpdir):
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,
@@ -155,7 +150,6 @@ def test_trainer_arg_str(tmpdir):
 
 
 def test_call_to_trainer_method(tmpdir):
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightTrainDataloader,

tests/trainer/test_optimizers.py

@@ -12,13 +12,12 @@ from tests.base import (
     LightTestMultipleOptimizersWithSchedulingMixin,
     LightTestOptimizersWithMixedSchedulingMixin,
     LightTestReduceLROnPlateauMixin,
-    LightTestNoneOptimizerMixin
+    LightTestNoneOptimizerMixin, EvalModelTemplate
 )
 
 
 def test_optimizer_with_scheduling(tmpdir):
     """ Verify that learning rate scheduling is working """
-    tutils.reset_seed()
 
     class CurrentTestModel(
             LightTestOptimizerWithSchedulingMixin,
@@ -54,7 +53,6 @@ def test_optimizer_with_scheduling(tmpdir):
 
 def test_multi_optimizer_with_scheduling(tmpdir):
     """ Verify that learning rate scheduling is working """
-    tutils.reset_seed()
 
     class CurrentTestModel(
             LightTestMultipleOptimizersWithSchedulingMixin,
@@ -94,7 +92,6 @@ def test_multi_optimizer_with_scheduling(tmpdir):
 
 
 def test_multi_optimizer_with_scheduling_stepping(tmpdir):
-    tutils.reset_seed()
 
     class CurrentTestModel(
             LightTestOptimizersWithMixedSchedulingMixin,
@@ -138,7 +135,6 @@ def test_multi_optimizer_with_scheduling_stepping(tmpdir):
 
 
 def test_reduce_lr_on_plateau_scheduling(tmpdir):
-    tutils.reset_seed()
 
     class CurrentTestModel(
             LightTestReduceLROnPlateauMixin,
@@ -168,10 +164,9 @@ def test_reduce_lr_on_plateau_scheduling(tmpdir):
 
 
 def test_optimizer_return_options():
-    tutils.reset_seed()
 
     trainer = Trainer()
-    model, hparams = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     # single optimizer
     opt_a = torch.optim.Adam(model.parameters(), lr=0.002)
@@ -226,11 +221,10 @@ def test_optimizer_return_options():
 
 
 def test_none_optimizer_warning():
-    tutils.reset_seed()
 
     trainer = Trainer()
-    model, hparams = tutils.get_default_model()
 
+    model = EvalModelTemplate(tutils.get_default_hparams())
     model.configure_optimizers = lambda: None
 
     with pytest.warns(UserWarning, match='will run with no optimizer'):
@@ -238,7 +232,6 @@ def test_none_optimizer_warning():
 
 
 def test_none_optimizer(tmpdir):
-    tutils.reset_seed()
 
     class CurrentTestModel(
             LightTestNoneOptimizerMixin,

tests/trainer/test_trainer.py

@@ -1,6 +1,7 @@
 import glob
 import math
 import os
+import types
 from argparse import Namespace
 
 import pytest
@@ -22,7 +23,7 @@ from tests.base import (
     LightValidationMultipleDataloadersMixin,
     LightTrainDataloader,
     LightTestDataloader,
-    LightValidationMixin,
+    LightValidationMixin, EvalModelTemplate,
 )
 
 
@@ -53,7 +54,6 @@ def test_hparams_save_load(tmpdir):
 
 def test_no_val_module(tmpdir):
     """Tests use case where trainer saves the model, and user loads it from tags independently."""
-    tutils.reset_seed()
 
     hparams = tutils.get_default_hparams()
 
@@ -92,7 +92,6 @@ def test_no_val_module(tmpdir):
 
 def test_no_val_end_module(tmpdir):
     """Tests use case where trainer saves the model, and user loads it from tags independently."""
-    tutils.reset_seed()
 
     class CurrentTestModel(LightTrainDataloader, LightValidationStepMixin, TestModelBase):
         pass
@@ -132,7 +131,6 @@ def test_gradient_accumulation_scheduling(tmpdir):
     """
     Test grad accumulation by the freq of optimizer updates
     """
-    tutils.reset_seed()
 
     # test incorrect configs
     with pytest.raises(IndexError):
@@ -205,7 +203,6 @@ def test_gradient_accumulation_scheduling(tmpdir):
 
 
 def test_loading_meta_tags(tmpdir):
-    tutils.reset_seed()
 
     hparams = tutils.get_default_hparams()
 
@@ -225,7 +222,6 @@ def test_loading_meta_tags(tmpdir):
 
 def test_dp_output_reduce():
     mixin = TrainerLoggingMixin()
-    tutils.reset_seed()
 
     # test identity when we have a single gpu
     out = torch.rand(3, 1)
@@ -291,7 +287,6 @@ def test_model_checkpoint_options(tmpdir, save_top_k, file_prefix, expected_file
 
 
 def test_model_freeze_unfreeze():
-    tutils.reset_seed()
 
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)
@@ -300,11 +295,8 @@ def test_model_freeze_unfreeze():
     model.unfreeze()
 
 
-def test_resume_from_checkpoint(tmpdir):
-    """Verify resuming from checkpoint (epoch, batch numbers and on_load_checkpoint())"""
-    import types
-
-    tutils.reset_seed()
+def test_resume_from_checkpoint_epoch_restored(tmpdir):
+    """Verify resuming from checkpoint runs the right number of epochs"""
 
     hparams = tutils.get_default_hparams()
 
@@ -371,8 +363,7 @@ def test_resume_from_checkpoint(tmpdir):
 
 def _init_steps_model():
     """private method for initializing a model with 5% train epochs"""
-    tutils.reset_seed()
-    model, _ = tutils.get_default_model()
+    model = EvalModelTemplate(tutils.get_default_hparams())
 
     # define train epoch to 5% of data
     train_percent = 0.5
@@ -460,7 +451,6 @@ def test_trainer_min_steps_and_epochs(tmpdir):
 
 def test_benchmark_option(tmpdir):
     """Verify benchmark option."""
-    tutils.reset_seed()
 
     class CurrentTestModel(
         LightValidationMultipleDataloadersMixin,
@@ -523,7 +513,6 @@ def test_testpass_overrides(tmpdir):
 
 def test_disabled_validation():
     """Verify that `val_percent_check=0` disables the validation loop unless `fast_dev_run=True`."""
-    tutils.reset_seed()
 
     class CurrentModel(LightTrainDataloader, LightValidationMixin, TestModelBase):
 
@@ -666,7 +655,6 @@ def test_gradient_clipping(tmpdir):
     """
     Test gradient clipping
     """
-    tutils.reset_seed()
 
     hparams = tutils.get_default_hparams()
     model = LightningTestModel(hparams)

tests/trainer/test_trainer_cli.py

@@ -13,7 +13,6 @@ from pytorch_lightning import Trainer
             return_value=Namespace(**Trainer.default_attributes()))
 def test_default_args(tmpdir):
     """Tests default argument parser for Trainer"""
-    tutils.reset_seed()
 
     # logger file to get meta
     logger = tutils.get_default_logger(tmpdir)