From f3800279512cd90ac29c3b0a9f5b859a4ba8f70a Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sat, 2 May 2020 14:38:22 +0200 Subject: [PATCH] refactor default model (#1652) * refactor default model * drop redundant seeds * formatting * path * formatting * rename --- tests/base/eval_model_optimizers.py | 12 ++--- tests/base/eval_model_template.py | 2 +- tests/base/eval_model_test_steps.py | 2 +- tests/base/eval_model_valid_steps.py | 2 +- tests/base/utils.py | 20 ++------ tests/callbacks/test_callbacks.py | 7 +-- tests/loggers/test_all.py | 7 +-- tests/loggers/test_base.py | 4 +- tests/loggers/test_neptune.py | 5 +- tests/loggers/test_trains.py | 4 -- tests/loggers/test_wandb.py | 4 -- .../data/horovod/train_default_model.py | 12 ++--- tests/models/test_amp.py | 16 ++---- tests/models/test_cpu.py | 50 +++++-------------- tests/models/test_gpu.py | 40 ++++++--------- tests/models/test_horovod.py | 8 ++- tests/models/test_restore.py | 11 ---- tests/trainer/test_checks.py | 5 -- tests/trainer/test_dataloaders.py | 12 ----- tests/trainer/test_lr_finder.py | 12 ++--- tests/trainer/test_optimizers.py | 13 ++--- tests/trainer/test_trainer.py | 22 ++------ tests/trainer/test_trainer_cli.py | 1 - 23 files changed, 77 insertions(+), 194 deletions(-) diff --git a/tests/base/eval_model_optimizers.py b/tests/base/eval_model_optimizers.py index 1666e26e..bcce319d 100644 --- a/tests/base/eval_model_optimizers.py +++ b/tests/base/eval_model_optimizers.py @@ -15,7 +15,7 @@ class ConfigureOptimizersPool(ABC): def configure_optimizers_empty(self): return None - def configure_optimizers_lbfgs(self): + def configure_optimizers__lbfgs(self): """ return whatever optimizers we want here. :return: list of optimizers @@ -23,7 +23,7 @@ class ConfigureOptimizersPool(ABC): optimizer = optim.LBFGS(self.parameters(), lr=self.hparams.learning_rate) return optimizer - def configure_optimizers_multiple_optimizers(self): + def configure_optimizers__multiple_optimizers(self): """ return whatever optimizers we want here. :return: list of optimizers @@ -33,12 +33,12 @@ class ConfigureOptimizersPool(ABC): optimizer2 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) return optimizer1, optimizer2 - def configure_optimizers_single_scheduler(self): + def configure_optimizers__single_scheduler(self): optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) lr_scheduler = optim.lr_scheduler.StepLR(optimizer, 1, gamma=0.1) return [optimizer], [lr_scheduler] - def configure_optimizers_multiple_schedulers(self): + def configure_optimizers__multiple_schedulers(self): optimizer1 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) optimizer2 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) lr_scheduler1 = optim.lr_scheduler.StepLR(optimizer1, 1, gamma=0.1) @@ -46,7 +46,7 @@ class ConfigureOptimizersPool(ABC): return [optimizer1, optimizer2], [lr_scheduler1, lr_scheduler2] - def configure_optimizers_mixed_scheduling(self): + def configure_optimizers__mixed_scheduling(self): optimizer1 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) optimizer2 = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) lr_scheduler1 = optim.lr_scheduler.StepLR(optimizer1, 4, gamma=0.1) @@ -55,7 +55,7 @@ class ConfigureOptimizersPool(ABC): return [optimizer1, optimizer2], \ [{'scheduler': lr_scheduler1, 'interval': 'step'}, lr_scheduler2] - def configure_optimizers_reduce_lr_on_plateau(self): + def configure_optimizers__reduce_lr_on_plateau(self): optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) lr_scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer) return [optimizer], [lr_scheduler] diff --git a/tests/base/eval_model_template.py b/tests/base/eval_model_template.py index 77d83b48..37f4dfbd 100644 --- a/tests/base/eval_model_template.py +++ b/tests/base/eval_model_template.py @@ -33,7 +33,7 @@ class EvalModelTemplate( """ This template houses all combinations of model configurations we want to test """ - def __init__(self, hparams): + def __init__(self, hparams: object) -> object: """Pass in parsed HyperOptArgumentParser to the model.""" # init superclass super().__init__() diff --git a/tests/base/eval_model_test_steps.py b/tests/base/eval_model_test_steps.py index ed8fe75c..b4c80cff 100644 --- a/tests/base/eval_model_test_steps.py +++ b/tests/base/eval_model_test_steps.py @@ -45,7 +45,7 @@ class TestStepVariations(ABC): }) return output - def test_step_multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs): + def test_step__multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs): """ Default, baseline test_step :param batch: diff --git a/tests/base/eval_model_valid_steps.py b/tests/base/eval_model_valid_steps.py index 1f40b45f..d6c9a847 100644 --- a/tests/base/eval_model_valid_steps.py +++ b/tests/base/eval_model_valid_steps.py @@ -51,7 +51,7 @@ class ValidationStepVariations(ABC): }) return output - def validation_step_multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs): + def validation_step__multiple_dataloaders(self, batch, batch_idx, dataloader_idx, **kwargs): """ Lightning calls this inside the validation loop :param batch: diff --git a/tests/base/utils.py b/tests/base/utils.py index 42e6d17d..f27d0bbd 100644 --- a/tests/base/utils.py +++ b/tests/base/utils.py @@ -9,7 +9,7 @@ from pytorch_lightning import Trainer from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.loggers import TensorBoardLogger from tests import TEMP_PATH, RANDOM_PORTS, RANDOM_SEEDS -from tests.base import LightningTestModel +from tests.base import LightningTestModel, EvalModelTemplate from tests.base.datasets import PATH_DATASETS @@ -27,6 +27,8 @@ def assert_speed_parity(pl_times, pt_times, num_epochs): def run_model_test_without_loggers(trainer_options, model, min_acc=0.50): + reset_seed() + # fit model trainer = Trainer(**trainer_options) result = trainer.fit(model) @@ -54,6 +56,7 @@ def run_model_test_without_loggers(trainer_options, model, min_acc=0.50): def run_model_test(trainer_options, model, on_gpu=True, version=None, with_hpc=True): + reset_seed() save_dir = trainer_options['default_root_dir'] # logger file to get meta @@ -95,8 +98,6 @@ def run_model_test(trainer_options, model, on_gpu=True, version=None, with_hpc=T def get_default_hparams(continue_training=False, hpc_exp_number=0): - _ = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - args = { 'drop_prob': 0.2, 'batch_size': 32, @@ -120,18 +121,6 @@ def get_default_hparams(continue_training=False, hpc_exp_number=0): return hparams -def get_default_model(lbfgs=False): - # set up model with these hyperparams - hparams = get_default_hparams() - if lbfgs: - setattr(hparams, 'optimizer_name', 'lbfgs') - setattr(hparams, 'learning_rate', 0.005) - - model = LightningTestModel(hparams) - - return model, hparams - - def get_default_logger(save_dir, version=None): # set up logger object without actually saving logs logger = TensorBoardLogger(save_dir, name='lightning_logs', version=version) @@ -229,6 +218,7 @@ def reset_seed(): def set_random_master_port(): + reset_seed() port = RANDOM_PORTS.pop() os.environ['MASTER_PORT'] = str(port) diff --git a/tests/callbacks/test_callbacks.py b/tests/callbacks/test_callbacks.py index fcd0836f..2bbcfaea 100644 --- a/tests/callbacks/test_callbacks.py +++ b/tests/callbacks/test_callbacks.py @@ -214,8 +214,6 @@ def test_trainer_callback_system(tmpdir): def test_early_stopping_no_val_step(tmpdir): """Test that early stopping callback falls back to training metrics when no validation defined.""" - tutils.reset_seed() - class ModelWithoutValStep(LightTrainDataloader, TestModelBase): def training_step(self, *args, **kwargs): @@ -224,8 +222,7 @@ def test_early_stopping_no_val_step(tmpdir): output.update({'my_train_metric': loss}) return output - hparams = tutils.get_default_hparams() - model = ModelWithoutValStep(hparams) + model = ModelWithoutValStep(tutils.get_default_hparams()) stopping = EarlyStopping(monitor='my_train_metric', min_delta=0.1) @@ -269,7 +266,7 @@ def test_model_checkpoint_with_non_string_input(tmpdir, save_top_k): overfit_pct=0.20, max_epochs=5 ) - result = trainer.fit(model) + trainer.fit(model) # These should be different if the dirpath has be overridden assert trainer.ckpt_path != trainer.default_root_dir diff --git a/tests/loggers/test_all.py b/tests/loggers/test_all.py index 383ca263..06e93fa6 100644 --- a/tests/loggers/test_all.py +++ b/tests/loggers/test_all.py @@ -7,6 +7,7 @@ import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.loggers import ( TensorBoardLogger, MLFlowLogger, NeptuneLogger, TestTubeLogger, CometLogger) +from tests.base import EvalModelTemplate def _get_logger_args(logger_class, save_dir): @@ -29,14 +30,12 @@ def _get_logger_args(logger_class, save_dir): ]) def test_loggers_fit_test(tmpdir, monkeypatch, logger_class): """Verify that basic functionality of all loggers.""" - tutils.reset_seed() - # prevent comet logger from trying to print at exit, since # pytest's stdout/stderr redirection breaks it import atexit monkeypatch.setattr(atexit, 'register', lambda _: None) - model, _ = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) class StoreHistoryLogger(logger_class): def __init__(self, *args, **kwargs): @@ -78,8 +77,6 @@ def test_loggers_fit_test(tmpdir, monkeypatch, logger_class): ]) def test_loggers_pickle(tmpdir, monkeypatch, logger_class): """Verify that pickling trainer with logger works.""" - tutils.reset_seed() - # prevent comet logger from trying to print at exit, since # pytest's stdout/stderr redirection breaks it import atexit diff --git a/tests/loggers/test_base.py b/tests/loggers/test_base.py index 56f6b97b..1a52dadf 100644 --- a/tests/loggers/test_base.py +++ b/tests/loggers/test_base.py @@ -7,7 +7,7 @@ import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.loggers import LightningLoggerBase, LoggerCollection from pytorch_lightning.utilities import rank_zero_only -from tests.base import LightningTestModel +from tests.base import LightningTestModel, EvalModelTemplate def test_logger_collection(): @@ -139,7 +139,7 @@ def test_adding_step_key(tmpdir): return decorated - model, hparams = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) model.validation_epoch_end = _validation_epoch_end model.training_epoch_end = _training_epoch_end trainer = Trainer( diff --git a/tests/loggers/test_neptune.py b/tests/loggers/test_neptune.py index 4cfdd673..11961234 100644 --- a/tests/loggers/test_neptune.py +++ b/tests/loggers/test_neptune.py @@ -61,10 +61,7 @@ def test_neptune_additional_methods(neptune): def test_neptune_leave_open_experiment_after_fit(tmpdir): """Verify that neptune experiment was closed after training""" - tutils.reset_seed() - - hparams = tutils.get_default_hparams() - model = LightningTestModel(hparams) + model = LightningTestModel(tutils.get_default_hparams()) def _run_training(logger): logger._experiment = MagicMock() diff --git a/tests/loggers/test_trains.py b/tests/loggers/test_trains.py index e4ee78c6..305d0707 100644 --- a/tests/loggers/test_trains.py +++ b/tests/loggers/test_trains.py @@ -8,8 +8,6 @@ from tests.base import LightningTestModel def test_trains_logger(tmpdir): """Verify that basic functionality of TRAINS logger works.""" - tutils.reset_seed() - hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) TrainsLogger.set_bypass_mode(True) @@ -33,8 +31,6 @@ def test_trains_logger(tmpdir): def test_trains_pickle(tmpdir): """Verify that pickling trainer with TRAINS logger works.""" - tutils.reset_seed() - # hparams = tutils.get_default_hparams() # model = LightningTestModel(hparams) TrainsLogger.set_bypass_mode(True) diff --git a/tests/loggers/test_wandb.py b/tests/loggers/test_wandb.py index bb2739f9..82555f88 100644 --- a/tests/loggers/test_wandb.py +++ b/tests/loggers/test_wandb.py @@ -11,8 +11,6 @@ from pytorch_lightning.loggers import WandbLogger def test_wandb_logger(wandb): """Verify that basic functionality of wandb logger works. Wandb doesn't work well with pytest so we have to mock it out here.""" - tutils.reset_seed() - logger = WandbLogger(anonymous=True, offline=True) logger.log_metrics({'acc': 1.0}) @@ -38,8 +36,6 @@ def test_wandb_pickle(wandb): Wandb doesn't work well with pytest so we have to mock it out here. """ - tutils.reset_seed() - class Experiment: id = 'the_id' diff --git a/tests/models/data/horovod/train_default_model.py b/tests/models/data/horovod/train_default_model.py index 3410cdc1..5b9a08c6 100644 --- a/tests/models/data/horovod/train_default_model.py +++ b/tests/models/data/horovod/train_default_model.py @@ -29,7 +29,8 @@ sys.path.insert(0, os.path.abspath(PATH_ROOT)) from pytorch_lightning import Trainer # noqa: E402 from pytorch_lightning.callbacks import ModelCheckpoint # noqa: E402 -import tests.base.utils as tutils # noqa: E402 +from tests.base import EvalModelTemplate # noqa: E402 +from tests.base.utils import set_random_master_port, get_default_hparams, run_model_test # noqa: E402 parser = argparse.ArgumentParser() @@ -39,14 +40,13 @@ parser.add_argument('--on-gpu', action='store_true', default=False) def run_test_from_config(trainer_options): """Trains the default model with the given config.""" - tutils.reset_seed() - tutils.set_random_master_port() + set_random_master_port() ckpt_path = trainer_options['default_root_dir'] - trainer_options['checkpoint_callback'] = ModelCheckpoint(ckpt_path) + trainer_options.update(checkpoint_callback=ModelCheckpoint(ckpt_path)) - model, hparams = tutils.get_default_model() - tutils.run_model_test(trainer_options, model, on_gpu=args.on_gpu, version=0, with_hpc=False) + model = EvalModelTemplate(get_default_hparams()) + run_model_test(trainer_options, model, on_gpu=args.on_gpu, version=0, with_hpc=False) # Horovod should be initialized following training. If not, this will raise an exception. assert hvd.size() == 2 diff --git a/tests/models/test_amp.py b/tests/models/test_amp.py index 81a2325c..f4f1d9c2 100644 --- a/tests/models/test_amp.py +++ b/tests/models/test_amp.py @@ -6,9 +6,7 @@ import torch import tests.base.utils as tutils from pytorch_lightning import Trainer from pytorch_lightning.utilities.exceptions import MisconfigurationException -from tests.base import ( - LightningTestModel, -) +from tests.base import LightningTestModel, EvalModelTemplate @pytest.mark.spawn @@ -18,8 +16,6 @@ def test_amp_single_gpu(tmpdir, backend): """Make sure DP/DDP + AMP work.""" tutils.reset_seed() - model, hparams = tutils.get_default_model() - trainer = Trainer( default_root_dir=tmpdir, max_epochs=1, @@ -28,6 +24,7 @@ def test_amp_single_gpu(tmpdir, backend): precision=16 ) + model = EvalModelTemplate(tutils.get_default_hparams()) # tutils.run_model_test(trainer_options, model) result = trainer.fit(model) @@ -39,10 +36,9 @@ def test_amp_single_gpu(tmpdir, backend): @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_amp_multi_gpu(tmpdir, backend): """Make sure DP/DDP + AMP work.""" - tutils.reset_seed() tutils.set_random_master_port() - model, hparams = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) trainer_options = dict( default_root_dir=tmpdir, @@ -63,8 +59,6 @@ def test_amp_multi_gpu(tmpdir, backend): @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_amp_gpu_ddp_slurm_managed(tmpdir): """Make sure DDP + AMP work.""" - tutils.reset_seed() - # simulate setting slurm flags tutils.set_random_master_port() os.environ['SLURM_LOCALID'] = str(0) @@ -102,8 +96,6 @@ def test_amp_gpu_ddp_slurm_managed(tmpdir): def test_cpu_model_with_amp(tmpdir): """Make sure model trains on CPU.""" - tutils.reset_seed() - trainer_options = dict( default_root_dir=tmpdir, progress_bar_refresh_rate=0, @@ -113,7 +105,7 @@ def test_cpu_model_with_amp(tmpdir): precision=16 ) - model, hparams = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) with pytest.raises((MisconfigurationException, ModuleNotFoundError)): tutils.run_model_test(trainer_options, model, on_gpu=False) diff --git a/tests/models/test_cpu.py b/tests/models/test_cpu.py index e7b422dc..46d1ba6e 100644 --- a/tests/models/test_cpu.py +++ b/tests/models/test_cpu.py @@ -15,13 +15,12 @@ from tests.base import ( LightTrainDataloader, LightningTestModel, LightTestMixin, + EvalModelTemplate, ) def test_early_stopping_cpu_model(tmpdir): """Test each of the trainer options.""" - tutils.reset_seed() - stopping = EarlyStopping(monitor='val_loss', min_delta=0.1) trainer_options = dict( default_root_dir=tmpdir, @@ -33,7 +32,7 @@ def test_early_stopping_cpu_model(tmpdir): val_percent_check=0.1, ) - model, hparams = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) tutils.run_model_test(trainer_options, model, on_gpu=False) # test freeze on cpu @@ -49,10 +48,8 @@ def test_early_stopping_cpu_model(tmpdir): reason="Distributed training is not supported on MacOS before Torch 1.3.0") def test_multi_cpu_model_ddp(tmpdir): """Make sure DDP works.""" - tutils.reset_seed() tutils.set_random_master_port() - model, hparams = tutils.get_default_model() trainer_options = dict( default_root_dir=tmpdir, progress_bar_refresh_rate=0, @@ -64,13 +61,12 @@ def test_multi_cpu_model_ddp(tmpdir): distributed_backend='ddp_cpu' ) + model = EvalModelTemplate(tutils.get_default_hparams()) tutils.run_model_test(trainer_options, model, on_gpu=False) def test_lbfgs_cpu_model(tmpdir): """Test each of the trainer options.""" - tutils.reset_seed() - trainer_options = dict( default_root_dir=tmpdir, max_epochs=2, @@ -80,15 +76,16 @@ def test_lbfgs_cpu_model(tmpdir): val_percent_check=0.2, ) - model, hparams = tutils.get_default_model(lbfgs=True) - # the test is there for the closure not the performance - tutils.run_model_test_without_loggers(trainer_options, model, min_acc=0.) + hparams = tutils.get_default_hparams() + setattr(hparams, 'optimizer_name', 'lbfgs') + setattr(hparams, 'learning_rate', 0.002) + model = EvalModelTemplate(hparams) + model.configure_optimizers = model.configure_optimizers__lbfgs + tutils.run_model_test_without_loggers(trainer_options, model, min_acc=0.5) def test_default_logger_callbacks_cpu_model(tmpdir): """Test each of the trainer options.""" - tutils.reset_seed() - trainer_options = dict( default_root_dir=tmpdir, max_epochs=1, @@ -99,7 +96,7 @@ def test_default_logger_callbacks_cpu_model(tmpdir): val_percent_check=0.01, ) - model, hparams = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) tutils.run_model_test_without_loggers(trainer_options, model) # test freeze on cpu @@ -109,8 +106,6 @@ def test_default_logger_callbacks_cpu_model(tmpdir): def test_running_test_after_fitting(tmpdir): """Verify test() on fitted model.""" - tutils.reset_seed() - hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) @@ -143,8 +138,6 @@ def test_running_test_after_fitting(tmpdir): def test_running_test_no_val(tmpdir): """Verify `test()` works on a model with no `val_loader`.""" - tutils.reset_seed() - class CurrentTestModel(LightTrainDataloader, LightTestMixin, TestModelBase): pass @@ -180,8 +173,6 @@ def test_running_test_no_val(tmpdir): @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine") def test_single_gpu_batch_parse(): - tutils.reset_seed() - trainer = Trainer() # batch is just a tensor @@ -229,8 +220,6 @@ def test_single_gpu_batch_parse(): def test_simple_cpu(tmpdir): """Verify continue training session on CPU.""" - tutils.reset_seed() - hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) @@ -249,8 +238,6 @@ def test_simple_cpu(tmpdir): def test_cpu_model(tmpdir): """Make sure model trains on CPU.""" - tutils.reset_seed() - trainer_options = dict( default_root_dir=tmpdir, progress_bar_refresh_rate=0, @@ -259,15 +246,13 @@ def test_cpu_model(tmpdir): val_percent_check=0.4 ) - model, hparams = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) tutils.run_model_test(trainer_options, model, on_gpu=False) def test_all_features_cpu_model(tmpdir): """Test each of the trainer options.""" - tutils.reset_seed() - trainer_options = dict( default_root_dir=tmpdir, gradient_clip_val=1.0, @@ -280,14 +265,12 @@ def test_all_features_cpu_model(tmpdir): val_percent_check=0.4 ) - model, hparams = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) tutils.run_model_test(trainer_options, model, on_gpu=False) def test_tbptt_cpu_model(tmpdir): """Test truncated back propagation through time works.""" - tutils.reset_seed() - truncated_bptt_steps = 2 sequence_size = 30 batch_size = 30 @@ -358,10 +341,6 @@ def test_tbptt_cpu_model(tmpdir): @pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU machine") def test_single_gpu_model(tmpdir): """Make sure single GPU works (DP mode).""" - tutils.reset_seed() - - model, hparams = tutils.get_default_model() - trainer_options = dict( default_root_dir=tmpdir, progress_bar_refresh_rate=0, @@ -371,8 +350,5 @@ def test_single_gpu_model(tmpdir): gpus=1 ) + model = EvalModelTemplate(tutils.get_default_hparams()) tutils.run_model_test(trainer_options, model) - - -# if __name__ == '__main__': -# pytest.main([__file__]) diff --git a/tests/models/test_gpu.py b/tests/models/test_gpu.py index dcd90b08..dbaf4db8 100644 --- a/tests/models/test_gpu.py +++ b/tests/models/test_gpu.py @@ -9,7 +9,7 @@ from pytorch_lightning.callbacks import ModelCheckpoint from pytorch_lightning.core import memory from pytorch_lightning.trainer.distrib_parts import parse_gpu_ids, determine_root_gpu_device from pytorch_lightning.utilities.exceptions import MisconfigurationException -from tests.base import LightningTestModel +from tests.base import LightningTestModel, EvalModelTemplate PRETEND_N_OF_GPUS = 16 @@ -19,11 +19,8 @@ PRETEND_N_OF_GPUS = 16 @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_multi_gpu_model(tmpdir, backend): """Make sure DDP works.""" - - tutils.reset_seed() tutils.set_random_master_port() - model, hparams = tutils.get_default_model() trainer_options = dict( default_root_dir=tmpdir, max_epochs=1, @@ -33,6 +30,7 @@ def test_multi_gpu_model(tmpdir, backend): distributed_backend=backend, ) + model = EvalModelTemplate(tutils.get_default_hparams()) # tutils.run_model_test(trainer_options, model) trainer = Trainer(**trainer_options) result = trainer.fit(model) @@ -45,31 +43,27 @@ def test_multi_gpu_model(tmpdir, backend): @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_ddp_all_dataloaders_passed_to_fit(tmpdir): """Make sure DDP works with dataloaders passed to fit()""" - - tutils.reset_seed() tutils.set_random_master_port() - model, hparams = tutils.get_default_model() + trainer_options = dict(default_root_dir=tmpdir, + progress_bar_refresh_rate=0, + max_epochs=1, + train_percent_check=0.4, + val_percent_check=0.2, + gpus=[0, 1], + distributed_backend='ddp') - trainer = Trainer( - default_root_dir=tmpdir, - progress_bar_refresh_rate=0, - max_epochs=1, - train_percent_check=0.4, - val_percent_check=0.2, - gpus=[0, 1], - distributed_backend='ddp' - ) - result = trainer.fit(model, - train_dataloader=model.train_dataloader(), - val_dataloaders=model.val_dataloader()) + model = EvalModelTemplate(tutils.get_default_hparams()) + fit_options = dict(train_dataloader=model.train_dataloader(), + val_dataloaders=model.val_dataloader()) + + trainer = Trainer(**trainer_options) + result = trainer.fit(model, **fit_options) assert result == 1, "DDP doesn't work with dataloaders passed to fit()." def test_cpu_slurm_save_load(tmpdir): """Verify model save/load/checkpoint on CPU.""" - tutils.reset_seed() - hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) @@ -139,9 +133,6 @@ def test_cpu_slurm_save_load(tmpdir): @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_multi_gpu_none_backend(tmpdir): """Make sure when using multiple GPUs the user can't use `distributed_backend = None`.""" - tutils.reset_seed() - - model, hparams = tutils.get_default_model() trainer_options = dict( default_root_dir=tmpdir, progress_bar_refresh_rate=0, @@ -151,6 +142,7 @@ def test_multi_gpu_none_backend(tmpdir): gpus='-1' ) + model = EvalModelTemplate(tutils.get_default_hparams()) with pytest.warns(UserWarning): tutils.run_model_test(trainer_options, model) diff --git a/tests/models/test_horovod.py b/tests/models/test_horovod.py index 21a90c19..0f41dee6 100644 --- a/tests/models/test_horovod.py +++ b/tests/models/test_horovod.py @@ -40,8 +40,12 @@ def _nccl_available(): def _run_horovod(trainer_options, on_gpu=False): """Execute the training script across multiple workers in parallel.""" - cmdline = ['horovodrun', '-np', '2', sys.executable, TEST_SCRIPT, - '--trainer-options', shlex.quote(json.dumps(trainer_options))] + cmdline = [ + 'horovodrun', + '-np', '2', + sys.executable, TEST_SCRIPT, + '--trainer-options', shlex.quote(json.dumps(trainer_options)) + ] if on_gpu: cmdline += ['--on-gpu'] exit_code = subprocess.call(' '.join(cmdline), shell=True, env=os.environ.copy()) diff --git a/tests/models/test_restore.py b/tests/models/test_restore.py index 0921a3a8..af0165d4 100644 --- a/tests/models/test_restore.py +++ b/tests/models/test_restore.py @@ -21,8 +21,6 @@ from tests.base import ( @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_running_test_pretrained_model_distrib(tmpdir, backend): """Verify `test()` on pretrained model.""" - - tutils.reset_seed() tutils.set_random_master_port() hparams = tutils.get_default_hparams() @@ -74,8 +72,6 @@ def test_running_test_pretrained_model_distrib(tmpdir, backend): def test_running_test_pretrained_model_cpu(tmpdir): """Verify test() on pretrained model.""" - tutils.reset_seed() - hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) @@ -113,8 +109,6 @@ def test_running_test_pretrained_model_cpu(tmpdir): def test_load_model_from_checkpoint(tmpdir): """Verify test() on pretrained model.""" - tutils.reset_seed() - hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) @@ -157,9 +151,6 @@ def test_load_model_from_checkpoint(tmpdir): @pytest.mark.skipif(torch.cuda.device_count() < 2, reason="test requires multi-GPU machine") def test_dp_resume(tmpdir): """Make sure DP continues training correctly.""" - - tutils.reset_seed() - hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) @@ -232,8 +223,6 @@ def test_dp_resume(tmpdir): def test_model_saving_loading(tmpdir): """Tests use case where trainer saves the model, and user loads it from tags independently.""" - tutils.reset_seed() - hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) diff --git a/tests/trainer/test_checks.py b/tests/trainer/test_checks.py index d69ec1e6..7a68b2ab 100755 --- a/tests/trainer/test_checks.py +++ b/tests/trainer/test_checks.py @@ -15,7 +15,6 @@ from tests.base import ( def test_error_on_no_train_step(tmpdir): """ Test that an error is thrown when no `training_step()` is defined """ - tutils.reset_seed() class CurrentTestModel(LightningModule): def forward(self, x): @@ -30,7 +29,6 @@ def test_error_on_no_train_step(tmpdir): def test_error_on_no_train_dataloader(tmpdir): """ Test that an error is thrown when no `training_dataloader()` is defined """ - tutils.reset_seed() hparams = tutils.get_default_hparams() class CurrentTestModel(TestModelBase): @@ -45,7 +43,6 @@ def test_error_on_no_train_dataloader(tmpdir): def test_error_on_no_configure_optimizers(tmpdir): """ Test that an error is thrown when no `configure_optimizers()` is defined """ - tutils.reset_seed() class CurrentTestModel(LightTrainDataloader, LightningModule): def forward(self, x): @@ -68,7 +65,6 @@ def test_warning_on_wrong_validation_settings(tmpdir): throw warning if `val_epoch_end()` is not defined * error if `validation_step()` is overriden but `val_dataloader()` is not """ - tutils.reset_seed() hparams = tutils.get_default_hparams() trainer = Trainer(default_root_dir=tmpdir, max_epochs=1) @@ -111,7 +107,6 @@ def test_warning_on_wrong_test_settigs(tmpdir): throw warning if `test_epoch_end()` is not defined * error if `test_step()` is overriden but `test_dataloader()` is not """ - tutils.reset_seed() hparams = tutils.get_default_hparams() trainer = Trainer(default_root_dir=tmpdir, max_epochs=1) diff --git a/tests/trainer/test_dataloaders.py b/tests/trainer/test_dataloaders.py index 83ff481d..0d528474 100644 --- a/tests/trainer/test_dataloaders.py +++ b/tests/trainer/test_dataloaders.py @@ -55,7 +55,6 @@ def test_dataloader_config_errors(tmpdir, dataloader_options): def test_multiple_val_dataloader(tmpdir): """Verify multiple val_dataloader.""" - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader, @@ -90,7 +89,6 @@ def test_multiple_val_dataloader(tmpdir): def test_multiple_test_dataloader(tmpdir): """Verify multiple test_dataloader.""" - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader, @@ -127,7 +125,6 @@ def test_multiple_test_dataloader(tmpdir): def test_train_dataloaders_passed_to_fit(tmpdir): """Verify that train dataloader can be passed to fit """ - tutils.reset_seed() class CurrentTestModel(LightTrainDataloader, TestModelBase): pass @@ -149,7 +146,6 @@ def test_train_dataloaders_passed_to_fit(tmpdir): def test_train_val_dataloaders_passed_to_fit(tmpdir): """ Verify that train & val dataloader can be passed to fit """ - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader, @@ -178,7 +174,6 @@ def test_train_val_dataloaders_passed_to_fit(tmpdir): def test_all_dataloaders_passed_to_fit(tmpdir): """Verify train, val & test dataloader can be passed to fit """ - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader, @@ -215,7 +210,6 @@ def test_all_dataloaders_passed_to_fit(tmpdir): def test_multiple_dataloaders_passed_to_fit(tmpdir): """Verify that multiple val & test dataloaders can be passed to fit.""" - tutils.reset_seed() class CurrentTestModel( LightningTestModel, @@ -252,7 +246,6 @@ def test_multiple_dataloaders_passed_to_fit(tmpdir): def test_mixing_of_dataloader_options(tmpdir): """Verify that dataloaders can be passed to fit""" - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader, @@ -294,7 +287,6 @@ def test_mixing_of_dataloader_options(tmpdir): def test_inf_train_dataloader(tmpdir): """Test inf train data loader (e.g. IterableDataset)""" - tutils.reset_seed() class CurrentTestModel( LightInfTrainDataloader, @@ -336,7 +328,6 @@ def test_inf_train_dataloader(tmpdir): def test_inf_val_dataloader(tmpdir): """Test inf val data loader (e.g. IterableDataset)""" - tutils.reset_seed() class CurrentTestModel( LightInfValDataloader, @@ -369,7 +360,6 @@ def test_inf_val_dataloader(tmpdir): def test_inf_test_dataloader(tmpdir): """Test inf test data loader (e.g. IterableDataset)""" - tutils.reset_seed() class CurrentTestModel( LightInfTestDataloader, @@ -404,7 +394,6 @@ def test_inf_test_dataloader(tmpdir): def test_error_on_zero_len_dataloader(tmpdir): """ Test that error is raised if a zero-length dataloader is defined """ - tutils.reset_seed() class CurrentTestModel( LightZeroLenDataloader, @@ -428,7 +417,6 @@ def test_error_on_zero_len_dataloader(tmpdir): @pytest.mark.skipif(platform.system() == 'Windows', reason='Does not apply to Windows platform.') def test_warning_with_few_workers(tmpdir): """ Test that error is raised if dataloader with only a few workers is used """ - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader, diff --git a/tests/trainer/test_lr_finder.py b/tests/trainer/test_lr_finder.py index 96fe461e..ba6e9c33 100755 --- a/tests/trainer/test_lr_finder.py +++ b/tests/trainer/test_lr_finder.py @@ -12,8 +12,7 @@ from tests.base import ( def test_error_on_more_than_1_optimizer(tmpdir): - ''' Check that error is thrown when more than 1 optimizer is passed ''' - tutils.reset_seed() + """ Check that error is thrown when more than 1 optimizer is passed """ class CurrentTestModel( LightTestMultipleOptimizersWithSchedulingMixin, @@ -36,8 +35,7 @@ def test_error_on_more_than_1_optimizer(tmpdir): def test_model_reset_correctly(tmpdir): - ''' Check that model weights are correctly reset after lr_find() ''' - tutils.reset_seed() + """ Check that model weights are correctly reset after lr_find() """ class CurrentTestModel( LightTrainDataloader, @@ -66,8 +64,7 @@ def test_model_reset_correctly(tmpdir): def test_trainer_reset_correctly(tmpdir): - ''' Check that all trainer parameters are reset correctly after lr_find() ''' - tutils.reset_seed() + """ Check that all trainer parameters are reset correctly after lr_find() """ class CurrentTestModel( LightTrainDataloader, @@ -104,7 +101,6 @@ def test_trainer_reset_correctly(tmpdir): def test_trainer_arg_bool(tmpdir): - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader, @@ -129,7 +125,6 @@ def test_trainer_arg_bool(tmpdir): def test_trainer_arg_str(tmpdir): - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader, @@ -155,7 +150,6 @@ def test_trainer_arg_str(tmpdir): def test_call_to_trainer_method(tmpdir): - tutils.reset_seed() class CurrentTestModel( LightTrainDataloader, diff --git a/tests/trainer/test_optimizers.py b/tests/trainer/test_optimizers.py index b445dcb2..be0ac547 100644 --- a/tests/trainer/test_optimizers.py +++ b/tests/trainer/test_optimizers.py @@ -12,13 +12,12 @@ from tests.base import ( LightTestMultipleOptimizersWithSchedulingMixin, LightTestOptimizersWithMixedSchedulingMixin, LightTestReduceLROnPlateauMixin, - LightTestNoneOptimizerMixin + LightTestNoneOptimizerMixin, EvalModelTemplate ) def test_optimizer_with_scheduling(tmpdir): """ Verify that learning rate scheduling is working """ - tutils.reset_seed() class CurrentTestModel( LightTestOptimizerWithSchedulingMixin, @@ -54,7 +53,6 @@ def test_optimizer_with_scheduling(tmpdir): def test_multi_optimizer_with_scheduling(tmpdir): """ Verify that learning rate scheduling is working """ - tutils.reset_seed() class CurrentTestModel( LightTestMultipleOptimizersWithSchedulingMixin, @@ -94,7 +92,6 @@ def test_multi_optimizer_with_scheduling(tmpdir): def test_multi_optimizer_with_scheduling_stepping(tmpdir): - tutils.reset_seed() class CurrentTestModel( LightTestOptimizersWithMixedSchedulingMixin, @@ -138,7 +135,6 @@ def test_multi_optimizer_with_scheduling_stepping(tmpdir): def test_reduce_lr_on_plateau_scheduling(tmpdir): - tutils.reset_seed() class CurrentTestModel( LightTestReduceLROnPlateauMixin, @@ -168,10 +164,9 @@ def test_reduce_lr_on_plateau_scheduling(tmpdir): def test_optimizer_return_options(): - tutils.reset_seed() trainer = Trainer() - model, hparams = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) # single optimizer opt_a = torch.optim.Adam(model.parameters(), lr=0.002) @@ -226,11 +221,10 @@ def test_optimizer_return_options(): def test_none_optimizer_warning(): - tutils.reset_seed() trainer = Trainer() - model, hparams = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) model.configure_optimizers = lambda: None with pytest.warns(UserWarning, match='will run with no optimizer'): @@ -238,7 +232,6 @@ def test_none_optimizer_warning(): def test_none_optimizer(tmpdir): - tutils.reset_seed() class CurrentTestModel( LightTestNoneOptimizerMixin, diff --git a/tests/trainer/test_trainer.py b/tests/trainer/test_trainer.py index b7344a70..d72c04ed 100644 --- a/tests/trainer/test_trainer.py +++ b/tests/trainer/test_trainer.py @@ -1,6 +1,7 @@ import glob import math import os +import types from argparse import Namespace import pytest @@ -22,7 +23,7 @@ from tests.base import ( LightValidationMultipleDataloadersMixin, LightTrainDataloader, LightTestDataloader, - LightValidationMixin, + LightValidationMixin, EvalModelTemplate, ) @@ -53,7 +54,6 @@ def test_hparams_save_load(tmpdir): def test_no_val_module(tmpdir): """Tests use case where trainer saves the model, and user loads it from tags independently.""" - tutils.reset_seed() hparams = tutils.get_default_hparams() @@ -92,7 +92,6 @@ def test_no_val_module(tmpdir): def test_no_val_end_module(tmpdir): """Tests use case where trainer saves the model, and user loads it from tags independently.""" - tutils.reset_seed() class CurrentTestModel(LightTrainDataloader, LightValidationStepMixin, TestModelBase): pass @@ -132,7 +131,6 @@ def test_gradient_accumulation_scheduling(tmpdir): """ Test grad accumulation by the freq of optimizer updates """ - tutils.reset_seed() # test incorrect configs with pytest.raises(IndexError): @@ -205,7 +203,6 @@ def test_gradient_accumulation_scheduling(tmpdir): def test_loading_meta_tags(tmpdir): - tutils.reset_seed() hparams = tutils.get_default_hparams() @@ -225,7 +222,6 @@ def test_loading_meta_tags(tmpdir): def test_dp_output_reduce(): mixin = TrainerLoggingMixin() - tutils.reset_seed() # test identity when we have a single gpu out = torch.rand(3, 1) @@ -291,7 +287,6 @@ def test_model_checkpoint_options(tmpdir, save_top_k, file_prefix, expected_file def test_model_freeze_unfreeze(): - tutils.reset_seed() hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) @@ -300,11 +295,8 @@ def test_model_freeze_unfreeze(): model.unfreeze() -def test_resume_from_checkpoint(tmpdir): - """Verify resuming from checkpoint (epoch, batch numbers and on_load_checkpoint())""" - import types - - tutils.reset_seed() +def test_resume_from_checkpoint_epoch_restored(tmpdir): + """Verify resuming from checkpoint runs the right number of epochs""" hparams = tutils.get_default_hparams() @@ -371,8 +363,7 @@ def test_resume_from_checkpoint(tmpdir): def _init_steps_model(): """private method for initializing a model with 5% train epochs""" - tutils.reset_seed() - model, _ = tutils.get_default_model() + model = EvalModelTemplate(tutils.get_default_hparams()) # define train epoch to 5% of data train_percent = 0.5 @@ -460,7 +451,6 @@ def test_trainer_min_steps_and_epochs(tmpdir): def test_benchmark_option(tmpdir): """Verify benchmark option.""" - tutils.reset_seed() class CurrentTestModel( LightValidationMultipleDataloadersMixin, @@ -523,7 +513,6 @@ def test_testpass_overrides(tmpdir): def test_disabled_validation(): """Verify that `val_percent_check=0` disables the validation loop unless `fast_dev_run=True`.""" - tutils.reset_seed() class CurrentModel(LightTrainDataloader, LightValidationMixin, TestModelBase): @@ -666,7 +655,6 @@ def test_gradient_clipping(tmpdir): """ Test gradient clipping """ - tutils.reset_seed() hparams = tutils.get_default_hparams() model = LightningTestModel(hparams) diff --git a/tests/trainer/test_trainer_cli.py b/tests/trainer/test_trainer_cli.py index b2d1da95..c4c23d0f 100644 --- a/tests/trainer/test_trainer_cli.py +++ b/tests/trainer/test_trainer_cli.py @@ -13,7 +13,6 @@ from pytorch_lightning import Trainer return_value=Namespace(**Trainer.default_attributes())) def test_default_args(tmpdir): """Tests default argument parser for Trainer""" - tutils.reset_seed() # logger file to get meta logger = tutils.get_default_logger(tmpdir)