diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index b3e4edb5..0ab91b07 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -40,7 +40,7 @@ jobs: - name: Cache datasets uses: actions/cache@v1 with: - path: tests/models/mnist # This path is specific to Ubuntu + path: tests/datasets # This path is specific to Ubuntu # Look to see if there is a cache hit for the corresponding requirements file key: mnist-dataset diff --git a/pl_examples/basic_examples/lightning_module_template.py b/pl_examples/basic_examples/lightning_module_template.py index edc482c7..4caf251a 100644 --- a/pl_examples/basic_examples/lightning_module_template.py +++ b/pl_examples/basic_examples/lightning_module_template.py @@ -188,6 +188,8 @@ class LightningTemplateModel(pl.LightningModule): return [optimizer], [scheduler] def __dataloader(self, train): + # this is neede when you want some info about dataset before binding to trainer + self.prepare_data() # init data generators transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) @@ -208,10 +210,8 @@ class LightningTemplateModel(pl.LightningModule): def prepare_data(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) - dataset = MNIST(root=self.hparams.data_root, train=True, - transform=transform, download=True) - dataset = MNIST(root=self.hparams.data_root, train=False, - transform=transform, download=True) + _ = MNIST(root=self.hparams.data_root, train=True, + transform=transform, download=True) def train_dataloader(self): log.info('Training data loader called.') diff --git a/pytorch_lightning/core/lightning.py b/pytorch_lightning/core/lightning.py index fb3e2d0e..de596da4 100644 --- a/pytorch_lightning/core/lightning.py +++ b/pytorch_lightning/core/lightning.py @@ -1,7 +1,6 @@ import collections import inspect import logging as log -import csv import os import warnings from abc import ABC, abstractmethod @@ -13,7 +12,7 @@ import torch.distributed as dist from pytorch_lightning.core.decorators import data_loader from pytorch_lightning.core.grads import GradInformation from pytorch_lightning.core.hooks import ModelHooks -from pytorch_lightning.core.saving import ModelIO +from pytorch_lightning.core.saving import ModelIO, load_hparams_from_tags_csv from pytorch_lightning.core.memory import ModelSummary from pytorch_lightning.overrides.data_parallel import LightningDistributedDataParallel from pytorch_lightning.utilities.debugging import MisconfigurationException @@ -1316,34 +1315,3 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks): tqdm_dict['v_num'] = self.trainer.logger.version return tqdm_dict - - -def load_hparams_from_tags_csv(tags_csv): - if not os.path.isfile(tags_csv): - log.warning(f'Missing Tags: {tags_csv}.') - return Namespace() - - tags = {} - with open(tags_csv) as f: - csv_reader = csv.reader(f, delimiter=',') - for row in list(csv_reader)[1:]: - tags[row[0]] = convert(row[1]) - ns = Namespace(**tags) - return ns - - -def convert(val): - constructors = [int, float, str] - - if isinstance(val, str): - if val.lower() == 'true': - return True - if val.lower() == 'false': - return False - - for c in constructors: - try: - return c(val) - except ValueError: - pass - return val diff --git a/pytorch_lightning/core/saving.py b/pytorch_lightning/core/saving.py index ef44c7a5..e8a1fac7 100644 --- a/pytorch_lightning/core/saving.py +++ b/pytorch_lightning/core/saving.py @@ -1,3 +1,9 @@ +import os +import csv +import logging as log +from argparse import Namespace + + class ModelIO(object): def on_load_checkpoint(self, checkpoint): @@ -28,3 +34,34 @@ class ModelIO(object): Hook to do whatever you need right before Slurm manager loads the model :return: """ + + +def load_hparams_from_tags_csv(tags_csv): + if not os.path.isfile(tags_csv): + log.warning(f'Missing Tags: {tags_csv}.') + return Namespace() + + tags = {} + with open(tags_csv) as f: + csv_reader = csv.reader(f, delimiter=',') + for row in list(csv_reader)[1:]: + tags[row[0]] = convert(row[1]) + ns = Namespace(**tags) + return ns + + +def convert(val): + constructors = [int, float, str] + + if isinstance(val, str): + if val.lower() == 'true': + return True + if val.lower() == 'false': + return False + + for c in constructors: + try: + return c(val) + except ValueError: + pass + return val diff --git a/pytorch_lightning/loggers/comet.py b/pytorch_lightning/loggers/comet.py index fd8d456f..112b89a1 100644 --- a/pytorch_lightning/loggers/comet.py +++ b/pytorch_lightning/loggers/comet.py @@ -22,8 +22,8 @@ except ImportError: from torch import is_tensor +from pytorch_lightning.utilities.debugging import MisconfigurationException from .base import LightningLoggerBase, rank_zero_only -from ..utilities.debugging import MisconfigurationException logger = getLogger(__name__) diff --git a/pytorch_lightning/trainer/data_loading.py b/pytorch_lightning/trainer/data_loading.py index c355aef4..d1e548ee 100644 --- a/pytorch_lightning/trainer/data_loading.py +++ b/pytorch_lightning/trainer/data_loading.py @@ -177,7 +177,7 @@ class TrainerDataLoadingMixin(ABC): self.is_iterable_train_dataloader = ( EXIST_ITER_DATASET and isinstance(self.train_dataloader.dataset, IterableDataset) ) - if self.is_iterable_train_dataloader and not isinstance(self.val_check_interval, int): + if self.is_iterable_dataloader(self.train_dataloader) and not isinstance(self.val_check_interval, int): m = ''' When using an iterableDataset for `train_dataloader`, `Trainer(val_check_interval)` must be an int. @@ -185,6 +185,11 @@ class TrainerDataLoadingMixin(ABC): ''' raise MisconfigurationException(m) + def is_iterable_dataloader(self, dataloader): + return ( + EXIST_ITER_DATASET and isinstance(dataloader.dataset, IterableDataset) + ) + def reset_val_dataloader(self, model): """ Dataloaders are provided by the model @@ -200,9 +205,8 @@ class TrainerDataLoadingMixin(ABC): self.num_val_batches = 0 # add samplers - for i, dataloader in enumerate(self.val_dataloaders): - dl = self.auto_add_sampler(dataloader, train=False) - self.val_dataloaders[i] = dl + self.val_dataloaders = [self.auto_add_sampler(dl, train=False) + for dl in self.val_dataloaders if dl] # determine number of validation batches # val datasets could be none, 1 or 2+ @@ -227,9 +231,8 @@ class TrainerDataLoadingMixin(ABC): self.num_test_batches = 0 # add samplers - for i, dataloader in enumerate(self.test_dataloaders): - dl = self.auto_add_sampler(dataloader, train=False) - self.test_dataloaders[i] = dl + self.test_dataloaders = [self.auto_add_sampler(dl, train=False) + for dl in self.test_dataloaders if dl] # determine number of test batches if self.test_dataloaders is not None: diff --git a/pytorch_lightning/trainer/evaluation_loop.py b/pytorch_lightning/trainer/evaluation_loop.py index c03c13dd..1bb0e885 100644 --- a/pytorch_lightning/trainer/evaluation_loop.py +++ b/pytorch_lightning/trainer/evaluation_loop.py @@ -216,13 +216,13 @@ class TrainerEvaluationLoopMixin(ABC): # this is just empty shell for code from other class pass - def evaluate(self, model, dataloaders, max_batches, test=False): + def evaluate(self, model, dataloaders, max_batches, test_mode: bool = False): """Run evaluation code. :param model: PT model :param dataloaders: list of PT dataloaders :param max_batches: Scalar - :param test: boolean + :param test_mode :return: """ # enable eval mode @@ -260,18 +260,14 @@ class TrainerEvaluationLoopMixin(ABC): # ----------------- # RUN EVALUATION STEP # ----------------- - output = self.evaluation_forward(model, - batch, - batch_idx, - dataloader_idx, - test) + output = self.evaluation_forward(model, batch, batch_idx, dataloader_idx, test_mode) # track outputs for collation dl_outputs.append(output) # batch done if batch_idx % self.progress_bar_refresh_rate == 0: - if test: + if test_mode: self.test_progress_bar.update(self.progress_bar_refresh_rate) else: self.val_progress_bar.update(self.progress_bar_refresh_rate) @@ -286,7 +282,7 @@ class TrainerEvaluationLoopMixin(ABC): # give model a chance to do something with the outputs (and method defined) model = self.get_model() - if test and self.is_overriden('test_end'): + if test_mode and self.is_overriden('test_end'): eval_results = model.test_end(outputs) elif self.is_overriden('validation_end'): eval_results = model.validation_end(outputs) @@ -299,11 +295,11 @@ class TrainerEvaluationLoopMixin(ABC): return eval_results - def run_evaluation(self, test=False): + def run_evaluation(self, test_mode: bool = False): # when testing make sure user defined a test step - if test and not self.is_overriden('test_step'): - m = '''You called `.test()` without defining model's `.test_step()`. - Please define and try again''' + if test_mode and not self.is_overriden('test_step'): + m = "You called `.test()` without defining model's `.test_step()`." \ + " Please define and try again" raise MisconfigurationException(m) # hook @@ -311,7 +307,7 @@ class TrainerEvaluationLoopMixin(ABC): model.on_pre_performance_check() # select dataloaders - if test: + if test_mode: if self.reload_dataloaders_every_epoch or self.test_dataloaders is None: self.reset_test_dataloader(model) @@ -331,18 +327,15 @@ class TrainerEvaluationLoopMixin(ABC): # init validation or test progress bar # main progress bar will already be closed when testing so initial position is free - position = 2 * self.process_position + (not test) - desc = 'Testing' if test else 'Validating' - pbar = tqdm(desc=desc, total=max_batches, leave=test, position=position, + position = 2 * self.process_position + (not test_mode) + desc = 'Testing' if test_mode else 'Validating' + pbar = tqdm(desc=desc, total=max_batches, leave=test_mode, position=position, disable=not self.show_progress_bar, dynamic_ncols=True, file=sys.stdout) - setattr(self, f'{"test" if test else "val"}_progress_bar', pbar) + setattr(self, f'{"test" if test_mode else "val"}_progress_bar', pbar) # run evaluation - eval_results = self.evaluate(self.model, - dataloaders, - max_batches, - test) + eval_results = self.evaluate(self.model, dataloaders, max_batches, test_mode) _, prog_bar_metrics, log_metrics, callback_metrics, _ = self.process_output( eval_results) @@ -359,27 +352,27 @@ class TrainerEvaluationLoopMixin(ABC): model.on_post_performance_check() # add model specific metrics - if not test: + if not test_mode: self.main_progress_bar.set_postfix(**self.training_tqdm_dict) # close progress bar - if test: + if test_mode: self.test_progress_bar.close() else: self.val_progress_bar.close() # model checkpointing - if self.proc_rank == 0 and self.checkpoint_callback is not None and not test: + if self.proc_rank == 0 and self.checkpoint_callback is not None and not test_mode: self.checkpoint_callback.on_validation_end() - def evaluation_forward(self, model, batch, batch_idx, dataloader_idx, test=False): + def evaluation_forward(self, model, batch, batch_idx, dataloader_idx, test_mode: bool = False): # make dataloader_idx arg in validation_step optional args = [batch, batch_idx] - if test and len(self.test_dataloaders) > 1: + if test_mode and len(self.test_dataloaders) > 1: args.append(dataloader_idx) - elif not test and len(self.val_dataloaders) > 1: + elif not test_mode and len(self.val_dataloaders) > 1: args.append(dataloader_idx) # handle DP, DDP forward @@ -402,7 +395,7 @@ class TrainerEvaluationLoopMixin(ABC): args[0] = batch # CPU - if test: + if test_mode: output = model.test_step(*args) else: output = model.validation_step(*args) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 85516390..038cdc80 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -84,7 +84,7 @@ class Trainer(TrainerIOMixin, track_grad_norm: int = -1, check_val_every_n_epoch: int = 1, fast_dev_run: bool = False, - accumulate_grad_batches: Union[int, Dict[int, int]] = 1, + accumulate_grad_batches: Union[int, Dict[int, int], List[list]] = 1, max_nb_epochs=None, # backward compatible, todo: remove in v0.8.0 min_nb_epochs=None, # backward compatible, todo: remove in v0.8.0 max_epochs: int = 1000, @@ -681,7 +681,6 @@ class Trainer(TrainerIOMixin, self.train_dataloader = None self.test_dataloaders = None self.val_dataloaders = None - self.is_iterable_train_dataloader = False # training state self.model = None @@ -1068,7 +1067,7 @@ class Trainer(TrainerIOMixin, if self.testing: # only load test dataloader for testing self.reset_test_dataloader(ref_model) - self.run_evaluation(test=True) + self.run_evaluation(test_mode=True) return # load the dataloaders @@ -1087,15 +1086,17 @@ class Trainer(TrainerIOMixin, if not self.disable_validation and self.num_sanity_val_steps > 0: # init progress bars for validation sanity check pbar = tqdm(desc='Validation sanity check', - total=self.num_sanity_val_steps * len(self.val_dataloaders), - leave=False, position=2 * self.process_position, - disable=not self.show_progress_bar, dynamic_ncols=True) + total=self.num_sanity_val_steps * len(self.val_dataloaders), + leave=False, position=2 * self.process_position, + disable=not self.show_progress_bar, dynamic_ncols=True) self.main_progress_bar = pbar # dummy validation progress bar self.val_progress_bar = tqdm(disable=True) - eval_results = self.evaluate(model, self.val_dataloaders, - self.num_sanity_val_steps, False) + eval_results = self.evaluate(model, + self.val_dataloaders, + self.num_sanity_val_steps, + False) _, _, _, callback_metrics, _ = self.process_output(eval_results) # close progress bars @@ -1145,5 +1146,4 @@ class Trainer(TrainerIOMixin, self.testing = True if model is not None: self.fit(model) - else: - self.run_evaluation(test=True) + self.run_evaluation(test_mode=True) diff --git a/pytorch_lightning/trainer/training_loop.py b/pytorch_lightning/trainer/training_loop.py index 0542d0a8..93466ae6 100644 --- a/pytorch_lightning/trainer/training_loop.py +++ b/pytorch_lightning/trainer/training_loop.py @@ -197,7 +197,6 @@ class TrainerTrainLoopMixin(ABC): self.num_val_batches = None self.disable_validation = None self.fast_dev_run = None - self.is_iterable_train_dataloader = None self.main_progress_bar = None self.accumulation_scheduler = None self.lr_schedulers = None @@ -227,6 +226,8 @@ class TrainerTrainLoopMixin(ABC): self.train_dataloader = None self.reload_dataloaders_every_epoch = None self.progress_bar_refresh_rate = None + self.max_steps = ... + self.max_steps = ... @property def max_nb_epochs(self): @@ -257,7 +258,12 @@ class TrainerTrainLoopMixin(ABC): pass @abstractmethod - def run_evaluation(self, test): + def is_iterable_dataloader(self, dataloader): + # this is just empty shell for code from other class + pass + + @abstractmethod + def run_evaluation(self, test_mode): # this is just empty shell for code from other class pass @@ -306,6 +312,11 @@ class TrainerTrainLoopMixin(ABC): # this is just empty shell for code from other class pass + @abstractmethod + def has_arg(self, f_name, arg_name): + # this is just empty shell for code from other class + pass + def train(self): warnings.warn('Displayed epoch numbers in the progress bar start from "1" until v0.6.x,' ' but will start from "0" in v0.8.0.', DeprecationWarning) @@ -342,7 +353,7 @@ class TrainerTrainLoopMixin(ABC): if self.fast_dev_run: # limit the number of batches to 2 (1 train and 1 val) in fast_dev_run num_iterations = 2 - elif self.is_iterable_train_dataloader: + elif self.is_iterable_dataloader(self.train_dataloader): # for iterable train loader, the progress bar never ends num_iterations = None else: @@ -352,7 +363,7 @@ class TrainerTrainLoopMixin(ABC): # .reset() doesn't work on disabled progress bar so we should check if not self.main_progress_bar.disable: self.main_progress_bar.reset(num_iterations) - desc = f'Epoch {epoch + 1}' if not self.is_iterable_train_dataloader else '' + desc = f'Epoch {epoch + 1}' if not self.is_iterable_dataloader(self.train_dataloader) else '' self.main_progress_bar.set_description(desc) # changing gradient according accumulation_scheduler @@ -449,7 +460,7 @@ class TrainerTrainLoopMixin(ABC): # fast_dev_run always forces val checking after train batch if self.fast_dev_run or should_check_val: - self.run_evaluation(test=self.testing) + self.run_evaluation(test_mode=self.testing) if self.enable_early_stop: self.early_stop_callback.check_metrics(self.callback_metrics) diff --git a/tests/models/__init__.py b/tests/models/__init__.py index ad926b60..ae27a1bc 100644 --- a/tests/models/__init__.py +++ b/tests/models/__init__.py @@ -2,27 +2,31 @@ import torch -from .base import LightningTestModelBase, LightningTestModelBaseWithoutDataloader +from .base import TestModelBase from .mixins import ( - LightningValidationStepMixin, - LightningValidationMixin, - LightningValidationStepMultipleDataloadersMixin, - LightningValidationMultipleDataloadersMixin, - LightningTestStepMixin, - LightningTestMixin, - LightningTestStepMultipleDataloadersMixin, - LightningTestMultipleDataloadersMixin, - LightningTestFitSingleTestDataloadersMixin, - LightningTestFitMultipleTestDataloadersMixin, - LightningValStepFitSingleDataloaderMixin, - LightningValStepFitMultipleDataloadersMixin + LightEmptyTestStep, + LightValidationStepMixin, + LightValidationMixin, + LightValidationStepMultipleDataloadersMixin, + LightValidationMultipleDataloadersMixin, + LightTestStepMixin, + LightTestMixin, + LightTestStepMultipleDataloadersMixin, + LightTestMultipleDataloadersMixin, + LightTestFitSingleTestDataloadersMixin, + LightTestFitMultipleTestDataloadersMixin, + LightValStepFitSingleDataloaderMixin, + LightValStepFitMultipleDataloadersMixin, + LightTrainDataloader, + LightTestDataloader, ) -class LightningTestModel(LightningValidationMixin, LightningTestMixin, LightningTestModelBase): - """ - Most common test case. Validation and test dataloaders. - """ +class LightningTestModel(LightTrainDataloader, + LightValidationMixin, + LightTestMixin, + TestModelBase): + """Most common test case. Validation and test dataloaders.""" def on_training_metrics(self, logs): logs['some_tensor_to_test'] = torch.rand(1) diff --git a/tests/models/base.py b/tests/models/base.py index 2c174026..e0677da6 100644 --- a/tests/models/base.py +++ b/tests/models/base.py @@ -24,7 +24,7 @@ class TestingMNIST(MNIST): def __init__(self, root, train=True, transform=None, target_transform=None, download=False, num_samples=8000): - super(TestingMNIST, self).__init__( + super().__init__( root, train=train, transform=transform, @@ -48,7 +48,7 @@ class TestModelBase(LightningModule): :param hparams: """ # init superclass - super(TestModelBase, self).__init__() + super().__init__() self.hparams = hparams self.batch_size = hparams.batch_size @@ -87,7 +87,6 @@ class TestModelBase(LightningModule): :param x: :return: """ - x = self.c_d1(x) x = torch.tanh(x) x = self.c_d1_bn(x) @@ -153,10 +152,8 @@ class TestModelBase(LightningModule): def prepare_data(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) - dataset = TestingMNIST(root=self.hparams.data_root, train=True, - transform=transform, download=True, num_samples=2000) - dataset = TestingMNIST(root=self.hparams.data_root, train=False, - transform=transform, download=True, num_samples=2000) + _ = TestingMNIST(root=self.hparams.data_root, train=True, + transform=transform, download=True, num_samples=2000) def _dataloader(self, train): # init data generators @@ -194,31 +191,17 @@ class TestModelBase(LightningModule): parser.add_argument('--out_features', default=10, type=int) # use 500 for CPU, 50000 for GPU to see speed difference parser.add_argument('--hidden_dim', default=50000, type=int) - # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) - # training params (opt) parser.opt_list('--learning_rate', default=0.001 * 8, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) - # if using 2 nodes with 4 gpus each the batch size here # (256) will be 256 / (2*8) = 16 per gpu parser.opt_list('--batch_size', default=256 * 8, type=int, options=[32, 64, 128, 256], tunable=False, - help='batch size will be divided over all gpus being used across all nodes') + help='batch size will be divided over all GPUs being used across all nodes') return parser - - -class LightningTestModelBase(TestModelBase): - """ with pre-defined train dataloader """ - def train_dataloader(self): - return self._dataloader(train=True) - - -class LightningTestModelBaseWithoutDataloader(TestModelBase): - """ without pre-defined train dataloader """ - pass diff --git a/tests/models/mixins.py b/tests/models/mixins.py index 940f5e30..69c0d235 100644 --- a/tests/models/mixins.py +++ b/tests/models/mixins.py @@ -5,7 +5,7 @@ import torch from pytorch_lightning.core.decorators import data_loader -class LightningValidationStepMixin: +class LightValidationStepMixin: """ Add val_dataloader and validation_step methods for the case when val_dataloader returns a single dataloader @@ -14,7 +14,7 @@ class LightningValidationStepMixin: def val_dataloader(self): return self._dataloader(train=False) - def validation_step(self, batch, batch_idx): + def validation_step(self, batch, batch_idx, *args, **kwargs): """ Lightning calls this inside the validation loop :param batch: @@ -58,7 +58,7 @@ class LightningValidationStepMixin: return output -class LightningValidationMixin(LightningValidationStepMixin): +class LightValidationMixin(LightValidationStepMixin): """ Add val_dataloader, validation_step, and validation_end methods for the case when val_dataloader returns a single dataloader @@ -76,7 +76,7 @@ class LightningValidationMixin(LightningValidationStepMixin): val_loss_mean = 0 val_acc_mean = 0 for output in outputs: - val_loss = output['val_loss'] + val_loss = _get_output_metric(output, 'val_loss') # reduce manually when using dp if self.trainer.use_dp or self.trainer.use_ddp2: @@ -84,7 +84,7 @@ class LightningValidationMixin(LightningValidationStepMixin): val_loss_mean += val_loss # reduce manually when using dp - val_acc = output['val_acc'] + val_acc = _get_output_metric(output, 'val_acc') if self.trainer.use_dp or self.trainer.use_ddp2: val_acc = torch.mean(val_acc) @@ -98,7 +98,7 @@ class LightningValidationMixin(LightningValidationStepMixin): return results -class LightningValidationStepMultipleDataloadersMixin: +class LightValidationStepMultipleDataloadersMixin: """ Add val_dataloader and validation_step methods for the case when val_dataloader returns multiple dataloaders @@ -107,7 +107,7 @@ class LightningValidationStepMultipleDataloadersMixin: def val_dataloader(self): return [self._dataloader(train=False), self._dataloader(train=False)] - def validation_step(self, batch, batch_idx, dataloader_idx): + def validation_step(self, batch, batch_idx, dataloader_idx, **kwargs): """ Lightning calls this inside the validation loop :param batch: @@ -157,7 +157,7 @@ class LightningValidationStepMultipleDataloadersMixin: return output -class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipleDataloadersMixin): +class LightValidationMultipleDataloadersMixin(LightValidationStepMultipleDataloadersMixin): """ Add val_dataloader, validation_step, and validation_end methods for the case when val_dataloader returns multiple dataloaders @@ -200,12 +200,31 @@ class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipl return result -class LightningTestStepMixin: +class LightTrainDataloader: + """Simple train dataloader.""" + + def train_dataloader(self): + return self._dataloader(train=True) + + +class LightTestDataloader: + """Simple test dataloader.""" def test_dataloader(self): return self._dataloader(train=False) - def test_step(self, batch, batch_idx): + +class LightEmptyTestStep: + """Empty test step.""" + + def test_step(self, *args, **kwargs): + return dict() + + +class LightTestStepMixin(LightTestDataloader): + """Test step mixin.""" + + def test_step(self, batch, batch_idx, *args, **kwargs): """ Lightning calls this inside the validation loop :param batch: @@ -249,7 +268,9 @@ class LightningTestStepMixin: return output -class LightningTestMixin(LightningTestStepMixin): +class LightTestMixin(LightTestStepMixin): + """Ritch test mixin.""" + def test_end(self, outputs): """ Called at the end of validation to aggregate outputs @@ -262,7 +283,7 @@ class LightningTestMixin(LightningTestStepMixin): test_loss_mean = 0 test_acc_mean = 0 for output in outputs: - test_loss = output['test_loss'] + test_loss = _get_output_metric(output, 'test_loss') # reduce manually when using dp if self.trainer.use_dp: @@ -270,7 +291,7 @@ class LightningTestMixin(LightningTestStepMixin): test_loss_mean += test_loss # reduce manually when using dp - test_acc = output['test_acc'] + test_acc = _get_output_metric(output, 'test_acc') if self.trainer.use_dp: test_acc = torch.mean(test_acc) @@ -284,12 +305,13 @@ class LightningTestMixin(LightningTestStepMixin): return result -class LightningTestStepMultipleDataloadersMixin: +class LightTestStepMultipleDataloadersMixin: + """Test step multiple dataloaders mixin.""" def test_dataloader(self): return [self._dataloader(train=False), self._dataloader(train=False)] - def test_step(self, batch, batch_idx, dataloader_idx): + def test_step(self, batch, batch_idx, dataloader_idx, **kwargs): """ Lightning calls this inside the validation loop :param batch: @@ -339,8 +361,10 @@ class LightningTestStepMultipleDataloadersMixin: return output -class LightningTestFitSingleTestDataloadersMixin: - def test_step(self, batch, batch_idx): +class LightTestFitSingleTestDataloadersMixin: + """Test fit single test dataloaders mixin.""" + + def test_step(self, batch, batch_idx, *args, **kwargs): """ Lightning calls this inside the validation loop :param batch: @@ -384,8 +408,10 @@ class LightningTestFitSingleTestDataloadersMixin: return output -class LightningTestFitMultipleTestDataloadersMixin: - def test_step(self, batch, batch_idx, dataloader_idx): +class LightTestFitMultipleTestDataloadersMixin: + """Test fit multiple test dataloaders mixin.""" + + def test_step(self, batch, batch_idx, dataloader_idx, **kwargs): """ Lightning calls this inside the validation loop :param batch: @@ -435,8 +461,9 @@ class LightningTestFitMultipleTestDataloadersMixin: return output -class LightningValStepFitSingleDataloaderMixin: - def validation_step(self, batch, batch_idx): +class LightValStepFitSingleDataloaderMixin: + + def validation_step(self, batch, batch_idx, *args, **kwargs): """ Lightning calls this inside the validation loop :param batch: @@ -480,8 +507,9 @@ class LightningValStepFitSingleDataloaderMixin: return output -class LightningValStepFitMultipleDataloadersMixin: - def validation_step(self, batch, batch_idx, dataloader_idx): +class LightValStepFitMultipleDataloadersMixin: + + def validation_step(self, batch, batch_idx, dataloader_idx, **kwargs): """ Lightning calls this inside the validation loop :param batch: @@ -531,7 +559,8 @@ class LightningValStepFitMultipleDataloadersMixin: return output -class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloadersMixin): +class LightTestMultipleDataloadersMixin(LightTestStepMultipleDataloadersMixin): + def test_end(self, outputs): """ Called at the end of validation to aggregate outputs @@ -567,3 +596,11 @@ class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloaders tqdm_dict = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()} result = {'progress_bar': tqdm_dict} return result + + +def _get_output_metric(output, name): + if isinstance(output, dict): + val = output[name] + else: # if it is 2level deep -> per dataloader and per batch + val = sum(out[name] for out in output) / len(output) + return val diff --git a/tests/models/utils.py b/tests/models/utils.py index 644efb9c..75be02ef 100644 --- a/tests/models/utils.py +++ b/tests/models/utils.py @@ -90,7 +90,7 @@ def run_model_test(trainer_options, model, on_gpu=True): def get_hparams(continue_training=False, hpc_exp_number=0): - root_dir = os.path.dirname(os.path.realpath(__file__)) + tests_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) args = { 'drop_prob': 0.2, @@ -98,7 +98,7 @@ def get_hparams(continue_training=False, hpc_exp_number=0): 'in_features': 28 * 28, 'learning_rate': 0.001 * 8, 'optimizer_name': 'adam', - 'data_root': os.path.join(root_dir, 'mnist'), + 'data_root': os.path.join(tests_dir, 'datasets'), 'out_features': 10, 'hidden_dim': 1000, } diff --git a/tests/test_cpu_models.py b/tests/test_cpu_models.py index 5d5a20bb..193e5632 100644 --- a/tests/test_cpu_models.py +++ b/tests/test_cpu_models.py @@ -8,9 +8,10 @@ from pytorch_lightning.callbacks import ( EarlyStopping, ) from tests.models import ( + TestModelBase, + LightTrainDataloader, LightningTestModel, - LightningTestModelBase, - LightningTestMixin, + LightTestMixin, ) @@ -121,7 +122,7 @@ def test_running_test_without_val(tmpdir): """Verify `test()` works on a model with no `val_loader`.""" tutils.reset_seed() - class CurrentTestModel(LightningTestMixin, LightningTestModelBase): + class CurrentTestModel(LightTrainDataloader, LightTestMixin, TestModelBase): pass hparams = tutils.get_hparams() @@ -281,7 +282,7 @@ def test_tbptt_cpu_model(tmpdir): def __len__(self): return 1 - class BpttTestModel(LightningTestModelBase): + class BpttTestModel(LightTrainDataloader, TestModelBase): def __init__(self, hparams): super().__init__(hparams) self.test_hidden = None diff --git a/tests/test_trainer.py b/tests/test_trainer.py index 8fcc1900..e5d7f68c 100644 --- a/tests/test_trainer.py +++ b/tests/test_trainer.py @@ -11,19 +11,22 @@ from pytorch_lightning.callbacks import ( ModelCheckpoint, ) from tests.models import ( + TestModelBase, LightningTestModel, - LightningTestModelBase, - LightningTestModelBaseWithoutDataloader, - LightningValidationStepMixin, - LightningValidationMultipleDataloadersMixin, - LightningTestMultipleDataloadersMixin, - LightningTestFitSingleTestDataloadersMixin, - LightningTestFitMultipleTestDataloadersMixin, - LightningValStepFitMultipleDataloadersMixin, - LightningValStepFitSingleDataloaderMixin + LightEmptyTestStep, + LightValidationStepMixin, + LightValidationMultipleDataloadersMixin, + LightTestMultipleDataloadersMixin, + LightTestFitSingleTestDataloadersMixin, + LightTestFitMultipleTestDataloadersMixin, + LightValStepFitMultipleDataloadersMixin, + LightValStepFitSingleDataloaderMixin, + LightTrainDataloader, + LightTestDataloader, ) from pytorch_lightning.core.lightning import load_hparams_from_tags_csv from pytorch_lightning.trainer.logging import TrainerLoggingMixin +from pytorch_lightning.utilities.debugging import MisconfigurationException def test_no_val_module(tmpdir): @@ -32,7 +35,7 @@ def test_no_val_module(tmpdir): hparams = tutils.get_hparams() - class CurrentTestModel(LightningTestModelBase): + class CurrentTestModel(LightTrainDataloader, TestModelBase): pass model = CurrentTestModel(hparams) @@ -69,7 +72,7 @@ def test_no_val_end_module(tmpdir): """Tests use case where trainer saves the model, and user loads it from tags independently.""" tutils.reset_seed() - class CurrentTestModel(LightningValidationStepMixin, LightningTestModelBase): + class CurrentTestModel(LightTrainDataloader, LightValidationStepMixin, TestModelBase): pass hparams = tutils.get_hparams() @@ -385,8 +388,9 @@ def test_multiple_val_dataloader(tmpdir): tutils.reset_seed() class CurrentTestModel( - LightningValidationMultipleDataloadersMixin, - LightningTestModelBase + LightTrainDataloader, + LightValidationMultipleDataloadersMixin, + TestModelBase, ): pass @@ -490,8 +494,10 @@ def test_multiple_test_dataloader(tmpdir): tutils.reset_seed() class CurrentTestModel( - LightningTestMultipleDataloadersMixin, - LightningTestModelBase + LightTrainDataloader, + LightTestMultipleDataloadersMixin, + LightEmptyTestStep, + TestModelBase, ): pass @@ -508,8 +514,7 @@ def test_multiple_test_dataloader(tmpdir): # fit model trainer = Trainer(**trainer_options) - result = trainer.fit(model) - + trainer.fit(model) trainer.test() # verify there are 2 val loaders @@ -528,9 +533,7 @@ def test_train_dataloaders_passed_to_fit(tmpdir): """ Verify that train dataloader can be passed to fit """ tutils.reset_seed() - class CurrentTestModel( - LightningTestModelBaseWithoutDataloader, - ): + class CurrentTestModel(LightTrainDataloader, TestModelBase): pass hparams = tutils.get_hparams() @@ -555,8 +558,9 @@ def test_train_val_dataloaders_passed_to_fit(tmpdir): tutils.reset_seed() class CurrentTestModel( - LightningValStepFitSingleDataloaderMixin, - LightningTestModelBaseWithoutDataloader, + LightTrainDataloader, + LightValStepFitSingleDataloaderMixin, + TestModelBase, ): pass @@ -586,9 +590,11 @@ def test_all_dataloaders_passed_to_fit(tmpdir): tutils.reset_seed() class CurrentTestModel( - LightningValStepFitSingleDataloaderMixin, - LightningTestFitSingleTestDataloadersMixin, - LightningTestModelBaseWithoutDataloader, + LightTrainDataloader, + LightValStepFitSingleDataloaderMixin, + LightTestFitSingleTestDataloadersMixin, + LightEmptyTestStep, + TestModelBase, ): pass @@ -624,9 +630,9 @@ def test_multiple_dataloaders_passed_to_fit(tmpdir): tutils.reset_seed() class CurrentTestModel( - LightningValStepFitMultipleDataloadersMixin, - LightningTestFitMultipleTestDataloadersMixin, - LightningTestModelBaseWithoutDataloader, + LightningTestModel, + LightValStepFitMultipleDataloadersMixin, + LightTestFitMultipleTestDataloadersMixin, ): pass @@ -662,9 +668,10 @@ def test_mixing_of_dataloader_options(tmpdir): tutils.reset_seed() class CurrentTestModel( - LightningValStepFitSingleDataloaderMixin, - LightningTestFitSingleTestDataloadersMixin, - LightningTestModelBase, + LightTrainDataloader, + LightValStepFitSingleDataloaderMixin, + LightTestFitSingleTestDataloadersMixin, + TestModelBase, ): pass @@ -688,7 +695,7 @@ def test_mixing_of_dataloader_options(tmpdir): trainer = Trainer(**trainer_options) fit_options = dict(val_dataloaders=model._dataloader(train=False), test_dataloaders=model._dataloader(train=False)) - results = trainer.fit(model, **fit_options) + _ = trainer.fit(model, **fit_options) trainer.test() assert len(trainer.val_dataloaders) == 1, \ @@ -719,6 +726,7 @@ def test_trainer_max_steps_and_epochs(tmpdir): # define less train steps than epochs trainer_options.update(dict( + default_save_path=tmpdir, max_epochs=5, max_steps=num_train_samples + 10 )) @@ -732,8 +740,10 @@ def test_trainer_max_steps_and_epochs(tmpdir): assert trainer.global_step == trainer.max_steps, "Model did not stop at max_steps" # define less train epochs than steps - trainer_options['max_epochs'] = 2 - trainer_options['max_steps'] = trainer_options['max_epochs'] * 2 * num_train_samples + trainer_options.update(dict( + max_epochs=2, + max_steps=trainer_options['max_epochs'] * 2 * num_train_samples + )) # fit model trainer = Trainer(**trainer_options) @@ -741,8 +751,8 @@ def test_trainer_max_steps_and_epochs(tmpdir): assert result == 1, "Training did not complete" # check training stopped at max_epochs - assert trainer.global_step == num_train_samples * trainer.max_nb_epochs \ - and trainer.current_epoch == trainer.max_nb_epochs - 1, "Model did not stop at max_epochs" + assert trainer.global_step == num_train_samples * trainer.max_epochs \ + and trainer.current_epoch == trainer.max_epochs - 1, "Model did not stop at max_epochs" def test_trainer_min_steps_and_epochs(tmpdir): @@ -750,12 +760,13 @@ def test_trainer_min_steps_and_epochs(tmpdir): model, trainer_options, num_train_samples = _init_steps_model() # define callback for stopping the model and default epochs - trainer_options.update({ - 'early_stop_callback': EarlyStopping(monitor='val_loss', min_delta=1.0), - 'val_check_interval': 20, - 'min_epochs': 1, - 'max_epochs': 10 - }) + trainer_options.update(dict( + default_save_path=tmpdir, + early_stop_callback=EarlyStopping(monitor='val_loss', min_delta=1.0), + val_check_interval=20, + min_epochs=1, + max_epochs=10 + )) # define less min steps than 1 epoch trainer_options['min_steps'] = math.floor(num_train_samples / 2) @@ -784,22 +795,29 @@ def test_trainer_min_steps_and_epochs(tmpdir): def test_testpass_overrides(tmpdir): hparams = tutils.get_hparams() - from pytorch_lightning.utilities.debugging import MisconfigurationException - class TestModelNoEnd(LightningTestModelBase): - def test_step(self, *args, **kwargs): + class LocalModel(LightTrainDataloader, TestModelBase): + pass + + class LocalModelNoEnd(LightTrainDataloader, LightTestDataloader, LightEmptyTestStep, TestModelBase): + pass + + class LocalModelNoStep(LightTrainDataloader, TestModelBase): + def test_end(self, outputs): return {} - def test_dataloader(self): - return self.train_dataloader() - # Misconfig when neither test_step or test_end is implemented with pytest.raises(MisconfigurationException): - model = LightningTestModelBase(hparams) + model = LocalModel(hparams) + Trainer().test(model) + + # Misconfig when neither test_step or test_end is implemented + with pytest.raises(MisconfigurationException): + model = LocalModelNoStep(hparams) Trainer().test(model) # No exceptions when one or both of test_step or test_end are implemented - model = TestModelNoEnd(hparams) + model = LocalModelNoEnd(hparams) Trainer().test(model) model = LightningTestModel(hparams)