From 1733dba7355c406e6a9b55fadb730859e5ab1cb6 Mon Sep 17 00:00:00 2001 From: Nic Eggert Date: Fri, 6 Sep 2019 04:37:25 -0700 Subject: [PATCH] Pass outputs from all dataloaders to test_end and validation_end (#203) * Pass outputs from all dataloaders to test_end and validation_end * Update tests * Update docs * Update trainer.py * Update test_models.py --- .../RequiredTrainerInterface.md | 62 +++++++++++++++-- pytorch_lightning/models/trainer.py | 67 ++++++++++--------- .../testing/lm_test_module_mixins.py | 58 +++++++++------- tests/test_models.py | 6 +- 4 files changed, 127 insertions(+), 66 deletions(-) diff --git a/docs/LightningModule/RequiredTrainerInterface.md b/docs/LightningModule/RequiredTrainerInterface.md index e53e4d05..ca305bbb 100644 --- a/docs/LightningModule/RequiredTrainerInterface.md +++ b/docs/LightningModule/RequiredTrainerInterface.md @@ -316,7 +316,7 @@ def validation_end(self, outputs) ``` If you didn't define a validation_step, this won't be called. -Called at the end of the validation loop with the output of each validation_step. Called once per validation dataset. +Called at the end of the validation loop with the outputs of validation_step. The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything. @@ -324,7 +324,7 @@ The outputs here are strictly for the progress bar. If you don't need to display | Param | description | |---|---| -| outputs | List of outputs you defined in validation_step | +| outputs | List of outputs you defined in validation_step, or if there are multiple dataloaders, a list containing a list of outputs for each dataloader | **Return** @@ -334,6 +334,8 @@ The outputs here are strictly for the progress bar. If you don't need to display **Example** +With a single dataloader + ``` {.python} def validation_end(self, outputs): """ @@ -353,6 +355,32 @@ def validation_end(self, outputs): return tqdm_dic ``` +With multiple dataloaders, `outputs` will be a list of lists. The outer list contains +one entry per dataloader, while the inner list contains the individual outputs of +each validation step for that dataloader. + +``` {.python} +def validation_end(self, outputs): + """ + Called at the end of validation to aggregate outputs + :param outputs: list of list of individual outputs of each validation step + :return: + """ + val_loss_mean = 0 + val_acc_mean = 0 + i = 0 + for dataloader_outputs in outputs: + for output in dataloader_outputs: + val_loss_mean += output['val_loss'] + val_acc_mean += output['val_acc'] + i += 1 + + val_loss_mean /= i + val_acc_mean /= i + tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} + return tqdm_dic +``` + ### test_step ``` {.python} @@ -429,7 +457,7 @@ def test_end(self, outputs) ``` If you didn't define a test_step, this won't be called. -Called at the end of the test step with the output of each test_step. Called once per test dataset. +Called at the end of the test step with the output of each test_step. The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything. @@ -437,7 +465,7 @@ The outputs here are strictly for the progress bar. If you don't need to display | Param | description | |---|---| -| outputs | List of outputs you defined test_step | +| outputs | List of outputs you defined in test_step, or if there are multiple dataloaders, a list containing a list of outputs for each dataloader | **Return** @@ -466,6 +494,32 @@ def test_end(self, outputs): return tqdm_dic ``` +With multiple dataloaders, `outputs` will be a list of lists. The outer list contains +one entry per dataloader, while the inner list contains the individual outputs of +each validation step for that dataloader. + +``` {.python} +def test_end(self, outputs): + """ + Called at the end of test to aggregate outputs + :param outputs: list of individual outputs of each test step + :return: + """ + test_loss_mean = 0 + test_acc_mean = 0 + i = 0 + for dataloader_outputs in outputs: + for output in dataloader_outputs: + test_loss_mean += output['test_loss'] + test_acc_mean += output['test_acc'] + i += 1 + + test_loss_mean /= i + test_acc_mean /= i + tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()} + return tqdm_dic +``` + --- ### on_save_checkpoint diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py index 4d9a5122..14bec3aa 100644 --- a/pytorch_lightning/models/trainer.py +++ b/pytorch_lightning/models/trainer.py @@ -441,11 +441,11 @@ class Trainer(TrainerIO): return output - def evaluate(self, model, dataloader, max_batches, dataloader_i, test=False): + def evaluate(self, model, dataloaders, max_batches, test=False): """ Run evaluation code :param model: PT model - :param dataloader: PT dataloader + :param dataloaders: list of PT dataloaders :param max_batches: Scalar :param dataloader_i: :param test: boolean @@ -462,32 +462,37 @@ class Trainer(TrainerIO): outputs = [] # run training - for batch_i, data_batch in enumerate(dataloader): + for dataloader_i, dl in enumerate(dataloaders): + dl_outputs = [] + for batch_i, data_batch in enumerate(dl): - if data_batch is None: # pragma: no cover - continue + if data_batch is None: # pragma: no cover + continue - # stop short when on fast_dev_run (sets max_batch=1) - if batch_i >= max_batches: - break + # stop short when on fast_dev_run (sets max_batch=1) + if batch_i >= max_batches: + break - # ----------------- - # RUN EVALUATION STEP - # ----------------- - output = self.__evaluation_forward(model, data_batch, batch_i, dataloader_i, - test) + # ----------------- + # RUN EVALUATION STEP + # ----------------- + output = self.__evaluation_forward(model, data_batch, batch_i, dataloader_i, + test) - # track outputs for collation - outputs.append(output) + # track outputs for collation + dl_outputs.append(output) - # batch done - if self.show_progress_bar: - self.progress_bar.update(1) + # batch done + if self.show_progress_bar: + self.progress_bar.update(1) + outputs.append(dl_outputs) eval_results = {} # give model a chance to do something with the outputs (and method defined) model = self.__get_model() + if len(dataloaders) == 1: + outputs = outputs[0] if test and self.__is_overriden('test_end'): eval_results = model.test_end(outputs) elif self.__is_overriden('validation_end'): @@ -855,13 +860,11 @@ class Trainer(TrainerIO): # to make sure program won't crash during val ref_model.on_sanity_check_start() if self.val_dataloader is not None and self.nb_sanity_val_steps > 0: - for ds_i, dataloader in enumerate(self.val_dataloader): + # reset progress_bar limit for sanity check + if self.show_progress_bar: + self.progress_bar.reset(self.nb_sanity_val_steps) - # reset progress_bar limit for sanity check - if self.show_progress_bar: - self.progress_bar.reset(self.nb_sanity_val_steps) - - self.evaluate(model, dataloader, self.nb_sanity_val_steps, ds_i, self.testing) + self.evaluate(model, self.val_dataloader, self.nb_sanity_val_steps, self.testing) # --------------------------- # CORE TRAINING LOOP @@ -1221,17 +1224,15 @@ class Trainer(TrainerIO): if self.fast_dev_run: max_batches = 1 - for ds_i, dataloader in enumerate(dataloaders): - eval_out_metrics = self.evaluate(self.model, - dataloader, - max_batches, - ds_i, - test) + eval_out_metrics = self.evaluate(self.model, + dataloaders, + max_batches, + test) - self.__add_tqdm_metrics(eval_out_metrics) + self.__add_tqdm_metrics(eval_out_metrics) - # hook - model.on_post_performance_check() + # hook + model.on_post_performance_check() if self.show_progress_bar: # add model specific metrics diff --git a/pytorch_lightning/testing/lm_test_module_mixins.py b/pytorch_lightning/testing/lm_test_module_mixins.py index c4842685..8c6bd020 100644 --- a/pytorch_lightning/testing/lm_test_module_mixins.py +++ b/pytorch_lightning/testing/lm_test_module_mixins.py @@ -185,23 +185,26 @@ class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipl # return torch.stack(outputs).mean() val_loss_mean = 0 val_acc_mean = 0 - for output in outputs: - val_loss = output['val_loss'] + i = 0 + for dl_output in outputs: + for output in dl_output: + val_loss = output['val_loss'] - # reduce manually when using dp - if self.trainer.use_dp: - val_loss = torch.mean(val_loss) - val_loss_mean += val_loss + # reduce manually when using dp + if self.trainer.use_dp: + val_loss = torch.mean(val_loss) + val_loss_mean += val_loss - # reduce manually when using dp - val_acc = output['val_acc'] - if self.trainer.use_dp: - val_acc = torch.mean(val_acc) + # reduce manually when using dp + val_acc = output['val_acc'] + if self.trainer.use_dp: + val_acc = torch.mean(val_acc) - val_acc_mean += val_acc + val_acc_mean += val_acc + i += 1 - val_loss_mean /= len(outputs) - val_acc_mean /= len(outputs) + val_loss_mean /= i + val_acc_mean /= i tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic @@ -359,23 +362,26 @@ class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloaders # return torch.stack(outputs).mean() test_loss_mean = 0 test_acc_mean = 0 - for output in outputs: - test_loss = output['test_loss'] + i = 0 + for dl_output in outputs: + for output in dl_output: + test_loss = output['test_loss'] - # reduce manually when using dp - if self.trainer.use_dp: - test_loss = torch.mean(test_loss) - test_loss_mean += test_loss + # reduce manually when using dp + if self.trainer.use_dp: + test_loss = torch.mean(test_loss) + test_loss_mean += test_loss - # reduce manually when using dp - test_acc = output['test_acc'] - if self.trainer.use_dp: - test_acc = torch.mean(test_acc) + # reduce manually when using dp + test_acc = output['test_acc'] + if self.trainer.use_dp: + test_acc = torch.mean(test_acc) - test_acc_mean += test_acc + test_acc_mean += test_acc + i += 1 - test_loss_mean /= len(outputs) - test_acc_mean /= len(outputs) + test_loss_mean /= i + test_acc_mean /= i tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()} return tqdm_dic diff --git a/tests/test_models.py b/tests/test_models.py index 9b2a06a8..049ecf1e 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1162,15 +1162,15 @@ def test_multiple_test_dataloader(): trainer = Trainer(**trainer_options) result = trainer.fit(model) - # verify tng completed - assert result == 1 - # verify there are 2 val loaders assert len(trainer.test_dataloader) == 2, 'Multiple test_dataloaders not initiated properly' # make sure predictions are good for each test set [run_prediction(dataloader, trainer.model) for dataloader in trainer.test_dataloader] + # run the test method + trainer.test() + # ------------------------------------------------------------------------ # UTILS