wassname/pytorch-lightning
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-lightning.git synced 2026-09-10 12:21:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Pass outputs from all dataloaders to test_end and validation_end (#203)

Browse Source 
* Pass outputs from all dataloaders to test_end and validation_end

* Update tests

* Update docs

* Update trainer.py

* Update test_models.py
This commit is contained in:
Nic Eggert
2019-09-06 07:37:25 -04:00
committed by William Falcon
parent 447ed30716
commit 1733dba735
4 changed files with 127 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/LightningModule/RequiredTrainerInterface.md
+58 -4
View File
@@ -316,7 +316,7 @@ def validation_end(self, outputs)
```
If you didn't define a validation_step, this won't be called.
Called at the end of the validation loop with the output of each validation_step. Called once per validation dataset.
Called at the end of the validation loop with the outputs of validation_step.
The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything.
@@ -324,7 +324,7 @@ The outputs here are strictly for the progress bar. If you don't need to display
| Param | description |
|---|---|
| outputs | List of outputs you defined in validation_step |
| outputs | List of outputs you defined in validation_step, or if there are multiple dataloaders, a list containing a list of outputs for each dataloader |
**Return**
@@ -334,6 +334,8 @@ The outputs here are strictly for the progress bar. If you don't need to display
**Example**
With a single dataloader
``` {.python}
def validation_end(self, outputs):
"""
@@ -353,6 +355,32 @@ def validation_end(self, outputs):
return tqdm_dic
```
With multiple dataloaders, `outputs` will be a list of lists. The outer list contains
one entry per dataloader, while the inner list contains the individual outputs of
each validation step for that dataloader.
``` {.python}
def validation_end(self, outputs):
"""
Called at the end of validation to aggregate outputs
:param outputs: list of list of individual outputs of each validation step
:return:
"""
val_loss_mean = 0
val_acc_mean = 0
i = 0
for dataloader_outputs in outputs:
for output in dataloader_outputs:
val_loss_mean += output['val_loss']
val_acc_mean += output['val_acc']
i += 1
val_loss_mean /= i
val_acc_mean /= i
tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
return tqdm_dic
```
### test_step
``` {.python}
@@ -429,7 +457,7 @@ def test_end(self, outputs)
```
If you didn't define a test_step, this won't be called.
Called at the end of the test step with the output of each test_step. Called once per test dataset.
Called at the end of the test step with the output of each test_step.
The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything.
@@ -437,7 +465,7 @@ The outputs here are strictly for the progress bar. If you don't need to display
| Param | description |
|---|---|
| outputs | List of outputs you defined test_step |
| outputs | List of outputs you defined in test_step, or if there are multiple dataloaders, a list containing a list of outputs for each dataloader |
**Return**
@@ -466,6 +494,32 @@ def test_end(self, outputs):
return tqdm_dic
```
With multiple dataloaders, `outputs` will be a list of lists. The outer list contains
one entry per dataloader, while the inner list contains the individual outputs of
each validation step for that dataloader.
``` {.python}
def test_end(self, outputs):
"""
Called at the end of test to aggregate outputs
:param outputs: list of individual outputs of each test step
:return:
"""
test_loss_mean = 0
test_acc_mean = 0
i = 0
for dataloader_outputs in outputs:
for output in dataloader_outputs:
test_loss_mean += output['test_loss']
test_acc_mean += output['test_acc']
i += 1
test_loss_mean /= i
test_acc_mean /= i
tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
return tqdm_dic
```
---
### on_save_checkpoint
pytorch_lightning/models/trainer.py
+34 -33
View File
@@ -441,11 +441,11 @@ class Trainer(TrainerIO):
return output
def evaluate(self, model, dataloader, max_batches, dataloader_i, test=False):
def evaluate(self, model, dataloaders, max_batches, test=False):
"""
Run evaluation code
:param model: PT model
:param dataloader: PT dataloader
:param dataloaders: list of PT dataloaders
:param max_batches: Scalar
:param dataloader_i:
:param test: boolean
@@ -462,32 +462,37 @@ class Trainer(TrainerIO):
outputs = []
# run training
for batch_i, data_batch in enumerate(dataloader):
for dataloader_i, dl in enumerate(dataloaders):
dl_outputs = []
for batch_i, data_batch in enumerate(dl):
if data_batch is None: # pragma: no cover
continue
if data_batch is None: # pragma: no cover
continue
# stop short when on fast_dev_run (sets max_batch=1)
if batch_i >= max_batches:
break
# stop short when on fast_dev_run (sets max_batch=1)
if batch_i >= max_batches:
break
# -----------------
# RUN EVALUATION STEP
# -----------------
output = self.__evaluation_forward(model, data_batch, batch_i, dataloader_i,
test)
# -----------------
# RUN EVALUATION STEP
# -----------------
output = self.__evaluation_forward(model, data_batch, batch_i, dataloader_i,
test)
# track outputs for collation
outputs.append(output)
# track outputs for collation
dl_outputs.append(output)
# batch done
if self.show_progress_bar:
self.progress_bar.update(1)
# batch done
if self.show_progress_bar:
self.progress_bar.update(1)
outputs.append(dl_outputs)
eval_results = {}
# give model a chance to do something with the outputs (and method defined)
model = self.__get_model()
if len(dataloaders) == 1:
outputs = outputs[0]
if test and self.__is_overriden('test_end'):
eval_results = model.test_end(outputs)
elif self.__is_overriden('validation_end'):
@@ -855,13 +860,11 @@ class Trainer(TrainerIO):
# to make sure program won't crash during val
ref_model.on_sanity_check_start()
if self.val_dataloader is not None and self.nb_sanity_val_steps > 0:
for ds_i, dataloader in enumerate(self.val_dataloader):
# reset progress_bar limit for sanity check
if self.show_progress_bar:
self.progress_bar.reset(self.nb_sanity_val_steps)
# reset progress_bar limit for sanity check
if self.show_progress_bar:
self.progress_bar.reset(self.nb_sanity_val_steps)
self.evaluate(model, dataloader, self.nb_sanity_val_steps, ds_i, self.testing)
self.evaluate(model, self.val_dataloader, self.nb_sanity_val_steps, self.testing)
# ---------------------------
# CORE TRAINING LOOP
@@ -1221,17 +1224,15 @@ class Trainer(TrainerIO):
if self.fast_dev_run:
max_batches = 1
for ds_i, dataloader in enumerate(dataloaders):
eval_out_metrics = self.evaluate(self.model,
dataloader,
max_batches,
ds_i,
test)
eval_out_metrics = self.evaluate(self.model,
dataloaders,
max_batches,
test)
self.__add_tqdm_metrics(eval_out_metrics)
self.__add_tqdm_metrics(eval_out_metrics)
# hook
model.on_post_performance_check()
# hook
model.on_post_performance_check()
if self.show_progress_bar:
# add model specific metrics
pytorch_lightning/testing/lm_test_module_mixins.py
+32 -26
View File
@@ -185,23 +185,26 @@ class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipl
# return torch.stack(outputs).mean()
val_loss_mean = 0
val_acc_mean = 0
for output in outputs:
val_loss = output['val_loss']
i = 0
for dl_output in outputs:
for output in dl_output:
val_loss = output['val_loss']
# reduce manually when using dp
if self.trainer.use_dp:
val_loss = torch.mean(val_loss)
val_loss_mean += val_loss
# reduce manually when using dp
if self.trainer.use_dp:
val_loss = torch.mean(val_loss)
val_loss_mean += val_loss
# reduce manually when using dp
val_acc = output['val_acc']
if self.trainer.use_dp:
val_acc = torch.mean(val_acc)
# reduce manually when using dp
val_acc = output['val_acc']
if self.trainer.use_dp:
val_acc = torch.mean(val_acc)
val_acc_mean += val_acc
val_acc_mean += val_acc
i += 1
val_loss_mean /= len(outputs)
val_acc_mean /= len(outputs)
val_loss_mean /= i
val_acc_mean /= i
tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
return tqdm_dic
@@ -359,23 +362,26 @@ class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloaders
# return torch.stack(outputs).mean()
test_loss_mean = 0
test_acc_mean = 0
for output in outputs:
test_loss = output['test_loss']
i = 0
for dl_output in outputs:
for output in dl_output:
test_loss = output['test_loss']
# reduce manually when using dp
if self.trainer.use_dp:
test_loss = torch.mean(test_loss)
test_loss_mean += test_loss
# reduce manually when using dp
if self.trainer.use_dp:
test_loss = torch.mean(test_loss)
test_loss_mean += test_loss
# reduce manually when using dp
test_acc = output['test_acc']
if self.trainer.use_dp:
test_acc = torch.mean(test_acc)
# reduce manually when using dp
test_acc = output['test_acc']
if self.trainer.use_dp:
test_acc = torch.mean(test_acc)
test_acc_mean += test_acc
test_acc_mean += test_acc
i += 1
test_loss_mean /= len(outputs)
test_acc_mean /= len(outputs)
test_loss_mean /= i
test_acc_mean /= i
tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
return tqdm_dic
tests/test_models.py
+3 -3
View File
@@ -1162,15 +1162,15 @@ def test_multiple_test_dataloader():
trainer = Trainer(**trainer_options)
result = trainer.fit(model)
# verify tng completed
assert result == 1
# verify there are 2 val loaders
assert len(trainer.test_dataloader) == 2, 'Multiple test_dataloaders not initiated properly'
# make sure predictions are good for each test set
[run_prediction(dataloader, trainer.model) for dataloader in trainer.test_dataloader]
# run the test method
trainer.test()
# ------------------------------------------------------------------------
# UTILS