wassname/pytorch-lightning
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-lightning.git synced 2026-09-09 11:32:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixing tests (#936)

Browse Source 
* abs import

* rename test model

* update trainer

* revert test_step check

* move tags

* fix test_step

* clean tests

* fix template

* update dataset path

* fix parent order
This commit is contained in:
Jirka Borovec
2020-02-25 13:06:24 -05:00
committed by GitHub
parent 20d15c8023
commit 5dd2afeab1
15 changed files with 264 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/models/__init__.py
+21 -17
View File
@@ -2,27 +2,31 @@
import torch
from .base import LightningTestModelBase, LightningTestModelBaseWithoutDataloader
from .base import TestModelBase
from .mixins import (
LightningValidationStepMixin,
LightningValidationMixin,
LightningValidationStepMultipleDataloadersMixin,
LightningValidationMultipleDataloadersMixin,
LightningTestStepMixin,
LightningTestMixin,
LightningTestStepMultipleDataloadersMixin,
LightningTestMultipleDataloadersMixin,
LightningTestFitSingleTestDataloadersMixin,
LightningTestFitMultipleTestDataloadersMixin,
LightningValStepFitSingleDataloaderMixin,
LightningValStepFitMultipleDataloadersMixin
LightEmptyTestStep,
LightValidationStepMixin,
LightValidationMixin,
LightValidationStepMultipleDataloadersMixin,
LightValidationMultipleDataloadersMixin,
LightTestStepMixin,
LightTestMixin,
LightTestStepMultipleDataloadersMixin,
LightTestMultipleDataloadersMixin,
LightTestFitSingleTestDataloadersMixin,
LightTestFitMultipleTestDataloadersMixin,
LightValStepFitSingleDataloaderMixin,
LightValStepFitMultipleDataloadersMixin,
LightTrainDataloader,
LightTestDataloader,
)
class LightningTestModel(LightningValidationMixin, LightningTestMixin, LightningTestModelBase):
"""
Most common test case. Validation and test dataloaders.
"""
class LightningTestModel(LightTrainDataloader,
LightValidationMixin,
LightTestMixin,
TestModelBase):
"""Most common test case. Validation and test dataloaders."""
def on_training_metrics(self, logs):
logs['some_tensor_to_test'] = torch.rand(1)
tests/models/base.py
+5 -22
View File
@@ -24,7 +24,7 @@ class TestingMNIST(MNIST):
def __init__(self, root, train=True, transform=None, target_transform=None,
download=False, num_samples=8000):
super(TestingMNIST, self).__init__(
super().__init__(
root,
train=train,
transform=transform,
@@ -48,7 +48,7 @@ class TestModelBase(LightningModule):
:param hparams:
"""
# init superclass
super(TestModelBase, self).__init__()
super().__init__()
self.hparams = hparams
self.batch_size = hparams.batch_size
@@ -87,7 +87,6 @@ class TestModelBase(LightningModule):
:param x:
:return:
"""
x = self.c_d1(x)
x = torch.tanh(x)
x = self.c_d1_bn(x)
@@ -153,10 +152,8 @@ class TestModelBase(LightningModule):
def prepare_data(self):
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.5,), (1.0,))])
dataset = TestingMNIST(root=self.hparams.data_root, train=True,
transform=transform, download=True, num_samples=2000)
dataset = TestingMNIST(root=self.hparams.data_root, train=False,
transform=transform, download=True, num_samples=2000)
_ = TestingMNIST(root=self.hparams.data_root, train=True,
transform=transform, download=True, num_samples=2000)
def _dataloader(self, train):
# init data generators
@@ -194,31 +191,17 @@ class TestModelBase(LightningModule):
parser.add_argument('--out_features', default=10, type=int)
# use 500 for CPU, 50000 for GPU to see speed difference
parser.add_argument('--hidden_dim', default=50000, type=int)
# data
parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
# training params (opt)
parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
options=[0.0001, 0.0005, 0.001, 0.005],
tunable=False)
parser.opt_list('--optimizer_name', default='adam', type=str,
options=['adam'], tunable=False)
# if using 2 nodes with 4 gpus each the batch size here
# (256) will be 256 / (2*8) = 16 per gpu
parser.opt_list('--batch_size', default=256 * 8, type=int,
options=[32, 64, 128, 256], tunable=False,
help='batch size will be divided over all gpus being used across all nodes')
help='batch size will be divided over all GPUs being used across all nodes')
return parser
class LightningTestModelBase(TestModelBase):
""" with pre-defined train dataloader """
def train_dataloader(self):
return self._dataloader(train=True)
class LightningTestModelBaseWithoutDataloader(TestModelBase):
""" without pre-defined train dataloader """
pass
tests/models/mixins.py
+61 -24
View File
@@ -5,7 +5,7 @@ import torch
from pytorch_lightning.core.decorators import data_loader
class LightningValidationStepMixin:
class LightValidationStepMixin:
"""
Add val_dataloader and validation_step methods for the case
when val_dataloader returns a single dataloader
@@ -14,7 +14,7 @@ class LightningValidationStepMixin:
def val_dataloader(self):
return self._dataloader(train=False)
def validation_step(self, batch, batch_idx):
def validation_step(self, batch, batch_idx, *args, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -58,7 +58,7 @@ class LightningValidationStepMixin:
return output
class LightningValidationMixin(LightningValidationStepMixin):
class LightValidationMixin(LightValidationStepMixin):
"""
Add val_dataloader, validation_step, and validation_end methods for the case
when val_dataloader returns a single dataloader
@@ -76,7 +76,7 @@ class LightningValidationMixin(LightningValidationStepMixin):
val_loss_mean = 0
val_acc_mean = 0
for output in outputs:
val_loss = output['val_loss']
val_loss = _get_output_metric(output, 'val_loss')
# reduce manually when using dp
if self.trainer.use_dp or self.trainer.use_ddp2:
@@ -84,7 +84,7 @@ class LightningValidationMixin(LightningValidationStepMixin):
val_loss_mean += val_loss
# reduce manually when using dp
val_acc = output['val_acc']
val_acc = _get_output_metric(output, 'val_acc')
if self.trainer.use_dp or self.trainer.use_ddp2:
val_acc = torch.mean(val_acc)
@@ -98,7 +98,7 @@ class LightningValidationMixin(LightningValidationStepMixin):
return results
class LightningValidationStepMultipleDataloadersMixin:
class LightValidationStepMultipleDataloadersMixin:
"""
Add val_dataloader and validation_step methods for the case
when val_dataloader returns multiple dataloaders
@@ -107,7 +107,7 @@ class LightningValidationStepMultipleDataloadersMixin:
def val_dataloader(self):
return [self._dataloader(train=False), self._dataloader(train=False)]
def validation_step(self, batch, batch_idx, dataloader_idx):
def validation_step(self, batch, batch_idx, dataloader_idx, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -157,7 +157,7 @@ class LightningValidationStepMultipleDataloadersMixin:
return output
class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipleDataloadersMixin):
class LightValidationMultipleDataloadersMixin(LightValidationStepMultipleDataloadersMixin):
"""
Add val_dataloader, validation_step, and validation_end methods for the case
when val_dataloader returns multiple dataloaders
@@ -200,12 +200,31 @@ class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipl
return result
class LightningTestStepMixin:
class LightTrainDataloader:
"""Simple train dataloader."""
def train_dataloader(self):
return self._dataloader(train=True)
class LightTestDataloader:
"""Simple test dataloader."""
def test_dataloader(self):
return self._dataloader(train=False)
def test_step(self, batch, batch_idx):
class LightEmptyTestStep:
"""Empty test step."""
def test_step(self, *args, **kwargs):
return dict()
class LightTestStepMixin(LightTestDataloader):
"""Test step mixin."""
def test_step(self, batch, batch_idx, *args, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -249,7 +268,9 @@ class LightningTestStepMixin:
return output
class LightningTestMixin(LightningTestStepMixin):
class LightTestMixin(LightTestStepMixin):
"""Ritch test mixin."""
def test_end(self, outputs):
"""
Called at the end of validation to aggregate outputs
@@ -262,7 +283,7 @@ class LightningTestMixin(LightningTestStepMixin):
test_loss_mean = 0
test_acc_mean = 0
for output in outputs:
test_loss = output['test_loss']
test_loss = _get_output_metric(output, 'test_loss')
# reduce manually when using dp
if self.trainer.use_dp:
@@ -270,7 +291,7 @@ class LightningTestMixin(LightningTestStepMixin):
test_loss_mean += test_loss
# reduce manually when using dp
test_acc = output['test_acc']
test_acc = _get_output_metric(output, 'test_acc')
if self.trainer.use_dp:
test_acc = torch.mean(test_acc)
@@ -284,12 +305,13 @@ class LightningTestMixin(LightningTestStepMixin):
return result
class LightningTestStepMultipleDataloadersMixin:
class LightTestStepMultipleDataloadersMixin:
"""Test step multiple dataloaders mixin."""
def test_dataloader(self):
return [self._dataloader(train=False), self._dataloader(train=False)]
def test_step(self, batch, batch_idx, dataloader_idx):
def test_step(self, batch, batch_idx, dataloader_idx, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -339,8 +361,10 @@ class LightningTestStepMultipleDataloadersMixin:
return output
class LightningTestFitSingleTestDataloadersMixin:
def test_step(self, batch, batch_idx):
class LightTestFitSingleTestDataloadersMixin:
"""Test fit single test dataloaders mixin."""
def test_step(self, batch, batch_idx, *args, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -384,8 +408,10 @@ class LightningTestFitSingleTestDataloadersMixin:
return output
class LightningTestFitMultipleTestDataloadersMixin:
def test_step(self, batch, batch_idx, dataloader_idx):
class LightTestFitMultipleTestDataloadersMixin:
"""Test fit multiple test dataloaders mixin."""
def test_step(self, batch, batch_idx, dataloader_idx, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -435,8 +461,9 @@ class LightningTestFitMultipleTestDataloadersMixin:
return output
class LightningValStepFitSingleDataloaderMixin:
def validation_step(self, batch, batch_idx):
class LightValStepFitSingleDataloaderMixin:
def validation_step(self, batch, batch_idx, *args, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -480,8 +507,9 @@ class LightningValStepFitSingleDataloaderMixin:
return output
class LightningValStepFitMultipleDataloadersMixin:
def validation_step(self, batch, batch_idx, dataloader_idx):
class LightValStepFitMultipleDataloadersMixin:
def validation_step(self, batch, batch_idx, dataloader_idx, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -531,7 +559,8 @@ class LightningValStepFitMultipleDataloadersMixin:
return output
class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloadersMixin):
class LightTestMultipleDataloadersMixin(LightTestStepMultipleDataloadersMixin):
def test_end(self, outputs):
"""
Called at the end of validation to aggregate outputs
@@ -567,3 +596,11 @@ class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloaders
tqdm_dict = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
result = {'progress_bar': tqdm_dict}
return result
def _get_output_metric(output, name):
if isinstance(output, dict):
val = output[name]
else: # if it is 2level deep -> per dataloader and per batch
val = sum(out[name] for out in output) / len(output)
return val
tests/models/utils.py
+2 -2
View File
@@ -90,7 +90,7 @@ def run_model_test(trainer_options, model, on_gpu=True):
def get_hparams(continue_training=False, hpc_exp_number=0):
root_dir = os.path.dirname(os.path.realpath(__file__))
tests_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
args = {
'drop_prob': 0.2,
@@ -98,7 +98,7 @@ def get_hparams(continue_training=False, hpc_exp_number=0):
'in_features': 28 * 28,
'learning_rate': 0.001 * 8,
'optimizer_name': 'adam',
'data_root': os.path.join(root_dir, 'mnist'),
'data_root': os.path.join(tests_dir, 'datasets'),
'out_features': 10,
'hidden_dim': 1000,
}