mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-10 12:21:57 +08:00
Refactor test modules (#180)
* Expectopatronum implement #89 (#182) * rename validate -> evaluate; implement test logic; allow multiple test_loaders * add test_step and test_end to LightningModule * add in_test_mode to pretraining to implement case 2 (test pretrained model) * fix code style issues * LightningTestModel: add optional second test set, implement test_step and test_end * implemented test for multiple test_dataloaders; fixed typo * add two test cases for #89 * add documentation for test_step, test_end; fix computation of loss in validation_step example * Update trainer.py * Update trainer.py * Update trainer.py * Update trainer.py * Update trainer.py * Update trainer.py * Added proper dp ddp routing calls for test mode * Update trainer.py * Update test_models.py * Update trainer.py * Update trainer.py * Update override_data_parallel.py * Update test_models.py * Update test_models.py * Update trainer.py * Update trainer.py * Update trainer.py * Update test_models.py * Update test_models.py * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * debug * Update trainer.py * Update override_data_parallel.py * Update debug.py * Update lm_test_module.py * Update test_models.py * release v0.4.8 * Update README.md * add training loop docs * testing loop docs * testing loop docs * Convert __dataloader to _dataloader This will let inherited classes use it * Factor common test model setup into base class * Specialized test modules inherit from LightningTestModelBase * Fix __is_overriden so that it works with more complicated inheritance * Use mixins to add functionality to test models * Fix test with no val_dataloader * Remove unused imports * Get rid of wild card import * Update trainer.py * Update lm_test_module.py
This commit is contained in:
Nic Eggert
committed by
William Falcon
William Falcon
parent
c4ce347f3e
commit
64688e1e15
@@ -14,6 +14,7 @@ import torch.multiprocessing as mp
|
||||
import torch.distributed as dist
|
||||
from torch.optim.optimizer import Optimizer
|
||||
|
||||
from pytorch_lightning.root_module.root_module import LightningModule
|
||||
from pytorch_lightning.root_module.memory import get_gpu_memory_map
|
||||
from pytorch_lightning.root_module.model_saving import TrainerIO
|
||||
from pytorch_lightning.pt_overrides.override_data_parallel import (
|
||||
@@ -326,7 +327,7 @@ class Trainer(TrainerIO):
|
||||
|
||||
def __is_overriden(self, f_name):
|
||||
model = self.__get_model()
|
||||
super_object = super(model.__class__, model)
|
||||
super_object = LightningModule
|
||||
|
||||
# when code pointers are different, it was overriden
|
||||
is_overriden = getattr(model, f_name).__code__ is not getattr(super_object, f_name).__code__
|
||||
@@ -395,7 +396,7 @@ class Trainer(TrainerIO):
|
||||
if test and len(self.test_dataloader) > 1:
|
||||
args.append(dataloader_i)
|
||||
|
||||
elif len(self.val_dataloader) > 1:
|
||||
elif not test and len(self.val_dataloader) > 1:
|
||||
args.append(dataloader_i)
|
||||
|
||||
# handle DP, DDP forward
|
||||
|
||||
@@ -1,3 +1,12 @@
|
||||
from .lm_test_module import LightningTestModel
|
||||
from .no_val_end_module import NoValEndTestModel
|
||||
from .no_val_module import NoValModel
|
||||
from .lm_test_module_base import LightningTestModelBase
|
||||
from .lm_test_module_mixins import (
|
||||
LightningValidationStepMixin,
|
||||
LightningValidationMixin,
|
||||
LightningValidationStepMultipleDataloadersMixin,
|
||||
LightningValidationMultipleDataloadersMixin,
|
||||
LightningTestStepMixin,
|
||||
LightningTestMixin,
|
||||
LightningTestStepMultipleDataloadersMixin,
|
||||
LightningTestMultipleDataloadersMixin,
|
||||
)
|
||||
|
||||
@@ -14,356 +14,14 @@ from test_tube import HyperOptArgumentParser
|
||||
from pytorch_lightning.root_module.root_module import LightningModule
|
||||
from pytorch_lightning import data_loader
|
||||
|
||||
from .lm_test_module_base import LightningTestModelBase
|
||||
from .lm_test_module_mixins import LightningValidationMixin, LightningTestMixin
|
||||
|
||||
class LightningTestModel(LightningModule):
|
||||
|
||||
class LightningTestModel(LightningValidationMixin, LightningTestMixin, LightningTestModelBase):
|
||||
"""
|
||||
Sample model to show how to define a template
|
||||
Most common test case. Validation and test dataloaders
|
||||
"""
|
||||
|
||||
def __init__(self, hparams, force_remove_distributed_sampler=False, use_two_test_sets=False):
|
||||
"""
|
||||
Pass in parsed HyperOptArgumentParser to the model
|
||||
:param hparams:
|
||||
"""
|
||||
# init superclass
|
||||
super(LightningTestModel, self).__init__()
|
||||
self.hparams = hparams
|
||||
self.use_two_test_sets = use_two_test_sets # for some tests regarding testing
|
||||
|
||||
self.batch_size = hparams.batch_size
|
||||
|
||||
# if you specify an example input, the summary will show input/output for each layer
|
||||
self.example_input_array = torch.rand(5, 28 * 28)
|
||||
|
||||
# remove to test warning for dist sampler
|
||||
self.force_remove_distributed_sampler = force_remove_distributed_sampler
|
||||
|
||||
# build model
|
||||
self.__build_model()
|
||||
|
||||
# ---------------------
|
||||
# MODEL SETUP
|
||||
# ---------------------
|
||||
def __build_model(self):
|
||||
"""
|
||||
Layout model
|
||||
:return:
|
||||
"""
|
||||
self.c_d1 = nn.Linear(in_features=self.hparams.in_features,
|
||||
out_features=self.hparams.hidden_dim)
|
||||
self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
|
||||
self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
|
||||
|
||||
self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim,
|
||||
out_features=self.hparams.out_features)
|
||||
|
||||
# ---------------------
|
||||
# TRAINING
|
||||
# ---------------------
|
||||
def forward(self, x):
|
||||
"""
|
||||
No special modification required for lightning, define as you normally would
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
|
||||
x = self.c_d1(x)
|
||||
x = torch.tanh(x)
|
||||
x = self.c_d1_bn(x)
|
||||
x = self.c_d1_drop(x)
|
||||
|
||||
x = self.c_d2(x)
|
||||
logits = F.log_softmax(x, dim=1)
|
||||
|
||||
return logits
|
||||
|
||||
def loss(self, labels, logits):
|
||||
nll = F.nll_loss(logits, labels)
|
||||
return nll
|
||||
|
||||
def training_step(self, data_batch, batch_i):
|
||||
"""
|
||||
Lightning calls this inside the training loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
# forward pass
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
|
||||
y_hat = self.forward(x)
|
||||
|
||||
# calculate loss
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
|
||||
if self.trainer.use_dp:
|
||||
loss_val = loss_val.unsqueeze(0)
|
||||
|
||||
# alternate possible outputs to test
|
||||
if self.trainer.batch_nb % 1 == 0:
|
||||
output = OrderedDict({
|
||||
'loss': loss_val,
|
||||
'prog': {'some_val': loss_val * loss_val}
|
||||
})
|
||||
return output
|
||||
if self.trainer.batch_nb % 2 == 0:
|
||||
return loss_val
|
||||
|
||||
def validation_step(self, data_batch, batch_i, dataloader_i):
|
||||
"""
|
||||
Lightning calls this inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
val_acc = torch.tensor(val_acc)
|
||||
|
||||
if self.on_gpu:
|
||||
val_acc = val_acc.cuda(loss_val.device.index)
|
||||
|
||||
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
|
||||
if self.trainer.use_dp:
|
||||
loss_val = loss_val.unsqueeze(0)
|
||||
val_acc = val_acc.unsqueeze(0)
|
||||
|
||||
# alternate possible outputs to test
|
||||
if batch_i % 1 == 0:
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': val_acc,
|
||||
})
|
||||
return output
|
||||
if batch_i % 2 == 0:
|
||||
return val_acc
|
||||
|
||||
if batch_i % 3 == 0:
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': val_acc,
|
||||
'test_dic': {'val_loss_a': loss_val}
|
||||
})
|
||||
return output
|
||||
if batch_i % 5 == 0:
|
||||
output = OrderedDict({
|
||||
f'val_loss_{dataloader_i}': loss_val,
|
||||
f'val_acc_{dataloader_i}': val_acc,
|
||||
})
|
||||
return output
|
||||
|
||||
def validation_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
# if returned a scalar from validation_step, outputs is a list of tensor scalars
|
||||
# we return just the average in this case (if we want)
|
||||
# return torch.stack(outputs).mean()
|
||||
val_loss_mean = 0
|
||||
val_acc_mean = 0
|
||||
for output in outputs:
|
||||
val_loss = output['val_loss']
|
||||
|
||||
# reduce manually when using dp
|
||||
if self.trainer.use_dp:
|
||||
val_loss = torch.mean(val_loss)
|
||||
val_loss_mean += val_loss
|
||||
|
||||
# reduce manually when using dp
|
||||
val_acc = output['val_acc']
|
||||
if self.trainer.use_dp:
|
||||
val_acc = torch.mean(val_acc)
|
||||
|
||||
val_acc_mean += val_acc
|
||||
|
||||
val_loss_mean /= len(outputs)
|
||||
val_acc_mean /= len(outputs)
|
||||
|
||||
tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
|
||||
def test_step(self, data_batch, batch_i, dataloader_i):
|
||||
"""
|
||||
Lightning calls this inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_test = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
test_acc = torch.tensor(test_acc)
|
||||
|
||||
if self.on_gpu:
|
||||
test_acc = test_acc.cuda(loss_test.device.index)
|
||||
|
||||
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
|
||||
if self.trainer.use_dp:
|
||||
loss_test = loss_test.unsqueeze(0)
|
||||
test_acc = test_acc.unsqueeze(0)
|
||||
|
||||
# alternate possible outputs to test
|
||||
if batch_i % 1 == 0:
|
||||
output = OrderedDict({
|
||||
'test_loss': loss_test,
|
||||
'test_acc': test_acc,
|
||||
})
|
||||
return output
|
||||
if batch_i % 2 == 0:
|
||||
return test_acc
|
||||
|
||||
if batch_i % 3 == 0:
|
||||
output = OrderedDict({
|
||||
'test_loss': loss_test,
|
||||
'test_acc': test_acc,
|
||||
'test_dic': {'test_loss_a': loss_test}
|
||||
})
|
||||
return output
|
||||
if batch_i % 5 == 0:
|
||||
output = OrderedDict({
|
||||
f'test_loss_{dataloader_i}': loss_test,
|
||||
f'test_acc_{dataloader_i}': test_acc,
|
||||
})
|
||||
return output
|
||||
|
||||
def test_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
# if returned a scalar from test_step, outputs is a list of tensor scalars
|
||||
# we return just the average in this case (if we want)
|
||||
# return torch.stack(outputs).mean()
|
||||
test_loss_mean = 0
|
||||
test_acc_mean = 0
|
||||
for output in outputs:
|
||||
test_loss = output['test_loss']
|
||||
|
||||
# reduce manually when using dp
|
||||
if self.trainer.use_dp:
|
||||
test_loss = torch.mean(test_loss)
|
||||
test_loss_mean += test_loss
|
||||
|
||||
# reduce manually when using dp
|
||||
test_acc = output['test_acc']
|
||||
if self.trainer.use_dp:
|
||||
test_acc = torch.mean(test_acc)
|
||||
|
||||
test_acc_mean += test_acc
|
||||
|
||||
test_loss_mean /= len(outputs)
|
||||
test_acc_mean /= len(outputs)
|
||||
|
||||
tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
|
||||
def on_tng_metrics(self, logs):
|
||||
logs['some_tensor_to_test'] = torch.rand(1)
|
||||
|
||||
# ---------------------
|
||||
# TRAINING SETUP
|
||||
# ---------------------
|
||||
def configure_optimizers(self):
|
||||
"""
|
||||
return whatever optimizers we want here
|
||||
:return: list of optimizers
|
||||
"""
|
||||
# try no scheduler for this model (testing purposes)
|
||||
optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
|
||||
|
||||
# test returning only 1 list instead of 2
|
||||
return optimizer
|
||||
|
||||
def __dataloader(self, train):
|
||||
# init data generators
|
||||
transform = transforms.Compose([transforms.ToTensor(),
|
||||
transforms.Normalize((0.5,), (1.0,))])
|
||||
dataset = MNIST(root=self.hparams.data_root, train=train,
|
||||
transform=transform, download=True)
|
||||
|
||||
# when using multi-node we need to add the datasampler
|
||||
train_sampler = None
|
||||
batch_size = self.hparams.batch_size
|
||||
|
||||
try:
|
||||
if self.use_ddp and not self.force_remove_distributed_sampler:
|
||||
train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
|
||||
batch_size = batch_size // self.trainer.world_size # scale batch size
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
should_shuffle = train_sampler is None
|
||||
loader = DataLoader(
|
||||
dataset=dataset,
|
||||
batch_size=batch_size,
|
||||
shuffle=should_shuffle,
|
||||
sampler=train_sampler
|
||||
)
|
||||
|
||||
return loader
|
||||
|
||||
@data_loader
|
||||
def tng_dataloader(self):
|
||||
return self.__dataloader(train=True)
|
||||
|
||||
@data_loader
|
||||
def val_dataloader(self):
|
||||
return [self.__dataloader(train=False), self.__dataloader(train=False)]
|
||||
|
||||
@data_loader
|
||||
def test_dataloader(self):
|
||||
if self.use_two_test_sets:
|
||||
return [self.__dataloader(train=False), self.__dataloader(train=False)]
|
||||
return self.__dataloader(train=False)
|
||||
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser, root_dir): # pragma: no cover
|
||||
"""
|
||||
Parameters you define here will be available to your model through self.hparams
|
||||
:param parent_parser:
|
||||
:param root_dir:
|
||||
:return:
|
||||
"""
|
||||
parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
|
||||
|
||||
# param overwrites
|
||||
# parser.set_defaults(gradient_clip=5.0)
|
||||
|
||||
# network params
|
||||
parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
|
||||
parser.add_argument('--in_features', default=28 * 28, type=int)
|
||||
parser.add_argument('--out_features', default=10, type=int)
|
||||
# use 500 for CPU, 50000 for GPU to see speed difference
|
||||
parser.add_argument('--hidden_dim', default=50000, type=int)
|
||||
|
||||
# data
|
||||
parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
|
||||
|
||||
# training params (opt)
|
||||
parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
|
||||
options=[0.0001, 0.0005, 0.001, 0.005],
|
||||
tunable=False)
|
||||
parser.opt_list('--optimizer_name', default='adam', type=str,
|
||||
options=['adam'], tunable=False)
|
||||
|
||||
# if using 2 nodes with 4 gpus each the batch size here
|
||||
# (256) will be 256 / (2*8) = 16 per gpu
|
||||
parser.opt_list('--batch_size', default=256 * 8, type=int,
|
||||
options=[32, 64, 128, 256], tunable=False,
|
||||
help='batch size will be divided over all gpus being used across all nodes')
|
||||
return parser
|
||||
|
||||
+7
-9
@@ -15,9 +15,10 @@ from pytorch_lightning.root_module.root_module import LightningModule
|
||||
from pytorch_lightning import data_loader
|
||||
|
||||
|
||||
class NoValModel(LightningModule):
|
||||
class LightningTestModelBase(LightningModule):
|
||||
"""
|
||||
Sample model to show how to define a template
|
||||
Base LightningModule for testing. Implements only the required
|
||||
interface
|
||||
"""
|
||||
|
||||
def __init__(self, hparams, force_remove_distributed_sampler=False):
|
||||
@@ -26,7 +27,7 @@ class NoValModel(LightningModule):
|
||||
:param hparams:
|
||||
"""
|
||||
# init superclass
|
||||
super(NoValModel, self).__init__()
|
||||
super(LightningTestModelBase, self).__init__()
|
||||
self.hparams = hparams
|
||||
|
||||
self.batch_size = hparams.batch_size
|
||||
@@ -109,9 +110,6 @@ class NoValModel(LightningModule):
|
||||
if self.trainer.batch_nb % 2 == 0:
|
||||
return loss_val
|
||||
|
||||
def on_tng_metrics(self, logs):
|
||||
logs['some_tensor_to_test'] = torch.rand(1)
|
||||
|
||||
# ---------------------
|
||||
# TRAINING SETUP
|
||||
# ---------------------
|
||||
@@ -124,9 +122,9 @@ class NoValModel(LightningModule):
|
||||
optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
|
||||
|
||||
# test returning only 1 list instead of 2
|
||||
return [optimizer]
|
||||
return optimizer
|
||||
|
||||
def __dataloader(self, train):
|
||||
def _dataloader(self, train):
|
||||
# init data generators
|
||||
transform = transforms.Compose([transforms.ToTensor(),
|
||||
transforms.Normalize((0.5,), (1.0,))])
|
||||
@@ -156,7 +154,7 @@ class NoValModel(LightningModule):
|
||||
|
||||
@data_loader
|
||||
def tng_dataloader(self):
|
||||
return self.__dataloader(train=True)
|
||||
return self._dataloader(train=True)
|
||||
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser, root_dir): # pragma: no cover
|
||||
@@ -0,0 +1,381 @@
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch import optim
|
||||
from torch.utils.data import DataLoader
|
||||
from torch.utils.data.distributed import DistributedSampler
|
||||
from torchvision.datasets import MNIST
|
||||
from torchvision import transforms
|
||||
from test_tube import HyperOptArgumentParser
|
||||
|
||||
from pytorch_lightning.root_module.root_module import LightningModule
|
||||
from pytorch_lightning import data_loader
|
||||
|
||||
|
||||
class LightningValidationStepMixin:
|
||||
"""
|
||||
Add val_dataloader and validation_step methods for the case
|
||||
when val_dataloader returns a single dataloader
|
||||
"""
|
||||
|
||||
@data_loader
|
||||
def val_dataloader(self):
|
||||
return self._dataloader(train=False)
|
||||
|
||||
def validation_step(self, data_batch, batch_i):
|
||||
"""
|
||||
Lightning calls this inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
val_acc = torch.tensor(val_acc)
|
||||
|
||||
if self.on_gpu:
|
||||
val_acc = val_acc.cuda(loss_val.device.index)
|
||||
|
||||
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
|
||||
if self.trainer.use_dp:
|
||||
loss_val = loss_val.unsqueeze(0)
|
||||
val_acc = val_acc.unsqueeze(0)
|
||||
|
||||
# alternate possible outputs to test
|
||||
if batch_i % 1 == 0:
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': val_acc,
|
||||
})
|
||||
return output
|
||||
if batch_i % 2 == 0:
|
||||
return val_acc
|
||||
|
||||
if batch_i % 3 == 0:
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': val_acc,
|
||||
'test_dic': {'val_loss_a': loss_val}
|
||||
})
|
||||
return output
|
||||
|
||||
|
||||
class LightningValidationMixin(LightningValidationStepMixin):
|
||||
"""
|
||||
Add val_dataloader, validation_step, and validation_end methods for the case
|
||||
when val_dataloader returns a single dataloader
|
||||
"""
|
||||
|
||||
def validation_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
# if returned a scalar from validation_step, outputs is a list of tensor scalars
|
||||
# we return just the average in this case (if we want)
|
||||
# return torch.stack(outputs).mean()
|
||||
val_loss_mean = 0
|
||||
val_acc_mean = 0
|
||||
for output in outputs:
|
||||
val_loss = output['val_loss']
|
||||
|
||||
# reduce manually when using dp
|
||||
if self.trainer.use_dp:
|
||||
val_loss = torch.mean(val_loss)
|
||||
val_loss_mean += val_loss
|
||||
|
||||
# reduce manually when using dp
|
||||
val_acc = output['val_acc']
|
||||
if self.trainer.use_dp:
|
||||
val_acc = torch.mean(val_acc)
|
||||
|
||||
val_acc_mean += val_acc
|
||||
|
||||
val_loss_mean /= len(outputs)
|
||||
val_acc_mean /= len(outputs)
|
||||
|
||||
tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
|
||||
|
||||
class LightningValidationStepMultipleDataloadersMixin:
|
||||
"""
|
||||
Add val_dataloader and validation_step methods for the case
|
||||
when val_dataloader returns multiple dataloaders
|
||||
"""
|
||||
|
||||
@data_loader
|
||||
def val_dataloader(self):
|
||||
return [self._dataloader(train=False), self._dataloader(train=False)]
|
||||
|
||||
def validation_step(self, data_batch, batch_i, dataloader_i):
|
||||
"""
|
||||
Lightning calls this inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
val_acc = torch.tensor(val_acc)
|
||||
|
||||
if self.on_gpu:
|
||||
val_acc = val_acc.cuda(loss_val.device.index)
|
||||
|
||||
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
|
||||
if self.trainer.use_dp:
|
||||
loss_val = loss_val.unsqueeze(0)
|
||||
val_acc = val_acc.unsqueeze(0)
|
||||
|
||||
# alternate possible outputs to test
|
||||
if batch_i % 1 == 0:
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': val_acc,
|
||||
})
|
||||
return output
|
||||
if batch_i % 2 == 0:
|
||||
return val_acc
|
||||
|
||||
if batch_i % 3 == 0:
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': val_acc,
|
||||
'test_dic': {'val_loss_a': loss_val}
|
||||
})
|
||||
return output
|
||||
if batch_i % 5 == 0:
|
||||
output = OrderedDict({
|
||||
f'val_loss_{dataloader_i}': loss_val,
|
||||
f'val_acc_{dataloader_i}': val_acc,
|
||||
})
|
||||
return output
|
||||
|
||||
|
||||
class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipleDataloadersMixin):
|
||||
"""
|
||||
Add val_dataloader, validation_step, and validation_end methods for the case
|
||||
when val_dataloader returns multiple dataloaders
|
||||
"""
|
||||
|
||||
def validation_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
# if returned a scalar from validation_step, outputs is a list of tensor scalars
|
||||
# we return just the average in this case (if we want)
|
||||
# return torch.stack(outputs).mean()
|
||||
val_loss_mean = 0
|
||||
val_acc_mean = 0
|
||||
for output in outputs:
|
||||
val_loss = output['val_loss']
|
||||
|
||||
# reduce manually when using dp
|
||||
if self.trainer.use_dp:
|
||||
val_loss = torch.mean(val_loss)
|
||||
val_loss_mean += val_loss
|
||||
|
||||
# reduce manually when using dp
|
||||
val_acc = output['val_acc']
|
||||
if self.trainer.use_dp:
|
||||
val_acc = torch.mean(val_acc)
|
||||
|
||||
val_acc_mean += val_acc
|
||||
|
||||
val_loss_mean /= len(outputs)
|
||||
val_acc_mean /= len(outputs)
|
||||
|
||||
tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
|
||||
|
||||
class LightningTestStepMixin:
|
||||
|
||||
@data_loader
|
||||
def test_dataloader(self):
|
||||
return self._dataloader(train=False)
|
||||
|
||||
def test_step(self, data_batch, batch_i):
|
||||
"""
|
||||
Lightning calls this inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_test = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
test_acc = torch.tensor(test_acc)
|
||||
|
||||
if self.on_gpu:
|
||||
test_acc = test_acc.cuda(loss_test.device.index)
|
||||
|
||||
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
|
||||
if self.trainer.use_dp:
|
||||
loss_test = loss_test.unsqueeze(0)
|
||||
test_acc = test_acc.unsqueeze(0)
|
||||
|
||||
# alternate possible outputs to test
|
||||
if batch_i % 1 == 0:
|
||||
output = OrderedDict({
|
||||
'test_loss': loss_test,
|
||||
'test_acc': test_acc,
|
||||
})
|
||||
return output
|
||||
if batch_i % 2 == 0:
|
||||
return test_acc
|
||||
|
||||
if batch_i % 3 == 0:
|
||||
output = OrderedDict({
|
||||
'test_loss': loss_test,
|
||||
'test_acc': test_acc,
|
||||
'test_dic': {'test_loss_a': loss_test}
|
||||
})
|
||||
return output
|
||||
|
||||
|
||||
class LightningTestMixin(LightningTestStepMixin):
|
||||
def test_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
# if returned a scalar from test_step, outputs is a list of tensor scalars
|
||||
# we return just the average in this case (if we want)
|
||||
# return torch.stack(outputs).mean()
|
||||
test_loss_mean = 0
|
||||
test_acc_mean = 0
|
||||
for output in outputs:
|
||||
test_loss = output['test_loss']
|
||||
|
||||
# reduce manually when using dp
|
||||
if self.trainer.use_dp:
|
||||
test_loss = torch.mean(test_loss)
|
||||
test_loss_mean += test_loss
|
||||
|
||||
# reduce manually when using dp
|
||||
test_acc = output['test_acc']
|
||||
if self.trainer.use_dp:
|
||||
test_acc = torch.mean(test_acc)
|
||||
|
||||
test_acc_mean += test_acc
|
||||
|
||||
test_loss_mean /= len(outputs)
|
||||
test_acc_mean /= len(outputs)
|
||||
|
||||
tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
|
||||
|
||||
class LightningTestStepMultipleDataloadersMixin:
|
||||
|
||||
@data_loader
|
||||
def test_dataloader(self):
|
||||
return [self._dataloader(train=False), self._dataloader(train=False)]
|
||||
|
||||
def test_step(self, data_batch, batch_i, dataloader_i):
|
||||
"""
|
||||
Lightning calls this inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_test = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
test_acc = torch.tensor(test_acc)
|
||||
|
||||
if self.on_gpu:
|
||||
test_acc = test_acc.cuda(loss_test.device.index)
|
||||
|
||||
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
|
||||
if self.trainer.use_dp:
|
||||
loss_test = loss_test.unsqueeze(0)
|
||||
test_acc = test_acc.unsqueeze(0)
|
||||
|
||||
# alternate possible outputs to test
|
||||
if batch_i % 1 == 0:
|
||||
output = OrderedDict({
|
||||
'test_loss': loss_test,
|
||||
'test_acc': test_acc,
|
||||
})
|
||||
return output
|
||||
if batch_i % 2 == 0:
|
||||
return test_acc
|
||||
|
||||
if batch_i % 3 == 0:
|
||||
output = OrderedDict({
|
||||
'test_loss': loss_test,
|
||||
'test_acc': test_acc,
|
||||
'test_dic': {'test_loss_a': loss_test}
|
||||
})
|
||||
return output
|
||||
if batch_i % 5 == 0:
|
||||
output = OrderedDict({
|
||||
f'test_loss_{dataloader_i}': loss_test,
|
||||
f'test_acc_{dataloader_i}': test_acc,
|
||||
})
|
||||
return output
|
||||
|
||||
|
||||
class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloadersMixin):
|
||||
def test_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
# if returned a scalar from test_step, outputs is a list of tensor scalars
|
||||
# we return just the average in this case (if we want)
|
||||
# return torch.stack(outputs).mean()
|
||||
test_loss_mean = 0
|
||||
test_acc_mean = 0
|
||||
for output in outputs:
|
||||
test_loss = output['test_loss']
|
||||
|
||||
# reduce manually when using dp
|
||||
if self.trainer.use_dp:
|
||||
test_loss = torch.mean(test_loss)
|
||||
test_loss_mean += test_loss
|
||||
|
||||
# reduce manually when using dp
|
||||
test_acc = output['test_acc']
|
||||
if self.trainer.use_dp:
|
||||
test_acc = torch.mean(test_acc)
|
||||
|
||||
test_acc_mean += test_acc
|
||||
|
||||
test_loss_mean /= len(outputs)
|
||||
test_acc_mean /= len(outputs)
|
||||
|
||||
tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
@@ -1,247 +0,0 @@
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from torch import optim
|
||||
from torch.utils.data import DataLoader
|
||||
from torch.utils.data.distributed import DistributedSampler
|
||||
from torchvision.datasets import MNIST
|
||||
from torchvision import transforms
|
||||
from test_tube import HyperOptArgumentParser
|
||||
|
||||
from pytorch_lightning.root_module.root_module import LightningModule
|
||||
from pytorch_lightning import data_loader
|
||||
|
||||
|
||||
class NoValEndTestModel(LightningModule):
|
||||
"""
|
||||
Sample model to show how to define a template
|
||||
"""
|
||||
|
||||
def __init__(self, hparams, force_remove_distributed_sampler=False):
|
||||
"""
|
||||
Pass in parsed HyperOptArgumentParser to the model
|
||||
:param hparams:
|
||||
"""
|
||||
# init superclass
|
||||
super(NoValEndTestModel, self).__init__()
|
||||
self.hparams = hparams
|
||||
|
||||
self.batch_size = hparams.batch_size
|
||||
|
||||
# if you specify an example input, the summary will show input/output for each layer
|
||||
self.example_input_array = torch.rand(5, 28 * 28)
|
||||
|
||||
# remove to test warning for dist sampler
|
||||
self.force_remove_distributed_sampler = force_remove_distributed_sampler
|
||||
|
||||
# build model
|
||||
self.__build_model()
|
||||
|
||||
# ---------------------
|
||||
# MODEL SETUP
|
||||
# ---------------------
|
||||
def __build_model(self):
|
||||
"""
|
||||
Layout model
|
||||
:return:
|
||||
"""
|
||||
self.c_d1 = nn.Linear(in_features=self.hparams.in_features,
|
||||
out_features=self.hparams.hidden_dim)
|
||||
self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
|
||||
self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
|
||||
|
||||
self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim,
|
||||
out_features=self.hparams.out_features)
|
||||
|
||||
# ---------------------
|
||||
# TRAINING
|
||||
# ---------------------
|
||||
def forward(self, x):
|
||||
"""
|
||||
No special modification required for lightning, define as you normally would
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
|
||||
x = self.c_d1(x)
|
||||
x = torch.tanh(x)
|
||||
x = self.c_d1_bn(x)
|
||||
x = self.c_d1_drop(x)
|
||||
|
||||
x = self.c_d2(x)
|
||||
logits = F.log_softmax(x, dim=1)
|
||||
|
||||
return logits
|
||||
|
||||
def loss(self, labels, logits):
|
||||
nll = F.nll_loss(logits, labels)
|
||||
return nll
|
||||
|
||||
def training_step(self, data_batch, batch_i):
|
||||
"""
|
||||
Lightning calls this inside the training loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
# forward pass
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
|
||||
y_hat = self.forward(x)
|
||||
|
||||
# calculate loss
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
|
||||
if self.trainer.use_dp:
|
||||
loss_val = loss_val.unsqueeze(0)
|
||||
|
||||
# alternate possible outputs to test
|
||||
if self.trainer.batch_nb % 1 == 0:
|
||||
output = OrderedDict({
|
||||
'loss': loss_val,
|
||||
'prog': {'some_val': loss_val * loss_val}
|
||||
})
|
||||
return output
|
||||
if self.trainer.batch_nb % 2 == 0:
|
||||
return loss_val
|
||||
|
||||
def validation_step(self, data_batch, batch_nb):
|
||||
"""
|
||||
Lightning calls this inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
val_acc = torch.tensor(val_acc)
|
||||
|
||||
if self.on_gpu:
|
||||
val_acc = val_acc.cuda(loss_val.device.index)
|
||||
|
||||
# in DP mode (default) make sure if result is scalar, there's another dim in the beginning
|
||||
if self.trainer.use_dp:
|
||||
loss_val = loss_val.unsqueeze(0)
|
||||
val_acc = val_acc.unsqueeze(0)
|
||||
|
||||
# alternate possible outputs to test
|
||||
if batch_nb % 1 == 0:
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': val_acc,
|
||||
})
|
||||
return output
|
||||
if batch_nb % 2 == 0:
|
||||
return val_acc
|
||||
|
||||
if batch_nb % 3 == 0:
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': val_acc,
|
||||
'test_dic': {'val_loss_a': loss_val}
|
||||
})
|
||||
return output
|
||||
|
||||
def on_tng_metrics(self, logs):
|
||||
logs['some_tensor_to_test'] = torch.rand(1)
|
||||
|
||||
# ---------------------
|
||||
# TRAINING SETUP
|
||||
# ---------------------
|
||||
def configure_optimizers(self):
|
||||
"""
|
||||
return whatever optimizers we want here
|
||||
:return: list of optimizers
|
||||
"""
|
||||
# try no scheduler for this model (testing purposes)
|
||||
optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
|
||||
|
||||
# test returning only 1 list instead of 2
|
||||
return [optimizer]
|
||||
|
||||
def __dataloader(self, train):
|
||||
# init data generators
|
||||
transform = transforms.Compose([transforms.ToTensor(),
|
||||
transforms.Normalize((0.5,), (1.0,))])
|
||||
dataset = MNIST(root=self.hparams.data_root, train=train,
|
||||
transform=transform, download=True)
|
||||
|
||||
# when using multi-node we need to add the datasampler
|
||||
train_sampler = None
|
||||
batch_size = self.hparams.batch_size
|
||||
|
||||
try:
|
||||
if self.use_ddp and not self.force_remove_distributed_sampler:
|
||||
train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
|
||||
batch_size = batch_size // self.trainer.world_size # scale batch size
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
should_shuffle = train_sampler is None
|
||||
loader = DataLoader(
|
||||
dataset=dataset,
|
||||
batch_size=batch_size,
|
||||
shuffle=should_shuffle,
|
||||
sampler=train_sampler
|
||||
)
|
||||
|
||||
return loader
|
||||
|
||||
@data_loader
|
||||
def tng_dataloader(self):
|
||||
return self.__dataloader(train=True)
|
||||
|
||||
@data_loader
|
||||
def val_dataloader(self):
|
||||
return self.__dataloader(train=False)
|
||||
|
||||
@data_loader
|
||||
def test_dataloader(self):
|
||||
return self.__dataloader(train=False)
|
||||
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser, root_dir): # pragma: no cover
|
||||
"""
|
||||
Parameters you define here will be available to your model through self.hparams
|
||||
:param parent_parser:
|
||||
:param root_dir:
|
||||
:return:
|
||||
"""
|
||||
parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
|
||||
|
||||
# param overwrites
|
||||
# parser.set_defaults(gradient_clip=5.0)
|
||||
|
||||
# network params
|
||||
parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
|
||||
parser.add_argument('--in_features', default=28 * 28, type=int)
|
||||
parser.add_argument('--out_features', default=10, type=int)
|
||||
# use 500 for CPU, 50000 for GPU to see speed difference
|
||||
parser.add_argument('--hidden_dim', default=50000, type=int)
|
||||
|
||||
# data
|
||||
parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
|
||||
|
||||
# training params (opt)
|
||||
parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
|
||||
options=[0.0001, 0.0005, 0.001, 0.005],
|
||||
tunable=False)
|
||||
parser.opt_list('--optimizer_name', default='adam', type=str,
|
||||
options=['adam'], tunable=False)
|
||||
|
||||
# if using 2 nodes with 4 gpus each the batch size here
|
||||
# (256) will be 256 / (2*8) = 16 per gpu
|
||||
parser.opt_list('--batch_size', default=256 * 8, type=int,
|
||||
options=[32, 64, 128, 256], tunable=False,
|
||||
help='batch size will be divided over all gpus being used across all nodes')
|
||||
return parser
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
from pytorch_lightning import Trainer
|
||||
from examples import LightningTemplateModel
|
||||
from pytorch_lightning.testing import LightningTestModel, NoValEndTestModel, NoValModel
|
||||
from pytorch_lightning.testing import LightningTestModel
|
||||
from argparse import Namespace
|
||||
from test_tube import Experiment
|
||||
from pytorch_lightning.callbacks import ModelCheckpoint
|
||||
|
||||
+73
-6
@@ -10,7 +10,15 @@ from test_tube import Experiment, SlurmCluster
|
||||
|
||||
# sys.path += [os.path.abspath('..'), os.path.abspath('../..')]
|
||||
from pytorch_lightning import Trainer
|
||||
from pytorch_lightning.testing import LightningTestModel, NoValEndTestModel, NoValModel
|
||||
from pytorch_lightning.testing import (
|
||||
LightningTestModel,
|
||||
LightningTestModelBase,
|
||||
LightningValidationMixin,
|
||||
LightningValidationStepMixin,
|
||||
LightningValidationMultipleDataloadersMixin,
|
||||
LightningTestMixin,
|
||||
LightningTestMultipleDataloadersMixin,
|
||||
)
|
||||
from pytorch_lightning.callbacks import (
|
||||
ModelCheckpoint,
|
||||
EarlyStopping,
|
||||
@@ -116,6 +124,47 @@ def test_running_test_after_fitting():
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_running_test_without_val():
|
||||
"""Verify test() works on a model with no val_loader"""
|
||||
class CurrentTestModel(LightningTestMixin, LightningTestModelBase):
|
||||
pass
|
||||
hparams = get_hparams()
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
# exp file to get meta
|
||||
exp = get_exp(False)
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
|
||||
# exp file to get weights
|
||||
checkpoint = ModelCheckpoint(save_dir)
|
||||
|
||||
trainer_options = dict(
|
||||
show_progress_bar=False,
|
||||
max_nb_epochs=1,
|
||||
train_percent_check=0.4,
|
||||
val_percent_check=0.2,
|
||||
test_percent_check=0.2,
|
||||
checkpoint_callback=checkpoint,
|
||||
experiment=exp
|
||||
)
|
||||
|
||||
# fit model
|
||||
trainer = Trainer(**trainer_options)
|
||||
result = trainer.fit(model)
|
||||
|
||||
assert result == 1, 'training failed to complete'
|
||||
|
||||
trainer.test()
|
||||
|
||||
# test we have good test accuracy
|
||||
assert_ok_test_acc(trainer)
|
||||
|
||||
clear_save_dir()
|
||||
|
||||
|
||||
def test_running_test_pretrained_model():
|
||||
"""Verify test() on pretrained model"""
|
||||
hparams = get_hparams()
|
||||
@@ -146,7 +195,9 @@ def test_running_test_pretrained_model():
|
||||
|
||||
# correct result and ok accuracy
|
||||
assert result == 1, 'training failed to complete'
|
||||
pretrained_model = load_model(exp, save_dir, on_gpu=False, module_class=LightningTestModel)
|
||||
pretrained_model = load_model(
|
||||
exp, save_dir, on_gpu=False, module_class=LightningTestModel
|
||||
)
|
||||
|
||||
new_trainer = Trainer(**trainer_options)
|
||||
new_trainer.test(pretrained_model)
|
||||
@@ -400,7 +451,10 @@ def test_no_val_module():
|
||||
:return:
|
||||
"""
|
||||
hparams = get_hparams()
|
||||
model = NoValModel(hparams)
|
||||
|
||||
class CurrentTestModel(LightningTestModelBase):
|
||||
pass
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
@@ -443,8 +497,11 @@ def test_no_val_end_module():
|
||||
Tests use case where trainer saves the model, and user loads it from tags independently
|
||||
:return:
|
||||
"""
|
||||
|
||||
class CurrentTestModel(LightningValidationStepMixin, LightningTestModelBase):
|
||||
pass
|
||||
hparams = get_hparams()
|
||||
model = NoValEndTestModel(hparams)
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
save_dir = init_save_dir()
|
||||
|
||||
@@ -1052,8 +1109,13 @@ def test_multiple_val_dataloader():
|
||||
Verify multiple val_dataloader
|
||||
:return:
|
||||
"""
|
||||
class CurrentTestModel(
|
||||
LightningValidationMultipleDataloadersMixin,
|
||||
LightningTestModelBase
|
||||
):
|
||||
pass
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams)
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
# exp file to get meta
|
||||
trainer_options = dict(
|
||||
@@ -1081,8 +1143,13 @@ def test_multiple_test_dataloader():
|
||||
Verify multiple test_dataloader
|
||||
:return:
|
||||
"""
|
||||
class CurrentTestModel(
|
||||
LightningTestMultipleDataloadersMixin,
|
||||
LightningTestModelBase
|
||||
):
|
||||
pass
|
||||
hparams = get_hparams()
|
||||
model = LightningTestModel(hparams, use_two_test_sets=True)
|
||||
model = CurrentTestModel(hparams)
|
||||
|
||||
# exp file to get meta
|
||||
trainer_options = dict(
|
||||
|
||||
Reference in New Issue
Block a user