wassname/pytorch-lightning
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-lightning.git synced 2026-09-09 11:32:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fixing tests (#936)

Browse Source 
* abs import

* rename test model

* update trainer

* revert test_step check

* move tags

* fix test_step

* clean tests

* fix template

* update dataset path

* fix parent order
This commit is contained in:
Jirka Borovec
2020-02-25 13:06:24 -05:00
committed by GitHub
parent 20d15c8023
commit 5dd2afeab1
15 changed files with 264 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci-testing.yml
+1 -1
View File
@@ -40,7 +40,7 @@ jobs:
- name: Cache datasets
uses: actions/cache@v1
with:
path: tests/models/mnist # This path is specific to Ubuntu
path: tests/datasets # This path is specific to Ubuntu
# Look to see if there is a cache hit for the corresponding requirements file
key: mnist-dataset
pl_examples/basic_examples/lightning_module_template.py
+4 -4
View File
@@ -188,6 +188,8 @@ class LightningTemplateModel(pl.LightningModule):
return [optimizer], [scheduler]
def __dataloader(self, train):
# this is neede when you want some info about dataset before binding to trainer
self.prepare_data()
# init data generators
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.5,), (1.0,))])
@@ -208,10 +210,8 @@ class LightningTemplateModel(pl.LightningModule):
def prepare_data(self):
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.5,), (1.0,))])
dataset = MNIST(root=self.hparams.data_root, train=True,
transform=transform, download=True)
dataset = MNIST(root=self.hparams.data_root, train=False,
transform=transform, download=True)
_ = MNIST(root=self.hparams.data_root, train=True,
transform=transform, download=True)
def train_dataloader(self):
log.info('Training data loader called.')
pytorch_lightning/core/lightning.py
+1 -33
View File
@@ -1,7 +1,6 @@
import collections
import inspect
import logging as log
import csv
import os
import warnings
from abc import ABC, abstractmethod
@@ -13,7 +12,7 @@ import torch.distributed as dist
from pytorch_lightning.core.decorators import data_loader
from pytorch_lightning.core.grads import GradInformation
from pytorch_lightning.core.hooks import ModelHooks
from pytorch_lightning.core.saving import ModelIO
from pytorch_lightning.core.saving import ModelIO, load_hparams_from_tags_csv
from pytorch_lightning.core.memory import ModelSummary
from pytorch_lightning.overrides.data_parallel import LightningDistributedDataParallel
from pytorch_lightning.utilities.debugging import MisconfigurationException
@@ -1316,34 +1315,3 @@ class LightningModule(ABC, GradInformation, ModelIO, ModelHooks):
tqdm_dict['v_num'] = self.trainer.logger.version
return tqdm_dict
def load_hparams_from_tags_csv(tags_csv):
if not os.path.isfile(tags_csv):
log.warning(f'Missing Tags: {tags_csv}.')
return Namespace()
tags = {}
with open(tags_csv) as f:
csv_reader = csv.reader(f, delimiter=',')
for row in list(csv_reader)[1:]:
tags[row[0]] = convert(row[1])
ns = Namespace(**tags)
return ns
def convert(val):
constructors = [int, float, str]
if isinstance(val, str):
if val.lower() == 'true':
return True
if val.lower() == 'false':
return False
for c in constructors:
try:
return c(val)
except ValueError:
pass
return val
pytorch_lightning/core/saving.py
+37
View File
@@ -1,3 +1,9 @@
import os
import csv
import logging as log
from argparse import Namespace
class ModelIO(object):
def on_load_checkpoint(self, checkpoint):
@@ -28,3 +34,34 @@ class ModelIO(object):
Hook to do whatever you need right before Slurm manager loads the model
:return:
"""
def load_hparams_from_tags_csv(tags_csv):
if not os.path.isfile(tags_csv):
log.warning(f'Missing Tags: {tags_csv}.')
return Namespace()
tags = {}
with open(tags_csv) as f:
csv_reader = csv.reader(f, delimiter=',')
for row in list(csv_reader)[1:]:
tags[row[0]] = convert(row[1])
ns = Namespace(**tags)
return ns
def convert(val):
constructors = [int, float, str]
if isinstance(val, str):
if val.lower() == 'true':
return True
if val.lower() == 'false':
return False
for c in constructors:
try:
return c(val)
except ValueError:
pass
return val
pytorch_lightning/loggers/comet.py
+1 -1
View File
@@ -22,8 +22,8 @@ except ImportError:
from torch import is_tensor
from pytorch_lightning.utilities.debugging import MisconfigurationException
from .base import LightningLoggerBase, rank_zero_only
from ..utilities.debugging import MisconfigurationException
logger = getLogger(__name__)
pytorch_lightning/trainer/data_loading.py
+10 -7
View File
@@ -177,7 +177,7 @@ class TrainerDataLoadingMixin(ABC):
self.is_iterable_train_dataloader = (
EXIST_ITER_DATASET and isinstance(self.train_dataloader.dataset, IterableDataset)
)
if self.is_iterable_train_dataloader and not isinstance(self.val_check_interval, int):
if self.is_iterable_dataloader(self.train_dataloader) and not isinstance(self.val_check_interval, int):
m = '''
When using an iterableDataset for `train_dataloader`,
`Trainer(val_check_interval)` must be an int.
@@ -185,6 +185,11 @@ class TrainerDataLoadingMixin(ABC):
'''
raise MisconfigurationException(m)
def is_iterable_dataloader(self, dataloader):
return (
EXIST_ITER_DATASET and isinstance(dataloader.dataset, IterableDataset)
)
def reset_val_dataloader(self, model):
"""
Dataloaders are provided by the model
@@ -200,9 +205,8 @@ class TrainerDataLoadingMixin(ABC):
self.num_val_batches = 0
# add samplers
for i, dataloader in enumerate(self.val_dataloaders):
dl = self.auto_add_sampler(dataloader, train=False)
self.val_dataloaders[i] = dl
self.val_dataloaders = [self.auto_add_sampler(dl, train=False)
for dl in self.val_dataloaders if dl]
# determine number of validation batches
# val datasets could be none, 1 or 2+
@@ -227,9 +231,8 @@ class TrainerDataLoadingMixin(ABC):
self.num_test_batches = 0
# add samplers
for i, dataloader in enumerate(self.test_dataloaders):
dl = self.auto_add_sampler(dataloader, train=False)
self.test_dataloaders[i] = dl
self.test_dataloaders = [self.auto_add_sampler(dl, train=False)
for dl in self.test_dataloaders if dl]
# determine number of test batches
if self.test_dataloaders is not None:
pytorch_lightning/trainer/evaluation_loop.py
+22 -29
View File
@@ -216,13 +216,13 @@ class TrainerEvaluationLoopMixin(ABC):
# this is just empty shell for code from other class
pass
def evaluate(self, model, dataloaders, max_batches, test=False):
def evaluate(self, model, dataloaders, max_batches, test_mode: bool = False):
"""Run evaluation code.
:param model: PT model
:param dataloaders: list of PT dataloaders
:param max_batches: Scalar
:param test: boolean
:param test_mode
:return:
"""
# enable eval mode
@@ -260,18 +260,14 @@ class TrainerEvaluationLoopMixin(ABC):
# -----------------
# RUN EVALUATION STEP
# -----------------
output = self.evaluation_forward(model,
batch,
batch_idx,
dataloader_idx,
test)
output = self.evaluation_forward(model, batch, batch_idx, dataloader_idx, test_mode)
# track outputs for collation
dl_outputs.append(output)
# batch done
if batch_idx % self.progress_bar_refresh_rate == 0:
if test:
if test_mode:
self.test_progress_bar.update(self.progress_bar_refresh_rate)
else:
self.val_progress_bar.update(self.progress_bar_refresh_rate)
@@ -286,7 +282,7 @@ class TrainerEvaluationLoopMixin(ABC):
# give model a chance to do something with the outputs (and method defined)
model = self.get_model()
if test and self.is_overriden('test_end'):
if test_mode and self.is_overriden('test_end'):
eval_results = model.test_end(outputs)
elif self.is_overriden('validation_end'):
eval_results = model.validation_end(outputs)
@@ -299,11 +295,11 @@ class TrainerEvaluationLoopMixin(ABC):
return eval_results
def run_evaluation(self, test=False):
def run_evaluation(self, test_mode: bool = False):
# when testing make sure user defined a test step
if test and not self.is_overriden('test_step'):
m = '''You called `.test()` without defining model's `.test_step()`.
Please define and try again'''
if test_mode and not self.is_overriden('test_step'):
m = "You called `.test()` without defining model's `.test_step()`." \
" Please define and try again"
raise MisconfigurationException(m)
# hook
@@ -311,7 +307,7 @@ class TrainerEvaluationLoopMixin(ABC):
model.on_pre_performance_check()
# select dataloaders
if test:
if test_mode:
if self.reload_dataloaders_every_epoch or self.test_dataloaders is None:
self.reset_test_dataloader(model)
@@ -331,18 +327,15 @@ class TrainerEvaluationLoopMixin(ABC):
# init validation or test progress bar
# main progress bar will already be closed when testing so initial position is free
position = 2 * self.process_position + (not test)
desc = 'Testing' if test else 'Validating'
pbar = tqdm(desc=desc, total=max_batches, leave=test, position=position,
position = 2 * self.process_position + (not test_mode)
desc = 'Testing' if test_mode else 'Validating'
pbar = tqdm(desc=desc, total=max_batches, leave=test_mode, position=position,
disable=not self.show_progress_bar, dynamic_ncols=True,
file=sys.stdout)
setattr(self, f'{"test" if test else "val"}_progress_bar', pbar)
setattr(self, f'{"test" if test_mode else "val"}_progress_bar', pbar)
# run evaluation
eval_results = self.evaluate(self.model,
dataloaders,
max_batches,
test)
eval_results = self.evaluate(self.model, dataloaders, max_batches, test_mode)
_, prog_bar_metrics, log_metrics, callback_metrics, _ = self.process_output(
eval_results)
@@ -359,27 +352,27 @@ class TrainerEvaluationLoopMixin(ABC):
model.on_post_performance_check()
# add model specific metrics
if not test:
if not test_mode:
self.main_progress_bar.set_postfix(**self.training_tqdm_dict)
# close progress bar
if test:
if test_mode:
self.test_progress_bar.close()
else:
self.val_progress_bar.close()
# model checkpointing
if self.proc_rank == 0 and self.checkpoint_callback is not None and not test:
if self.proc_rank == 0 and self.checkpoint_callback is not None and not test_mode:
self.checkpoint_callback.on_validation_end()
def evaluation_forward(self, model, batch, batch_idx, dataloader_idx, test=False):
def evaluation_forward(self, model, batch, batch_idx, dataloader_idx, test_mode: bool = False):
# make dataloader_idx arg in validation_step optional
args = [batch, batch_idx]
if test and len(self.test_dataloaders) > 1:
if test_mode and len(self.test_dataloaders) > 1:
args.append(dataloader_idx)
elif not test and len(self.val_dataloaders) > 1:
elif not test_mode and len(self.val_dataloaders) > 1:
args.append(dataloader_idx)
# handle DP, DDP forward
@@ -402,7 +395,7 @@ class TrainerEvaluationLoopMixin(ABC):
args[0] = batch
# CPU
if test:
if test_mode:
output = model.test_step(*args)
else:
output = model.validation_step(*args)
pytorch_lightning/trainer/trainer.py
+10 -10
View File
@@ -84,7 +84,7 @@ class Trainer(TrainerIOMixin,
track_grad_norm: int = -1,
check_val_every_n_epoch: int = 1,
fast_dev_run: bool = False,
accumulate_grad_batches: Union[int, Dict[int, int]] = 1,
accumulate_grad_batches: Union[int, Dict[int, int], List[list]] = 1,
max_nb_epochs=None, # backward compatible, todo: remove in v0.8.0
min_nb_epochs=None, # backward compatible, todo: remove in v0.8.0
max_epochs: int = 1000,
@@ -681,7 +681,6 @@ class Trainer(TrainerIOMixin,
self.train_dataloader = None
self.test_dataloaders = None
self.val_dataloaders = None
self.is_iterable_train_dataloader = False
# training state
self.model = None
@@ -1068,7 +1067,7 @@ class Trainer(TrainerIOMixin,
if self.testing:
# only load test dataloader for testing
self.reset_test_dataloader(ref_model)
self.run_evaluation(test=True)
self.run_evaluation(test_mode=True)
return
# load the dataloaders
@@ -1087,15 +1086,17 @@ class Trainer(TrainerIOMixin,
if not self.disable_validation and self.num_sanity_val_steps > 0:
# init progress bars for validation sanity check
pbar = tqdm(desc='Validation sanity check',
total=self.num_sanity_val_steps * len(self.val_dataloaders),
leave=False, position=2 * self.process_position,
disable=not self.show_progress_bar, dynamic_ncols=True)
total=self.num_sanity_val_steps * len(self.val_dataloaders),
leave=False, position=2 * self.process_position,
disable=not self.show_progress_bar, dynamic_ncols=True)
self.main_progress_bar = pbar
# dummy validation progress bar
self.val_progress_bar = tqdm(disable=True)
eval_results = self.evaluate(model, self.val_dataloaders,
self.num_sanity_val_steps, False)
eval_results = self.evaluate(model,
self.val_dataloaders,
self.num_sanity_val_steps,
False)
_, _, _, callback_metrics, _ = self.process_output(eval_results)
# close progress bars
@@ -1145,5 +1146,4 @@ class Trainer(TrainerIOMixin,
self.testing = True
if model is not None:
self.fit(model)
else:
self.run_evaluation(test=True)
self.run_evaluation(test_mode=True)
pytorch_lightning/trainer/training_loop.py
+16 -5
View File
@@ -197,7 +197,6 @@ class TrainerTrainLoopMixin(ABC):
self.num_val_batches = None
self.disable_validation = None
self.fast_dev_run = None
self.is_iterable_train_dataloader = None
self.main_progress_bar = None
self.accumulation_scheduler = None
self.lr_schedulers = None
@@ -227,6 +226,8 @@ class TrainerTrainLoopMixin(ABC):
self.train_dataloader = None
self.reload_dataloaders_every_epoch = None
self.progress_bar_refresh_rate = None
self.max_steps = ...
self.max_steps = ...
@property
def max_nb_epochs(self):
@@ -257,7 +258,12 @@ class TrainerTrainLoopMixin(ABC):
pass
@abstractmethod
def run_evaluation(self, test):
def is_iterable_dataloader(self, dataloader):
# this is just empty shell for code from other class
pass
@abstractmethod
def run_evaluation(self, test_mode):
# this is just empty shell for code from other class
pass
@@ -306,6 +312,11 @@ class TrainerTrainLoopMixin(ABC):
# this is just empty shell for code from other class
pass
@abstractmethod
def has_arg(self, f_name, arg_name):
# this is just empty shell for code from other class
pass
def train(self):
warnings.warn('Displayed epoch numbers in the progress bar start from "1" until v0.6.x,'
' but will start from "0" in v0.8.0.', DeprecationWarning)
@@ -342,7 +353,7 @@ class TrainerTrainLoopMixin(ABC):
if self.fast_dev_run:
# limit the number of batches to 2 (1 train and 1 val) in fast_dev_run
num_iterations = 2
elif self.is_iterable_train_dataloader:
elif self.is_iterable_dataloader(self.train_dataloader):
# for iterable train loader, the progress bar never ends
num_iterations = None
else:
@@ -352,7 +363,7 @@ class TrainerTrainLoopMixin(ABC):
# .reset() doesn't work on disabled progress bar so we should check
if not self.main_progress_bar.disable:
self.main_progress_bar.reset(num_iterations)
desc = f'Epoch {epoch + 1}' if not self.is_iterable_train_dataloader else ''
desc = f'Epoch {epoch + 1}' if not self.is_iterable_dataloader(self.train_dataloader) else ''
self.main_progress_bar.set_description(desc)
# changing gradient according accumulation_scheduler
@@ -449,7 +460,7 @@ class TrainerTrainLoopMixin(ABC):
# fast_dev_run always forces val checking after train batch
if self.fast_dev_run or should_check_val:
self.run_evaluation(test=self.testing)
self.run_evaluation(test_mode=self.testing)
if self.enable_early_stop:
self.early_stop_callback.check_metrics(self.callback_metrics)
tests/models/__init__.py
+21 -17
View File
@@ -2,27 +2,31 @@
import torch
from .base import LightningTestModelBase, LightningTestModelBaseWithoutDataloader
from .base import TestModelBase
from .mixins import (
LightningValidationStepMixin,
LightningValidationMixin,
LightningValidationStepMultipleDataloadersMixin,
LightningValidationMultipleDataloadersMixin,
LightningTestStepMixin,
LightningTestMixin,
LightningTestStepMultipleDataloadersMixin,
LightningTestMultipleDataloadersMixin,
LightningTestFitSingleTestDataloadersMixin,
LightningTestFitMultipleTestDataloadersMixin,
LightningValStepFitSingleDataloaderMixin,
LightningValStepFitMultipleDataloadersMixin
LightEmptyTestStep,
LightValidationStepMixin,
LightValidationMixin,
LightValidationStepMultipleDataloadersMixin,
LightValidationMultipleDataloadersMixin,
LightTestStepMixin,
LightTestMixin,
LightTestStepMultipleDataloadersMixin,
LightTestMultipleDataloadersMixin,
LightTestFitSingleTestDataloadersMixin,
LightTestFitMultipleTestDataloadersMixin,
LightValStepFitSingleDataloaderMixin,
LightValStepFitMultipleDataloadersMixin,
LightTrainDataloader,
LightTestDataloader,
)
class LightningTestModel(LightningValidationMixin, LightningTestMixin, LightningTestModelBase):
"""
Most common test case. Validation and test dataloaders.
"""
class LightningTestModel(LightTrainDataloader,
LightValidationMixin,
LightTestMixin,
TestModelBase):
"""Most common test case. Validation and test dataloaders."""
def on_training_metrics(self, logs):
logs['some_tensor_to_test'] = torch.rand(1)
tests/models/base.py
+5 -22
View File
@@ -24,7 +24,7 @@ class TestingMNIST(MNIST):
def __init__(self, root, train=True, transform=None, target_transform=None,
download=False, num_samples=8000):
super(TestingMNIST, self).__init__(
super().__init__(
root,
train=train,
transform=transform,
@@ -48,7 +48,7 @@ class TestModelBase(LightningModule):
:param hparams:
"""
# init superclass
super(TestModelBase, self).__init__()
super().__init__()
self.hparams = hparams
self.batch_size = hparams.batch_size
@@ -87,7 +87,6 @@ class TestModelBase(LightningModule):
:param x:
:return:
"""
x = self.c_d1(x)
x = torch.tanh(x)
x = self.c_d1_bn(x)
@@ -153,10 +152,8 @@ class TestModelBase(LightningModule):
def prepare_data(self):
transform = transforms.Compose([transforms.ToTensor(),
transforms.Normalize((0.5,), (1.0,))])
dataset = TestingMNIST(root=self.hparams.data_root, train=True,
transform=transform, download=True, num_samples=2000)
dataset = TestingMNIST(root=self.hparams.data_root, train=False,
transform=transform, download=True, num_samples=2000)
_ = TestingMNIST(root=self.hparams.data_root, train=True,
transform=transform, download=True, num_samples=2000)
def _dataloader(self, train):
# init data generators
@@ -194,31 +191,17 @@ class TestModelBase(LightningModule):
parser.add_argument('--out_features', default=10, type=int)
# use 500 for CPU, 50000 for GPU to see speed difference
parser.add_argument('--hidden_dim', default=50000, type=int)
# data
parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
# training params (opt)
parser.opt_list('--learning_rate', default=0.001 * 8, type=float,
options=[0.0001, 0.0005, 0.001, 0.005],
tunable=False)
parser.opt_list('--optimizer_name', default='adam', type=str,
options=['adam'], tunable=False)
# if using 2 nodes with 4 gpus each the batch size here
# (256) will be 256 / (2*8) = 16 per gpu
parser.opt_list('--batch_size', default=256 * 8, type=int,
options=[32, 64, 128, 256], tunable=False,
help='batch size will be divided over all gpus being used across all nodes')
help='batch size will be divided over all GPUs being used across all nodes')
return parser
class LightningTestModelBase(TestModelBase):
""" with pre-defined train dataloader """
def train_dataloader(self):
return self._dataloader(train=True)
class LightningTestModelBaseWithoutDataloader(TestModelBase):
""" without pre-defined train dataloader """
pass
tests/models/mixins.py
+61 -24
View File
@@ -5,7 +5,7 @@ import torch
from pytorch_lightning.core.decorators import data_loader
class LightningValidationStepMixin:
class LightValidationStepMixin:
"""
Add val_dataloader and validation_step methods for the case
when val_dataloader returns a single dataloader
@@ -14,7 +14,7 @@ class LightningValidationStepMixin:
def val_dataloader(self):
return self._dataloader(train=False)
def validation_step(self, batch, batch_idx):
def validation_step(self, batch, batch_idx, *args, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -58,7 +58,7 @@ class LightningValidationStepMixin:
return output
class LightningValidationMixin(LightningValidationStepMixin):
class LightValidationMixin(LightValidationStepMixin):
"""
Add val_dataloader, validation_step, and validation_end methods for the case
when val_dataloader returns a single dataloader
@@ -76,7 +76,7 @@ class LightningValidationMixin(LightningValidationStepMixin):
val_loss_mean = 0
val_acc_mean = 0
for output in outputs:
val_loss = output['val_loss']
val_loss = _get_output_metric(output, 'val_loss')
# reduce manually when using dp
if self.trainer.use_dp or self.trainer.use_ddp2:
@@ -84,7 +84,7 @@ class LightningValidationMixin(LightningValidationStepMixin):
val_loss_mean += val_loss
# reduce manually when using dp
val_acc = output['val_acc']
val_acc = _get_output_metric(output, 'val_acc')
if self.trainer.use_dp or self.trainer.use_ddp2:
val_acc = torch.mean(val_acc)
@@ -98,7 +98,7 @@ class LightningValidationMixin(LightningValidationStepMixin):
return results
class LightningValidationStepMultipleDataloadersMixin:
class LightValidationStepMultipleDataloadersMixin:
"""
Add val_dataloader and validation_step methods for the case
when val_dataloader returns multiple dataloaders
@@ -107,7 +107,7 @@ class LightningValidationStepMultipleDataloadersMixin:
def val_dataloader(self):
return [self._dataloader(train=False), self._dataloader(train=False)]
def validation_step(self, batch, batch_idx, dataloader_idx):
def validation_step(self, batch, batch_idx, dataloader_idx, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -157,7 +157,7 @@ class LightningValidationStepMultipleDataloadersMixin:
return output
class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipleDataloadersMixin):
class LightValidationMultipleDataloadersMixin(LightValidationStepMultipleDataloadersMixin):
"""
Add val_dataloader, validation_step, and validation_end methods for the case
when val_dataloader returns multiple dataloaders
@@ -200,12 +200,31 @@ class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipl
return result
class LightningTestStepMixin:
class LightTrainDataloader:
"""Simple train dataloader."""
def train_dataloader(self):
return self._dataloader(train=True)
class LightTestDataloader:
"""Simple test dataloader."""
def test_dataloader(self):
return self._dataloader(train=False)
def test_step(self, batch, batch_idx):
class LightEmptyTestStep:
"""Empty test step."""
def test_step(self, *args, **kwargs):
return dict()
class LightTestStepMixin(LightTestDataloader):
"""Test step mixin."""
def test_step(self, batch, batch_idx, *args, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -249,7 +268,9 @@ class LightningTestStepMixin:
return output
class LightningTestMixin(LightningTestStepMixin):
class LightTestMixin(LightTestStepMixin):
"""Ritch test mixin."""
def test_end(self, outputs):
"""
Called at the end of validation to aggregate outputs
@@ -262,7 +283,7 @@ class LightningTestMixin(LightningTestStepMixin):
test_loss_mean = 0
test_acc_mean = 0
for output in outputs:
test_loss = output['test_loss']
test_loss = _get_output_metric(output, 'test_loss')
# reduce manually when using dp
if self.trainer.use_dp:
@@ -270,7 +291,7 @@ class LightningTestMixin(LightningTestStepMixin):
test_loss_mean += test_loss
# reduce manually when using dp
test_acc = output['test_acc']
test_acc = _get_output_metric(output, 'test_acc')
if self.trainer.use_dp:
test_acc = torch.mean(test_acc)
@@ -284,12 +305,13 @@ class LightningTestMixin(LightningTestStepMixin):
return result
class LightningTestStepMultipleDataloadersMixin:
class LightTestStepMultipleDataloadersMixin:
"""Test step multiple dataloaders mixin."""
def test_dataloader(self):
return [self._dataloader(train=False), self._dataloader(train=False)]
def test_step(self, batch, batch_idx, dataloader_idx):
def test_step(self, batch, batch_idx, dataloader_idx, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -339,8 +361,10 @@ class LightningTestStepMultipleDataloadersMixin:
return output
class LightningTestFitSingleTestDataloadersMixin:
def test_step(self, batch, batch_idx):
class LightTestFitSingleTestDataloadersMixin:
"""Test fit single test dataloaders mixin."""
def test_step(self, batch, batch_idx, *args, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -384,8 +408,10 @@ class LightningTestFitSingleTestDataloadersMixin:
return output
class LightningTestFitMultipleTestDataloadersMixin:
def test_step(self, batch, batch_idx, dataloader_idx):
class LightTestFitMultipleTestDataloadersMixin:
"""Test fit multiple test dataloaders mixin."""
def test_step(self, batch, batch_idx, dataloader_idx, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -435,8 +461,9 @@ class LightningTestFitMultipleTestDataloadersMixin:
return output
class LightningValStepFitSingleDataloaderMixin:
def validation_step(self, batch, batch_idx):
class LightValStepFitSingleDataloaderMixin:
def validation_step(self, batch, batch_idx, *args, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -480,8 +507,9 @@ class LightningValStepFitSingleDataloaderMixin:
return output
class LightningValStepFitMultipleDataloadersMixin:
def validation_step(self, batch, batch_idx, dataloader_idx):
class LightValStepFitMultipleDataloadersMixin:
def validation_step(self, batch, batch_idx, dataloader_idx, **kwargs):
"""
Lightning calls this inside the validation loop
:param batch:
@@ -531,7 +559,8 @@ class LightningValStepFitMultipleDataloadersMixin:
return output
class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloadersMixin):
class LightTestMultipleDataloadersMixin(LightTestStepMultipleDataloadersMixin):
def test_end(self, outputs):
"""
Called at the end of validation to aggregate outputs
@@ -567,3 +596,11 @@ class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloaders
tqdm_dict = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()}
result = {'progress_bar': tqdm_dict}
return result
def _get_output_metric(output, name):
if isinstance(output, dict):
val = output[name]
else: # if it is 2level deep -> per dataloader and per batch
val = sum(out[name] for out in output) / len(output)
return val
tests/models/utils.py
+2 -2
View File
@@ -90,7 +90,7 @@ def run_model_test(trainer_options, model, on_gpu=True):
def get_hparams(continue_training=False, hpc_exp_number=0):
root_dir = os.path.dirname(os.path.realpath(__file__))
tests_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
args = {
'drop_prob': 0.2,
@@ -98,7 +98,7 @@ def get_hparams(continue_training=False, hpc_exp_number=0):
'in_features': 28 * 28,
'learning_rate': 0.001 * 8,
'optimizer_name': 'adam',
'data_root': os.path.join(root_dir, 'mnist'),
'data_root': os.path.join(tests_dir, 'datasets'),
'out_features': 10,
'hidden_dim': 1000,
}
tests/test_cpu_models.py
+5 -4
View File
@@ -8,9 +8,10 @@ from pytorch_lightning.callbacks import (
EarlyStopping,
)
from tests.models import (
TestModelBase,
LightTrainDataloader,
LightningTestModel,
LightningTestModelBase,
LightningTestMixin,
LightTestMixin,
)
@@ -121,7 +122,7 @@ def test_running_test_without_val(tmpdir):
"""Verify `test()` works on a model with no `val_loader`."""
tutils.reset_seed()
class CurrentTestModel(LightningTestMixin, LightningTestModelBase):
class CurrentTestModel(LightTrainDataloader, LightTestMixin, TestModelBase):
pass
hparams = tutils.get_hparams()
@@ -281,7 +282,7 @@ def test_tbptt_cpu_model(tmpdir):
def __len__(self):
return 1
class BpttTestModel(LightningTestModelBase):
class BpttTestModel(LightTrainDataloader, TestModelBase):
def __init__(self, hparams):
super().__init__(hparams)
self.test_hidden = None
tests/test_trainer.py
+68 -50
View File
@@ -11,19 +11,22 @@ from pytorch_lightning.callbacks import (
ModelCheckpoint,
)
from tests.models import (
TestModelBase,
LightningTestModel,
LightningTestModelBase,
LightningTestModelBaseWithoutDataloader,
LightningValidationStepMixin,
LightningValidationMultipleDataloadersMixin,
LightningTestMultipleDataloadersMixin,
LightningTestFitSingleTestDataloadersMixin,
LightningTestFitMultipleTestDataloadersMixin,
LightningValStepFitMultipleDataloadersMixin,
LightningValStepFitSingleDataloaderMixin
LightEmptyTestStep,
LightValidationStepMixin,
LightValidationMultipleDataloadersMixin,
LightTestMultipleDataloadersMixin,
LightTestFitSingleTestDataloadersMixin,
LightTestFitMultipleTestDataloadersMixin,
LightValStepFitMultipleDataloadersMixin,
LightValStepFitSingleDataloaderMixin,
LightTrainDataloader,
LightTestDataloader,
)
from pytorch_lightning.core.lightning import load_hparams_from_tags_csv
from pytorch_lightning.trainer.logging import TrainerLoggingMixin
from pytorch_lightning.utilities.debugging import MisconfigurationException
def test_no_val_module(tmpdir):
@@ -32,7 +35,7 @@ def test_no_val_module(tmpdir):
hparams = tutils.get_hparams()
class CurrentTestModel(LightningTestModelBase):
class CurrentTestModel(LightTrainDataloader, TestModelBase):
pass
model = CurrentTestModel(hparams)
@@ -69,7 +72,7 @@ def test_no_val_end_module(tmpdir):
"""Tests use case where trainer saves the model, and user loads it from tags independently."""
tutils.reset_seed()
class CurrentTestModel(LightningValidationStepMixin, LightningTestModelBase):
class CurrentTestModel(LightTrainDataloader, LightValidationStepMixin, TestModelBase):
pass
hparams = tutils.get_hparams()
@@ -385,8 +388,9 @@ def test_multiple_val_dataloader(tmpdir):
tutils.reset_seed()
class CurrentTestModel(
LightningValidationMultipleDataloadersMixin,
LightningTestModelBase
LightTrainDataloader,
LightValidationMultipleDataloadersMixin,
TestModelBase,
):
pass
@@ -490,8 +494,10 @@ def test_multiple_test_dataloader(tmpdir):
tutils.reset_seed()
class CurrentTestModel(
LightningTestMultipleDataloadersMixin,
LightningTestModelBase
LightTrainDataloader,
LightTestMultipleDataloadersMixin,
LightEmptyTestStep,
TestModelBase,
):
pass
@@ -508,8 +514,7 @@ def test_multiple_test_dataloader(tmpdir):
# fit model
trainer = Trainer(**trainer_options)
result = trainer.fit(model)
trainer.fit(model)
trainer.test()
# verify there are 2 val loaders
@@ -528,9 +533,7 @@ def test_train_dataloaders_passed_to_fit(tmpdir):
""" Verify that train dataloader can be passed to fit """
tutils.reset_seed()
class CurrentTestModel(
LightningTestModelBaseWithoutDataloader,
):
class CurrentTestModel(LightTrainDataloader, TestModelBase):
pass
hparams = tutils.get_hparams()
@@ -555,8 +558,9 @@ def test_train_val_dataloaders_passed_to_fit(tmpdir):
tutils.reset_seed()
class CurrentTestModel(
LightningValStepFitSingleDataloaderMixin,
LightningTestModelBaseWithoutDataloader,
LightTrainDataloader,
LightValStepFitSingleDataloaderMixin,
TestModelBase,
):
pass
@@ -586,9 +590,11 @@ def test_all_dataloaders_passed_to_fit(tmpdir):
tutils.reset_seed()
class CurrentTestModel(
LightningValStepFitSingleDataloaderMixin,
LightningTestFitSingleTestDataloadersMixin,
LightningTestModelBaseWithoutDataloader,
LightTrainDataloader,
LightValStepFitSingleDataloaderMixin,
LightTestFitSingleTestDataloadersMixin,
LightEmptyTestStep,
TestModelBase,
):
pass
@@ -624,9 +630,9 @@ def test_multiple_dataloaders_passed_to_fit(tmpdir):
tutils.reset_seed()
class CurrentTestModel(
LightningValStepFitMultipleDataloadersMixin,
LightningTestFitMultipleTestDataloadersMixin,
LightningTestModelBaseWithoutDataloader,
LightningTestModel,
LightValStepFitMultipleDataloadersMixin,
LightTestFitMultipleTestDataloadersMixin,
):
pass
@@ -662,9 +668,10 @@ def test_mixing_of_dataloader_options(tmpdir):
tutils.reset_seed()
class CurrentTestModel(
LightningValStepFitSingleDataloaderMixin,
LightningTestFitSingleTestDataloadersMixin,
LightningTestModelBase,
LightTrainDataloader,
LightValStepFitSingleDataloaderMixin,
LightTestFitSingleTestDataloadersMixin,
TestModelBase,
):
pass
@@ -688,7 +695,7 @@ def test_mixing_of_dataloader_options(tmpdir):
trainer = Trainer(**trainer_options)
fit_options = dict(val_dataloaders=model._dataloader(train=False),
test_dataloaders=model._dataloader(train=False))
results = trainer.fit(model, **fit_options)
_ = trainer.fit(model, **fit_options)
trainer.test()
assert len(trainer.val_dataloaders) == 1, \
@@ -719,6 +726,7 @@ def test_trainer_max_steps_and_epochs(tmpdir):
# define less train steps than epochs
trainer_options.update(dict(
default_save_path=tmpdir,
max_epochs=5,
max_steps=num_train_samples + 10
))
@@ -732,8 +740,10 @@ def test_trainer_max_steps_and_epochs(tmpdir):
assert trainer.global_step == trainer.max_steps, "Model did not stop at max_steps"
# define less train epochs than steps
trainer_options['max_epochs'] = 2
trainer_options['max_steps'] = trainer_options['max_epochs'] * 2 * num_train_samples
trainer_options.update(dict(
max_epochs=2,
max_steps=trainer_options['max_epochs'] * 2 * num_train_samples
))
# fit model
trainer = Trainer(**trainer_options)
@@ -741,8 +751,8 @@ def test_trainer_max_steps_and_epochs(tmpdir):
assert result == 1, "Training did not complete"
# check training stopped at max_epochs
assert trainer.global_step == num_train_samples * trainer.max_nb_epochs \
and trainer.current_epoch == trainer.max_nb_epochs - 1, "Model did not stop at max_epochs"
assert trainer.global_step == num_train_samples * trainer.max_epochs \
and trainer.current_epoch == trainer.max_epochs - 1, "Model did not stop at max_epochs"
def test_trainer_min_steps_and_epochs(tmpdir):
@@ -750,12 +760,13 @@ def test_trainer_min_steps_and_epochs(tmpdir):
model, trainer_options, num_train_samples = _init_steps_model()
# define callback for stopping the model and default epochs
trainer_options.update({
'early_stop_callback': EarlyStopping(monitor='val_loss', min_delta=1.0),
'val_check_interval': 20,
'min_epochs': 1,
'max_epochs': 10
})
trainer_options.update(dict(
default_save_path=tmpdir,
early_stop_callback=EarlyStopping(monitor='val_loss', min_delta=1.0),
val_check_interval=20,
min_epochs=1,
max_epochs=10
))
# define less min steps than 1 epoch
trainer_options['min_steps'] = math.floor(num_train_samples / 2)
@@ -784,22 +795,29 @@ def test_trainer_min_steps_and_epochs(tmpdir):
def test_testpass_overrides(tmpdir):
hparams = tutils.get_hparams()
from pytorch_lightning.utilities.debugging import MisconfigurationException
class TestModelNoEnd(LightningTestModelBase):
def test_step(self, *args, **kwargs):
class LocalModel(LightTrainDataloader, TestModelBase):
pass
class LocalModelNoEnd(LightTrainDataloader, LightTestDataloader, LightEmptyTestStep, TestModelBase):
pass
class LocalModelNoStep(LightTrainDataloader, TestModelBase):
def test_end(self, outputs):
return {}
def test_dataloader(self):
return self.train_dataloader()
# Misconfig when neither test_step or test_end is implemented
with pytest.raises(MisconfigurationException):
model = LightningTestModelBase(hparams)
model = LocalModel(hparams)
Trainer().test(model)
# Misconfig when neither test_step or test_end is implemented
with pytest.raises(MisconfigurationException):
model = LocalModelNoStep(hparams)
Trainer().test(model)
# No exceptions when one or both of test_step or test_end are implemented
model = TestModelNoEnd(hparams)
model = LocalModelNoEnd(hparams)
Trainer().test(model)
model = LightningTestModel(hparams)