diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py index ed73f2e2..4a7ae48f 100644 --- a/pytorch_lightning/models/trainer.py +++ b/pytorch_lightning/models/trainer.py @@ -14,6 +14,7 @@ import torch.multiprocessing as mp import torch.distributed as dist from torch.optim.optimizer import Optimizer +from pytorch_lightning.root_module.root_module import LightningModule from pytorch_lightning.root_module.memory import get_gpu_memory_map from pytorch_lightning.root_module.model_saving import TrainerIO from pytorch_lightning.pt_overrides.override_data_parallel import ( @@ -326,7 +327,7 @@ class Trainer(TrainerIO): def __is_overriden(self, f_name): model = self.__get_model() - super_object = super(model.__class__, model) + super_object = LightningModule # when code pointers are different, it was overriden is_overriden = getattr(model, f_name).__code__ is not getattr(super_object, f_name).__code__ @@ -395,7 +396,7 @@ class Trainer(TrainerIO): if test and len(self.test_dataloader) > 1: args.append(dataloader_i) - elif len(self.val_dataloader) > 1: + elif not test and len(self.val_dataloader) > 1: args.append(dataloader_i) # handle DP, DDP forward diff --git a/pytorch_lightning/testing/__init__.py b/pytorch_lightning/testing/__init__.py index b3289a1c..8097c0b4 100644 --- a/pytorch_lightning/testing/__init__.py +++ b/pytorch_lightning/testing/__init__.py @@ -1,3 +1,12 @@ from .lm_test_module import LightningTestModel -from .no_val_end_module import NoValEndTestModel -from .no_val_module import NoValModel +from .lm_test_module_base import LightningTestModelBase +from .lm_test_module_mixins import ( + LightningValidationStepMixin, + LightningValidationMixin, + LightningValidationStepMultipleDataloadersMixin, + LightningValidationMultipleDataloadersMixin, + LightningTestStepMixin, + LightningTestMixin, + LightningTestStepMultipleDataloadersMixin, + LightningTestMultipleDataloadersMixin, +) diff --git a/pytorch_lightning/testing/lm_test_module.py b/pytorch_lightning/testing/lm_test_module.py index 16566053..c3b7d9b4 100644 --- a/pytorch_lightning/testing/lm_test_module.py +++ b/pytorch_lightning/testing/lm_test_module.py @@ -14,356 +14,14 @@ from test_tube import HyperOptArgumentParser from pytorch_lightning.root_module.root_module import LightningModule from pytorch_lightning import data_loader +from .lm_test_module_base import LightningTestModelBase +from .lm_test_module_mixins import LightningValidationMixin, LightningTestMixin -class LightningTestModel(LightningModule): + +class LightningTestModel(LightningValidationMixin, LightningTestMixin, LightningTestModelBase): """ - Sample model to show how to define a template + Most common test case. Validation and test dataloaders """ - def __init__(self, hparams, force_remove_distributed_sampler=False, use_two_test_sets=False): - """ - Pass in parsed HyperOptArgumentParser to the model - :param hparams: - """ - # init superclass - super(LightningTestModel, self).__init__() - self.hparams = hparams - self.use_two_test_sets = use_two_test_sets # for some tests regarding testing - - self.batch_size = hparams.batch_size - - # if you specify an example input, the summary will show input/output for each layer - self.example_input_array = torch.rand(5, 28 * 28) - - # remove to test warning for dist sampler - self.force_remove_distributed_sampler = force_remove_distributed_sampler - - # build model - self.__build_model() - - # --------------------- - # MODEL SETUP - # --------------------- - def __build_model(self): - """ - Layout model - :return: - """ - self.c_d1 = nn.Linear(in_features=self.hparams.in_features, - out_features=self.hparams.hidden_dim) - self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim) - self.c_d1_drop = nn.Dropout(self.hparams.drop_prob) - - self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, - out_features=self.hparams.out_features) - - # --------------------- - # TRAINING - # --------------------- - def forward(self, x): - """ - No special modification required for lightning, define as you normally would - :param x: - :return: - """ - - x = self.c_d1(x) - x = torch.tanh(x) - x = self.c_d1_bn(x) - x = self.c_d1_drop(x) - - x = self.c_d2(x) - logits = F.log_softmax(x, dim=1) - - return logits - - def loss(self, labels, logits): - nll = F.nll_loss(logits, labels) - return nll - - def training_step(self, data_batch, batch_i): - """ - Lightning calls this inside the training loop - :param data_batch: - :return: - """ - # forward pass - x, y = data_batch - x = x.view(x.size(0), -1) - - y_hat = self.forward(x) - - # calculate loss - loss_val = self.loss(y, y_hat) - - # in DP mode (default) make sure if result is scalar, there's another dim in the beginning - if self.trainer.use_dp: - loss_val = loss_val.unsqueeze(0) - - # alternate possible outputs to test - if self.trainer.batch_nb % 1 == 0: - output = OrderedDict({ - 'loss': loss_val, - 'prog': {'some_val': loss_val * loss_val} - }) - return output - if self.trainer.batch_nb % 2 == 0: - return loss_val - - def validation_step(self, data_batch, batch_i, dataloader_i): - """ - Lightning calls this inside the validation loop - :param data_batch: - :return: - """ - x, y = data_batch - x = x.view(x.size(0), -1) - y_hat = self.forward(x) - - loss_val = self.loss(y, y_hat) - - # acc - labels_hat = torch.argmax(y_hat, dim=1) - val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) - val_acc = torch.tensor(val_acc) - - if self.on_gpu: - val_acc = val_acc.cuda(loss_val.device.index) - - # in DP mode (default) make sure if result is scalar, there's another dim in the beginning - if self.trainer.use_dp: - loss_val = loss_val.unsqueeze(0) - val_acc = val_acc.unsqueeze(0) - - # alternate possible outputs to test - if batch_i % 1 == 0: - output = OrderedDict({ - 'val_loss': loss_val, - 'val_acc': val_acc, - }) - return output - if batch_i % 2 == 0: - return val_acc - - if batch_i % 3 == 0: - output = OrderedDict({ - 'val_loss': loss_val, - 'val_acc': val_acc, - 'test_dic': {'val_loss_a': loss_val} - }) - return output - if batch_i % 5 == 0: - output = OrderedDict({ - f'val_loss_{dataloader_i}': loss_val, - f'val_acc_{dataloader_i}': val_acc, - }) - return output - - def validation_end(self, outputs): - """ - Called at the end of validation to aggregate outputs - :param outputs: list of individual outputs of each validation step - :return: - """ - # if returned a scalar from validation_step, outputs is a list of tensor scalars - # we return just the average in this case (if we want) - # return torch.stack(outputs).mean() - val_loss_mean = 0 - val_acc_mean = 0 - for output in outputs: - val_loss = output['val_loss'] - - # reduce manually when using dp - if self.trainer.use_dp: - val_loss = torch.mean(val_loss) - val_loss_mean += val_loss - - # reduce manually when using dp - val_acc = output['val_acc'] - if self.trainer.use_dp: - val_acc = torch.mean(val_acc) - - val_acc_mean += val_acc - - val_loss_mean /= len(outputs) - val_acc_mean /= len(outputs) - - tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} - return tqdm_dic - - def test_step(self, data_batch, batch_i, dataloader_i): - """ - Lightning calls this inside the validation loop - :param data_batch: - :return: - """ - x, y = data_batch - x = x.view(x.size(0), -1) - y_hat = self.forward(x) - - loss_test = self.loss(y, y_hat) - - # acc - labels_hat = torch.argmax(y_hat, dim=1) - test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) - test_acc = torch.tensor(test_acc) - - if self.on_gpu: - test_acc = test_acc.cuda(loss_test.device.index) - - # in DP mode (default) make sure if result is scalar, there's another dim in the beginning - if self.trainer.use_dp: - loss_test = loss_test.unsqueeze(0) - test_acc = test_acc.unsqueeze(0) - - # alternate possible outputs to test - if batch_i % 1 == 0: - output = OrderedDict({ - 'test_loss': loss_test, - 'test_acc': test_acc, - }) - return output - if batch_i % 2 == 0: - return test_acc - - if batch_i % 3 == 0: - output = OrderedDict({ - 'test_loss': loss_test, - 'test_acc': test_acc, - 'test_dic': {'test_loss_a': loss_test} - }) - return output - if batch_i % 5 == 0: - output = OrderedDict({ - f'test_loss_{dataloader_i}': loss_test, - f'test_acc_{dataloader_i}': test_acc, - }) - return output - - def test_end(self, outputs): - """ - Called at the end of validation to aggregate outputs - :param outputs: list of individual outputs of each validation step - :return: - """ - # if returned a scalar from test_step, outputs is a list of tensor scalars - # we return just the average in this case (if we want) - # return torch.stack(outputs).mean() - test_loss_mean = 0 - test_acc_mean = 0 - for output in outputs: - test_loss = output['test_loss'] - - # reduce manually when using dp - if self.trainer.use_dp: - test_loss = torch.mean(test_loss) - test_loss_mean += test_loss - - # reduce manually when using dp - test_acc = output['test_acc'] - if self.trainer.use_dp: - test_acc = torch.mean(test_acc) - - test_acc_mean += test_acc - - test_loss_mean /= len(outputs) - test_acc_mean /= len(outputs) - - tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()} - return tqdm_dic - def on_tng_metrics(self, logs): logs['some_tensor_to_test'] = torch.rand(1) - - # --------------------- - # TRAINING SETUP - # --------------------- - def configure_optimizers(self): - """ - return whatever optimizers we want here - :return: list of optimizers - """ - # try no scheduler for this model (testing purposes) - optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) - - # test returning only 1 list instead of 2 - return optimizer - - def __dataloader(self, train): - # init data generators - transform = transforms.Compose([transforms.ToTensor(), - transforms.Normalize((0.5,), (1.0,))]) - dataset = MNIST(root=self.hparams.data_root, train=train, - transform=transform, download=True) - - # when using multi-node we need to add the datasampler - train_sampler = None - batch_size = self.hparams.batch_size - - try: - if self.use_ddp and not self.force_remove_distributed_sampler: - train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank) - batch_size = batch_size // self.trainer.world_size # scale batch size - except Exception: - pass - - should_shuffle = train_sampler is None - loader = DataLoader( - dataset=dataset, - batch_size=batch_size, - shuffle=should_shuffle, - sampler=train_sampler - ) - - return loader - - @data_loader - def tng_dataloader(self): - return self.__dataloader(train=True) - - @data_loader - def val_dataloader(self): - return [self.__dataloader(train=False), self.__dataloader(train=False)] - - @data_loader - def test_dataloader(self): - if self.use_two_test_sets: - return [self.__dataloader(train=False), self.__dataloader(train=False)] - return self.__dataloader(train=False) - - @staticmethod - def add_model_specific_args(parent_parser, root_dir): # pragma: no cover - """ - Parameters you define here will be available to your model through self.hparams - :param parent_parser: - :param root_dir: - :return: - """ - parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) - - # param overwrites - # parser.set_defaults(gradient_clip=5.0) - - # network params - parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) - parser.add_argument('--in_features', default=28 * 28, type=int) - parser.add_argument('--out_features', default=10, type=int) - # use 500 for CPU, 50000 for GPU to see speed difference - parser.add_argument('--hidden_dim', default=50000, type=int) - - # data - parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) - - # training params (opt) - parser.opt_list('--learning_rate', default=0.001 * 8, type=float, - options=[0.0001, 0.0005, 0.001, 0.005], - tunable=False) - parser.opt_list('--optimizer_name', default='adam', type=str, - options=['adam'], tunable=False) - - # if using 2 nodes with 4 gpus each the batch size here - # (256) will be 256 / (2*8) = 16 per gpu - parser.opt_list('--batch_size', default=256 * 8, type=int, - options=[32, 64, 128, 256], tunable=False, - help='batch size will be divided over all gpus being used across all nodes') - return parser diff --git a/pytorch_lightning/testing/no_val_module.py b/pytorch_lightning/testing/lm_test_module_base.py similarity index 95% rename from pytorch_lightning/testing/no_val_module.py rename to pytorch_lightning/testing/lm_test_module_base.py index 17cd9c77..47350d0c 100644 --- a/pytorch_lightning/testing/no_val_module.py +++ b/pytorch_lightning/testing/lm_test_module_base.py @@ -15,9 +15,10 @@ from pytorch_lightning.root_module.root_module import LightningModule from pytorch_lightning import data_loader -class NoValModel(LightningModule): +class LightningTestModelBase(LightningModule): """ - Sample model to show how to define a template + Base LightningModule for testing. Implements only the required + interface """ def __init__(self, hparams, force_remove_distributed_sampler=False): @@ -26,7 +27,7 @@ class NoValModel(LightningModule): :param hparams: """ # init superclass - super(NoValModel, self).__init__() + super(LightningTestModelBase, self).__init__() self.hparams = hparams self.batch_size = hparams.batch_size @@ -109,9 +110,6 @@ class NoValModel(LightningModule): if self.trainer.batch_nb % 2 == 0: return loss_val - def on_tng_metrics(self, logs): - logs['some_tensor_to_test'] = torch.rand(1) - # --------------------- # TRAINING SETUP # --------------------- @@ -124,9 +122,9 @@ class NoValModel(LightningModule): optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) # test returning only 1 list instead of 2 - return [optimizer] + return optimizer - def __dataloader(self, train): + def _dataloader(self, train): # init data generators transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) @@ -156,7 +154,7 @@ class NoValModel(LightningModule): @data_loader def tng_dataloader(self): - return self.__dataloader(train=True) + return self._dataloader(train=True) @staticmethod def add_model_specific_args(parent_parser, root_dir): # pragma: no cover diff --git a/pytorch_lightning/testing/lm_test_module_mixins.py b/pytorch_lightning/testing/lm_test_module_mixins.py new file mode 100644 index 00000000..c4842685 --- /dev/null +++ b/pytorch_lightning/testing/lm_test_module_mixins.py @@ -0,0 +1,381 @@ +import os +from collections import OrderedDict + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import optim +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler +from torchvision.datasets import MNIST +from torchvision import transforms +from test_tube import HyperOptArgumentParser + +from pytorch_lightning.root_module.root_module import LightningModule +from pytorch_lightning import data_loader + + +class LightningValidationStepMixin: + """ + Add val_dataloader and validation_step methods for the case + when val_dataloader returns a single dataloader + """ + + @data_loader + def val_dataloader(self): + return self._dataloader(train=False) + + def validation_step(self, data_batch, batch_i): + """ + Lightning calls this inside the validation loop + :param data_batch: + :return: + """ + x, y = data_batch + x = x.view(x.size(0), -1) + y_hat = self.forward(x) + + loss_val = self.loss(y, y_hat) + + # acc + labels_hat = torch.argmax(y_hat, dim=1) + val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) + val_acc = torch.tensor(val_acc) + + if self.on_gpu: + val_acc = val_acc.cuda(loss_val.device.index) + + # in DP mode (default) make sure if result is scalar, there's another dim in the beginning + if self.trainer.use_dp: + loss_val = loss_val.unsqueeze(0) + val_acc = val_acc.unsqueeze(0) + + # alternate possible outputs to test + if batch_i % 1 == 0: + output = OrderedDict({ + 'val_loss': loss_val, + 'val_acc': val_acc, + }) + return output + if batch_i % 2 == 0: + return val_acc + + if batch_i % 3 == 0: + output = OrderedDict({ + 'val_loss': loss_val, + 'val_acc': val_acc, + 'test_dic': {'val_loss_a': loss_val} + }) + return output + + +class LightningValidationMixin(LightningValidationStepMixin): + """ + Add val_dataloader, validation_step, and validation_end methods for the case + when val_dataloader returns a single dataloader + """ + + def validation_end(self, outputs): + """ + Called at the end of validation to aggregate outputs + :param outputs: list of individual outputs of each validation step + :return: + """ + # if returned a scalar from validation_step, outputs is a list of tensor scalars + # we return just the average in this case (if we want) + # return torch.stack(outputs).mean() + val_loss_mean = 0 + val_acc_mean = 0 + for output in outputs: + val_loss = output['val_loss'] + + # reduce manually when using dp + if self.trainer.use_dp: + val_loss = torch.mean(val_loss) + val_loss_mean += val_loss + + # reduce manually when using dp + val_acc = output['val_acc'] + if self.trainer.use_dp: + val_acc = torch.mean(val_acc) + + val_acc_mean += val_acc + + val_loss_mean /= len(outputs) + val_acc_mean /= len(outputs) + + tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} + return tqdm_dic + + +class LightningValidationStepMultipleDataloadersMixin: + """ + Add val_dataloader and validation_step methods for the case + when val_dataloader returns multiple dataloaders + """ + + @data_loader + def val_dataloader(self): + return [self._dataloader(train=False), self._dataloader(train=False)] + + def validation_step(self, data_batch, batch_i, dataloader_i): + """ + Lightning calls this inside the validation loop + :param data_batch: + :return: + """ + x, y = data_batch + x = x.view(x.size(0), -1) + y_hat = self.forward(x) + + loss_val = self.loss(y, y_hat) + + # acc + labels_hat = torch.argmax(y_hat, dim=1) + val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) + val_acc = torch.tensor(val_acc) + + if self.on_gpu: + val_acc = val_acc.cuda(loss_val.device.index) + + # in DP mode (default) make sure if result is scalar, there's another dim in the beginning + if self.trainer.use_dp: + loss_val = loss_val.unsqueeze(0) + val_acc = val_acc.unsqueeze(0) + + # alternate possible outputs to test + if batch_i % 1 == 0: + output = OrderedDict({ + 'val_loss': loss_val, + 'val_acc': val_acc, + }) + return output + if batch_i % 2 == 0: + return val_acc + + if batch_i % 3 == 0: + output = OrderedDict({ + 'val_loss': loss_val, + 'val_acc': val_acc, + 'test_dic': {'val_loss_a': loss_val} + }) + return output + if batch_i % 5 == 0: + output = OrderedDict({ + f'val_loss_{dataloader_i}': loss_val, + f'val_acc_{dataloader_i}': val_acc, + }) + return output + + +class LightningValidationMultipleDataloadersMixin(LightningValidationStepMultipleDataloadersMixin): + """ + Add val_dataloader, validation_step, and validation_end methods for the case + when val_dataloader returns multiple dataloaders + """ + + def validation_end(self, outputs): + """ + Called at the end of validation to aggregate outputs + :param outputs: list of individual outputs of each validation step + :return: + """ + # if returned a scalar from validation_step, outputs is a list of tensor scalars + # we return just the average in this case (if we want) + # return torch.stack(outputs).mean() + val_loss_mean = 0 + val_acc_mean = 0 + for output in outputs: + val_loss = output['val_loss'] + + # reduce manually when using dp + if self.trainer.use_dp: + val_loss = torch.mean(val_loss) + val_loss_mean += val_loss + + # reduce manually when using dp + val_acc = output['val_acc'] + if self.trainer.use_dp: + val_acc = torch.mean(val_acc) + + val_acc_mean += val_acc + + val_loss_mean /= len(outputs) + val_acc_mean /= len(outputs) + + tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} + return tqdm_dic + + +class LightningTestStepMixin: + + @data_loader + def test_dataloader(self): + return self._dataloader(train=False) + + def test_step(self, data_batch, batch_i): + """ + Lightning calls this inside the validation loop + :param data_batch: + :return: + """ + x, y = data_batch + x = x.view(x.size(0), -1) + y_hat = self.forward(x) + + loss_test = self.loss(y, y_hat) + + # acc + labels_hat = torch.argmax(y_hat, dim=1) + test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) + test_acc = torch.tensor(test_acc) + + if self.on_gpu: + test_acc = test_acc.cuda(loss_test.device.index) + + # in DP mode (default) make sure if result is scalar, there's another dim in the beginning + if self.trainer.use_dp: + loss_test = loss_test.unsqueeze(0) + test_acc = test_acc.unsqueeze(0) + + # alternate possible outputs to test + if batch_i % 1 == 0: + output = OrderedDict({ + 'test_loss': loss_test, + 'test_acc': test_acc, + }) + return output + if batch_i % 2 == 0: + return test_acc + + if batch_i % 3 == 0: + output = OrderedDict({ + 'test_loss': loss_test, + 'test_acc': test_acc, + 'test_dic': {'test_loss_a': loss_test} + }) + return output + + +class LightningTestMixin(LightningTestStepMixin): + def test_end(self, outputs): + """ + Called at the end of validation to aggregate outputs + :param outputs: list of individual outputs of each validation step + :return: + """ + # if returned a scalar from test_step, outputs is a list of tensor scalars + # we return just the average in this case (if we want) + # return torch.stack(outputs).mean() + test_loss_mean = 0 + test_acc_mean = 0 + for output in outputs: + test_loss = output['test_loss'] + + # reduce manually when using dp + if self.trainer.use_dp: + test_loss = torch.mean(test_loss) + test_loss_mean += test_loss + + # reduce manually when using dp + test_acc = output['test_acc'] + if self.trainer.use_dp: + test_acc = torch.mean(test_acc) + + test_acc_mean += test_acc + + test_loss_mean /= len(outputs) + test_acc_mean /= len(outputs) + + tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()} + return tqdm_dic + + +class LightningTestStepMultipleDataloadersMixin: + + @data_loader + def test_dataloader(self): + return [self._dataloader(train=False), self._dataloader(train=False)] + + def test_step(self, data_batch, batch_i, dataloader_i): + """ + Lightning calls this inside the validation loop + :param data_batch: + :return: + """ + x, y = data_batch + x = x.view(x.size(0), -1) + y_hat = self.forward(x) + + loss_test = self.loss(y, y_hat) + + # acc + labels_hat = torch.argmax(y_hat, dim=1) + test_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) + test_acc = torch.tensor(test_acc) + + if self.on_gpu: + test_acc = test_acc.cuda(loss_test.device.index) + + # in DP mode (default) make sure if result is scalar, there's another dim in the beginning + if self.trainer.use_dp: + loss_test = loss_test.unsqueeze(0) + test_acc = test_acc.unsqueeze(0) + + # alternate possible outputs to test + if batch_i % 1 == 0: + output = OrderedDict({ + 'test_loss': loss_test, + 'test_acc': test_acc, + }) + return output + if batch_i % 2 == 0: + return test_acc + + if batch_i % 3 == 0: + output = OrderedDict({ + 'test_loss': loss_test, + 'test_acc': test_acc, + 'test_dic': {'test_loss_a': loss_test} + }) + return output + if batch_i % 5 == 0: + output = OrderedDict({ + f'test_loss_{dataloader_i}': loss_test, + f'test_acc_{dataloader_i}': test_acc, + }) + return output + + +class LightningTestMultipleDataloadersMixin(LightningTestStepMultipleDataloadersMixin): + def test_end(self, outputs): + """ + Called at the end of validation to aggregate outputs + :param outputs: list of individual outputs of each validation step + :return: + """ + # if returned a scalar from test_step, outputs is a list of tensor scalars + # we return just the average in this case (if we want) + # return torch.stack(outputs).mean() + test_loss_mean = 0 + test_acc_mean = 0 + for output in outputs: + test_loss = output['test_loss'] + + # reduce manually when using dp + if self.trainer.use_dp: + test_loss = torch.mean(test_loss) + test_loss_mean += test_loss + + # reduce manually when using dp + test_acc = output['test_acc'] + if self.trainer.use_dp: + test_acc = torch.mean(test_acc) + + test_acc_mean += test_acc + + test_loss_mean /= len(outputs) + test_acc_mean /= len(outputs) + + tqdm_dic = {'test_loss': test_loss_mean.item(), 'test_acc': test_acc_mean.item()} + return tqdm_dic diff --git a/pytorch_lightning/testing/no_val_end_module.py b/pytorch_lightning/testing/no_val_end_module.py deleted file mode 100644 index bf6b553c..00000000 --- a/pytorch_lightning/testing/no_val_end_module.py +++ /dev/null @@ -1,247 +0,0 @@ -import os -from collections import OrderedDict - -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch import optim -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler -from torchvision.datasets import MNIST -from torchvision import transforms -from test_tube import HyperOptArgumentParser - -from pytorch_lightning.root_module.root_module import LightningModule -from pytorch_lightning import data_loader - - -class NoValEndTestModel(LightningModule): - """ - Sample model to show how to define a template - """ - - def __init__(self, hparams, force_remove_distributed_sampler=False): - """ - Pass in parsed HyperOptArgumentParser to the model - :param hparams: - """ - # init superclass - super(NoValEndTestModel, self).__init__() - self.hparams = hparams - - self.batch_size = hparams.batch_size - - # if you specify an example input, the summary will show input/output for each layer - self.example_input_array = torch.rand(5, 28 * 28) - - # remove to test warning for dist sampler - self.force_remove_distributed_sampler = force_remove_distributed_sampler - - # build model - self.__build_model() - - # --------------------- - # MODEL SETUP - # --------------------- - def __build_model(self): - """ - Layout model - :return: - """ - self.c_d1 = nn.Linear(in_features=self.hparams.in_features, - out_features=self.hparams.hidden_dim) - self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim) - self.c_d1_drop = nn.Dropout(self.hparams.drop_prob) - - self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, - out_features=self.hparams.out_features) - - # --------------------- - # TRAINING - # --------------------- - def forward(self, x): - """ - No special modification required for lightning, define as you normally would - :param x: - :return: - """ - - x = self.c_d1(x) - x = torch.tanh(x) - x = self.c_d1_bn(x) - x = self.c_d1_drop(x) - - x = self.c_d2(x) - logits = F.log_softmax(x, dim=1) - - return logits - - def loss(self, labels, logits): - nll = F.nll_loss(logits, labels) - return nll - - def training_step(self, data_batch, batch_i): - """ - Lightning calls this inside the training loop - :param data_batch: - :return: - """ - # forward pass - x, y = data_batch - x = x.view(x.size(0), -1) - - y_hat = self.forward(x) - - # calculate loss - loss_val = self.loss(y, y_hat) - - # in DP mode (default) make sure if result is scalar, there's another dim in the beginning - if self.trainer.use_dp: - loss_val = loss_val.unsqueeze(0) - - # alternate possible outputs to test - if self.trainer.batch_nb % 1 == 0: - output = OrderedDict({ - 'loss': loss_val, - 'prog': {'some_val': loss_val * loss_val} - }) - return output - if self.trainer.batch_nb % 2 == 0: - return loss_val - - def validation_step(self, data_batch, batch_nb): - """ - Lightning calls this inside the validation loop - :param data_batch: - :return: - """ - x, y = data_batch - x = x.view(x.size(0), -1) - y_hat = self.forward(x) - - loss_val = self.loss(y, y_hat) - - # acc - labels_hat = torch.argmax(y_hat, dim=1) - val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) - val_acc = torch.tensor(val_acc) - - if self.on_gpu: - val_acc = val_acc.cuda(loss_val.device.index) - - # in DP mode (default) make sure if result is scalar, there's another dim in the beginning - if self.trainer.use_dp: - loss_val = loss_val.unsqueeze(0) - val_acc = val_acc.unsqueeze(0) - - # alternate possible outputs to test - if batch_nb % 1 == 0: - output = OrderedDict({ - 'val_loss': loss_val, - 'val_acc': val_acc, - }) - return output - if batch_nb % 2 == 0: - return val_acc - - if batch_nb % 3 == 0: - output = OrderedDict({ - 'val_loss': loss_val, - 'val_acc': val_acc, - 'test_dic': {'val_loss_a': loss_val} - }) - return output - - def on_tng_metrics(self, logs): - logs['some_tensor_to_test'] = torch.rand(1) - - # --------------------- - # TRAINING SETUP - # --------------------- - def configure_optimizers(self): - """ - return whatever optimizers we want here - :return: list of optimizers - """ - # try no scheduler for this model (testing purposes) - optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate) - - # test returning only 1 list instead of 2 - return [optimizer] - - def __dataloader(self, train): - # init data generators - transform = transforms.Compose([transforms.ToTensor(), - transforms.Normalize((0.5,), (1.0,))]) - dataset = MNIST(root=self.hparams.data_root, train=train, - transform=transform, download=True) - - # when using multi-node we need to add the datasampler - train_sampler = None - batch_size = self.hparams.batch_size - - try: - if self.use_ddp and not self.force_remove_distributed_sampler: - train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank) - batch_size = batch_size // self.trainer.world_size # scale batch size - except Exception: - pass - - should_shuffle = train_sampler is None - loader = DataLoader( - dataset=dataset, - batch_size=batch_size, - shuffle=should_shuffle, - sampler=train_sampler - ) - - return loader - - @data_loader - def tng_dataloader(self): - return self.__dataloader(train=True) - - @data_loader - def val_dataloader(self): - return self.__dataloader(train=False) - - @data_loader - def test_dataloader(self): - return self.__dataloader(train=False) - - @staticmethod - def add_model_specific_args(parent_parser, root_dir): # pragma: no cover - """ - Parameters you define here will be available to your model through self.hparams - :param parent_parser: - :param root_dir: - :return: - """ - parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) - - # param overwrites - # parser.set_defaults(gradient_clip=5.0) - - # network params - parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) - parser.add_argument('--in_features', default=28 * 28, type=int) - parser.add_argument('--out_features', default=10, type=int) - # use 500 for CPU, 50000 for GPU to see speed difference - parser.add_argument('--hidden_dim', default=50000, type=int) - - # data - parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) - - # training params (opt) - parser.opt_list('--learning_rate', default=0.001 * 8, type=float, - options=[0.0001, 0.0005, 0.001, 0.005], - tunable=False) - parser.opt_list('--optimizer_name', default='adam', type=str, - options=['adam'], tunable=False) - - # if using 2 nodes with 4 gpus each the batch size here - # (256) will be 256 / (2*8) = 16 per gpu - parser.opt_list('--batch_size', default=256 * 8, type=int, - options=[32, 64, 128, 256], tunable=False, - help='batch size will be divided over all gpus being used across all nodes') - return parser diff --git a/tests/debug.py b/tests/debug.py index 856a1b06..5efa25da 100644 --- a/tests/debug.py +++ b/tests/debug.py @@ -1,6 +1,6 @@ from pytorch_lightning import Trainer from examples import LightningTemplateModel -from pytorch_lightning.testing import LightningTestModel, NoValEndTestModel, NoValModel +from pytorch_lightning.testing import LightningTestModel from argparse import Namespace from test_tube import Experiment from pytorch_lightning.callbacks import ModelCheckpoint diff --git a/tests/test_models.py b/tests/test_models.py index df2a41c3..9b2a06a8 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -10,7 +10,15 @@ from test_tube import Experiment, SlurmCluster # sys.path += [os.path.abspath('..'), os.path.abspath('../..')] from pytorch_lightning import Trainer -from pytorch_lightning.testing import LightningTestModel, NoValEndTestModel, NoValModel +from pytorch_lightning.testing import ( + LightningTestModel, + LightningTestModelBase, + LightningValidationMixin, + LightningValidationStepMixin, + LightningValidationMultipleDataloadersMixin, + LightningTestMixin, + LightningTestMultipleDataloadersMixin, +) from pytorch_lightning.callbacks import ( ModelCheckpoint, EarlyStopping, @@ -116,6 +124,47 @@ def test_running_test_after_fitting(): clear_save_dir() +def test_running_test_without_val(): + """Verify test() works on a model with no val_loader""" + class CurrentTestModel(LightningTestMixin, LightningTestModelBase): + pass + hparams = get_hparams() + model = CurrentTestModel(hparams) + + save_dir = init_save_dir() + + # exp file to get meta + exp = get_exp(False) + exp.argparse(hparams) + exp.save() + + # exp file to get weights + checkpoint = ModelCheckpoint(save_dir) + + trainer_options = dict( + show_progress_bar=False, + max_nb_epochs=1, + train_percent_check=0.4, + val_percent_check=0.2, + test_percent_check=0.2, + checkpoint_callback=checkpoint, + experiment=exp + ) + + # fit model + trainer = Trainer(**trainer_options) + result = trainer.fit(model) + + assert result == 1, 'training failed to complete' + + trainer.test() + + # test we have good test accuracy + assert_ok_test_acc(trainer) + + clear_save_dir() + + def test_running_test_pretrained_model(): """Verify test() on pretrained model""" hparams = get_hparams() @@ -146,7 +195,9 @@ def test_running_test_pretrained_model(): # correct result and ok accuracy assert result == 1, 'training failed to complete' - pretrained_model = load_model(exp, save_dir, on_gpu=False, module_class=LightningTestModel) + pretrained_model = load_model( + exp, save_dir, on_gpu=False, module_class=LightningTestModel + ) new_trainer = Trainer(**trainer_options) new_trainer.test(pretrained_model) @@ -400,7 +451,10 @@ def test_no_val_module(): :return: """ hparams = get_hparams() - model = NoValModel(hparams) + + class CurrentTestModel(LightningTestModelBase): + pass + model = CurrentTestModel(hparams) save_dir = init_save_dir() @@ -443,8 +497,11 @@ def test_no_val_end_module(): Tests use case where trainer saves the model, and user loads it from tags independently :return: """ + + class CurrentTestModel(LightningValidationStepMixin, LightningTestModelBase): + pass hparams = get_hparams() - model = NoValEndTestModel(hparams) + model = CurrentTestModel(hparams) save_dir = init_save_dir() @@ -1052,8 +1109,13 @@ def test_multiple_val_dataloader(): Verify multiple val_dataloader :return: """ + class CurrentTestModel( + LightningValidationMultipleDataloadersMixin, + LightningTestModelBase + ): + pass hparams = get_hparams() - model = LightningTestModel(hparams) + model = CurrentTestModel(hparams) # exp file to get meta trainer_options = dict( @@ -1081,8 +1143,13 @@ def test_multiple_test_dataloader(): Verify multiple test_dataloader :return: """ + class CurrentTestModel( + LightningTestMultipleDataloadersMixin, + LightningTestModelBase + ): + pass hparams = get_hparams() - model = LightningTestModel(hparams, use_two_test_sets=True) + model = CurrentTestModel(hparams) # exp file to get meta trainer_options = dict(