From 7fb868bfd80f261f556c53d5e957097f5ec71bfe Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 24 Oct 2019 06:23:00 -0400 Subject: [PATCH 01/17] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 257c02a3..4a22b0c7 100644 --- a/README.md +++ b/README.md @@ -394,7 +394,7 @@ Nope. Please use anaconda or miniconda. # install latest Lightning version without upgrading deps pip install -U --no-deps pytorch-lightning ``` -- **PyTorch 1.2.0** +- **PyTorch 1.2.0, 1.3.0,** Install via pip as normal ## Custom installation From 48eabf07519b52410545481897f62347c771b445 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 24 Oct 2019 06:25:56 -0400 Subject: [PATCH 02/17] Update CONTRIBUTING.md --- .github/CONTRIBUTING.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 3fd3f248..e4347f09 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,10 +1,15 @@ # Contributing Welcome to the PyTorch Lightning community! We're building the most advanced research platform on the planet to implement the latest, best practices that the amazing PyTorch team rolls out! -## One less thing to remember +## Core value 1: One less thing to remember Simplify the API as much as possible from the user perspective. Any additions or improvements should minimize things the user needs to remember. -For example: One benefit of the validation_step is that the user doesn't have to remember to set the model to .eval(). This avoids all sorts of subtle errors the user could make. +For example: One benefit of the validation_step is that the user doesn't have to remember to set the model to .eval(). This avoids all sorts of subtle errors the user could make. + +## Core value 2: Backward-compatible API +We all hate updating our deep learning packages because we don't want to refactor a bunch of stuff. In Lightning, we make sure every change we make which could break an API is backwards compatible with good deprecation warnings. + +You shouldn't be afraid to upgrade Lightning :) ## Lightning Design Principles We encourage all sorts of contributions you're interested in adding! When coding for lightning, please follow these principles. From 28c3bcb0c018d9fadf82085939c8392bf2e1fa86 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 24 Oct 2019 06:26:39 -0400 Subject: [PATCH 03/17] Update CONTRIBUTING.md --- .github/CONTRIBUTING.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index e4347f09..6640f332 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,19 +1,13 @@ # Contributing Welcome to the PyTorch Lightning community! We're building the most advanced research platform on the planet to implement the latest, best practices that the amazing PyTorch team rolls out! -## Core value 1: One less thing to remember +## Main Core Value: One less thing to remember Simplify the API as much as possible from the user perspective. Any additions or improvements should minimize things the user needs to remember. For example: One benefit of the validation_step is that the user doesn't have to remember to set the model to .eval(). This avoids all sorts of subtle errors the user could make. -## Core value 2: Backward-compatible API -We all hate updating our deep learning packages because we don't want to refactor a bunch of stuff. In Lightning, we make sure every change we make which could break an API is backwards compatible with good deprecation warnings. - -You shouldn't be afraid to upgrade Lightning :) - ## Lightning Design Principles We encourage all sorts of contributions you're interested in adding! When coding for lightning, please follow these principles. - #### No PyTorch interference We don't want to add any abstractions on top of pure PyTorch. This gives researchers all the control they need without having to learn yet another framework. @@ -26,7 +20,12 @@ There are 1,000 ways to do something. However, something eventually becomes stan When something becomes a best practice, we add it to the framework. This likely looks like code in utils or in the model file that everyone keeps adding over and over again across projects. When this happens, bring that code inside the trainer and add a flag for it. #### Simple External API -What makes sense to you may not make sense to others. Create an issue with an API change suggestion and validate that it makes sense for others. Treat code changes how you treat a startup: validate that it's a needed feature, then add if it makes sense for many people. +What makes sense to you may not make sense to others. Create an issue with an API change suggestion and validate that it makes sense for others. Treat code changes how you treat a startup: validate that it's a needed feature, then add if it makes sense for many people. + +#### Backward-compatible API +We all hate updating our deep learning packages because we don't want to refactor a bunch of stuff. In Lightning, we make sure every change we make which could break an API is backwards compatible with good deprecation warnings. + +You shouldn't be afraid to upgrade Lightning :) #### Gain User Trust As a researcher you can't have any part of your code going wrong. So, make thorough tests that ensure an implementation of a new trick or subbtle change is correct. From a4b43ce09539edba78cee42ddd3892b0060476a9 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 24 Oct 2019 06:43:35 -0400 Subject: [PATCH 04/17] Loaders (#422) * refactor dataloading * refactor dataloading * refactor dataloading * refactor dataloading * refactor dataloading * refactor dataloading * refactor dataloading * refactor dataloading --- .../trainer/data_loading_mixin.py | 209 ++++++++++-------- pytorch_lightning/trainer/trainer.py | 4 +- 2 files changed, 119 insertions(+), 94 deletions(-) diff --git a/pytorch_lightning/trainer/data_loading_mixin.py b/pytorch_lightning/trainer/data_loading_mixin.py index 066c21dc..7755bc7d 100644 --- a/pytorch_lightning/trainer/data_loading_mixin.py +++ b/pytorch_lightning/trainer/data_loading_mixin.py @@ -15,8 +15,13 @@ except ImportError: class TrainerDataLoadingMixin(object): - - def layout_bookeeping(self): + def init_train_dataloader(self, model): + """ + Dataloaders are provided by the model + :param model: + :return: + """ + self.get_train_dataloader = model.train_dataloader # determine number of training batches if isinstance(self.get_train_dataloader(), IterableDataset): @@ -25,21 +30,6 @@ class TrainerDataLoadingMixin(object): self.nb_training_batches = len(self.get_train_dataloader()) self.nb_training_batches = int(self.nb_training_batches * self.train_percent_check) - # determine number of validation batches - # val datasets could be none, 1 or 2+ - if self.get_val_dataloaders() is not None: - self.nb_val_batches = sum(len(dataloader) for dataloader in self.get_val_dataloaders()) - self.nb_val_batches = int(self.nb_val_batches * self.val_percent_check) - self.nb_val_batches = max(1, self.nb_val_batches) - - # determine number of test batches - if self.get_test_dataloaders() is not None: - self.nb_test_batches = sum( - len(dataloader) for dataloader in self.get_test_dataloaders() - ) - self.nb_test_batches = int(self.nb_test_batches * self.test_percent_check) - self.nb_test_batches = max(1, self.nb_test_batches) - # determine when to check validation # if int passed in, val checks that often # otherwise, it checks in [0, 1.0] % range of a training epoch @@ -49,86 +39,123 @@ class TrainerDataLoadingMixin(object): self.val_check_batch = int(self.nb_training_batches * self.val_check_interval) self.val_check_batch = max(1, self.val_check_batch) + on_ddp = self.use_ddp or self.use_ddp2 + if on_ddp and not isinstance(self.get_train_dataloader().sampler, DistributedSampler): + msg = """ + You're using multiple gpus and multiple nodes without using a DistributedSampler + to assign a subset of your data to each process. To silence this warning, pass a + DistributedSampler to your DataLoader. + + ie: this: + dataset = myDataset() + dataloader = Dataloader(dataset) + + becomes: + dataset = myDataset() + dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) + dataloader = Dataloader(dataset, sampler=dist_sampler) + + If you want each process to load the full dataset, ignore this warning. + """ + if msg not in self.shown_warnings and self.proc_rank == 0: + self.shown_warnings.add(msg) + warnings.warn(msg) + + def init_val_dataloader(self, model): + """ + Dataloaders are provided by the model + :param model: + :return: + """ + self.get_val_dataloaders = model.val_dataloader + + # determine number of validation batches + # val datasets could be none, 1 or 2+ + if self.get_val_dataloaders() is not None: + self.nb_val_batches = sum(len(dataloader) for dataloader in self.get_val_dataloaders()) + self.nb_val_batches = int(self.nb_val_batches * self.val_percent_check) + self.nb_val_batches = max(1, self.nb_val_batches) + + on_ddp = self.use_ddp or self.use_ddp2 + if on_ddp and self.get_val_dataloaders() is not None: + for dataloader in self.get_val_dataloaders(): + if not isinstance(dataloader.sampler, DistributedSampler): + msg = """ + Your val_dataloader(s) don't use DistributedSampler. + + You're using multiple gpus and multiple nodes without using a + DistributedSampler to assign a subset of your data to each process. + To silence this warning, pass a DistributedSampler to your DataLoader. + + ie: this: + dataset = myDataset() + dataloader = Dataloader(dataset) + + becomes: + dataset = myDataset() + dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) + dataloader = Dataloader(dataset, sampler=dist_sampler) + + If you want each process to load the full dataset, ignore this warning. + """ + if msg not in self.shown_warnings and self.proc_rank == 0: + self.shown_warnings.add(msg) + warnings.warn(msg) + break + + def init_test_dataloader(self, model): + """ + Dataloaders are provided by the model + :param model: + :return: + """ + + self.get_test_dataloaders = model.test_dataloader + + # determine number of test batches + if self.get_test_dataloaders() is not None: + len_sum = sum(len(dataloader) for dataloader in self.get_test_dataloaders()) + self.nb_test_batches = len_sum + self.nb_test_batches = int(self.nb_test_batches * self.test_percent_check) + self.nb_test_batches = max(1, self.nb_test_batches) + + on_ddp = self.use_ddp or self.use_ddp2 + if on_ddp and self.get_test_dataloaders() is not None: + for dataloader in self.get_test_dataloaders(): + if not isinstance(dataloader.sampler, DistributedSampler): + msg = """ + Your test_dataloader(s) don't use DistributedSampler. + + You're using multiple gpus and multiple nodes without using a + DistributedSampler to assign a subset of your data to each process. + To silence this warning, pass a DistributedSampler to your DataLoader. + + ie: this: + dataset = myDataset() + dataloader = Dataloader(dataset) + + becomes: + dataset = myDataset() + dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) + dataloader = Dataloader(dataset, sampler=dist_sampler) + + If you want each process to load the full dataset, ignore this warning. + """ + if msg not in self.shown_warnings and self.proc_rank == 0: + self.shown_warnings.add(msg) + warnings.warn(msg) + break + def get_dataloaders(self, model): """ Dataloaders are provided by the model :param model: :return: """ - self.get_train_dataloader = model.train_dataloader - self.get_test_dataloaders = model.test_dataloader - self.get_val_dataloaders = model.val_dataloader - # call warnings from proc zero only which triggers dataloaders - # if those have to download data it will only happen on proc 0 - if self.proc_rank == 0: - on_ddp = self.use_ddp or self.use_ddp2 - if on_ddp and not isinstance(self.get_train_dataloader().sampler, DistributedSampler): - msg = """ - You're using multiple gpus and multiple nodes without using a DistributedSampler - to assign a subset of your data to each process. To silence this warning, pass a - DistributedSampler to your DataLoader. - - ie: this: - dataset = myDataset() - dataloader = Dataloader(dataset) - - becomes: - dataset = myDataset() - dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) - dataloader = Dataloader(dataset, sampler=dist_sampler) - - If you want each process to load the full dataset, ignore this warning. - """ - warnings.warn(msg) - - if on_ddp and self.get_val_dataloaders() is not None: - for dataloader in self.get_val_dataloaders(): - if not isinstance(dataloader.sampler, DistributedSampler): - msg = """ - Your val_dataloader(s) don't use DistributedSampler. - - You're using multiple gpus and multiple nodes without using a - DistributedSampler to assign a subset of your data to each process. - To silence this warning, pass a DistributedSampler to your DataLoader. - - ie: this: - dataset = myDataset() - dataloader = Dataloader(dataset) - - becomes: - dataset = myDataset() - dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) - dataloader = Dataloader(dataset, sampler=dist_sampler) - - If you want each process to load the full dataset, ignore this warning. - """ - warnings.warn(msg) - break - - if on_ddp and self.get_test_dataloaders() is not None: - for dataloader in self.get_test_dataloaders(): - if not isinstance(dataloader.sampler, DistributedSampler): - msg = """ - Your test_dataloader(s) don't use DistributedSampler. - - You're using multiple gpus and multiple nodes without using a - DistributedSampler to assign a subset of your data to each process. - To silence this warning, pass a DistributedSampler to your DataLoader. - - ie: this: - dataset = myDataset() - dataloader = Dataloader(dataset) - - becomes: - dataset = myDataset() - dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) - dataloader = Dataloader(dataset, sampler=dist_sampler) - - If you want each process to load the full dataset, ignore this warning. - """ - warnings.warn(msg) - break + self.init_train_dataloader(model) + self.init_test_dataloader(model) + self.init_val_dataloader(model) if self.use_ddp or self.use_ddp2: # wait for all processes to catch up @@ -147,7 +174,7 @@ class TrainerDataLoadingMixin(object): Trainer(val_check_interval) must be an int. An int k specifies checking validation every k training batches ''' - raise MisconfigurationException('when using ') + raise MisconfigurationException(m) def determine_data_use_amount(self, train_percent_check, val_percent_check, test_percent_check, overfit_pct): diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 19f31034..cbe9af37 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -135,6 +135,7 @@ class Trainer(TrainerIOMixin, self.min_nb_epochs = min_nb_epochs self.nb_sanity_val_steps = nb_sanity_val_steps self.print_nan_grads = print_nan_grads + self.shown_warnings = set() self.fast_dev_run = fast_dev_run if self.fast_dev_run: @@ -410,9 +411,6 @@ class Trainer(TrainerIOMixin, # transfer data loaders from model self.get_dataloaders(ref_model) - # init training constants - self.layout_bookeeping() - # print model summary if self.proc_rank == 0 and self.weights_summary is not None: if self.weights_summary in ['full', 'top']: From d5ca464cc645a63c3dd92548785cb8e5a7222caf Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 24 Oct 2019 07:56:56 -0400 Subject: [PATCH 05/17] Back hook (#424) * Fixes #356 * Fixes #356 * Fixes #356 * Fixes #356 * Fixes #356 * Fixes #356 --- docs/Trainer/hooks.md | 22 +++++++++++++++++++ pytorch_lightning/root_module/hooks.py | 22 +++++++++++++++++++ pytorch_lightning/trainer/train_loop_mixin.py | 8 +++---- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/docs/Trainer/hooks.md b/docs/Trainer/hooks.md index 6fd8df3d..98ef6b8b 100644 --- a/docs/Trainer/hooks.md +++ b/docs/Trainer/hooks.md @@ -115,6 +115,28 @@ def on_before_zero_grad(self, optimizer): # do something with the optimizer or inspect it. ``` +--- +#### backward +Called to perform backward step. +Feel free to override as needed. + +The loss passed in has already been scaled for accumulated gradients if requested. +```python +def backward(self, use_amp, loss, optimizer): + """ + Override backward with your own implementation if you need to + :param use_amp: Whether amp was requested or not + :param loss: Loss is already scaled by accumulated grads + :param optimizer: Current optimizer being used + :return: + """ + if use_amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() +``` + --- #### on_after_backward Called in the training loop after model.backward() diff --git a/pytorch_lightning/root_module/hooks.py b/pytorch_lightning/root_module/hooks.py index 37d41eb5..580d06e3 100644 --- a/pytorch_lightning/root_module/hooks.py +++ b/pytorch_lightning/root_module/hooks.py @@ -1,6 +1,14 @@ import torch +try: + from apex import amp + + APEX_AVAILABLE = True +except ImportError: + APEX_AVAILABLE = False + + class ModelHooks(torch.nn.Module): def on_sanity_check_start(self): @@ -48,3 +56,17 @@ class ModelHooks(torch.nn.Module): :return: """ pass + + def backward(self, use_amp, loss, optimizer): + """ + Override backward with your own implementation if you need to + :param use_amp: Whether amp was requested or not + :param loss: Loss is already scaled by accumulated grads + :param optimizer: Current optimizer being used + :return: + """ + if use_amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() diff --git a/pytorch_lightning/trainer/train_loop_mixin.py b/pytorch_lightning/trainer/train_loop_mixin.py index 1dad28ae..8cebe7a1 100644 --- a/pytorch_lightning/trainer/train_loop_mixin.py +++ b/pytorch_lightning/trainer/train_loop_mixin.py @@ -174,11 +174,9 @@ class TrainerTrainLoopMixin(object): closure_loss = closure_loss / self.accumulate_grad_batches # backward pass - if self.use_amp: - with amp.scale_loss(closure_loss, optimizer) as scaled_loss: - scaled_loss.backward() - else: - closure_loss.backward() + # done in hook so user can overwrite if needed + model_ref = self.get_model() + model_ref.backward(self.use_amp, closure_loss, optimizer) # insert after step hook if self.is_function_implemented('on_after_backward'): From b86d223889992cccbb4ca68537a464df88dbfef7 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Fri, 25 Oct 2019 08:57:05 -0400 Subject: [PATCH 06/17] makes checkpoint process safe (#431) --- pytorch_lightning/trainer/trainer_io.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pytorch_lightning/trainer/trainer_io.py b/pytorch_lightning/trainer/trainer_io.py index 5b483760..b8de2bbf 100644 --- a/pytorch_lightning/trainer/trainer_io.py +++ b/pytorch_lightning/trainer/trainer_io.py @@ -137,7 +137,13 @@ class TrainerIOMixin(object): checkpoint = self.dump_checkpoint() # do the actual save - torch.save(checkpoint, filepath) + try: + torch.save(checkpoint, filepath) + except AttributeError: + if 'hparams' in checkpoint: + del checkpoint['hparams'] + + torch.save(checkpoint, filepath) def restore(self, checkpoint_path, on_gpu): @@ -283,7 +289,14 @@ class TrainerIOMixin(object): model.on_hpc_save(checkpoint) # do the actual save - torch.save(checkpoint, filepath) + # TODO: fix for anything with multiprocess DP, DDP, DDP2 + try: + torch.save(checkpoint, filepath) + except AttributeError: + if 'hparams' in checkpoint: + del checkpoint['hparams'] + + torch.save(checkpoint, filepath) return filepath From 37647d835ac626bb6d7e224c3b917debcd497d5e Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 28 Oct 2019 23:41:13 +0100 Subject: [PATCH 07/17] add package info (#395) * add package info #358 * Update __init__.py * wrap package info * update CI * fix package info * fix for #388 * prune duplicated configs * fix install * use req from file * move info to sep. module drop comments from req * add setup req. * add setup req. * update get info * refactor init * update pip * fix failing on buildins * fix failing open * fix test imports * fix tests * fix pep8 --- .travis.yml | 7 ++- pytorch_lightning/__init__.py | 43 +++++++++++++--- pytorch_lightning/logging/comet_logger.py | 7 +-- pytorch_lightning/logging/mlflow_logger.py | 5 +- pytorch_lightning/logging/test_tube_logger.py | 5 +- .../testing/lm_test_module_base.py | 6 ++- requirements.txt | 3 +- setup.cfg | 12 ++++- setup.py | 49 +++++++++++++------ tox.ini | 40 ++++++++------- 10 files changed, 124 insertions(+), 53 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8a97272e..1ad6eba5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,14 +51,13 @@ matrix: cache: pip install: - - pip install -r requirements.txt - - pip install -r ./tests/requirements.txt - - pip --version ; pip list + - pip install future # needed for `builtins` + - sudo pip install tox script: # integration - tox --sitepackages - - python setup.py install --dry-run + - pip install --editable . after_success: - coverage report diff --git a/pytorch_lightning/__init__.py b/pytorch_lightning/__init__.py index d65aff02..64829568 100644 --- a/pytorch_lightning/__init__.py +++ b/pytorch_lightning/__init__.py @@ -1,9 +1,36 @@ -from .root_module.decorators import data_loader -from .root_module.root_module import LightningModule -from .trainer.trainer import Trainer +"""Package info""" -__all__ = [ - 'Trainer', - 'LightningModule', - 'data_loader', -] +__version__ = '0.5.2.1' +__author__ = ' William Falcon et al.' +__author_email__ = 'waf2107@columbia.edu' +__license__ = 'Apache-2.0' +__homepage__ = 'https://github.com/williamFalcon/pytorch-lightning', +__docs__ = """# PyTorch Lightning + +The lightweight PyTorch wrapper for ML researchers. Scale your models. Write less boilerplate. +""" + + +try: + # This variable is injected in the __builtins__ by the build + # process. It used to enable importing subpackages of skimage when + # the binaries are not built + __LIGHTNING_SETUP__ +except NameError: + __LIGHTNING_SETUP__ = False + +if __LIGHTNING_SETUP__: + import sys + sys.stderr.write('Partial import of skimage during the build process.\n') + # We are not importing the rest of the scikit during the build + # process, as it may not be compiled yet +else: + from .trainer.trainer import Trainer + from .root_module.root_module import LightningModule + from .root_module.decorators import data_loader + + __all__ = [ + 'Trainer', + 'LightningModule', + 'data_loader', + ] diff --git a/pytorch_lightning/logging/comet_logger.py b/pytorch_lightning/logging/comet_logger.py index dc05a9a9..5e1d281c 100644 --- a/pytorch_lightning/logging/comet_logger.py +++ b/pytorch_lightning/logging/comet_logger.py @@ -1,6 +1,7 @@ -from os import environ - -from comet_ml import Experiment as CometExperiment +try: + from comet_ml import Experiment as CometExperiment +except ImportError: + raise ImportError('Missing comet_ml package.') from .base import LightningLoggerBase, rank_zero_only diff --git a/pytorch_lightning/logging/mlflow_logger.py b/pytorch_lightning/logging/mlflow_logger.py index c38b7a16..6c0d460a 100644 --- a/pytorch_lightning/logging/mlflow_logger.py +++ b/pytorch_lightning/logging/mlflow_logger.py @@ -1,7 +1,10 @@ from logging import getLogger from time import time -import mlflow +try: + import mlflow +except ImportError: + raise ImportError('Missing mlflow package.') from .base import LightningLoggerBase, rank_zero_only diff --git a/pytorch_lightning/logging/test_tube_logger.py b/pytorch_lightning/logging/test_tube_logger.py index e8e9607f..0d74307b 100644 --- a/pytorch_lightning/logging/test_tube_logger.py +++ b/pytorch_lightning/logging/test_tube_logger.py @@ -1,4 +1,7 @@ -from test_tube import Experiment +try: + from test_tube import Experiment +except ImportError: + raise ImportError('Missing test-tube package.') from .base import LightningLoggerBase, rank_zero_only diff --git a/pytorch_lightning/testing/lm_test_module_base.py b/pytorch_lightning/testing/lm_test_module_base.py index 9a6e0ee6..89bc3f29 100644 --- a/pytorch_lightning/testing/lm_test_module_base.py +++ b/pytorch_lightning/testing/lm_test_module_base.py @@ -4,12 +4,16 @@ from collections import OrderedDict import torch import torch.nn as nn import torch.nn.functional as F -from test_tube import HyperOptArgumentParser from torch import optim from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler from torchvision import transforms from torchvision.datasets import MNIST +try: + from test_tube import HyperOptArgumentParser +except ImportError: + # TODO: this should be discussed and moved out of this package + raise ImportError('Missing test-tube package.') from pytorch_lightning import data_loader from pytorch_lightning.root_module.root_module import LightningModule diff --git a/requirements.txt b/requirements.txt index 4bb6acd6..62acf5ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ twine==1.13.0 numpy==1.16.4 torch>=1.2.0 torchvision>=0.3.0 -pandas +pandas>=0.20.3 +# future>=0.17.1 # required for buildins in setup.py \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 068788dc..f56376af 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,12 +11,14 @@ markers = slow remote_data filterwarnings + gpus_param_tests [pycodestyle] ignore = E731,W504 max-line-length = 120 [coverage:report] +# TODO: this looks suspicion, it should be reviewed exclude_lines = pragma: no cover def __repr__ @@ -39,7 +41,6 @@ exclude_lines = break pass os.makedirs - omit = pytorch_lightning/callbacks/pt_callbacks.py tests/test_models.py @@ -48,5 +49,12 @@ omit = examples/templates [flake8] -ignore = E731,W504,F401,F841 +# TODO: this should be 88 or 100 according PEP8 max-line-length = 120 +exclude = .tox,*.egg,build,temp,examples/* +select = E,W,F +doctests = True +verbose = 2 +# https://pep8.readthedocs.io/en/latest/intro.html#error-codes +format = pylint +ignore = E731,W504,F401,F841 diff --git a/setup.py b/setup.py index 883e71cd..2c57b0b1 100755 --- a/setup.py +++ b/setup.py @@ -1,12 +1,37 @@ #!/usr/bin/env python +import os +from io import open # Always prefer setuptools over distutils from setuptools import setup, find_packages -# https://packaging.python.org/guides/single-sourcing-package-version/ +try: + import builtins +except ImportError: + import __builtin__ as builtins +# https://packaging.python.org/guides/single-sourcing-package-version/ # http://blog.ionelmc.ro/2014/05/25/python-packaging/ +PATH_ROOT = os.path.dirname(__file__) +builtins.__LIGHTNING_SETUP__ = True + +import pytorch_lightning # noqa: E402 + + +def load_requirements(path_dir=PATH_ROOT, comment_char='#'): + with open(os.path.join(path_dir, 'requirements.txt'), 'r') as file: + lines = [ln.strip() for ln in file.readlines()] + reqs = [] + for ln in lines: + # filer all comments + if comment_char in ln: + ln = ln[:ln.index(comment_char)] + if ln: # if requirement is not empty + reqs.append(ln) + return reqs + + # https://packaging.python.org/discussions/install-requires-vs-requirements / # keep the meta-data here for simplicity in reading this file... it's not obvious # what happens and to non-engineers they won't know to look in init ... @@ -14,26 +39,22 @@ from setuptools import setup, find_packages # engineer specific practices setup( name='pytorch-lightning', - version='0.5.2.1', - description='The Keras for ML researchers using PyTorch', - author='William Falcon', - author_email='waf2107@columbia.edu', - url='https://github.com/williamFalcon/pytorch-lightning', + version=pytorch_lightning.__version__, + description=pytorch_lightning.__docs__, + author=pytorch_lightning.__author__, + author_email=pytorch_lightning.__author_email__, + url=pytorch_lightning.__homepage__, download_url='https://github.com/williamFalcon/pytorch-lightning', - license='Apache-2', - packages=find_packages(), + license=pytorch_lightning.__license__, + packages=find_packages(exclude=['examples']), long_description=open('README.md', encoding='utf-8').read(), long_description_content_type='text/markdown', include_package_data=True, zip_safe=False, keywords=['deep learning', 'pytorch', 'AI'], python_requires='>=3.6', - install_requires=[ - 'torch>=1.2.0', - 'tqdm>=4.35.0', - 'test-tube>=0.6.9', - 'pandas>=0.20.3', - ], + setup_requires=[], + install_requires=load_requirements(PATH_ROOT), classifiers=[ 'Environment :: Console', 'Natural Language :: English', diff --git a/tox.ini b/tox.ini index b3a62978..6d0c3171 100644 --- a/tox.ini +++ b/tox.ini @@ -12,36 +12,40 @@ # and also to help confirm pull requests to this project. [tox] -envlist = py{35,36,37} +envlist = py{35,36,37,38} -[pytest] -log_cli = 0 -log_cli_level = CRITICAL -log_cli_format = %(message)s -log_file = pytest.log -log_file_level = DEBUG -log_file_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) -log_file_date_format = %Y-%m-%d %H:%M:%S +# DROP, it is duplication of setup.cfg +# [pytest] +# log_cli = 0 +# log_cli_level = CRITICAL +# log_cli_format = %(message)s +# log_file = pytest.log +# log_file_level = DEBUG +# log_file_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) +# log_file_date_format=%Y-%m-%d %H:%M:%S [testenv] basepython = py35: python3.5 py36: python3.6 py37: python3.7 + py38: python3.8 deps = -r requirements.txt -r ./tests/requirements.txt commands = + pip list check-manifest --ignore tox.ini - python setup.py check -m -s - flake8 . + python setup.py check --metadata --strict coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules + flake8 . -[flake8] -exclude = .tox,*.egg,build,temp,examples/* -select = E,W,F -doctests = True -verbose = 2 +# DROP, it is duplication of setup.cfg +# [flake8] +# exclude = .tox,*.egg,build,temp,examples/* +# select = E,W,F +# doctests = True +# verbose = 2 # https://pep8.readthedocs.io/en/latest/intro.html#error-codes -format = pylint -max-line-length = 100 +# format = pylint +# max-line-length = 100 From 4df4d4cc03a687ae44ebe8d80669824a867b6d75 Mon Sep 17 00:00:00 2001 From: Nic Eggert Date: Wed, 30 Oct 2019 11:03:52 -0500 Subject: [PATCH 08/17] Catch exceptions when optional dependencies are missing (#442) --- pytorch_lightning/logging/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pytorch_lightning/logging/__init__.py b/pytorch_lightning/logging/__init__.py index 73960a48..dc47188a 100644 --- a/pytorch_lightning/logging/__init__.py +++ b/pytorch_lightning/logging/__init__.py @@ -3,16 +3,16 @@ from .base import LightningLoggerBase, rank_zero_only try: from .test_tube_logger import TestTubeLogger -except ModuleNotFoundError: +except ImportError: pass try: from .mlflow_logger import MLFlowLogger -except ModuleNotFoundError: +except ImportError: pass try: # needed to prevent ImportError and duplicated logs. environ["COMET_DISABLE_AUTO_LOGGING"] = "1" from .comet_logger import CometLogger -except ModuleNotFoundError: +except ImportError: del environ["COMET_DISABLE_AUTO_LOGGING"] From 8347a6c87e607a92fdc36938ec341d8f970c4322 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Wed, 30 Oct 2019 12:11:21 -0400 Subject: [PATCH 09/17] mem clear (#440) * mem clear * mem clear --- pytorch_lightning/trainer/trainer_io.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pytorch_lightning/trainer/trainer_io.py b/pytorch_lightning/trainer/trainer_io.py index b8de2bbf..dd38c051 100644 --- a/pytorch_lightning/trainer/trainer_io.py +++ b/pytorch_lightning/trainer/trainer_io.py @@ -32,10 +32,17 @@ class TrainerIOMixin(object): :param model: :return: """ + # clear cache before restore + if self.on_gpu: + torch.cuda.empty_cache() # if script called from hpc resubmit, load weights did_restore_hpc_weights = self.restore_hpc_weights_if_needed(model) + # clear cache after restore + if self.on_gpu: + torch.cuda.empty_cache() + if not did_restore_hpc_weights: # restore weights if same exp version self.restore_state_if_checkpoint_exists(model) From 9f8ab7c29e1fc42d1ab472a311e54cd052aa6582 Mon Sep 17 00:00:00 2001 From: Vadim Bereznyuk Date: Wed, 30 Oct 2019 19:13:40 +0300 Subject: [PATCH 10/17] Fixed total number of batches (#439) * Fixed total number of batches * Fixed flake8 warning * Update train_loop_mixin.py * Update train_loop_mixin.py --- pytorch_lightning/trainer/train_loop_mixin.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/trainer/train_loop_mixin.py b/pytorch_lightning/trainer/train_loop_mixin.py index 8cebe7a1..2683ae66 100644 --- a/pytorch_lightning/trainer/train_loop_mixin.py +++ b/pytorch_lightning/trainer/train_loop_mixin.py @@ -23,7 +23,15 @@ class TrainerTrainLoopMixin(object): # update training progress in trainer and model model.current_epoch = epoch_nb self.current_epoch = epoch_nb - self.total_batches = self.nb_training_batches + self.nb_val_batches + + # val can be checked multiple times in epoch + is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0 + val_checks_per_epoch = self.nb_training_batches // self.val_check_batch + val_checks_per_epoch = val_checks_per_epoch if is_val_epoch else 0 + + # total batches includes multiple val checks + self.total_batches = (self.nb_training_batches + + self.nb_val_batches * val_checks_per_epoch) self.batch_loss_value = 0 # accumulated grads # limit the number of batches to 1 in fast_dev_run From f79bdf232704e3d6454a32273f7bd6b2262029f4 Mon Sep 17 00:00:00 2001 From: Vadim Bereznyuk Date: Wed, 30 Oct 2019 19:14:28 +0300 Subject: [PATCH 11/17] Set total number of batches in progress bar while testing (#425) --- pytorch_lightning/trainer/trainer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index cbe9af37..a5beecc4 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -432,6 +432,9 @@ class Trainer(TrainerIOMixin, # when testing requested only run test and return if self.testing: + if self.show_progress_bar: + self.progress_bar.reset(self.nb_test_batches) + self.run_evaluation(test=True) return From f6b8b175bbf480e74586bf0fc5378e9efd221995 Mon Sep 17 00:00:00 2001 From: Joel Wong Date: Thu, 31 Oct 2019 21:40:32 +1100 Subject: [PATCH 12/17] Update Docs for current checkpointing behaviour (#445) Related issue #432 The old documentation suggested that the way to restore a training session is to use a test_tube Experiment. Trainer no longer takes an experiment as a parameter, so it seems the current way to restore a training session is to pass an experiment via a TestTubeLogger. Even if this is not the most elegant solution, updating the docs will at least point new users in the right direction. --- docs/Trainer/Checkpointing.md | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/Trainer/Checkpointing.md b/docs/Trainer/Checkpointing.md index 5d3b257b..791e72eb 100644 --- a/docs/Trainer/Checkpointing.md +++ b/docs/Trainer/Checkpointing.md @@ -32,12 +32,19 @@ You might want to not only load a model but also continue training it. Use this restore the trainer state as well. This will continue from the epoch and global step you last left off. However, the dataloaders will start from the first batch again (if you shuffled it shouldn't matter). -Lightning will restore the session if you pass an experiment with the same version and there's a saved checkpoint. +Lightning will restore the session if you pass a logger with the same version and there's a saved checkpoint. ``` {.python} -from test_tube import Experiment +from pytorch_lightning import Trainer +from pytorch_lightning.logging import TestTubeLogger -exp = Experiment(version=a_previous_version_with_a_saved_checkpoint) -trainer = Trainer(experiment=exp) +logger = TestTubeLogger( + save_dir='./savepath', + version=1 # An existing version with a saved checkpoint +) +trainer = Trainer( + logger=logger, + default_save_path='./savepath' +) # this fit call loads model weights and trainer state # the trainer continues seamlessly from where you left off From 248495b1d161695a64cf86dd0e7f9f9fd8c87107 Mon Sep 17 00:00:00 2001 From: Tullie Murrell Date: Thu, 31 Oct 2019 03:45:28 -0700 Subject: [PATCH 13/17] Add tbptt (#429) * Add truncated bptt * Fix rebase error * AutoPep8 * Address comments, incl default bptt_split impl * Add tbptt test * Add default split for lists/tuples * Add tbptt docs * Fix trainer spacing * Update RequiredTrainerInterface.md --- .../RequiredTrainerInterface.md | 7 + docs/Trainer/Training Loop.md | 26 +++- docs/Trainer/hooks.md | 24 ++++ docs/Trainer/index.md | 1 + pytorch_lightning/root_module/root_module.py | 30 ++++ .../trainer/evaluation_loop_mixin.py | 3 +- pytorch_lightning/trainer/logging_mixin.py | 9 +- pytorch_lightning/trainer/train_loop_mixin.py | 129 ++++++++++-------- pytorch_lightning/trainer/trainer.py | 8 +- tests/test_cpu_models.py | 77 ++++++++++- 10 files changed, 247 insertions(+), 67 deletions(-) diff --git a/docs/LightningModule/RequiredTrainerInterface.md b/docs/LightningModule/RequiredTrainerInterface.md index 8014b0b8..e0299615 100644 --- a/docs/LightningModule/RequiredTrainerInterface.md +++ b/docs/LightningModule/RequiredTrainerInterface.md @@ -168,6 +168,13 @@ def training_step(self, batch, batch_nb, optimizer_idx): # do training_step with decoder ``` +If you add truncated back propagation through time you will also get an additional argument with the hidden states of the previous step. +``` {.python} +# Truncated back-propagation through time +def training_step(self, batch, batch_nb, hiddens): + # hiddens are the hiddens from the previous truncated backprop step +``` + You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to break out of the current training epoch early. diff --git a/docs/Trainer/Training Loop.md b/docs/Trainer/Training Loop.md index abb4d920..f471c2a2 100644 --- a/docs/Trainer/Training Loop.md +++ b/docs/Trainer/Training Loop.md @@ -3,8 +3,8 @@ The lightning training loop handles everything except the actual computations of Below are all the things lightning automates for you in the training loop. --- -#### Accumulated gradients -Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. +#### Accumulated gradients +Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. ``` {.python} # DEFAULT (ie: no accumulated grads) @@ -21,7 +21,7 @@ trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000) --- #### Early stopping -The trainer already sets up default early stopping for you. +The trainer already sets up default early stopping for you. To modify this behavior, pass in your own EarlyStopping callback. ``` {.python} from pytorch_lightning.callbacks import EarlyStopping @@ -38,15 +38,15 @@ early_stop_callback = EarlyStopping( # without passing anything in, uses the default callback above trainer = Trainer() -# pass in your own to override the default callback +# pass in your own to override the default callback trainer = Trainer(early_stop_callback=early_stop_callback) -# pass in None to disable it +# pass in None to disable it trainer = Trainer(early_stop_callback=None) ``` --- -#### Force disable early stop +#### Force disable early stop To disable early stopping pass None to the early_stop_callback ``` {.python} # DEFAULT @@ -91,3 +91,17 @@ trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1) ``` + +--- +#### Truncated Back Propagation Through Time +There are times when multiple backwards passes are needed for each batch. For example, it may save memory to use Truncated Back Propagation Through Time when training RNNs. + +When this flag is enabled each batch is split into sequences of size truncated_bptt_steps and passed to training_step(...) separately. A default splitting function is provided, however, you can override it for more flexibility. See [tbptt_split_batch](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks#tbptt_split_batch). + +``` {.python} +# DEFAULT (single backwards pass per batch) +trainer = Trainer(truncated_bptt_steps=None) + +# (split batch into sequences of size 2) +trainer = Trainer(truncated_bptt_steps=2) +``` diff --git a/docs/Trainer/hooks.md b/docs/Trainer/hooks.md index 98ef6b8b..96c41e2b 100644 --- a/docs/Trainer/hooks.md +++ b/docs/Trainer/hooks.md @@ -151,3 +151,27 @@ def on_after_backward(self): name = k self.logger.experiment.add_histogram(tag=name, values=grads, global_step=self.trainer.global_step) ``` + +--- +#### tbptt_split_batch +Called in the training loop after on_batch_start if `truncated_bptt_steps > 0`. Each returned batch split is passed separately to training_step(...). + +```python +def tbptt_split_batch(self, batch, split_size): + splits = [] + for t in range(0, time_dims[0], split_size): + batch_split = [] + for i, x in enumerate(batch): + if isinstance(x, torch.Tensor): + split_x = x[:, t:t + split_size] + elif isinstance(x, collections.Sequence): + split_x = [None] * len(x) + for batch_idx in range(len(x)): + split_x[batch_idx] = x[batch_idx][t:t + split_size] + + batch_split.append(split_x) + + splits.append(batch_split) + + return splits +``` diff --git a/docs/Trainer/index.md b/docs/Trainer/index.md index 88779797..d71c07ed 100644 --- a/docs/Trainer/index.md +++ b/docs/Trainer/index.md @@ -71,6 +71,7 @@ But of course the fun is in all the advanced things it can do: - [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers) - [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check) - [Step optimizers at arbitrary intervals](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step) +- [Truncated Back Propagation Through Time](https://williamfalcon.github.io/pytorch-lightning//Training%20Loop/#truncated-back-propation-through-time) **Validation loop** diff --git a/pytorch_lightning/root_module/root_module.py b/pytorch_lightning/root_module/root_module.py index d037b038..ab5275dc 100644 --- a/pytorch_lightning/root_module/root_module.py +++ b/pytorch_lightning/root_module/root_module.py @@ -1,4 +1,5 @@ import warnings +import collections from argparse import Namespace import torch @@ -113,6 +114,35 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): # clear gradients optimizer.zero_grad() + def tbptt_split_batch(self, batch, split_size): + """ + Return list of batch splits. Each split will be passed to forward_step to enable truncated + back propagation through time. The default implementation splits root level Tensors and + Sequences at dim=1 (i.e. time dim). It assumes that each time dim is the same length. + :return: + """ + time_dims = [len(x[0]) for x in batch if isinstance( + x, torch.Tensor) or isinstance(x, collections.Sequence)] + assert len(time_dims) >= 1, "Unable to determine batch time dimension" + assert all(x == time_dims[0] for x in time_dims), "Batch time dimension length is ambiguous" + + splits = [] + for t in range(0, time_dims[0], split_size): + batch_split = [] + for i, x in enumerate(batch): + if isinstance(x, torch.Tensor): + split_x = x[:, t:t + split_size] + elif isinstance(x, collections.Sequence): + split_x = [None] * len(x) + for batch_idx in range(len(x)): + split_x[batch_idx] = x[batch_idx][t:t + split_size] + + batch_split.append(split_x) + + splits.append(batch_split) + + return splits + @data_loader def tng_dataloader(self): """ diff --git a/pytorch_lightning/trainer/evaluation_loop_mixin.py b/pytorch_lightning/trainer/evaluation_loop_mixin.py index c3cc51ef..ad4b6ad8 100644 --- a/pytorch_lightning/trainer/evaluation_loop_mixin.py +++ b/pytorch_lightning/trainer/evaluation_loop_mixin.py @@ -115,7 +115,8 @@ class TrainerEvaluationLoopMixin(object): dataloaders, max_batches, test) - _, prog_bar_metrics, log_metrics, callback_metrics = self.process_output(eval_results) + _, prog_bar_metrics, log_metrics, callback_metrics, _ = self.process_output( + eval_results) # add metrics to prog bar self.add_tqdm_metrics(prog_bar_metrics) diff --git a/pytorch_lightning/trainer/logging_mixin.py b/pytorch_lightning/trainer/logging_mixin.py index f437ee99..236e512d 100644 --- a/pytorch_lightning/trainer/logging_mixin.py +++ b/pytorch_lightning/trainer/logging_mixin.py @@ -64,7 +64,7 @@ class TrainerLoggingMixin(object): # all keys not progress_bar or log are candidates for callbacks callback_metrics = {} for k, v in output.items(): - if k not in ['progress_bar', 'log']: + if k not in ['progress_bar', 'log', 'hiddens']: callback_metrics[k] = v if train and (self.use_dp or self.use_ddp2): @@ -126,6 +126,11 @@ class TrainerLoggingMixin(object): if self.use_dp or self.use_ddp2: loss = self.reduce_distributed_output(loss, self.num_gpus) + # --------------- + # EXTRACT HIDDEN + # --------------- + hiddens = output.get('hiddens') + # use every metric passed in as a candidate for callback callback_metrics.update(progress_bar_metrics) callback_metrics.update(log_metrics) @@ -135,7 +140,7 @@ class TrainerLoggingMixin(object): if isinstance(v, torch.Tensor): callback_metrics[k] = v.item() - return loss, progress_bar_metrics, log_metrics, callback_metrics + return loss, progress_bar_metrics, log_metrics, callback_metrics, hiddens def reduce_distributed_output(self, output, nb_gpus): if nb_gpus <= 1: diff --git a/pytorch_lightning/trainer/train_loop_mixin.py b/pytorch_lightning/trainer/train_loop_mixin.py index 2683ae66..a10b8fc1 100644 --- a/pytorch_lightning/trainer/train_loop_mixin.py +++ b/pytorch_lightning/trainer/train_loop_mixin.py @@ -161,76 +161,91 @@ class TrainerTrainLoopMixin(object): if self.show_progress_bar: self.progress_bar.update(1) - # call training_step once per optimizer - for opt_idx, optimizer in enumerate(self.optimizers): + splits = [batch] + if self.truncated_bptt_steps is not None: + model_ref = self.get_model() + splits = model_ref.tbptt_split_batch(batch, self.truncated_bptt_steps) - # wrap the forward step in a closure so second order methods work - def optimizer_closure(): - # forward pass - output = self.training_forward(batch, batch_nb, opt_idx) - closure_loss, progress_bar_metrics, log_metrics, callback_metrics = output + self.hiddens = None + for split_nb, split_batch in enumerate(splits): + self.split_nb = split_nb - # track metrics for callbacks - all_callback_metrics.append(callback_metrics) + # call training_step once per optimizer + for opt_idx, optimizer in enumerate(self.optimizers): - # track progress bar metrics - self.add_tqdm_metrics(progress_bar_metrics) - all_log_metrics.append(log_metrics) + # wrap the forward step in a closure so second order methods work + def optimizer_closure(): + # forward pass + output = self.training_forward( + split_batch, batch_nb, opt_idx, self.hiddens) - # accumulate loss - # (if accumulate_grad_batches = 1 no effect) - closure_loss = closure_loss / self.accumulate_grad_batches + closure_loss = output[0] + progress_bar_metrics = output[1] + log_metrics = output[2] + callback_metrics = output[3] + self.hiddens = output[4] - # backward pass - # done in hook so user can overwrite if needed - model_ref = self.get_model() - model_ref.backward(self.use_amp, closure_loss, optimizer) + # track metrics for callbacks + all_callback_metrics.append(callback_metrics) - # insert after step hook - if self.is_function_implemented('on_after_backward'): + # track progress bar metrics + self.add_tqdm_metrics(progress_bar_metrics) + all_log_metrics.append(log_metrics) + + # accumulate loss + # (if accumulate_grad_batches = 1 no effect) + closure_loss = closure_loss / self.accumulate_grad_batches + + # backward pass model_ref = self.get_model() - model_ref.on_after_backward() + model_ref.backward(self.use_amp, closure_loss, optimizer) - return closure_loss + # insert after step hook + if self.is_function_implemented('on_after_backward'): + model_ref = self.get_model() + model_ref.on_after_backward() - # calculate loss - loss = optimizer_closure() + return closure_loss - # nan grads - if self.print_nan_grads: - self.print_nan_gradients() + # calculate loss + loss = optimizer_closure() - # track total loss for logging (avoid mem leaks) - self.batch_loss_value += loss.item() + # nan grads + if self.print_nan_grads: + self.print_nan_gradients() - # gradient update with accumulated gradients - if (self.batch_nb + 1) % self.accumulate_grad_batches == 0: + # track total loss for logging (avoid mem leaks) + self.batch_loss_value += loss.item() - # track gradient norms when requested - if batch_nb % self.row_log_interval == 0: - if self.track_grad_norm > 0: - model = self.get_model() - grad_norm_dic = model.grad_norm(self.track_grad_norm) + # gradient update with accumulated gradients + if (self.batch_nb + 1) % self.accumulate_grad_batches == 0: - # clip gradients - self.clip_gradients() + # track gradient norms when requested + if batch_nb % self.row_log_interval == 0: + if self.track_grad_norm > 0: + model = self.get_model() + grad_norm_dic = model.grad_norm( + self.track_grad_norm) - # calls .step(), .zero_grad() - # override function to modify this behavior - model = self.get_model() - model.optimizer_step(self.current_epoch, batch_nb, - optimizer, opt_idx, optimizer_closure) + # clip gradients + self.clip_gradients() - # calculate running loss for display - self.running_loss.append(self.batch_loss_value) - self.batch_loss_value = 0 - self.avg_loss = np.mean(self.running_loss[-100:]) + # calls .step(), .zero_grad() + # override function to modify this behavior + model = self.get_model() + model.optimizer_step(self.current_epoch, batch_nb, + optimizer, opt_idx, optimizer_closure) - # update progress bar - if self.show_progress_bar: - # add model specific metrics - tqdm_metrics = self.training_tqdm_dict - self.progress_bar.set_postfix(**tqdm_metrics) + # calculate running loss for display + self.running_loss.append(self.batch_loss_value) + self.batch_loss_value = 0 + self.avg_loss = np.mean(self.running_loss[-100:]) + + # update progress bar + if self.show_progress_bar: + # add model specific metrics + tqdm_metrics = self.training_tqdm_dict + self.progress_bar.set_postfix(**tqdm_metrics) # activate batch end hook if self.is_function_implemented('on_batch_end'): @@ -245,7 +260,7 @@ class TrainerTrainLoopMixin(object): return 0, grad_norm_dic, all_log_metrics - def training_forward(self, batch, batch_nb, opt_idx): + def training_forward(self, batch, batch_nb, opt_idx, hiddens): """ Handle forward for each training case (distributed, single gpu, etc...) :param batch: @@ -260,6 +275,9 @@ class TrainerTrainLoopMixin(object): if len(self.optimizers) > 1: args.append(opt_idx) + if self.truncated_bptt_steps is not None: + args.append(hiddens) + if self.use_ddp or self.use_ddp2: output = self.model(*args) elif self.use_dp: @@ -277,5 +295,4 @@ class TrainerTrainLoopMixin(object): # format and reduce outputs accordingly output = self.process_output(output, train=True) - loss, progress_bar_metrics, log_metrics, callback_metrics = output - return loss, progress_bar_metrics, log_metrics, callback_metrics + return output diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index a5beecc4..a61f1abb 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -80,7 +80,8 @@ class Trainer(TrainerIOMixin, weights_summary='full', weights_save_path=None, amp_level='O1', - nb_sanity_val_steps=5): + nb_sanity_val_steps=5, + truncated_bptt_steps=None): """ :param logger: Logger for experiment tracking @@ -116,6 +117,7 @@ class Trainer(TrainerIOMixin, :param weights_save_path: Bool. Where to save weights if on cluster :param amp_level: str. Check nvidia docs for level :param nb_sanity_val_steps: int. How many val steps before a full train loop. + :param truncated_bptt_steps: int. Enables multiple backward passes for each batch. """ # Transfer params self.nb_gpu_nodes = nb_gpu_nodes @@ -135,6 +137,7 @@ class Trainer(TrainerIOMixin, self.min_nb_epochs = min_nb_epochs self.nb_sanity_val_steps = nb_sanity_val_steps self.print_nan_grads = print_nan_grads + self.truncated_bptt_steps = truncated_bptt_steps self.shown_warnings = set() self.fast_dev_run = fast_dev_run @@ -297,6 +300,9 @@ class Trainer(TrainerIOMixin, 'batch_nb': '{}'.format(self.batch_nb), } + if self.truncated_bptt_steps is not None: + tqdm_dict['split_nb'] = self.split_nb + if self.logger is not None and self.logger.version is not None: tqdm_dict['v_nb'] = self.logger.version diff --git a/tests/test_cpu_models.py b/tests/test_cpu_models.py index 2c2be069..62f30404 100644 --- a/tests/test_cpu_models.py +++ b/tests/test_cpu_models.py @@ -3,7 +3,7 @@ import warnings import pytest import torch -from pytorch_lightning import Trainer +from pytorch_lightning import Trainer, data_loader from pytorch_lightning.callbacks import ( EarlyStopping, ) @@ -292,6 +292,81 @@ def test_all_features_cpu_model(): testing_utils.run_gpu_model_test(trainer_options, model, hparams, on_gpu=False) +def test_tbptt_cpu_model(): + """ + Test truncated back propagation through time works. + :return: + """ + testing_utils.reset_seed() + + truncated_bptt_steps = 2 + sequence_size = 30 + batch_size = 30 + + x_seq = torch.rand(batch_size, sequence_size, 1) + y_seq_list = torch.rand(batch_size, sequence_size, 1).tolist() + + class MockSeq2SeqDataset(torch.utils.data.Dataset): + def __getitem__(self, i): + return x_seq, y_seq_list + + def __len__(self): + return 1 + + class BpttTestModel(LightningTestModelBase): + def __init__(self, hparams): + super().__init__(hparams) + self.test_hidden = None + + def training_step(self, batch, batch_idx, hiddens): + assert hiddens == self.test_hidden, "Hidden state not persistent between tbptt steps" + self.test_hidden = torch.rand(1) + + x_tensor, y_list = batch + assert x_tensor.shape[1] == truncated_bptt_steps, "tbptt split Tensor failed" + + y_tensor = torch.tensor(y_list, dtype=x_tensor.dtype) + assert y_tensor.shape[1] == truncated_bptt_steps, "tbptt split list failed" + + pred = self.forward(x_tensor.view(batch_size, truncated_bptt_steps)) + loss_val = torch.nn.functional.mse_loss( + pred, y_tensor.view(batch_size, truncated_bptt_steps)) + return { + 'loss': loss_val, + 'hiddens': self.test_hidden, + } + + @data_loader + def train_dataloader(self): + return torch.utils.data.DataLoader( + dataset=MockSeq2SeqDataset(), + batch_size=batch_size, + shuffle=False, + sampler=None, + ) + + trainer_options = dict( + max_nb_epochs=1, + truncated_bptt_steps=truncated_bptt_steps, + val_percent_check=0, + weights_summary=None, + ) + + hparams = testing_utils.get_hparams() + hparams.batch_size = batch_size + hparams.in_features = truncated_bptt_steps + hparams.hidden_dim = truncated_bptt_steps + hparams.out_features = truncated_bptt_steps + + model = BpttTestModel(hparams) + + # fit model + trainer = Trainer(**trainer_options) + result = trainer.fit(model) + + assert result == 1, 'training failed to complete' + + def test_single_gpu_model(): """ Make sure single GPU works (DP mode) From 661a1c6fe66841769a7574a795572e7351e62b43 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 31 Oct 2019 10:49:07 -0400 Subject: [PATCH 14/17] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4a22b0c7..1797c2b2 100644 --- a/README.md +++ b/README.md @@ -337,8 +337,8 @@ Lightning also adds a text column with all the hyperparameters for this experime - [Run test set](https://williamfalcon.github.io/pytorch-lightning/Trainer/Testing%20loop/) ## Examples -- [GAN](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/domain_templates/gan.py) -- [MNIST](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/basic_examples) +- [GAN](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/domain_templates/gan.py) +- [MNIST](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/basic_examples) - [Other projects using Lightning](https://github.com/williamFalcon/pytorch-lightning/network/dependents?package_id=UGFja2FnZS0zNzE3NDU4OTM%3D) - [Multi-node](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/multi_node_examples) From 1865de1ff8fae62de613a3ae0ed0cf2b97ca6e8e Mon Sep 17 00:00:00 2001 From: Pattarawat Chormai Date: Fri, 1 Nov 2019 12:55:37 +0100 Subject: [PATCH 15/17] [WIP] Fix wrong example paths in README.md (#444) * Fix wrong example paths * correct dataloading wrong condition in Readme --- README.md | 4 ++-- docs/Trainer/Distributed training.md | 2 +- docs/examples/Examples.md | 4 ++-- docs/index.md | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1797c2b2..b91857a7 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ class CoolSystem(pl.LightningModule): @pl.data_loader def test_dataloader(self): # OPTIONAL - return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) + return DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=transforms.ToTensor()), batch_size=32) ``` 2. Fit with a [trainer](https://williamfalcon.github.io/pytorch-lightning/Trainer/) ```python @@ -340,7 +340,7 @@ Lightning also adds a text column with all the hyperparameters for this experime - [GAN](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/domain_templates/gan.py) - [MNIST](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/basic_examples) - [Other projects using Lightning](https://github.com/williamFalcon/pytorch-lightning/network/dependents?package_id=UGFja2FnZS0zNzE3NDU4OTM%3D) -- [Multi-node](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/multi_node_examples) +- [Multi-node](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/multi_node_examples) ## Tutorials - [Basic Lightning use](https://towardsdatascience.com/supercharge-your-ai-research-with-pytorch-lightning-337948a99eec) diff --git a/docs/Trainer/Distributed training.md b/docs/Trainer/Distributed training.md index 807bfc2c..8592525e 100644 --- a/docs/Trainer/Distributed training.md +++ b/docs/Trainer/Distributed training.md @@ -208,7 +208,7 @@ Instead of manually building SLURM scripts, you can use the [SlurmCluster object do this for you. The SlurmCluster can also run a grid search if you pass in a [HyperOptArgumentParser](https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/). Here is an example where you run a grid search of 9 combinations of hyperparams. -[The full examples are here](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/new_project_templates/multi_node_examples). +[The full examples are here](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/new_project_templates/multi_node_examples). ```python # grid search 3 values of learning rate and 3 values of number of layers for your net # this generates 9 experiments (lr=1e-3, layers=16), (lr=1e-3, layers=32), (lr=1e-3, layers=64), ... (lr=1e-1, layers=64) diff --git a/docs/examples/Examples.md b/docs/examples/Examples.md index f7c71365..5deda159 100644 --- a/docs/examples/Examples.md +++ b/docs/examples/Examples.md @@ -1,9 +1,9 @@ ### Template model definition -In 99% of cases you want to just copy [one of the examples](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples) to start a new lightningModule and change the core of what your model is actually trying to do. +In 99% of cases you want to just copy [one of the examples](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples) to start a new lightningModule and change the core of what your model is actually trying to do. ```bash # get a copy of the module template -wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/examples/new_project_templates/lightning_module_template.py +wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/pl_examples/new_project_templates/lightning_module_template.py ``` --- diff --git a/docs/index.md b/docs/index.md index 0a8e8cda..06a76fec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -60,8 +60,8 @@ Notice a few things about this flow: ###### Templates 1. [MNIST LightningModule](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#minimal-example) 2. [Trainer](https://williamfalcon.github.io/pytorch-lightning/Trainer/) - - [Basic CPU, GPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/basic_examples) - - [GPU cluster Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/multi_node_examples) + - [Basic CPU, GPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/basic_examples) + - [GPU cluster Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/multi_node_examples) ###### Docs shortcuts - [LightningModule](LightningModule/RequiredTrainerInterface/) From 4e9fd95f79f4d60868ad258d7a2ab38c55e0c4f3 Mon Sep 17 00:00:00 2001 From: s-rog <55400948+s-rog@users.noreply.github.com> Date: Sun, 3 Nov 2019 18:26:27 +0800 Subject: [PATCH 16/17] packed sequence clarification in train_dataloader (#443) * packed sequence clarification in train_dataloader * moved changes to training loop * removed changes from required interface * added index entry --- .../RequiredTrainerInterface.md | 2 +- docs/Trainer/Training Loop.md | 19 +++++++++++++++++++ docs/Trainer/index.md | 1 + 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/LightningModule/RequiredTrainerInterface.md b/docs/LightningModule/RequiredTrainerInterface.md index e0299615..53bfc821 100644 --- a/docs/LightningModule/RequiredTrainerInterface.md +++ b/docs/LightningModule/RequiredTrainerInterface.md @@ -186,7 +186,7 @@ break out of the current training epoch early. def train_dataloader(self) ``` Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. -If you want to change the data during every epoch DON'T use the data_loader decorator. +If you want to change the data during every epoch DON'T use the data_loader decorator. ##### Return PyTorch DataLoader diff --git a/docs/Trainer/Training Loop.md b/docs/Trainer/Training Loop.md index f471c2a2..bc0c83c8 100644 --- a/docs/Trainer/Training Loop.md +++ b/docs/Trainer/Training Loop.md @@ -92,6 +92,25 @@ trainer = Trainer(train_percent_check=1.0) trainer = Trainer(train_percent_check=0.1) ``` +--- +#### Packed sequences as inputs +When using PackedSequence, do 2 things: +1. return either a padded tensor in dataset or a list of variable length tensors in the dataloader collate_fn (example above shows the list implementation). +2. Pack the sequence in forward or training and validation steps depending on use case. + +``` {.python} +# For use in dataloader +def collate_fn(batch): + x = [item[0] for item in batch] + y = [item[1] for item in batch] + return x, y + +# In module +def training_step(self, batch, batch_nb): + x = rnn.pack_sequence(batch[0], enforce_sorted=False) + y = rnn.pack_sequence(batch[1], enforce_sorted=False) +``` + --- #### Truncated Back Propagation Through Time There are times when multiple backwards passes are needed for each batch. For example, it may save memory to use Truncated Back Propagation Through Time when training RNNs. diff --git a/docs/Trainer/index.md b/docs/Trainer/index.md index d71c07ed..13df0539 100644 --- a/docs/Trainer/index.md +++ b/docs/Trainer/index.md @@ -71,6 +71,7 @@ But of course the fun is in all the advanced things it can do: - [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers) - [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check) - [Step optimizers at arbitrary intervals](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step) +- [Packed sequences](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#packed-sequences-as-inputs) - [Truncated Back Propagation Through Time](https://williamfalcon.github.io/pytorch-lightning//Training%20Loop/#truncated-back-propation-through-time) **Validation loop** From 446a1b5d45c6345de90743513f552e9bbbcf172f Mon Sep 17 00:00:00 2001 From: Vadim Bereznyuk Date: Sun, 3 Nov 2019 13:42:53 +0300 Subject: [PATCH 17/17] Split progress bar (#449) * Splitted progress bars * Iterable dataset total batches fix * Use dynamic ncols and use batch as units * Count epochs from 1 in progress bar * Fix for disabled progress bar * Code simplifications --- .../trainer/evaluation_loop_mixin.py | 31 ++++++++++++--- pytorch_lightning/trainer/train_loop_mixin.py | 38 ++++++++++--------- pytorch_lightning/trainer/trainer.py | 27 +++++++------ 3 files changed, 61 insertions(+), 35 deletions(-) diff --git a/pytorch_lightning/trainer/evaluation_loop_mixin.py b/pytorch_lightning/trainer/evaluation_loop_mixin.py index ad4b6ad8..c2ba5d16 100644 --- a/pytorch_lightning/trainer/evaluation_loop_mixin.py +++ b/pytorch_lightning/trainer/evaluation_loop_mixin.py @@ -1,4 +1,5 @@ import torch +import tqdm from pytorch_lightning.utilities.debugging import MisconfigurationException @@ -52,8 +53,11 @@ class TrainerEvaluationLoopMixin(object): dl_outputs.append(output) # batch done - if self.show_progress_bar: - self.progress_bar.update(1) + if test: + self.test_progress_bar.update(1) + else: + self.val_progress_bar.update(1) + self.main_progress_bar.update(1) outputs.append(dl_outputs) eval_results = {} @@ -110,6 +114,15 @@ class TrainerEvaluationLoopMixin(object): if self.fast_dev_run: max_batches = 1 + # init validation or test progress bar + # main progress bar will already be closed when testing so initial position is free + position = 2 * self.process_position + (not test) + desc = 'Testing' if test else 'Validating' + pbar = tqdm.tqdm(desc=desc, total=max_batches, leave=test, position=position, + disable=not self.show_progress_bar, dynamic_ncols=True, + unit='batch') + setattr(self, f'{"test" if test else "val"}_progress_bar', pbar) + # run evaluation eval_results = self.evaluate(self.model, dataloaders, @@ -130,10 +143,16 @@ class TrainerEvaluationLoopMixin(object): # hook model.on_post_performance_check() - if self.show_progress_bar: - # add model specific metrics - tqdm_metrics = self.training_tqdm_dict - self.progress_bar.set_postfix(**tqdm_metrics) + # add model specific metrics + tqdm_metrics = self.training_tqdm_dict + if not test: + self.main_progress_bar.set_postfix(**tqdm_metrics) + + # close progress bar + if test: + self.test_progress_bar.close() + else: + self.val_progress_bar.close() # model checkpointing if self.proc_rank == 0 and self.checkpoint_callback is not None and not test: diff --git a/pytorch_lightning/trainer/train_loop_mixin.py b/pytorch_lightning/trainer/train_loop_mixin.py index a10b8fc1..d41fc7c1 100644 --- a/pytorch_lightning/trainer/train_loop_mixin.py +++ b/pytorch_lightning/trainer/train_loop_mixin.py @@ -1,4 +1,5 @@ import numpy as np +import tqdm try: from apex import amp @@ -34,18 +35,21 @@ class TrainerTrainLoopMixin(object): self.nb_val_batches * val_checks_per_epoch) self.batch_loss_value = 0 # accumulated grads - # limit the number of batches to 1 in fast_dev_run if self.fast_dev_run: - self.total_batches = 1 - - # init progress_bar when requested - if self.show_progress_bar: + # limit the number of batches to 2 (1 train and 1 val) in fast_dev_run + nb_iterations = 2 + elif self.is_iterable_train_dataloader: + # for iterable train loader, the progress bar never ends + nb_iterations = None + else: nb_iterations = self.total_batches - # for iterable train loader, the progress bar never ends - if self.is_iterable_train_dataloader: - nb_iterations = float('inf') - self.progress_bar.reset(nb_iterations) + # reset progress bar + # .reset() doesn't work on disabled progress bar so we should check + if not self.main_progress_bar.disable: + self.main_progress_bar.reset(nb_iterations) + desc = f'Epoch {epoch_nb + 1}' if not self.is_iterable_train_dataloader else '' + self.main_progress_bar.set_description(desc) # changing gradient according accumulation_scheduler self.accumulation_scheduler.on_epoch_begin(epoch_nb, self) @@ -68,8 +72,11 @@ class TrainerTrainLoopMixin(object): # stop training stop = should_stop and met_min_epochs if stop: + self.main_progress_bar.close() return + self.main_progress_bar.close() + if self.logger is not None: self.logger.finalize("success") @@ -158,9 +165,6 @@ class TrainerTrainLoopMixin(object): if response == -1: return -1, grad_norm_dic - if self.show_progress_bar: - self.progress_bar.update(1) - splits = [batch] if self.truncated_bptt_steps is not None: model_ref = self.get_model() @@ -241,17 +245,15 @@ class TrainerTrainLoopMixin(object): self.batch_loss_value = 0 self.avg_loss = np.mean(self.running_loss[-100:]) - # update progress bar - if self.show_progress_bar: - # add model specific metrics - tqdm_metrics = self.training_tqdm_dict - self.progress_bar.set_postfix(**tqdm_metrics) - # activate batch end hook if self.is_function_implemented('on_batch_end'): model = self.get_model() model.on_batch_end() + # update progress bar + self.main_progress_bar.update(1) + self.main_progress_bar.set_postfix(**self.training_tqdm_dict) + # collapse all metrics into one dict all_log_metrics = {k: v for d in all_log_metrics for k, v in d.items()} diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index a61f1abb..c53aeb7b 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -296,7 +296,6 @@ class Trainer(TrainerIOMixin, """ tqdm_dict = { 'loss': '{0:.3f}'.format(self.avg_loss), - 'epoch': '{}'.format(self.current_epoch), 'batch_nb': '{}'.format(self.batch_nb), } @@ -432,15 +431,8 @@ class Trainer(TrainerIOMixin, # restore training and model before hpc call self.restore_weights(model) - # progress bar init - if self.show_progress_bar: - self.progress_bar = tqdm.tqdm(0, position=self.process_position) - # when testing requested only run test and return if self.testing: - if self.show_progress_bar: - self.progress_bar.reset(self.nb_test_batches) - self.run_evaluation(test=True) return @@ -448,12 +440,25 @@ class Trainer(TrainerIOMixin, # to make sure program won't crash during val ref_model.on_sanity_check_start() if self.get_val_dataloaders() is not None and self.nb_sanity_val_steps > 0: - # reset progress_bar limit for sanity check - if self.show_progress_bar: - self.progress_bar.reset(self.nb_sanity_val_steps) + # init progress bars for validation sanity check + pbar = tqdm.tqdm(desc='Validation sanity check', total=self.nb_sanity_val_steps, + leave=False, position=2 * self.process_position, + disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch') + self.main_progress_bar = pbar + # dummy validation progress bar + self.val_progress_bar = tqdm.tqdm(disable=True) self.evaluate(model, self.get_val_dataloaders(), self.nb_sanity_val_steps, self.testing) + # close progress bars + self.main_progress_bar.close() + self.val_progress_bar.close() + + # init progress bar + pbar = tqdm.tqdm(leave=True, position=2 * self.process_position, + disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch') + self.main_progress_bar = pbar + # clear cache before training if self.on_gpu: torch.cuda.empty_cache()