diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 3fd3f248..6640f332 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,14 +1,13 @@ # Contributing Welcome to the PyTorch Lightning community! We're building the most advanced research platform on the planet to implement the latest, best practices that the amazing PyTorch team rolls out! -## One less thing to remember +## Main Core Value: One less thing to remember Simplify the API as much as possible from the user perspective. Any additions or improvements should minimize things the user needs to remember. -For example: One benefit of the validation_step is that the user doesn't have to remember to set the model to .eval(). This avoids all sorts of subtle errors the user could make. +For example: One benefit of the validation_step is that the user doesn't have to remember to set the model to .eval(). This avoids all sorts of subtle errors the user could make. ## Lightning Design Principles We encourage all sorts of contributions you're interested in adding! When coding for lightning, please follow these principles. - #### No PyTorch interference We don't want to add any abstractions on top of pure PyTorch. This gives researchers all the control they need without having to learn yet another framework. @@ -21,7 +20,12 @@ There are 1,000 ways to do something. However, something eventually becomes stan When something becomes a best practice, we add it to the framework. This likely looks like code in utils or in the model file that everyone keeps adding over and over again across projects. When this happens, bring that code inside the trainer and add a flag for it. #### Simple External API -What makes sense to you may not make sense to others. Create an issue with an API change suggestion and validate that it makes sense for others. Treat code changes how you treat a startup: validate that it's a needed feature, then add if it makes sense for many people. +What makes sense to you may not make sense to others. Create an issue with an API change suggestion and validate that it makes sense for others. Treat code changes how you treat a startup: validate that it's a needed feature, then add if it makes sense for many people. + +#### Backward-compatible API +We all hate updating our deep learning packages because we don't want to refactor a bunch of stuff. In Lightning, we make sure every change we make which could break an API is backwards compatible with good deprecation warnings. + +You shouldn't be afraid to upgrade Lightning :) #### Gain User Trust As a researcher you can't have any part of your code going wrong. So, make thorough tests that ensure an implementation of a new trick or subbtle change is correct. diff --git a/.travis.yml b/.travis.yml index 8a97272e..1ad6eba5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,14 +51,13 @@ matrix: cache: pip install: - - pip install -r requirements.txt - - pip install -r ./tests/requirements.txt - - pip --version ; pip list + - pip install future # needed for `builtins` + - sudo pip install tox script: # integration - tox --sitepackages - - python setup.py install --dry-run + - pip install --editable . after_success: - coverage report diff --git a/README.md b/README.md index 257c02a3..b91857a7 100644 --- a/README.md +++ b/README.md @@ -130,7 +130,7 @@ class CoolSystem(pl.LightningModule): @pl.data_loader def test_dataloader(self): # OPTIONAL - return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) + return DataLoader(MNIST(os.getcwd(), train=False, download=True, transform=transforms.ToTensor()), batch_size=32) ``` 2. Fit with a [trainer](https://williamfalcon.github.io/pytorch-lightning/Trainer/) ```python @@ -337,10 +337,10 @@ Lightning also adds a text column with all the hyperparameters for this experime - [Run test set](https://williamfalcon.github.io/pytorch-lightning/Trainer/Testing%20loop/) ## Examples -- [GAN](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/domain_templates/gan.py) -- [MNIST](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/basic_examples) +- [GAN](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/domain_templates/gan.py) +- [MNIST](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/basic_examples) - [Other projects using Lightning](https://github.com/williamFalcon/pytorch-lightning/network/dependents?package_id=UGFja2FnZS0zNzE3NDU4OTM%3D) -- [Multi-node](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/multi_node_examples) +- [Multi-node](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/multi_node_examples) ## Tutorials - [Basic Lightning use](https://towardsdatascience.com/supercharge-your-ai-research-with-pytorch-lightning-337948a99eec) @@ -394,7 +394,7 @@ Nope. Please use anaconda or miniconda. # install latest Lightning version without upgrading deps pip install -U --no-deps pytorch-lightning ``` -- **PyTorch 1.2.0** +- **PyTorch 1.2.0, 1.3.0,** Install via pip as normal ## Custom installation diff --git a/docs/LightningModule/RequiredTrainerInterface.md b/docs/LightningModule/RequiredTrainerInterface.md index 8014b0b8..53bfc821 100644 --- a/docs/LightningModule/RequiredTrainerInterface.md +++ b/docs/LightningModule/RequiredTrainerInterface.md @@ -168,6 +168,13 @@ def training_step(self, batch, batch_nb, optimizer_idx): # do training_step with decoder ``` +If you add truncated back propagation through time you will also get an additional argument with the hidden states of the previous step. +``` {.python} +# Truncated back-propagation through time +def training_step(self, batch, batch_nb, hiddens): + # hiddens are the hiddens from the previous truncated backprop step +``` + You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to break out of the current training epoch early. @@ -179,7 +186,7 @@ break out of the current training epoch early. def train_dataloader(self) ``` Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. -If you want to change the data during every epoch DON'T use the data_loader decorator. +If you want to change the data during every epoch DON'T use the data_loader decorator. ##### Return PyTorch DataLoader diff --git a/docs/Trainer/Checkpointing.md b/docs/Trainer/Checkpointing.md index 5d3b257b..791e72eb 100644 --- a/docs/Trainer/Checkpointing.md +++ b/docs/Trainer/Checkpointing.md @@ -32,12 +32,19 @@ You might want to not only load a model but also continue training it. Use this restore the trainer state as well. This will continue from the epoch and global step you last left off. However, the dataloaders will start from the first batch again (if you shuffled it shouldn't matter). -Lightning will restore the session if you pass an experiment with the same version and there's a saved checkpoint. +Lightning will restore the session if you pass a logger with the same version and there's a saved checkpoint. ``` {.python} -from test_tube import Experiment +from pytorch_lightning import Trainer +from pytorch_lightning.logging import TestTubeLogger -exp = Experiment(version=a_previous_version_with_a_saved_checkpoint) -trainer = Trainer(experiment=exp) +logger = TestTubeLogger( + save_dir='./savepath', + version=1 # An existing version with a saved checkpoint +) +trainer = Trainer( + logger=logger, + default_save_path='./savepath' +) # this fit call loads model weights and trainer state # the trainer continues seamlessly from where you left off diff --git a/docs/Trainer/Distributed training.md b/docs/Trainer/Distributed training.md index 807bfc2c..8592525e 100644 --- a/docs/Trainer/Distributed training.md +++ b/docs/Trainer/Distributed training.md @@ -208,7 +208,7 @@ Instead of manually building SLURM scripts, you can use the [SlurmCluster object do this for you. The SlurmCluster can also run a grid search if you pass in a [HyperOptArgumentParser](https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/). Here is an example where you run a grid search of 9 combinations of hyperparams. -[The full examples are here](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/new_project_templates/multi_node_examples). +[The full examples are here](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/new_project_templates/multi_node_examples). ```python # grid search 3 values of learning rate and 3 values of number of layers for your net # this generates 9 experiments (lr=1e-3, layers=16), (lr=1e-3, layers=32), (lr=1e-3, layers=64), ... (lr=1e-1, layers=64) diff --git a/docs/Trainer/Training Loop.md b/docs/Trainer/Training Loop.md index abb4d920..bc0c83c8 100644 --- a/docs/Trainer/Training Loop.md +++ b/docs/Trainer/Training Loop.md @@ -3,8 +3,8 @@ The lightning training loop handles everything except the actual computations of Below are all the things lightning automates for you in the training loop. --- -#### Accumulated gradients -Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. +#### Accumulated gradients +Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. ``` {.python} # DEFAULT (ie: no accumulated grads) @@ -21,7 +21,7 @@ trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000) --- #### Early stopping -The trainer already sets up default early stopping for you. +The trainer already sets up default early stopping for you. To modify this behavior, pass in your own EarlyStopping callback. ``` {.python} from pytorch_lightning.callbacks import EarlyStopping @@ -38,15 +38,15 @@ early_stop_callback = EarlyStopping( # without passing anything in, uses the default callback above trainer = Trainer() -# pass in your own to override the default callback +# pass in your own to override the default callback trainer = Trainer(early_stop_callback=early_stop_callback) -# pass in None to disable it +# pass in None to disable it trainer = Trainer(early_stop_callback=None) ``` --- -#### Force disable early stop +#### Force disable early stop To disable early stopping pass None to the early_stop_callback ``` {.python} # DEFAULT @@ -91,3 +91,36 @@ trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1) ``` + +--- +#### Packed sequences as inputs +When using PackedSequence, do 2 things: +1. return either a padded tensor in dataset or a list of variable length tensors in the dataloader collate_fn (example above shows the list implementation). +2. Pack the sequence in forward or training and validation steps depending on use case. + +``` {.python} +# For use in dataloader +def collate_fn(batch): + x = [item[0] for item in batch] + y = [item[1] for item in batch] + return x, y + +# In module +def training_step(self, batch, batch_nb): + x = rnn.pack_sequence(batch[0], enforce_sorted=False) + y = rnn.pack_sequence(batch[1], enforce_sorted=False) +``` + +--- +#### Truncated Back Propagation Through Time +There are times when multiple backwards passes are needed for each batch. For example, it may save memory to use Truncated Back Propagation Through Time when training RNNs. + +When this flag is enabled each batch is split into sequences of size truncated_bptt_steps and passed to training_step(...) separately. A default splitting function is provided, however, you can override it for more flexibility. See [tbptt_split_batch](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks#tbptt_split_batch). + +``` {.python} +# DEFAULT (single backwards pass per batch) +trainer = Trainer(truncated_bptt_steps=None) + +# (split batch into sequences of size 2) +trainer = Trainer(truncated_bptt_steps=2) +``` diff --git a/docs/Trainer/hooks.md b/docs/Trainer/hooks.md index 6fd8df3d..96c41e2b 100644 --- a/docs/Trainer/hooks.md +++ b/docs/Trainer/hooks.md @@ -115,6 +115,28 @@ def on_before_zero_grad(self, optimizer): # do something with the optimizer or inspect it. ``` +--- +#### backward +Called to perform backward step. +Feel free to override as needed. + +The loss passed in has already been scaled for accumulated gradients if requested. +```python +def backward(self, use_amp, loss, optimizer): + """ + Override backward with your own implementation if you need to + :param use_amp: Whether amp was requested or not + :param loss: Loss is already scaled by accumulated grads + :param optimizer: Current optimizer being used + :return: + """ + if use_amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() +``` + --- #### on_after_backward Called in the training loop after model.backward() @@ -129,3 +151,27 @@ def on_after_backward(self): name = k self.logger.experiment.add_histogram(tag=name, values=grads, global_step=self.trainer.global_step) ``` + +--- +#### tbptt_split_batch +Called in the training loop after on_batch_start if `truncated_bptt_steps > 0`. Each returned batch split is passed separately to training_step(...). + +```python +def tbptt_split_batch(self, batch, split_size): + splits = [] + for t in range(0, time_dims[0], split_size): + batch_split = [] + for i, x in enumerate(batch): + if isinstance(x, torch.Tensor): + split_x = x[:, t:t + split_size] + elif isinstance(x, collections.Sequence): + split_x = [None] * len(x) + for batch_idx in range(len(x)): + split_x[batch_idx] = x[batch_idx][t:t + split_size] + + batch_split.append(split_x) + + splits.append(batch_split) + + return splits +``` diff --git a/docs/Trainer/index.md b/docs/Trainer/index.md index 88779797..13df0539 100644 --- a/docs/Trainer/index.md +++ b/docs/Trainer/index.md @@ -71,6 +71,8 @@ But of course the fun is in all the advanced things it can do: - [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers) - [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check) - [Step optimizers at arbitrary intervals](https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step) +- [Packed sequences](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#packed-sequences-as-inputs) +- [Truncated Back Propagation Through Time](https://williamfalcon.github.io/pytorch-lightning//Training%20Loop/#truncated-back-propation-through-time) **Validation loop** diff --git a/docs/examples/Examples.md b/docs/examples/Examples.md index f7c71365..5deda159 100644 --- a/docs/examples/Examples.md +++ b/docs/examples/Examples.md @@ -1,9 +1,9 @@ ### Template model definition -In 99% of cases you want to just copy [one of the examples](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples) to start a new lightningModule and change the core of what your model is actually trying to do. +In 99% of cases you want to just copy [one of the examples](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples) to start a new lightningModule and change the core of what your model is actually trying to do. ```bash # get a copy of the module template -wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/examples/new_project_templates/lightning_module_template.py +wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/pl_examples/new_project_templates/lightning_module_template.py ``` --- diff --git a/docs/index.md b/docs/index.md index 0a8e8cda..06a76fec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -60,8 +60,8 @@ Notice a few things about this flow: ###### Templates 1. [MNIST LightningModule](https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#minimal-example) 2. [Trainer](https://williamfalcon.github.io/pytorch-lightning/Trainer/) - - [Basic CPU, GPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/basic_examples) - - [GPU cluster Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/multi_node_examples) + - [Basic CPU, GPU Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/basic_examples) + - [GPU cluster Trainer Template](https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/multi_node_examples) ###### Docs shortcuts - [LightningModule](LightningModule/RequiredTrainerInterface/) diff --git a/pytorch_lightning/__init__.py b/pytorch_lightning/__init__.py index d65aff02..64829568 100644 --- a/pytorch_lightning/__init__.py +++ b/pytorch_lightning/__init__.py @@ -1,9 +1,36 @@ -from .root_module.decorators import data_loader -from .root_module.root_module import LightningModule -from .trainer.trainer import Trainer +"""Package info""" -__all__ = [ - 'Trainer', - 'LightningModule', - 'data_loader', -] +__version__ = '0.5.2.1' +__author__ = ' William Falcon et al.' +__author_email__ = 'waf2107@columbia.edu' +__license__ = 'Apache-2.0' +__homepage__ = 'https://github.com/williamFalcon/pytorch-lightning', +__docs__ = """# PyTorch Lightning + +The lightweight PyTorch wrapper for ML researchers. Scale your models. Write less boilerplate. +""" + + +try: + # This variable is injected in the __builtins__ by the build + # process. It used to enable importing subpackages of skimage when + # the binaries are not built + __LIGHTNING_SETUP__ +except NameError: + __LIGHTNING_SETUP__ = False + +if __LIGHTNING_SETUP__: + import sys + sys.stderr.write('Partial import of skimage during the build process.\n') + # We are not importing the rest of the scikit during the build + # process, as it may not be compiled yet +else: + from .trainer.trainer import Trainer + from .root_module.root_module import LightningModule + from .root_module.decorators import data_loader + + __all__ = [ + 'Trainer', + 'LightningModule', + 'data_loader', + ] diff --git a/pytorch_lightning/logging/__init__.py b/pytorch_lightning/logging/__init__.py index 73960a48..dc47188a 100644 --- a/pytorch_lightning/logging/__init__.py +++ b/pytorch_lightning/logging/__init__.py @@ -3,16 +3,16 @@ from .base import LightningLoggerBase, rank_zero_only try: from .test_tube_logger import TestTubeLogger -except ModuleNotFoundError: +except ImportError: pass try: from .mlflow_logger import MLFlowLogger -except ModuleNotFoundError: +except ImportError: pass try: # needed to prevent ImportError and duplicated logs. environ["COMET_DISABLE_AUTO_LOGGING"] = "1" from .comet_logger import CometLogger -except ModuleNotFoundError: +except ImportError: del environ["COMET_DISABLE_AUTO_LOGGING"] diff --git a/pytorch_lightning/logging/comet_logger.py b/pytorch_lightning/logging/comet_logger.py index dc05a9a9..5e1d281c 100644 --- a/pytorch_lightning/logging/comet_logger.py +++ b/pytorch_lightning/logging/comet_logger.py @@ -1,6 +1,7 @@ -from os import environ - -from comet_ml import Experiment as CometExperiment +try: + from comet_ml import Experiment as CometExperiment +except ImportError: + raise ImportError('Missing comet_ml package.') from .base import LightningLoggerBase, rank_zero_only diff --git a/pytorch_lightning/logging/mlflow_logger.py b/pytorch_lightning/logging/mlflow_logger.py index c38b7a16..6c0d460a 100644 --- a/pytorch_lightning/logging/mlflow_logger.py +++ b/pytorch_lightning/logging/mlflow_logger.py @@ -1,7 +1,10 @@ from logging import getLogger from time import time -import mlflow +try: + import mlflow +except ImportError: + raise ImportError('Missing mlflow package.') from .base import LightningLoggerBase, rank_zero_only diff --git a/pytorch_lightning/logging/test_tube_logger.py b/pytorch_lightning/logging/test_tube_logger.py index e8e9607f..0d74307b 100644 --- a/pytorch_lightning/logging/test_tube_logger.py +++ b/pytorch_lightning/logging/test_tube_logger.py @@ -1,4 +1,7 @@ -from test_tube import Experiment +try: + from test_tube import Experiment +except ImportError: + raise ImportError('Missing test-tube package.') from .base import LightningLoggerBase, rank_zero_only diff --git a/pytorch_lightning/root_module/hooks.py b/pytorch_lightning/root_module/hooks.py index 37d41eb5..580d06e3 100644 --- a/pytorch_lightning/root_module/hooks.py +++ b/pytorch_lightning/root_module/hooks.py @@ -1,6 +1,14 @@ import torch +try: + from apex import amp + + APEX_AVAILABLE = True +except ImportError: + APEX_AVAILABLE = False + + class ModelHooks(torch.nn.Module): def on_sanity_check_start(self): @@ -48,3 +56,17 @@ class ModelHooks(torch.nn.Module): :return: """ pass + + def backward(self, use_amp, loss, optimizer): + """ + Override backward with your own implementation if you need to + :param use_amp: Whether amp was requested or not + :param loss: Loss is already scaled by accumulated grads + :param optimizer: Current optimizer being used + :return: + """ + if use_amp: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() diff --git a/pytorch_lightning/root_module/root_module.py b/pytorch_lightning/root_module/root_module.py index d037b038..ab5275dc 100644 --- a/pytorch_lightning/root_module/root_module.py +++ b/pytorch_lightning/root_module/root_module.py @@ -1,4 +1,5 @@ import warnings +import collections from argparse import Namespace import torch @@ -113,6 +114,35 @@ class LightningModule(GradInformation, ModelIO, ModelHooks): # clear gradients optimizer.zero_grad() + def tbptt_split_batch(self, batch, split_size): + """ + Return list of batch splits. Each split will be passed to forward_step to enable truncated + back propagation through time. The default implementation splits root level Tensors and + Sequences at dim=1 (i.e. time dim). It assumes that each time dim is the same length. + :return: + """ + time_dims = [len(x[0]) for x in batch if isinstance( + x, torch.Tensor) or isinstance(x, collections.Sequence)] + assert len(time_dims) >= 1, "Unable to determine batch time dimension" + assert all(x == time_dims[0] for x in time_dims), "Batch time dimension length is ambiguous" + + splits = [] + for t in range(0, time_dims[0], split_size): + batch_split = [] + for i, x in enumerate(batch): + if isinstance(x, torch.Tensor): + split_x = x[:, t:t + split_size] + elif isinstance(x, collections.Sequence): + split_x = [None] * len(x) + for batch_idx in range(len(x)): + split_x[batch_idx] = x[batch_idx][t:t + split_size] + + batch_split.append(split_x) + + splits.append(batch_split) + + return splits + @data_loader def tng_dataloader(self): """ diff --git a/pytorch_lightning/testing/lm_test_module_base.py b/pytorch_lightning/testing/lm_test_module_base.py index 9a6e0ee6..89bc3f29 100644 --- a/pytorch_lightning/testing/lm_test_module_base.py +++ b/pytorch_lightning/testing/lm_test_module_base.py @@ -4,12 +4,16 @@ from collections import OrderedDict import torch import torch.nn as nn import torch.nn.functional as F -from test_tube import HyperOptArgumentParser from torch import optim from torch.utils.data import DataLoader from torch.utils.data.distributed import DistributedSampler from torchvision import transforms from torchvision.datasets import MNIST +try: + from test_tube import HyperOptArgumentParser +except ImportError: + # TODO: this should be discussed and moved out of this package + raise ImportError('Missing test-tube package.') from pytorch_lightning import data_loader from pytorch_lightning.root_module.root_module import LightningModule diff --git a/pytorch_lightning/trainer/data_loading_mixin.py b/pytorch_lightning/trainer/data_loading_mixin.py index 066c21dc..7755bc7d 100644 --- a/pytorch_lightning/trainer/data_loading_mixin.py +++ b/pytorch_lightning/trainer/data_loading_mixin.py @@ -15,8 +15,13 @@ except ImportError: class TrainerDataLoadingMixin(object): - - def layout_bookeeping(self): + def init_train_dataloader(self, model): + """ + Dataloaders are provided by the model + :param model: + :return: + """ + self.get_train_dataloader = model.train_dataloader # determine number of training batches if isinstance(self.get_train_dataloader(), IterableDataset): @@ -25,21 +30,6 @@ class TrainerDataLoadingMixin(object): self.nb_training_batches = len(self.get_train_dataloader()) self.nb_training_batches = int(self.nb_training_batches * self.train_percent_check) - # determine number of validation batches - # val datasets could be none, 1 or 2+ - if self.get_val_dataloaders() is not None: - self.nb_val_batches = sum(len(dataloader) for dataloader in self.get_val_dataloaders()) - self.nb_val_batches = int(self.nb_val_batches * self.val_percent_check) - self.nb_val_batches = max(1, self.nb_val_batches) - - # determine number of test batches - if self.get_test_dataloaders() is not None: - self.nb_test_batches = sum( - len(dataloader) for dataloader in self.get_test_dataloaders() - ) - self.nb_test_batches = int(self.nb_test_batches * self.test_percent_check) - self.nb_test_batches = max(1, self.nb_test_batches) - # determine when to check validation # if int passed in, val checks that often # otherwise, it checks in [0, 1.0] % range of a training epoch @@ -49,86 +39,123 @@ class TrainerDataLoadingMixin(object): self.val_check_batch = int(self.nb_training_batches * self.val_check_interval) self.val_check_batch = max(1, self.val_check_batch) + on_ddp = self.use_ddp or self.use_ddp2 + if on_ddp and not isinstance(self.get_train_dataloader().sampler, DistributedSampler): + msg = """ + You're using multiple gpus and multiple nodes without using a DistributedSampler + to assign a subset of your data to each process. To silence this warning, pass a + DistributedSampler to your DataLoader. + + ie: this: + dataset = myDataset() + dataloader = Dataloader(dataset) + + becomes: + dataset = myDataset() + dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) + dataloader = Dataloader(dataset, sampler=dist_sampler) + + If you want each process to load the full dataset, ignore this warning. + """ + if msg not in self.shown_warnings and self.proc_rank == 0: + self.shown_warnings.add(msg) + warnings.warn(msg) + + def init_val_dataloader(self, model): + """ + Dataloaders are provided by the model + :param model: + :return: + """ + self.get_val_dataloaders = model.val_dataloader + + # determine number of validation batches + # val datasets could be none, 1 or 2+ + if self.get_val_dataloaders() is not None: + self.nb_val_batches = sum(len(dataloader) for dataloader in self.get_val_dataloaders()) + self.nb_val_batches = int(self.nb_val_batches * self.val_percent_check) + self.nb_val_batches = max(1, self.nb_val_batches) + + on_ddp = self.use_ddp or self.use_ddp2 + if on_ddp and self.get_val_dataloaders() is not None: + for dataloader in self.get_val_dataloaders(): + if not isinstance(dataloader.sampler, DistributedSampler): + msg = """ + Your val_dataloader(s) don't use DistributedSampler. + + You're using multiple gpus and multiple nodes without using a + DistributedSampler to assign a subset of your data to each process. + To silence this warning, pass a DistributedSampler to your DataLoader. + + ie: this: + dataset = myDataset() + dataloader = Dataloader(dataset) + + becomes: + dataset = myDataset() + dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) + dataloader = Dataloader(dataset, sampler=dist_sampler) + + If you want each process to load the full dataset, ignore this warning. + """ + if msg not in self.shown_warnings and self.proc_rank == 0: + self.shown_warnings.add(msg) + warnings.warn(msg) + break + + def init_test_dataloader(self, model): + """ + Dataloaders are provided by the model + :param model: + :return: + """ + + self.get_test_dataloaders = model.test_dataloader + + # determine number of test batches + if self.get_test_dataloaders() is not None: + len_sum = sum(len(dataloader) for dataloader in self.get_test_dataloaders()) + self.nb_test_batches = len_sum + self.nb_test_batches = int(self.nb_test_batches * self.test_percent_check) + self.nb_test_batches = max(1, self.nb_test_batches) + + on_ddp = self.use_ddp or self.use_ddp2 + if on_ddp and self.get_test_dataloaders() is not None: + for dataloader in self.get_test_dataloaders(): + if not isinstance(dataloader.sampler, DistributedSampler): + msg = """ + Your test_dataloader(s) don't use DistributedSampler. + + You're using multiple gpus and multiple nodes without using a + DistributedSampler to assign a subset of your data to each process. + To silence this warning, pass a DistributedSampler to your DataLoader. + + ie: this: + dataset = myDataset() + dataloader = Dataloader(dataset) + + becomes: + dataset = myDataset() + dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) + dataloader = Dataloader(dataset, sampler=dist_sampler) + + If you want each process to load the full dataset, ignore this warning. + """ + if msg not in self.shown_warnings and self.proc_rank == 0: + self.shown_warnings.add(msg) + warnings.warn(msg) + break + def get_dataloaders(self, model): """ Dataloaders are provided by the model :param model: :return: """ - self.get_train_dataloader = model.train_dataloader - self.get_test_dataloaders = model.test_dataloader - self.get_val_dataloaders = model.val_dataloader - # call warnings from proc zero only which triggers dataloaders - # if those have to download data it will only happen on proc 0 - if self.proc_rank == 0: - on_ddp = self.use_ddp or self.use_ddp2 - if on_ddp and not isinstance(self.get_train_dataloader().sampler, DistributedSampler): - msg = """ - You're using multiple gpus and multiple nodes without using a DistributedSampler - to assign a subset of your data to each process. To silence this warning, pass a - DistributedSampler to your DataLoader. - - ie: this: - dataset = myDataset() - dataloader = Dataloader(dataset) - - becomes: - dataset = myDataset() - dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) - dataloader = Dataloader(dataset, sampler=dist_sampler) - - If you want each process to load the full dataset, ignore this warning. - """ - warnings.warn(msg) - - if on_ddp and self.get_val_dataloaders() is not None: - for dataloader in self.get_val_dataloaders(): - if not isinstance(dataloader.sampler, DistributedSampler): - msg = """ - Your val_dataloader(s) don't use DistributedSampler. - - You're using multiple gpus and multiple nodes without using a - DistributedSampler to assign a subset of your data to each process. - To silence this warning, pass a DistributedSampler to your DataLoader. - - ie: this: - dataset = myDataset() - dataloader = Dataloader(dataset) - - becomes: - dataset = myDataset() - dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) - dataloader = Dataloader(dataset, sampler=dist_sampler) - - If you want each process to load the full dataset, ignore this warning. - """ - warnings.warn(msg) - break - - if on_ddp and self.get_test_dataloaders() is not None: - for dataloader in self.get_test_dataloaders(): - if not isinstance(dataloader.sampler, DistributedSampler): - msg = """ - Your test_dataloader(s) don't use DistributedSampler. - - You're using multiple gpus and multiple nodes without using a - DistributedSampler to assign a subset of your data to each process. - To silence this warning, pass a DistributedSampler to your DataLoader. - - ie: this: - dataset = myDataset() - dataloader = Dataloader(dataset) - - becomes: - dataset = myDataset() - dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) - dataloader = Dataloader(dataset, sampler=dist_sampler) - - If you want each process to load the full dataset, ignore this warning. - """ - warnings.warn(msg) - break + self.init_train_dataloader(model) + self.init_test_dataloader(model) + self.init_val_dataloader(model) if self.use_ddp or self.use_ddp2: # wait for all processes to catch up @@ -147,7 +174,7 @@ class TrainerDataLoadingMixin(object): Trainer(val_check_interval) must be an int. An int k specifies checking validation every k training batches ''' - raise MisconfigurationException('when using ') + raise MisconfigurationException(m) def determine_data_use_amount(self, train_percent_check, val_percent_check, test_percent_check, overfit_pct): diff --git a/pytorch_lightning/trainer/evaluation_loop_mixin.py b/pytorch_lightning/trainer/evaluation_loop_mixin.py index c3cc51ef..c2ba5d16 100644 --- a/pytorch_lightning/trainer/evaluation_loop_mixin.py +++ b/pytorch_lightning/trainer/evaluation_loop_mixin.py @@ -1,4 +1,5 @@ import torch +import tqdm from pytorch_lightning.utilities.debugging import MisconfigurationException @@ -52,8 +53,11 @@ class TrainerEvaluationLoopMixin(object): dl_outputs.append(output) # batch done - if self.show_progress_bar: - self.progress_bar.update(1) + if test: + self.test_progress_bar.update(1) + else: + self.val_progress_bar.update(1) + self.main_progress_bar.update(1) outputs.append(dl_outputs) eval_results = {} @@ -110,12 +114,22 @@ class TrainerEvaluationLoopMixin(object): if self.fast_dev_run: max_batches = 1 + # init validation or test progress bar + # main progress bar will already be closed when testing so initial position is free + position = 2 * self.process_position + (not test) + desc = 'Testing' if test else 'Validating' + pbar = tqdm.tqdm(desc=desc, total=max_batches, leave=test, position=position, + disable=not self.show_progress_bar, dynamic_ncols=True, + unit='batch') + setattr(self, f'{"test" if test else "val"}_progress_bar', pbar) + # run evaluation eval_results = self.evaluate(self.model, dataloaders, max_batches, test) - _, prog_bar_metrics, log_metrics, callback_metrics = self.process_output(eval_results) + _, prog_bar_metrics, log_metrics, callback_metrics, _ = self.process_output( + eval_results) # add metrics to prog bar self.add_tqdm_metrics(prog_bar_metrics) @@ -129,10 +143,16 @@ class TrainerEvaluationLoopMixin(object): # hook model.on_post_performance_check() - if self.show_progress_bar: - # add model specific metrics - tqdm_metrics = self.training_tqdm_dict - self.progress_bar.set_postfix(**tqdm_metrics) + # add model specific metrics + tqdm_metrics = self.training_tqdm_dict + if not test: + self.main_progress_bar.set_postfix(**tqdm_metrics) + + # close progress bar + if test: + self.test_progress_bar.close() + else: + self.val_progress_bar.close() # model checkpointing if self.proc_rank == 0 and self.checkpoint_callback is not None and not test: diff --git a/pytorch_lightning/trainer/logging_mixin.py b/pytorch_lightning/trainer/logging_mixin.py index f437ee99..236e512d 100644 --- a/pytorch_lightning/trainer/logging_mixin.py +++ b/pytorch_lightning/trainer/logging_mixin.py @@ -64,7 +64,7 @@ class TrainerLoggingMixin(object): # all keys not progress_bar or log are candidates for callbacks callback_metrics = {} for k, v in output.items(): - if k not in ['progress_bar', 'log']: + if k not in ['progress_bar', 'log', 'hiddens']: callback_metrics[k] = v if train and (self.use_dp or self.use_ddp2): @@ -126,6 +126,11 @@ class TrainerLoggingMixin(object): if self.use_dp or self.use_ddp2: loss = self.reduce_distributed_output(loss, self.num_gpus) + # --------------- + # EXTRACT HIDDEN + # --------------- + hiddens = output.get('hiddens') + # use every metric passed in as a candidate for callback callback_metrics.update(progress_bar_metrics) callback_metrics.update(log_metrics) @@ -135,7 +140,7 @@ class TrainerLoggingMixin(object): if isinstance(v, torch.Tensor): callback_metrics[k] = v.item() - return loss, progress_bar_metrics, log_metrics, callback_metrics + return loss, progress_bar_metrics, log_metrics, callback_metrics, hiddens def reduce_distributed_output(self, output, nb_gpus): if nb_gpus <= 1: diff --git a/pytorch_lightning/trainer/train_loop_mixin.py b/pytorch_lightning/trainer/train_loop_mixin.py index 1dad28ae..d41fc7c1 100644 --- a/pytorch_lightning/trainer/train_loop_mixin.py +++ b/pytorch_lightning/trainer/train_loop_mixin.py @@ -1,4 +1,5 @@ import numpy as np +import tqdm try: from apex import amp @@ -23,21 +24,32 @@ class TrainerTrainLoopMixin(object): # update training progress in trainer and model model.current_epoch = epoch_nb self.current_epoch = epoch_nb - self.total_batches = self.nb_training_batches + self.nb_val_batches + + # val can be checked multiple times in epoch + is_val_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0 + val_checks_per_epoch = self.nb_training_batches // self.val_check_batch + val_checks_per_epoch = val_checks_per_epoch if is_val_epoch else 0 + + # total batches includes multiple val checks + self.total_batches = (self.nb_training_batches + + self.nb_val_batches * val_checks_per_epoch) self.batch_loss_value = 0 # accumulated grads - # limit the number of batches to 1 in fast_dev_run if self.fast_dev_run: - self.total_batches = 1 - - # init progress_bar when requested - if self.show_progress_bar: + # limit the number of batches to 2 (1 train and 1 val) in fast_dev_run + nb_iterations = 2 + elif self.is_iterable_train_dataloader: + # for iterable train loader, the progress bar never ends + nb_iterations = None + else: nb_iterations = self.total_batches - # for iterable train loader, the progress bar never ends - if self.is_iterable_train_dataloader: - nb_iterations = float('inf') - self.progress_bar.reset(nb_iterations) + # reset progress bar + # .reset() doesn't work on disabled progress bar so we should check + if not self.main_progress_bar.disable: + self.main_progress_bar.reset(nb_iterations) + desc = f'Epoch {epoch_nb + 1}' if not self.is_iterable_train_dataloader else '' + self.main_progress_bar.set_description(desc) # changing gradient according accumulation_scheduler self.accumulation_scheduler.on_epoch_begin(epoch_nb, self) @@ -60,8 +72,11 @@ class TrainerTrainLoopMixin(object): # stop training stop = should_stop and met_min_epochs if stop: + self.main_progress_bar.close() return + self.main_progress_bar.close() + if self.logger is not None: self.logger.finalize("success") @@ -150,87 +165,95 @@ class TrainerTrainLoopMixin(object): if response == -1: return -1, grad_norm_dic - if self.show_progress_bar: - self.progress_bar.update(1) + splits = [batch] + if self.truncated_bptt_steps is not None: + model_ref = self.get_model() + splits = model_ref.tbptt_split_batch(batch, self.truncated_bptt_steps) - # call training_step once per optimizer - for opt_idx, optimizer in enumerate(self.optimizers): + self.hiddens = None + for split_nb, split_batch in enumerate(splits): + self.split_nb = split_nb - # wrap the forward step in a closure so second order methods work - def optimizer_closure(): - # forward pass - output = self.training_forward(batch, batch_nb, opt_idx) - closure_loss, progress_bar_metrics, log_metrics, callback_metrics = output + # call training_step once per optimizer + for opt_idx, optimizer in enumerate(self.optimizers): - # track metrics for callbacks - all_callback_metrics.append(callback_metrics) + # wrap the forward step in a closure so second order methods work + def optimizer_closure(): + # forward pass + output = self.training_forward( + split_batch, batch_nb, opt_idx, self.hiddens) - # track progress bar metrics - self.add_tqdm_metrics(progress_bar_metrics) - all_log_metrics.append(log_metrics) + closure_loss = output[0] + progress_bar_metrics = output[1] + log_metrics = output[2] + callback_metrics = output[3] + self.hiddens = output[4] - # accumulate loss - # (if accumulate_grad_batches = 1 no effect) - closure_loss = closure_loss / self.accumulate_grad_batches + # track metrics for callbacks + all_callback_metrics.append(callback_metrics) - # backward pass - if self.use_amp: - with amp.scale_loss(closure_loss, optimizer) as scaled_loss: - scaled_loss.backward() - else: - closure_loss.backward() + # track progress bar metrics + self.add_tqdm_metrics(progress_bar_metrics) + all_log_metrics.append(log_metrics) - # insert after step hook - if self.is_function_implemented('on_after_backward'): + # accumulate loss + # (if accumulate_grad_batches = 1 no effect) + closure_loss = closure_loss / self.accumulate_grad_batches + + # backward pass model_ref = self.get_model() - model_ref.on_after_backward() + model_ref.backward(self.use_amp, closure_loss, optimizer) - return closure_loss + # insert after step hook + if self.is_function_implemented('on_after_backward'): + model_ref = self.get_model() + model_ref.on_after_backward() - # calculate loss - loss = optimizer_closure() + return closure_loss - # nan grads - if self.print_nan_grads: - self.print_nan_gradients() + # calculate loss + loss = optimizer_closure() - # track total loss for logging (avoid mem leaks) - self.batch_loss_value += loss.item() + # nan grads + if self.print_nan_grads: + self.print_nan_gradients() - # gradient update with accumulated gradients - if (self.batch_nb + 1) % self.accumulate_grad_batches == 0: + # track total loss for logging (avoid mem leaks) + self.batch_loss_value += loss.item() - # track gradient norms when requested - if batch_nb % self.row_log_interval == 0: - if self.track_grad_norm > 0: - model = self.get_model() - grad_norm_dic = model.grad_norm(self.track_grad_norm) + # gradient update with accumulated gradients + if (self.batch_nb + 1) % self.accumulate_grad_batches == 0: - # clip gradients - self.clip_gradients() + # track gradient norms when requested + if batch_nb % self.row_log_interval == 0: + if self.track_grad_norm > 0: + model = self.get_model() + grad_norm_dic = model.grad_norm( + self.track_grad_norm) - # calls .step(), .zero_grad() - # override function to modify this behavior - model = self.get_model() - model.optimizer_step(self.current_epoch, batch_nb, - optimizer, opt_idx, optimizer_closure) + # clip gradients + self.clip_gradients() - # calculate running loss for display - self.running_loss.append(self.batch_loss_value) - self.batch_loss_value = 0 - self.avg_loss = np.mean(self.running_loss[-100:]) + # calls .step(), .zero_grad() + # override function to modify this behavior + model = self.get_model() + model.optimizer_step(self.current_epoch, batch_nb, + optimizer, opt_idx, optimizer_closure) - # update progress bar - if self.show_progress_bar: - # add model specific metrics - tqdm_metrics = self.training_tqdm_dict - self.progress_bar.set_postfix(**tqdm_metrics) + # calculate running loss for display + self.running_loss.append(self.batch_loss_value) + self.batch_loss_value = 0 + self.avg_loss = np.mean(self.running_loss[-100:]) # activate batch end hook if self.is_function_implemented('on_batch_end'): model = self.get_model() model.on_batch_end() + # update progress bar + self.main_progress_bar.update(1) + self.main_progress_bar.set_postfix(**self.training_tqdm_dict) + # collapse all metrics into one dict all_log_metrics = {k: v for d in all_log_metrics for k, v in d.items()} @@ -239,7 +262,7 @@ class TrainerTrainLoopMixin(object): return 0, grad_norm_dic, all_log_metrics - def training_forward(self, batch, batch_nb, opt_idx): + def training_forward(self, batch, batch_nb, opt_idx, hiddens): """ Handle forward for each training case (distributed, single gpu, etc...) :param batch: @@ -254,6 +277,9 @@ class TrainerTrainLoopMixin(object): if len(self.optimizers) > 1: args.append(opt_idx) + if self.truncated_bptt_steps is not None: + args.append(hiddens) + if self.use_ddp or self.use_ddp2: output = self.model(*args) elif self.use_dp: @@ -271,5 +297,4 @@ class TrainerTrainLoopMixin(object): # format and reduce outputs accordingly output = self.process_output(output, train=True) - loss, progress_bar_metrics, log_metrics, callback_metrics = output - return loss, progress_bar_metrics, log_metrics, callback_metrics + return output diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 19f31034..c53aeb7b 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -80,7 +80,8 @@ class Trainer(TrainerIOMixin, weights_summary='full', weights_save_path=None, amp_level='O1', - nb_sanity_val_steps=5): + nb_sanity_val_steps=5, + truncated_bptt_steps=None): """ :param logger: Logger for experiment tracking @@ -116,6 +117,7 @@ class Trainer(TrainerIOMixin, :param weights_save_path: Bool. Where to save weights if on cluster :param amp_level: str. Check nvidia docs for level :param nb_sanity_val_steps: int. How many val steps before a full train loop. + :param truncated_bptt_steps: int. Enables multiple backward passes for each batch. """ # Transfer params self.nb_gpu_nodes = nb_gpu_nodes @@ -135,6 +137,8 @@ class Trainer(TrainerIOMixin, self.min_nb_epochs = min_nb_epochs self.nb_sanity_val_steps = nb_sanity_val_steps self.print_nan_grads = print_nan_grads + self.truncated_bptt_steps = truncated_bptt_steps + self.shown_warnings = set() self.fast_dev_run = fast_dev_run if self.fast_dev_run: @@ -292,10 +296,12 @@ class Trainer(TrainerIOMixin, """ tqdm_dict = { 'loss': '{0:.3f}'.format(self.avg_loss), - 'epoch': '{}'.format(self.current_epoch), 'batch_nb': '{}'.format(self.batch_nb), } + if self.truncated_bptt_steps is not None: + tqdm_dict['split_nb'] = self.split_nb + if self.logger is not None and self.logger.version is not None: tqdm_dict['v_nb'] = self.logger.version @@ -410,9 +416,6 @@ class Trainer(TrainerIOMixin, # transfer data loaders from model self.get_dataloaders(ref_model) - # init training constants - self.layout_bookeeping() - # print model summary if self.proc_rank == 0 and self.weights_summary is not None: if self.weights_summary in ['full', 'top']: @@ -428,10 +431,6 @@ class Trainer(TrainerIOMixin, # restore training and model before hpc call self.restore_weights(model) - # progress bar init - if self.show_progress_bar: - self.progress_bar = tqdm.tqdm(0, position=self.process_position) - # when testing requested only run test and return if self.testing: self.run_evaluation(test=True) @@ -441,12 +440,25 @@ class Trainer(TrainerIOMixin, # to make sure program won't crash during val ref_model.on_sanity_check_start() if self.get_val_dataloaders() is not None and self.nb_sanity_val_steps > 0: - # reset progress_bar limit for sanity check - if self.show_progress_bar: - self.progress_bar.reset(self.nb_sanity_val_steps) + # init progress bars for validation sanity check + pbar = tqdm.tqdm(desc='Validation sanity check', total=self.nb_sanity_val_steps, + leave=False, position=2 * self.process_position, + disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch') + self.main_progress_bar = pbar + # dummy validation progress bar + self.val_progress_bar = tqdm.tqdm(disable=True) self.evaluate(model, self.get_val_dataloaders(), self.nb_sanity_val_steps, self.testing) + # close progress bars + self.main_progress_bar.close() + self.val_progress_bar.close() + + # init progress bar + pbar = tqdm.tqdm(leave=True, position=2 * self.process_position, + disable=not self.show_progress_bar, dynamic_ncols=True, unit='batch') + self.main_progress_bar = pbar + # clear cache before training if self.on_gpu: torch.cuda.empty_cache() diff --git a/pytorch_lightning/trainer/trainer_io.py b/pytorch_lightning/trainer/trainer_io.py index 5b483760..dd38c051 100644 --- a/pytorch_lightning/trainer/trainer_io.py +++ b/pytorch_lightning/trainer/trainer_io.py @@ -32,10 +32,17 @@ class TrainerIOMixin(object): :param model: :return: """ + # clear cache before restore + if self.on_gpu: + torch.cuda.empty_cache() # if script called from hpc resubmit, load weights did_restore_hpc_weights = self.restore_hpc_weights_if_needed(model) + # clear cache after restore + if self.on_gpu: + torch.cuda.empty_cache() + if not did_restore_hpc_weights: # restore weights if same exp version self.restore_state_if_checkpoint_exists(model) @@ -137,7 +144,13 @@ class TrainerIOMixin(object): checkpoint = self.dump_checkpoint() # do the actual save - torch.save(checkpoint, filepath) + try: + torch.save(checkpoint, filepath) + except AttributeError: + if 'hparams' in checkpoint: + del checkpoint['hparams'] + + torch.save(checkpoint, filepath) def restore(self, checkpoint_path, on_gpu): @@ -283,7 +296,14 @@ class TrainerIOMixin(object): model.on_hpc_save(checkpoint) # do the actual save - torch.save(checkpoint, filepath) + # TODO: fix for anything with multiprocess DP, DDP, DDP2 + try: + torch.save(checkpoint, filepath) + except AttributeError: + if 'hparams' in checkpoint: + del checkpoint['hparams'] + + torch.save(checkpoint, filepath) return filepath diff --git a/requirements.txt b/requirements.txt index 4bb6acd6..62acf5ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,5 @@ twine==1.13.0 numpy==1.16.4 torch>=1.2.0 torchvision>=0.3.0 -pandas +pandas>=0.20.3 +# future>=0.17.1 # required for buildins in setup.py \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 068788dc..f56376af 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,12 +11,14 @@ markers = slow remote_data filterwarnings + gpus_param_tests [pycodestyle] ignore = E731,W504 max-line-length = 120 [coverage:report] +# TODO: this looks suspicion, it should be reviewed exclude_lines = pragma: no cover def __repr__ @@ -39,7 +41,6 @@ exclude_lines = break pass os.makedirs - omit = pytorch_lightning/callbacks/pt_callbacks.py tests/test_models.py @@ -48,5 +49,12 @@ omit = examples/templates [flake8] -ignore = E731,W504,F401,F841 +# TODO: this should be 88 or 100 according PEP8 max-line-length = 120 +exclude = .tox,*.egg,build,temp,examples/* +select = E,W,F +doctests = True +verbose = 2 +# https://pep8.readthedocs.io/en/latest/intro.html#error-codes +format = pylint +ignore = E731,W504,F401,F841 diff --git a/setup.py b/setup.py index 883e71cd..2c57b0b1 100755 --- a/setup.py +++ b/setup.py @@ -1,12 +1,37 @@ #!/usr/bin/env python +import os +from io import open # Always prefer setuptools over distutils from setuptools import setup, find_packages -# https://packaging.python.org/guides/single-sourcing-package-version/ +try: + import builtins +except ImportError: + import __builtin__ as builtins +# https://packaging.python.org/guides/single-sourcing-package-version/ # http://blog.ionelmc.ro/2014/05/25/python-packaging/ +PATH_ROOT = os.path.dirname(__file__) +builtins.__LIGHTNING_SETUP__ = True + +import pytorch_lightning # noqa: E402 + + +def load_requirements(path_dir=PATH_ROOT, comment_char='#'): + with open(os.path.join(path_dir, 'requirements.txt'), 'r') as file: + lines = [ln.strip() for ln in file.readlines()] + reqs = [] + for ln in lines: + # filer all comments + if comment_char in ln: + ln = ln[:ln.index(comment_char)] + if ln: # if requirement is not empty + reqs.append(ln) + return reqs + + # https://packaging.python.org/discussions/install-requires-vs-requirements / # keep the meta-data here for simplicity in reading this file... it's not obvious # what happens and to non-engineers they won't know to look in init ... @@ -14,26 +39,22 @@ from setuptools import setup, find_packages # engineer specific practices setup( name='pytorch-lightning', - version='0.5.2.1', - description='The Keras for ML researchers using PyTorch', - author='William Falcon', - author_email='waf2107@columbia.edu', - url='https://github.com/williamFalcon/pytorch-lightning', + version=pytorch_lightning.__version__, + description=pytorch_lightning.__docs__, + author=pytorch_lightning.__author__, + author_email=pytorch_lightning.__author_email__, + url=pytorch_lightning.__homepage__, download_url='https://github.com/williamFalcon/pytorch-lightning', - license='Apache-2', - packages=find_packages(), + license=pytorch_lightning.__license__, + packages=find_packages(exclude=['examples']), long_description=open('README.md', encoding='utf-8').read(), long_description_content_type='text/markdown', include_package_data=True, zip_safe=False, keywords=['deep learning', 'pytorch', 'AI'], python_requires='>=3.6', - install_requires=[ - 'torch>=1.2.0', - 'tqdm>=4.35.0', - 'test-tube>=0.6.9', - 'pandas>=0.20.3', - ], + setup_requires=[], + install_requires=load_requirements(PATH_ROOT), classifiers=[ 'Environment :: Console', 'Natural Language :: English', diff --git a/tests/test_cpu_models.py b/tests/test_cpu_models.py index 2c2be069..62f30404 100644 --- a/tests/test_cpu_models.py +++ b/tests/test_cpu_models.py @@ -3,7 +3,7 @@ import warnings import pytest import torch -from pytorch_lightning import Trainer +from pytorch_lightning import Trainer, data_loader from pytorch_lightning.callbacks import ( EarlyStopping, ) @@ -292,6 +292,81 @@ def test_all_features_cpu_model(): testing_utils.run_gpu_model_test(trainer_options, model, hparams, on_gpu=False) +def test_tbptt_cpu_model(): + """ + Test truncated back propagation through time works. + :return: + """ + testing_utils.reset_seed() + + truncated_bptt_steps = 2 + sequence_size = 30 + batch_size = 30 + + x_seq = torch.rand(batch_size, sequence_size, 1) + y_seq_list = torch.rand(batch_size, sequence_size, 1).tolist() + + class MockSeq2SeqDataset(torch.utils.data.Dataset): + def __getitem__(self, i): + return x_seq, y_seq_list + + def __len__(self): + return 1 + + class BpttTestModel(LightningTestModelBase): + def __init__(self, hparams): + super().__init__(hparams) + self.test_hidden = None + + def training_step(self, batch, batch_idx, hiddens): + assert hiddens == self.test_hidden, "Hidden state not persistent between tbptt steps" + self.test_hidden = torch.rand(1) + + x_tensor, y_list = batch + assert x_tensor.shape[1] == truncated_bptt_steps, "tbptt split Tensor failed" + + y_tensor = torch.tensor(y_list, dtype=x_tensor.dtype) + assert y_tensor.shape[1] == truncated_bptt_steps, "tbptt split list failed" + + pred = self.forward(x_tensor.view(batch_size, truncated_bptt_steps)) + loss_val = torch.nn.functional.mse_loss( + pred, y_tensor.view(batch_size, truncated_bptt_steps)) + return { + 'loss': loss_val, + 'hiddens': self.test_hidden, + } + + @data_loader + def train_dataloader(self): + return torch.utils.data.DataLoader( + dataset=MockSeq2SeqDataset(), + batch_size=batch_size, + shuffle=False, + sampler=None, + ) + + trainer_options = dict( + max_nb_epochs=1, + truncated_bptt_steps=truncated_bptt_steps, + val_percent_check=0, + weights_summary=None, + ) + + hparams = testing_utils.get_hparams() + hparams.batch_size = batch_size + hparams.in_features = truncated_bptt_steps + hparams.hidden_dim = truncated_bptt_steps + hparams.out_features = truncated_bptt_steps + + model = BpttTestModel(hparams) + + # fit model + trainer = Trainer(**trainer_options) + result = trainer.fit(model) + + assert result == 1, 'training failed to complete' + + def test_single_gpu_model(): """ Make sure single GPU works (DP mode) diff --git a/tox.ini b/tox.ini index b3a62978..6d0c3171 100644 --- a/tox.ini +++ b/tox.ini @@ -12,36 +12,40 @@ # and also to help confirm pull requests to this project. [tox] -envlist = py{35,36,37} +envlist = py{35,36,37,38} -[pytest] -log_cli = 0 -log_cli_level = CRITICAL -log_cli_format = %(message)s -log_file = pytest.log -log_file_level = DEBUG -log_file_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) -log_file_date_format = %Y-%m-%d %H:%M:%S +# DROP, it is duplication of setup.cfg +# [pytest] +# log_cli = 0 +# log_cli_level = CRITICAL +# log_cli_format = %(message)s +# log_file = pytest.log +# log_file_level = DEBUG +# log_file_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s) +# log_file_date_format=%Y-%m-%d %H:%M:%S [testenv] basepython = py35: python3.5 py36: python3.6 py37: python3.7 + py38: python3.8 deps = -r requirements.txt -r ./tests/requirements.txt commands = + pip list check-manifest --ignore tox.ini - python setup.py check -m -s - flake8 . + python setup.py check --metadata --strict coverage run --source pytorch_lightning -m py.test pytorch_lightning tests pl_examples -v --doctest-modules + flake8 . -[flake8] -exclude = .tox,*.egg,build,temp,examples/* -select = E,W,F -doctests = True -verbose = 2 +# DROP, it is duplication of setup.cfg +# [flake8] +# exclude = .tox,*.egg,build,temp,examples/* +# select = E,W,F +# doctests = True +# verbose = 2 # https://pep8.readthedocs.io/en/latest/intro.html#error-codes -format = pylint -max-line-length = 100 +# format = pylint +# max-line-length = 100