diff --git a/Trainer/index.html b/Trainer/index.html
index 177927f4..81e0a48f 100644
--- a/Trainer/index.html
+++ b/Trainer/index.html
@@ -553,6 +553,7 @@ trainer.fit(model)
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers">Learning rate scheduling</a></li>
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers">Use multiple optimizers (like GANs)</a></li>
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check">Set how much of the training set to check (1-100%)</a></li>
+<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step">Step optimizers at arbitrary intervals</a></li>
 </ul>
 <p><strong>Validation loop</strong>    </p>
 <ul>
diff --git a/index.html b/index.html
index 29477a89..1e227aa9 100644
--- a/index.html
+++ b/index.html
@@ -732,6 +732,7 @@ one could be a seq-2-seq model, both (optionally) ran by the same trainer file.<
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers">Learning rate scheduling</a></li>
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers">Use multiple optimizers (like GANs)</a></li>
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check">Set how much of the training set to check (1-100%)</a></li>
+<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step">Step optimizers at arbitrary intervals</a></li>
 </ul>
 <h6 id="validation-loop">Validation loop</h6>
 <ul>
diff --git a/search/search_index.json b/search/search_index.json
index db48b97f..7e639974 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"New project Quick Start To start a new project you define two files, a LightningModule and a Trainer file. A separate trainer file allows to run many LightningModules. Each LightningModule has the core logic to a particular research project. For example, one lightningModule could be an image classifier, the other one could be a seq-2-seq model, both (optionally) ran by the same trainer file. MNIST LightningModule Trainer Basic CPU Trainer Template Multi-GPU Trainer Template GPU cluster Trainer Template Docs shortcuts LightningModule Trainer Quick start examples CPU example Hyperparameter search on single GPU Hyperparameter search on multiple GPUs on same node Hyperparameter search on a SLURM HPC cluster Checkpointing Model saving Model loading Restoring training session Computing cluster (SLURM) Running grid search on a cluster Walltime auto-resubmit Debugging Fast dev run Inspect gradient norms Log GPU usage Make model overfit on subset of data Print the parameter count by layer Pring which gradients are nan Print input and output size of every module in system Distributed training 16-bit mixed precision Multi-GPU Multi-node Single GPU Self-balancing architecture Experiment Logging Display metrics in progress bar Log metric row every k batches Process position Tensorboard support Save a snapshot of all hyperparameters Snapshot code for a training run Write logs file to csv every k batches Training loop Accumulate gradients Force training for min or max epochs Force disable early stop Gradient Clipping Hooks Learning rate scheduling Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%) Validation loop Check validation every n epochs Hooks Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps","title":"Home"},{"location":"#new-project-quick-start","text":"To start a new project you define two files, a LightningModule and a Trainer file. A separate trainer file allows to run many LightningModules. Each LightningModule has the core logic to a particular research project. For example, one lightningModule could be an image classifier, the other one could be a seq-2-seq model, both (optionally) ran by the same trainer file. MNIST LightningModule Trainer Basic CPU Trainer Template Multi-GPU Trainer Template GPU cluster Trainer Template","title":"New project Quick Start"},{"location":"#docs-shortcuts","text":"LightningModule Trainer","title":"Docs shortcuts"},{"location":"#quick-start-examples","text":"CPU example Hyperparameter search on single GPU Hyperparameter search on multiple GPUs on same node Hyperparameter search on a SLURM HPC cluster","title":"Quick start examples"},{"location":"#checkpointing","text":"Model saving Model loading Restoring training session","title":"Checkpointing"},{"location":"#computing-cluster-slurm","text":"Running grid search on a cluster Walltime auto-resubmit","title":"Computing cluster (SLURM)"},{"location":"#debugging","text":"Fast dev run Inspect gradient norms Log GPU usage Make model overfit on subset of data Print the parameter count by layer Pring which gradients are nan Print input and output size of every module in system","title":"Debugging"},{"location":"#distributed-training","text":"16-bit mixed precision Multi-GPU Multi-node Single GPU Self-balancing architecture","title":"Distributed training"},{"location":"#experiment-logging","text":"Display metrics in progress bar Log metric row every k batches Process position Tensorboard support Save a snapshot of all hyperparameters Snapshot code for a training run Write logs file to csv every k batches","title":"Experiment Logging"},{"location":"#training-loop","text":"Accumulate gradients Force training for min or max epochs Force disable early stop Gradient Clipping Hooks Learning rate scheduling Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%)","title":"Training loop"},{"location":"#validation-loop","text":"Check validation every n epochs Hooks Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps","title":"Validation loop"},{"location":"LightningModule/RequiredTrainerInterface/","text":"Lightning Module interface [ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. The easiest thing to do is copy the minimal example below and modify accordingly. Otherwise, to Define a Lightning Module, implement the following methods: Required : training_step tng_dataloader configure_optimizers Optional : - validation_step - validation_end - val_dataloader - test_dataloader on_save_checkpoint on_load_checkpoint update_tng_log_metrics add_model_specific_args Minimal example import os import torch from torch.nn import functional as F from torch.utils.data import DataLoader from torchvision.datasets import MNIST import torchvision.transforms as transforms import pytorch_lightning as pl class CoolModel(pl.LightningModule): def __init__(self): super(CoolModel, self).__init__() # not the best model... self.l1 = torch.nn.Linear(28 * 28, 10) def forward(self, x): return torch.relu(self.l1(x.view(x.size(0), -1))) def training_step(self, batch, batch_nb): # REQUIRED x, y = batch y_hat = self.forward(x) return {'loss': F.cross_entropy(y_hat, y)(y_hat, y)} def validation_step(self, batch, batch_nb): # OPTIONAL x, y = batch y_hat = self.forward(x) return {'val_loss': F.cross_entropy(y_hat, y)(y_hat, y)} def validation_end(self, outputs): # OPTIONAL avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() return {'avg_val_loss': avg_loss} def configure_optimizers(self): # REQUIRED return [torch.optim.Adam(self.parameters(), lr=0.02)] @pl.data_loader def tng_dataloader(self): return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) @pl.data_loader def val_dataloader(self): # OPTIONAL # can also return a list of val dataloaders return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) @pl.data_loader def test_dataloader(self): # OPTIONAL return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) How do these methods fit into the broader training? The LightningModule interface is on the right. Each method corresponds to a part of a research project. Lightning automates everything not in blue. Required Methods training_step def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output tng_dataloader @pl.data_loader def tng_dataloader(self) Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. Return PyTorch DataLoader Example @pl.data_loader def tng_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader configure_optimizers def configure_optimizers(self) Set up as many optimizers and (optionally) learning rate schedulers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one in every epoch. If you use 16 bit precision it will also handle that. Return List or Tuple - List of optimizers with an optional second list of learning-rate schedulers Example # most cases def configure_optimizers(self): opt = Adam(self.parameters(), lr=0.01) return [opt] # gan example, with scheduler for discriminator def configure_optimizers(self): generator_opt = Adam(self.model_gen.parameters(), lr=0.01) disriminator_opt = Adam(self.model_disc.parameters(), lr=0.02) discriminator_sched = CosineAnnealing(discriminator_opt, T_max=10) return [generator_opt, disriminator_opt], [discriminator_sched] If you need to control how often those optimizers step or override the default .step() schedule, override the optimizer_step hook. Optional Methods validation_step def validation_step(self, data_batch, batch_nb) # if have multiple val dataloaders: def validation_step(self, data_batch, batch_nb, dataloader_idx) OPTIONAL If you don't need to validate you don't need to implement this method. In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes, calculate accuracy, or save example outputs (using self.experiment or whatever you want). Really, anything you want. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is dataloader_i Integer displaying which dataloader this is (only if multiple val datasets used) Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example # CASE 1: A single validation dataset def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output If you pass in multiple validation datasets, validation_step will have an additional argument. # CASE 2: multiple validation datasets def validation_step(self, data_batch, batch_nb, dataset_idx): # dataset_idx tells you which dataset this is. The dataset_idx corresponds to the order of datasets returned in val_dataloader . validation_end def validation_end(self, outputs) If you didn't define a validation_step, this won't be called. Called at the end of the validation loop with the output of each validation_step. Called once per validation dataset. The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic on_save_checkpoint def on_save_checkpoint(self, checkpoint) Called by lightning to checkpoint your model. Lightning saves the training state (current epoch, global_step, etc) and also saves the model state_dict. If you want to save anything else, use this method to add your own key-value pair. Return Nothing Example def on_save_checkpoint(self, checkpoint): # 99% of use cases you don't need to implement this method checkpoint['something_cool_i_want_to_save'] = my_cool_pickable_object on_load_checkpoint def on_load_checkpoint(self, checkpoint) Called by lightning to restore your model. Lighting auto-restores global step, epoch, etc... It also restores the model state_dict. If you saved something with on_save_checkpoint this is your chance to restore this. Return Nothing Example def on_load_checkpoint(self, checkpoint): # 99% of the time you don't need to implement this method self.something_cool_i_want_to_save = checkpoint['something_cool_i_want_to_save'] val_dataloader @pl.data_loader def tng_dataloader(self) OPTIONAL If you don't need a validation dataset and a validation_step, you don't need to implement this method. Called by lightning during validation loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. Return PyTorch DataLoader or list of PyTorch Dataloaders. Example @pl.data_loader def val_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader # can also return multiple dataloaders @pl.data_loader def val_dataloader(self): return [loader_a, loader_b, ..., loader_n] In the case where you return multiple val_dataloaders, the validation_step will have an arguement dataset_idx which matches the order here. test_dataloader @pl.data_loader def test_dataloader(self) OPTIONAL If you don't need a test dataset and a test_step, you don't need to implement this method. Called by lightning during test loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. Return PyTorch DataLoader Example @pl.data_loader def test_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader update_tng_log_metrics def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged. Return Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs add_model_specific_args @staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The hyperparameter argument parser is available anywhere in your model by calling self.hparams. Return An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Lightning Module interface"},{"location":"LightningModule/RequiredTrainerInterface/#lightning-module-interface","text":"[ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. The easiest thing to do is copy the minimal example below and modify accordingly. Otherwise, to Define a Lightning Module, implement the following methods: Required : training_step tng_dataloader configure_optimizers Optional : - validation_step - validation_end - val_dataloader - test_dataloader on_save_checkpoint on_load_checkpoint update_tng_log_metrics add_model_specific_args","title":"Lightning Module interface"},{"location":"LightningModule/RequiredTrainerInterface/#minimal-example","text":"import os import torch from torch.nn import functional as F from torch.utils.data import DataLoader from torchvision.datasets import MNIST import torchvision.transforms as transforms import pytorch_lightning as pl class CoolModel(pl.LightningModule): def __init__(self): super(CoolModel, self).__init__() # not the best model... self.l1 = torch.nn.Linear(28 * 28, 10) def forward(self, x): return torch.relu(self.l1(x.view(x.size(0), -1))) def training_step(self, batch, batch_nb): # REQUIRED x, y = batch y_hat = self.forward(x) return {'loss': F.cross_entropy(y_hat, y)(y_hat, y)} def validation_step(self, batch, batch_nb): # OPTIONAL x, y = batch y_hat = self.forward(x) return {'val_loss': F.cross_entropy(y_hat, y)(y_hat, y)} def validation_end(self, outputs): # OPTIONAL avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() return {'avg_val_loss': avg_loss} def configure_optimizers(self): # REQUIRED return [torch.optim.Adam(self.parameters(), lr=0.02)] @pl.data_loader def tng_dataloader(self): return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) @pl.data_loader def val_dataloader(self): # OPTIONAL # can also return a list of val dataloaders return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) @pl.data_loader def test_dataloader(self): # OPTIONAL return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)","title":"Minimal example"},{"location":"LightningModule/RequiredTrainerInterface/#how-do-these-methods-fit-into-the-broader-training","text":"The LightningModule interface is on the right. Each method corresponds to a part of a research project. Lightning automates everything not in blue.","title":"How do these methods fit into the broader training?"},{"location":"LightningModule/RequiredTrainerInterface/#required-methods","text":"","title":"Required Methods"},{"location":"LightningModule/RequiredTrainerInterface/#training_step","text":"def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output","title":"training_step"},{"location":"LightningModule/RequiredTrainerInterface/#tng_dataloader","text":"@pl.data_loader def tng_dataloader(self) Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.","title":"tng_dataloader"},{"location":"LightningModule/RequiredTrainerInterface/#return","text":"PyTorch DataLoader Example @pl.data_loader def tng_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#configure_optimizers","text":"def configure_optimizers(self) Set up as many optimizers and (optionally) learning rate schedulers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one in every epoch. If you use 16 bit precision it will also handle that.","title":"configure_optimizers"},{"location":"LightningModule/RequiredTrainerInterface/#return_1","text":"List or Tuple - List of optimizers with an optional second list of learning-rate schedulers Example # most cases def configure_optimizers(self): opt = Adam(self.parameters(), lr=0.01) return [opt] # gan example, with scheduler for discriminator def configure_optimizers(self): generator_opt = Adam(self.model_gen.parameters(), lr=0.01) disriminator_opt = Adam(self.model_disc.parameters(), lr=0.02) discriminator_sched = CosineAnnealing(discriminator_opt, T_max=10) return [generator_opt, disriminator_opt], [discriminator_sched] If you need to control how often those optimizers step or override the default .step() schedule, override the optimizer_step hook.","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#optional-methods","text":"","title":"Optional Methods"},{"location":"LightningModule/RequiredTrainerInterface/#validation_step","text":"def validation_step(self, data_batch, batch_nb) # if have multiple val dataloaders: def validation_step(self, data_batch, batch_nb, dataloader_idx) OPTIONAL If you don't need to validate you don't need to implement this method. In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes, calculate accuracy, or save example outputs (using self.experiment or whatever you want). Really, anything you want. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is dataloader_i Integer displaying which dataloader this is (only if multiple val datasets used) Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example # CASE 1: A single validation dataset def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output If you pass in multiple validation datasets, validation_step will have an additional argument. # CASE 2: multiple validation datasets def validation_step(self, data_batch, batch_nb, dataset_idx): # dataset_idx tells you which dataset this is. The dataset_idx corresponds to the order of datasets returned in val_dataloader .","title":"validation_step"},{"location":"LightningModule/RequiredTrainerInterface/#validation_end","text":"def validation_end(self, outputs) If you didn't define a validation_step, this won't be called. Called at the end of the validation loop with the output of each validation_step. Called once per validation dataset. The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic","title":"validation_end"},{"location":"LightningModule/RequiredTrainerInterface/#on_save_checkpoint","text":"def on_save_checkpoint(self, checkpoint) Called by lightning to checkpoint your model. Lightning saves the training state (current epoch, global_step, etc) and also saves the model state_dict. If you want to save anything else, use this method to add your own key-value pair.","title":"on_save_checkpoint"},{"location":"LightningModule/RequiredTrainerInterface/#return_2","text":"Nothing Example def on_save_checkpoint(self, checkpoint): # 99% of use cases you don't need to implement this method checkpoint['something_cool_i_want_to_save'] = my_cool_pickable_object","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#on_load_checkpoint","text":"def on_load_checkpoint(self, checkpoint) Called by lightning to restore your model. Lighting auto-restores global step, epoch, etc... It also restores the model state_dict. If you saved something with on_save_checkpoint this is your chance to restore this.","title":"on_load_checkpoint"},{"location":"LightningModule/RequiredTrainerInterface/#return_3","text":"Nothing Example def on_load_checkpoint(self, checkpoint): # 99% of the time you don't need to implement this method self.something_cool_i_want_to_save = checkpoint['something_cool_i_want_to_save']","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#val_dataloader","text":"@pl.data_loader def tng_dataloader(self) OPTIONAL If you don't need a validation dataset and a validation_step, you don't need to implement this method. Called by lightning during validation loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.","title":"val_dataloader"},{"location":"LightningModule/RequiredTrainerInterface/#return_4","text":"PyTorch DataLoader or list of PyTorch Dataloaders. Example @pl.data_loader def val_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader # can also return multiple dataloaders @pl.data_loader def val_dataloader(self): return [loader_a, loader_b, ..., loader_n] In the case where you return multiple val_dataloaders, the validation_step will have an arguement dataset_idx which matches the order here.","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#test_dataloader","text":"@pl.data_loader def test_dataloader(self) OPTIONAL If you don't need a test dataset and a test_step, you don't need to implement this method. Called by lightning during test loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.","title":"test_dataloader"},{"location":"LightningModule/RequiredTrainerInterface/#return_5","text":"PyTorch DataLoader Example @pl.data_loader def test_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#update_tng_log_metrics","text":"def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged.","title":"update_tng_log_metrics"},{"location":"LightningModule/RequiredTrainerInterface/#return_6","text":"Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#add_model_specific_args","text":"@staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The hyperparameter argument parser is available anywhere in your model by calling self.hparams.","title":"add_model_specific_args"},{"location":"LightningModule/RequiredTrainerInterface/#return_7","text":"An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Return"},{"location":"LightningModule/methods/","text":"Lightning modules are strict superclasses of torch.nn.Module. A LightningModule offers the following in addition to that API. freeze Freeze all params for inference model = MyLightningModule(...) model.freeze() load_from_metrics This is the easiest/fastest way which uses the meta_tags.csv file from test-tube to rebuild the model. The meta_tags.csv file can be found in the test-tube experiment save_dir. pretrained_model = MyLightningModule.load_from_metrics( weights_path='/path/to/pytorch_checkpoint.ckpt', tags_csv='/path/to/test_tube/experiment/version/meta_tags.csv', on_gpu=True, map_location=None ) # predict pretrained_model.eval() pretrained_model.freeze() y_hat = pretrained_model(x) Params Param description weights_path Path to a PyTorch checkpoint tags_csv Path to meta_tags.csv file generated by the test-tube Experiment on_gpu if True, puts model on GPU. Make sure to use transforms option if model devices have changed map_location A dictionary mapping saved weight GPU devices to new GPU devices Returns LightningModule - The pretrained LightningModule unfreeze Unfreeze all params for inference model = MyLightningModule(...) model.unfreeze()","title":"Methods"},{"location":"LightningModule/methods/#freeze","text":"Freeze all params for inference model = MyLightningModule(...) model.freeze()","title":"freeze"},{"location":"LightningModule/methods/#load_from_metrics","text":"This is the easiest/fastest way which uses the meta_tags.csv file from test-tube to rebuild the model. The meta_tags.csv file can be found in the test-tube experiment save_dir. pretrained_model = MyLightningModule.load_from_metrics( weights_path='/path/to/pytorch_checkpoint.ckpt', tags_csv='/path/to/test_tube/experiment/version/meta_tags.csv', on_gpu=True, map_location=None ) # predict pretrained_model.eval() pretrained_model.freeze() y_hat = pretrained_model(x) Params Param description weights_path Path to a PyTorch checkpoint tags_csv Path to meta_tags.csv file generated by the test-tube Experiment on_gpu if True, puts model on GPU. Make sure to use transforms option if model devices have changed map_location A dictionary mapping saved weight GPU devices to new GPU devices Returns LightningModule - The pretrained LightningModule","title":"load_from_metrics"},{"location":"LightningModule/methods/#unfreeze","text":"Unfreeze all params for inference model = MyLightningModule(...) model.unfreeze()","title":"unfreeze"},{"location":"LightningModule/properties/","text":"A LightningModule has the following properties which you can access at any time current_epoch The current epoch dtype Current dtype experiment An instance of test-tube Experiment which you can use to log anything for tensorboarX. self.experiment.add_embedding(...) self.experiment.log({'val_loss': 0.9}) self.experiment.add_scalars(...) global_step Total training batches seen across all epochs gradient_clip The current gradient clip value on_gpu True if your model is currently running on GPUs. Useful to set flags around the LightningModule for different CPU vs GPU behavior. trainer Last resort access to any state the trainer has. Changing certain properties here could affect your training run. self.trainer.optimizers self.trainer.current_epoch ... Debugging The LightningModule also offers these tricks to help debug. example_input_array In the LightningModule init, you can set a dummy tensor for this property to get a print out of sizes coming into and out of every layer. def __init__(self): # put the dimensions of the first input to your system self.example_input_array = torch.rand(5, 28 * 28)","title":"Properties"},{"location":"LightningModule/properties/#current_epoch","text":"The current epoch","title":"current_epoch"},{"location":"LightningModule/properties/#dtype","text":"Current dtype","title":"dtype"},{"location":"LightningModule/properties/#experiment","text":"An instance of test-tube Experiment which you can use to log anything for tensorboarX. self.experiment.add_embedding(...) self.experiment.log({'val_loss': 0.9}) self.experiment.add_scalars(...)","title":"experiment"},{"location":"LightningModule/properties/#global_step","text":"Total training batches seen across all epochs","title":"global_step"},{"location":"LightningModule/properties/#gradient_clip","text":"The current gradient clip value","title":"gradient_clip"},{"location":"LightningModule/properties/#on_gpu","text":"True if your model is currently running on GPUs. Useful to set flags around the LightningModule for different CPU vs GPU behavior.","title":"on_gpu"},{"location":"LightningModule/properties/#trainer","text":"Last resort access to any state the trainer has. Changing certain properties here could affect your training run. self.trainer.optimizers self.trainer.current_epoch ...","title":"trainer"},{"location":"LightningModule/properties/#debugging","text":"The LightningModule also offers these tricks to help debug.","title":"Debugging"},{"location":"LightningModule/properties/#example_input_array","text":"In the LightningModule init, you can set a dummy tensor for this property to get a print out of sizes coming into and out of every layer. def __init__(self): # put the dimensions of the first input to your system self.example_input_array = torch.rand(5, 28 * 28)","title":"example_input_array"},{"location":"Trainer/","text":"Trainer [ Github Code ] The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. This is the basic use of the trainer: from pytorch_lightning import Trainer model = LightningTemplate() trainer = Trainer() trainer.fit(model) But of course the fun is in all the advanced things it can do: Checkpointing Model saving Model loading Restoring training session Computing cluster (SLURM) Running grid search on a cluster Walltime auto-resubmit Debugging Fast dev run Inspect gradient norms Log GPU usage Make model overfit on subset of data Print the parameter count by layer Pring which gradients are nan Print input and output size of every module in system Distributed training 16-bit mixed precision Multi-GPU Multi-node Single GPU Self-balancing architecture Experiment Logging Display metrics in progress bar Log metric row every k batches Process position Tensorboard support Save a snapshot of all hyperparameters Snapshot code for a training run Write logs file to csv every k batches Training loop Accumulate gradients Force training for min or max epochs Force disable early stop Gradient Clipping Hooks Learning rate scheduling Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%) Validation loop Check validation every n epochs Hooks Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps","title":"Trainer"},{"location":"Trainer/#trainer","text":"[ Github Code ] The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. This is the basic use of the trainer: from pytorch_lightning import Trainer model = LightningTemplate() trainer = Trainer() trainer.fit(model) But of course the fun is in all the advanced things it can do: Checkpointing Model saving Model loading Restoring training session Computing cluster (SLURM) Running grid search on a cluster Walltime auto-resubmit Debugging Fast dev run Inspect gradient norms Log GPU usage Make model overfit on subset of data Print the parameter count by layer Pring which gradients are nan Print input and output size of every module in system Distributed training 16-bit mixed precision Multi-GPU Multi-node Single GPU Self-balancing architecture Experiment Logging Display metrics in progress bar Log metric row every k batches Process position Tensorboard support Save a snapshot of all hyperparameters Snapshot code for a training run Write logs file to csv every k batches Training loop Accumulate gradients Force training for min or max epochs Force disable early stop Gradient Clipping Hooks Learning rate scheduling Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%) Validation loop Check validation every n epochs Hooks Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps","title":"Trainer"},{"location":"Trainer/Checkpointing/","text":"i Lightning can automate saving and loading checkpoints. Model saving To enable checkpointing, define the checkpoint callback and give it to the trainer. from pytorch_lightning.callbacks import ModelCheckpoint checkpoint_callback = ModelCheckpoint( filepath='/path/to/store/weights.ckpt', save_best_only=True, verbose=True, monitor='val_loss', mode='min' ) trainer = Trainer(checkpoint_callback=checkpoint_callback) Restoring training session You might want to not only load a model but also continue training it. Use this method to restore the trainer state as well. This will continue from the epoch and global step you last left off. However, the dataloaders will start from the first batch again (if you shuffled it shouldn't matter). Lightning will restore the session if you pass an experiment with the same version and there's a saved checkpoint. from test_tube import Experiment exp = Experiment(version=a_previous_version_with_a_saved_checkpoint) trainer = Trainer(experiment=exp) # this fit call loads model weights and trainer state # the trainer continues seamlessly from where you left off # without having to do anything else. trainer.fit(model) The trainer restores: - global_step - current_epoch - All optimizers - All lr_schedulers - Model weights You can even change the logic of your model as long as the weights and \"architecture\" of the system isn't different. If you add a layer, for instance, it might not work. At a rough level, here's what happens inside Trainer : self.global_step = checkpoint['global_step'] self.current_epoch = checkpoint['epoch'] # restore the optimizers optimizer_states = checkpoint['optimizer_states'] for optimizer, opt_state in zip(self.optimizers, optimizer_states): optimizer.load_state_dict(opt_state) # restore the lr schedulers lr_schedulers = checkpoint['lr_schedulers'] for scheduler, lrs_state in zip(self.lr_schedulers, lr_schedulers): scheduler.load_state_dict(lrs_state) # uses the model you passed into trainer model.load_state_dict(checkpoint['state_dict'])","title":"Checkpointing"},{"location":"Trainer/Checkpointing/#model-saving","text":"To enable checkpointing, define the checkpoint callback and give it to the trainer. from pytorch_lightning.callbacks import ModelCheckpoint checkpoint_callback = ModelCheckpoint( filepath='/path/to/store/weights.ckpt', save_best_only=True, verbose=True, monitor='val_loss', mode='min' ) trainer = Trainer(checkpoint_callback=checkpoint_callback)","title":"Model saving"},{"location":"Trainer/Checkpointing/#restoring-training-session","text":"You might want to not only load a model but also continue training it. Use this method to restore the trainer state as well. This will continue from the epoch and global step you last left off. However, the dataloaders will start from the first batch again (if you shuffled it shouldn't matter). Lightning will restore the session if you pass an experiment with the same version and there's a saved checkpoint. from test_tube import Experiment exp = Experiment(version=a_previous_version_with_a_saved_checkpoint) trainer = Trainer(experiment=exp) # this fit call loads model weights and trainer state # the trainer continues seamlessly from where you left off # without having to do anything else. trainer.fit(model) The trainer restores: - global_step - current_epoch - All optimizers - All lr_schedulers - Model weights You can even change the logic of your model as long as the weights and \"architecture\" of the system isn't different. If you add a layer, for instance, it might not work. At a rough level, here's what happens inside Trainer : self.global_step = checkpoint['global_step'] self.current_epoch = checkpoint['epoch'] # restore the optimizers optimizer_states = checkpoint['optimizer_states'] for optimizer, opt_state in zip(self.optimizers, optimizer_states): optimizer.load_state_dict(opt_state) # restore the lr schedulers lr_schedulers = checkpoint['lr_schedulers'] for scheduler, lrs_state in zip(self.lr_schedulers, lr_schedulers): scheduler.load_state_dict(lrs_state) # uses the model you passed into trainer model.load_state_dict(checkpoint['state_dict'])","title":"Restoring training session"},{"location":"Trainer/Distributed training/","text":"Lightning makes multi-gpu training and 16 bit training trivial. Note: None of the flags below require changing anything about your lightningModel definition. Choosing a backend Lightning supports two backends. DataParallel and DistributedDataParallel. Both can be used for single-node multi-GPU training. For multi-node training you must use DistributedDataParallel. You can toggle between each mode by setting this flag. # DEFAULT uses DataParallel trainer = Trainer(distributed_backend='dp') # change to distributed data parallel trainer = Trainer(distributed_backend='ddp') If you request multiple nodes, the back-end will auto-switch to ddp. We recommend you use DistributedDataparallel even for single-node multi-GPU training. It is MUCH faster than DP but may have configuration issues depending on your cluster. For a deeper understanding of what lightning is doing, feel free to read this guide . Distributed and 16-bit precision. Due to an issue with apex and DistributedDataParallel (PyTorch and NVIDIA issue), Lightning does not allow 16-bit and DP training. We tried to get this to work, but it's an issue on their end. Below are the possible configurations we support. 1 GPU 1+ GPUs DP DDP 16-bit command Y Trainer(gpus=[0]) Y Y Trainer(gpus=[0], use_amp=True) Y Y Trainer(gpus=[0, ...]) Y Y Trainer(gpus=[0, ...], distributed_backend='ddp') Y Y Y Trainer(gpus=[0, ...], distributed_backend='ddp', use_amp=True) CUDA flags CUDA flags make certain GPUs visible to your script. Lightning sets these for you automatically, there's NO NEED to do this yourself. # lightning will set according to what you give the trainer # os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\" # os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\" 16-bit mixed precision 16 bit precision can cut your memory footprint by half. If using volta architecture GPUs it can give a dramatic training speed-up as well. First, install apex (if install fails, look here ): $ git clone https://github.com/NVIDIA/apex $ cd apex $ pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./ then set this use_amp to True. # DEFAULT trainer = Trainer(amp_level='O2', use_amp=False) Single-gpu Make sure you're on a GPU machine. # DEFAULT trainer = Trainer(gpus=[0]) multi-gpu Make sure you're on a GPU machine. You can set as many GPUs as you want. In this setting, the model will run on all 8 GPUs at once using DataParallel under the hood. # to use DataParallel (default) trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='dp') # RECOMMENDED use DistributedDataParallel trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='ddp') Multi-node Multi-node training is easily done by specifying these flags. # train on 12*8 GPUs trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], nb_gpu_nodes=12) In addition, make sure to set up your SLURM job correctly via the SlurmClusterObject . In particular, specify the number of tasks per node correctly. cluster = SlurmCluster( hyperparam_optimizer=test_tube.HyperOptArgumentParser(), log_path='/some/path/to/save', ) # OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT # which interface your nodes use for communication cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo') # see output of the NCCL connection process # NCCL is how the nodes talk to each other cluster.add_command('export NCCL_DEBUG=INFO') # setting a master port here is a good idea. cluster.add_command('export MASTER_PORT=%r' % PORT) # good to load the latest NCCL version cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0']) # configure cluster cluster.per_experiment_nb_nodes = 12 cluster.per_experiment_nb_gpus = 8 cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu') Finally, make sure to add a distributed sampler to your dataset. The distributed sampler copies a portion of your dataset onto each GPU. (World_size = gpus_per_node * nb_nodes). # ie: this: dataset = myDataset() dataloader = Dataloader(dataset) # becomes: dataset = myDataset() dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) dataloader = Dataloader(dataset, sampler=dist_sampler) Self-balancing architecture Here lightning distributes parts of your module across available GPUs to optimize for speed and memory. COMING SOON.","title":"Distributed training"},{"location":"Trainer/Distributed training/#choosing-a-backend","text":"Lightning supports two backends. DataParallel and DistributedDataParallel. Both can be used for single-node multi-GPU training. For multi-node training you must use DistributedDataParallel. You can toggle between each mode by setting this flag. # DEFAULT uses DataParallel trainer = Trainer(distributed_backend='dp') # change to distributed data parallel trainer = Trainer(distributed_backend='ddp') If you request multiple nodes, the back-end will auto-switch to ddp. We recommend you use DistributedDataparallel even for single-node multi-GPU training. It is MUCH faster than DP but may have configuration issues depending on your cluster. For a deeper understanding of what lightning is doing, feel free to read this guide .","title":"Choosing a backend"},{"location":"Trainer/Distributed training/#distributed-and-16-bit-precision","text":"Due to an issue with apex and DistributedDataParallel (PyTorch and NVIDIA issue), Lightning does not allow 16-bit and DP training. We tried to get this to work, but it's an issue on their end. Below are the possible configurations we support. 1 GPU 1+ GPUs DP DDP 16-bit command Y Trainer(gpus=[0]) Y Y Trainer(gpus=[0], use_amp=True) Y Y Trainer(gpus=[0, ...]) Y Y Trainer(gpus=[0, ...], distributed_backend='ddp') Y Y Y Trainer(gpus=[0, ...], distributed_backend='ddp', use_amp=True)","title":"Distributed and 16-bit precision."},{"location":"Trainer/Distributed training/#cuda-flags","text":"CUDA flags make certain GPUs visible to your script. Lightning sets these for you automatically, there's NO NEED to do this yourself. # lightning will set according to what you give the trainer # os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\" # os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"","title":"CUDA flags"},{"location":"Trainer/Distributed training/#16-bit-mixed-precision","text":"16 bit precision can cut your memory footprint by half. If using volta architecture GPUs it can give a dramatic training speed-up as well. First, install apex (if install fails, look here ): $ git clone https://github.com/NVIDIA/apex $ cd apex $ pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./ then set this use_amp to True. # DEFAULT trainer = Trainer(amp_level='O2', use_amp=False)","title":"16-bit mixed precision"},{"location":"Trainer/Distributed training/#single-gpu","text":"Make sure you're on a GPU machine. # DEFAULT trainer = Trainer(gpus=[0])","title":"Single-gpu"},{"location":"Trainer/Distributed training/#multi-gpu","text":"Make sure you're on a GPU machine. You can set as many GPUs as you want. In this setting, the model will run on all 8 GPUs at once using DataParallel under the hood. # to use DataParallel (default) trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='dp') # RECOMMENDED use DistributedDataParallel trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='ddp')","title":"multi-gpu"},{"location":"Trainer/Distributed training/#multi-node","text":"Multi-node training is easily done by specifying these flags. # train on 12*8 GPUs trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], nb_gpu_nodes=12) In addition, make sure to set up your SLURM job correctly via the SlurmClusterObject . In particular, specify the number of tasks per node correctly. cluster = SlurmCluster( hyperparam_optimizer=test_tube.HyperOptArgumentParser(), log_path='/some/path/to/save', ) # OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT # which interface your nodes use for communication cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo') # see output of the NCCL connection process # NCCL is how the nodes talk to each other cluster.add_command('export NCCL_DEBUG=INFO') # setting a master port here is a good idea. cluster.add_command('export MASTER_PORT=%r' % PORT) # good to load the latest NCCL version cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0']) # configure cluster cluster.per_experiment_nb_nodes = 12 cluster.per_experiment_nb_gpus = 8 cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu') Finally, make sure to add a distributed sampler to your dataset. The distributed sampler copies a portion of your dataset onto each GPU. (World_size = gpus_per_node * nb_nodes). # ie: this: dataset = myDataset() dataloader = Dataloader(dataset) # becomes: dataset = myDataset() dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) dataloader = Dataloader(dataset, sampler=dist_sampler)","title":"Multi-node"},{"location":"Trainer/Distributed training/#self-balancing-architecture","text":"Here lightning distributes parts of your module across available GPUs to optimize for speed and memory. COMING SOON.","title":"Self-balancing architecture"},{"location":"Trainer/Logging/","text":"Lighting offers a few options for logging information about model, gpu usage, etc (via test-tube). It also offers printing options for training monitoring. Display metrics in progress bar # DEFAULT trainer = Trainer(progress_bar=True) Log metric row every k batches Every k batches lightning will make an entry in the metrics log # DEFAULT (ie: save a .csv log file every 10 batches) trainer = Trainer(add_log_row_interval=10) Process position When running multiple models on the same machine we want to decide which progress bar to use. Lightning will stack progress bars according to this value. # DEFAULT trainer = Trainer(process_position=0) # if this is the second model on the node, show the second progress bar below trainer = Trainer(process_position=1) Save a snapshot of all hyperparameters Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use. Give lightning a test-tube Experiment object to automate this for you. from test_tube import Experiment exp = Experiment(...) Trainer(experiment=exp) Snapshot code for a training run Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag. Give lightning a test-tube Experiment object to automate this for you. from test_tube import Experiment exp = Experiment(create_git_tag=True) Trainer(experiment=exp) Tensorboard support In the LightningModule you can access the experiment logger by doing: self.experiment # add image # Look at PyTorch SummaryWriter docs for what you can do. self.experiment.add_image(...) The experiment object is a strict subclass of PyTorch SummaryWriter. However, this class also snapshots every detail about the experiment (data folder paths, code, hyperparams), and allows you to visualize it using tensorboard. from test_tube import Experiment, HyperOptArgumentParser # exp hyperparams args = HyperOptArgumentParser() hparams = args.parse_args() # this is a summaryWriter with nicer logging structure exp = Experiment(save_dir='/some/path', create_git_tag=True) # track experiment details (must be ArgumentParser or HyperOptArgumentParser). # each option in the parser is tracked exp.argparse(hparams) exp.tag({'description': 'running demo'}) # trainer uses the exp object to log exp data trainer = Trainer(experiment=exp) trainer.fit(model) # view logs at: # tensorboard --logdir /some/path Write logs file to csv every k batches Every k batches, lightning will write the new logs to disk # DEFAULT (ie: save a .csv log file every 100 batches) trainer = Trainer(log_save_interval=100)","title":"Logging"},{"location":"Trainer/Logging/#display-metrics-in-progress-bar","text":"# DEFAULT trainer = Trainer(progress_bar=True)","title":"Display metrics in progress bar"},{"location":"Trainer/Logging/#log-metric-row-every-k-batches","text":"Every k batches lightning will make an entry in the metrics log # DEFAULT (ie: save a .csv log file every 10 batches) trainer = Trainer(add_log_row_interval=10)","title":"Log metric row every k batches"},{"location":"Trainer/Logging/#process-position","text":"When running multiple models on the same machine we want to decide which progress bar to use. Lightning will stack progress bars according to this value. # DEFAULT trainer = Trainer(process_position=0) # if this is the second model on the node, show the second progress bar below trainer = Trainer(process_position=1)","title":"Process position"},{"location":"Trainer/Logging/#save-a-snapshot-of-all-hyperparameters","text":"Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use. Give lightning a test-tube Experiment object to automate this for you. from test_tube import Experiment exp = Experiment(...) Trainer(experiment=exp)","title":"Save a snapshot of all hyperparameters"},{"location":"Trainer/Logging/#snapshot-code-for-a-training-run","text":"Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag. Give lightning a test-tube Experiment object to automate this for you. from test_tube import Experiment exp = Experiment(create_git_tag=True) Trainer(experiment=exp)","title":"Snapshot code for a training run"},{"location":"Trainer/Logging/#tensorboard-support","text":"In the LightningModule you can access the experiment logger by doing: self.experiment # add image # Look at PyTorch SummaryWriter docs for what you can do. self.experiment.add_image(...) The experiment object is a strict subclass of PyTorch SummaryWriter. However, this class also snapshots every detail about the experiment (data folder paths, code, hyperparams), and allows you to visualize it using tensorboard. from test_tube import Experiment, HyperOptArgumentParser # exp hyperparams args = HyperOptArgumentParser() hparams = args.parse_args() # this is a summaryWriter with nicer logging structure exp = Experiment(save_dir='/some/path', create_git_tag=True) # track experiment details (must be ArgumentParser or HyperOptArgumentParser). # each option in the parser is tracked exp.argparse(hparams) exp.tag({'description': 'running demo'}) # trainer uses the exp object to log exp data trainer = Trainer(experiment=exp) trainer.fit(model) # view logs at: # tensorboard --logdir /some/path","title":"Tensorboard support"},{"location":"Trainer/Logging/#write-logs-file-to-csv-every-k-batches","text":"Every k batches, lightning will write the new logs to disk # DEFAULT (ie: save a .csv log file every 100 batches) trainer = Trainer(log_save_interval=100)","title":"Write logs file to csv every k batches"},{"location":"Trainer/SLURM Managed Cluster/","text":"Lightning supports model training on a cluster managed by SLURM in the following cases: Training on single or multi-cpus only. Training on single or multi-gpus on the same node. Coming SOON: Training across multiple nodes. Running grid search on a cluster To use lightning to run a hyperparameter search (grid-search or random-search) on a cluster do 4 things: (1). Define the parameters for the grid search from test_tube import HyperOptArgumentParser # subclass of argparse parser = HyperOptArgumentParser(strategy='random_search') parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate') # let's enable optimizing over the number of layers in the network parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8]) hparams = parser.parse_args() (2). Define the cluster options in the SlurmCluster object (over 5 nodes and 8 gpus) from test_tube.hpc import SlurmCluster # hyperparameters is a test-tube hyper params object # see https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/ hyperparams = args.parse() # init cluster cluster = SlurmCluster( hyperparam_optimizer=hyperparams, log_path='/path/to/log/results/to', python_cmd='python3' ) # let the cluster know where to email for a change in job status (ie: complete, fail, etc...) cluster.notify_job_status(email='some@email.com', on_done=True, on_fail=True) # set the job options. In this instance, we'll run 20 different models # each with its own set of hyperparameters giving each one 1 GPU (ie: taking up 20 GPUs) cluster.per_experiment_nb_gpus = 8 cluster.per_experiment_nb_nodes = 5 # we'll request 10GB of memory per node cluster.memory_mb_per_node = 10000 # set a walltime of 10 minues cluster.job_time = '10:00' (3). Give trainer the cluster_manager in your main function: from pytorch_lightning import Trainer def train_fx(trial_hparams, cluster_manager, _): # hparams has a specific set of hyperparams my_model = MyLightningModel() # give the trainer the cluster object trainer = Trainer(cluster=cluster_manager) trainer.fit(my_model) (4). Start the grid search # run the models on the cluster cluster.optimize_parallel_cluster_gpu( train_fx, nb_trials=20, job_name='my_grid_search_exp_name', job_display_name='my_exp') That's it! The SlurmCluster object will automatically checkpoint the lightning model and resubmit if it runs into the walltime! Walltime auto-resubmit Lightning automatically resubmits jobs when they reach the walltime. You get this behavior for free if you give lightning a slurm cluster object. def my_main_fx(hparams, slurm_manager, _): trainer = Trainer(cluster=slurm_manager) (See the grid search example above for cluster configuration). With this feature lightning will: automatically checkpoint the model checkpoint the trainer session resubmit a continuation job. load the checkpoint and trainer session in the new model","title":"SLURM Managed Cluster"},{"location":"Trainer/SLURM Managed Cluster/#running-grid-search-on-a-cluster","text":"To use lightning to run a hyperparameter search (grid-search or random-search) on a cluster do 4 things: (1). Define the parameters for the grid search from test_tube import HyperOptArgumentParser # subclass of argparse parser = HyperOptArgumentParser(strategy='random_search') parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate') # let's enable optimizing over the number of layers in the network parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8]) hparams = parser.parse_args() (2). Define the cluster options in the SlurmCluster object (over 5 nodes and 8 gpus) from test_tube.hpc import SlurmCluster # hyperparameters is a test-tube hyper params object # see https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/ hyperparams = args.parse() # init cluster cluster = SlurmCluster( hyperparam_optimizer=hyperparams, log_path='/path/to/log/results/to', python_cmd='python3' ) # let the cluster know where to email for a change in job status (ie: complete, fail, etc...) cluster.notify_job_status(email='some@email.com', on_done=True, on_fail=True) # set the job options. In this instance, we'll run 20 different models # each with its own set of hyperparameters giving each one 1 GPU (ie: taking up 20 GPUs) cluster.per_experiment_nb_gpus = 8 cluster.per_experiment_nb_nodes = 5 # we'll request 10GB of memory per node cluster.memory_mb_per_node = 10000 # set a walltime of 10 minues cluster.job_time = '10:00' (3). Give trainer the cluster_manager in your main function: from pytorch_lightning import Trainer def train_fx(trial_hparams, cluster_manager, _): # hparams has a specific set of hyperparams my_model = MyLightningModel() # give the trainer the cluster object trainer = Trainer(cluster=cluster_manager) trainer.fit(my_model) (4). Start the grid search # run the models on the cluster cluster.optimize_parallel_cluster_gpu( train_fx, nb_trials=20, job_name='my_grid_search_exp_name', job_display_name='my_exp') That's it! The SlurmCluster object will automatically checkpoint the lightning model and resubmit if it runs into the walltime!","title":"Running grid search on a cluster"},{"location":"Trainer/SLURM Managed Cluster/#walltime-auto-resubmit","text":"Lightning automatically resubmits jobs when they reach the walltime. You get this behavior for free if you give lightning a slurm cluster object. def my_main_fx(hparams, slurm_manager, _): trainer = Trainer(cluster=slurm_manager) (See the grid search example above for cluster configuration). With this feature lightning will: automatically checkpoint the model checkpoint the trainer session resubmit a continuation job. load the checkpoint and trainer session in the new model","title":"Walltime auto-resubmit"},{"location":"Trainer/Training Loop/","text":"The lightning training loop handles everything except the actual computations of your model. To decide what will happen in your training loop, define the training_step function . Below are all the things lightning automates for you in the training loop. Accumulated gradients Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. # DEFAULT (ie: no accumulated grads) trainer = Trainer(accumulate_grad_batches=1) Force training for min or max epochs It can be useful to force training for a minimum number of epochs or limit to a max number # DEFAULT trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000) Force disable early stop Use this to turn off early stopping and run training to the max_epoch # DEFAULT trainer = Trainer(enable_early_stop=True) Gradient Clipping Gradient clipping may be enabled to avoid exploding gradients. Specifically, this will clip the gradient norm computed over all model parameters together . # DEFAULT (ie: don't clip) trainer = Trainer(gradient_clip=0) # clip gradients with norm above 0.5 trainer = Trainer(gradient_clip=0.5) Inspect gradient norms Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2) Set how much of the training set to check If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1)","title":"Training Loop"},{"location":"Trainer/Training Loop/#accumulated-gradients","text":"Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. # DEFAULT (ie: no accumulated grads) trainer = Trainer(accumulate_grad_batches=1)","title":"Accumulated gradients"},{"location":"Trainer/Training Loop/#force-training-for-min-or-max-epochs","text":"It can be useful to force training for a minimum number of epochs or limit to a max number # DEFAULT trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)","title":"Force training for min or max epochs"},{"location":"Trainer/Training Loop/#force-disable-early-stop","text":"Use this to turn off early stopping and run training to the max_epoch # DEFAULT trainer = Trainer(enable_early_stop=True)","title":"Force disable early stop"},{"location":"Trainer/Training Loop/#gradient-clipping","text":"Gradient clipping may be enabled to avoid exploding gradients. Specifically, this will clip the gradient norm computed over all model parameters together . # DEFAULT (ie: don't clip) trainer = Trainer(gradient_clip=0) # clip gradients with norm above 0.5 trainer = Trainer(gradient_clip=0.5)","title":"Gradient Clipping"},{"location":"Trainer/Training Loop/#inspect-gradient-norms","text":"Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2)","title":"Inspect gradient norms"},{"location":"Trainer/Training Loop/#set-how-much-of-the-training-set-to-check","text":"If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1)","title":"Set how much of the training set to check"},{"location":"Trainer/Validation loop/","text":"The lightning validation loop handles everything except the actual computations of your model. To decide what will happen in your validation loop, define the validation_step function . Below are all the things lightning automates for you in the validation loop. Note Lightning will run 5 steps of validation in the beginning of training as a sanity check so you don't have to wait until a full epoch to catch possible validation issues. Check validation every n epochs If you have a small dataset you might want to check validation every n epochs # DEFAULT trainer = Trainer(check_val_every_n_epoch=1) Set how much of the validation set to check If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(val_percent_check=1.0) # check 10% only trainer = Trainer(val_percent_check=0.1) Set how much of the test set to check If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(test_percent_check=1.0) # check 10% only trainer = Trainer(test_percent_check=0.1) Set validation check frequency within 1 training epoch For large datasets it's often desirable to check validation multiple times within a training loop # DEFAULT trainer = Trainer(val_check_interval=0.95) # check every .25 of an epoch trainer = Trainer(val_check_interval=0.25) Set the number of validation sanity steps Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop. # DEFAULT trainer = Trainer(nb_sanity_val_steps=5)","title":"Validation loop"},{"location":"Trainer/Validation loop/#check-validation-every-n-epochs","text":"If you have a small dataset you might want to check validation every n epochs # DEFAULT trainer = Trainer(check_val_every_n_epoch=1)","title":"Check validation every n epochs"},{"location":"Trainer/Validation loop/#set-how-much-of-the-validation-set-to-check","text":"If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(val_percent_check=1.0) # check 10% only trainer = Trainer(val_percent_check=0.1)","title":"Set how much of the validation set to check"},{"location":"Trainer/Validation loop/#set-how-much-of-the-test-set-to-check","text":"If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(test_percent_check=1.0) # check 10% only trainer = Trainer(test_percent_check=0.1)","title":"Set how much of the test set to check"},{"location":"Trainer/Validation loop/#set-validation-check-frequency-within-1-training-epoch","text":"For large datasets it's often desirable to check validation multiple times within a training loop # DEFAULT trainer = Trainer(val_check_interval=0.95) # check every .25 of an epoch trainer = Trainer(val_check_interval=0.25)","title":"Set validation check frequency within 1 training epoch"},{"location":"Trainer/Validation loop/#set-the-number-of-validation-sanity-steps","text":"Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop. # DEFAULT trainer = Trainer(nb_sanity_val_steps=5)","title":"Set the number of validation sanity steps"},{"location":"Trainer/debugging/","text":"These flags are useful to help debug a model. Fast dev run This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch. Use this to debug a full run of your program quickly # DEFAULT trainer = Trainer(fast_dev_run=False) Inspect gradient norms Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2) Make model overfit on subset of data A useful debugging trick is to make your model overfit a tiny fraction of the data. # DEFAULT don't overfit (ie: normal training) trainer = Trainer(overfit_pct=0.0) # overfit on 1% of data trainer = Trainer(overfit_pct=0.01) Print the parameter count by layer By default lightning prints a list of parameters and submodules when it starts training. Print which gradients are nan This option prints a list of tensors with nan gradients. # DEFAULT trainer = Trainer(print_nan_grads=False) Log GPU usage Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.","title":"Debugging"},{"location":"Trainer/debugging/#fast-dev-run","text":"This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch. Use this to debug a full run of your program quickly # DEFAULT trainer = Trainer(fast_dev_run=False)","title":"Fast dev run"},{"location":"Trainer/debugging/#inspect-gradient-norms","text":"Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2)","title":"Inspect gradient norms"},{"location":"Trainer/debugging/#make-model-overfit-on-subset-of-data","text":"A useful debugging trick is to make your model overfit a tiny fraction of the data. # DEFAULT don't overfit (ie: normal training) trainer = Trainer(overfit_pct=0.0) # overfit on 1% of data trainer = Trainer(overfit_pct=0.01)","title":"Make model overfit on subset of data"},{"location":"Trainer/debugging/#print-the-parameter-count-by-layer","text":"By default lightning prints a list of parameters and submodules when it starts training.","title":"Print the parameter count by layer"},{"location":"Trainer/debugging/#print-which-gradients-are-nan","text":"This option prints a list of tensors with nan gradients. # DEFAULT trainer = Trainer(print_nan_grads=False)","title":"Print which gradients are nan"},{"location":"Trainer/debugging/#log-gpu-usage","text":"Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.","title":"Log GPU usage"},{"location":"Trainer/hooks/","text":"Hooks [ Github Code ] There are cases when you might want to do something different at different parts of the training/validation loop. To enable a hook, simply override the method in your LightningModule and the trainer will call it at the correct time. Contributing If there's a hook you'd like to add, simply: 1. Fork PyTorchLightning. 2. Add the hook here . 3. Add the correct place in the Trainer where it should be called. on_epoch_start Called in the training loop at the very beginning of the epoch. def on_epoch_start(self): # do something when the epoch starts on_epoch_end Called in the training loop at the very end of the epoch. def on_epoch_end(self): # do something when the epoch ends on_batch_start Called in the training loop before anything happens for that batch. def on_batch_start(self): # do something when the batch starts on_batch_end Called in the training loop after the batch. def on_batch_end(self): # do something when the batch ends on_pre_performance_check Called at the very beginning of the validation loop. def on_pre_performance_check(self): # do something before validation starts on_post_performance_check Called at the very end of the validation loop. def on_post_performance_check(self): # do something before validation end on_tng_metrics Called in the training loop, right before metrics are logged. Although you can log at any time by using self.experiment, you can use this callback to modify what will be logged. def on_tng_metrics(self, metrics): # do something before validation end optimizer_step Calls .step() and .zero_grad for each optimizer. You can override this method to adjust how you do the optimizer step for each optimizer Called once per optimizer # DEFAULT def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i): optimizer.step() optimizer.zero_grad() # Alternating schedule for optimizer steps (ie: GANs) def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i): # update generator opt every 2 steps if optimizer_i == 0: if batch_nb % 2 == 0 : optimizer.step() optimizer.zero_grad() # update discriminator opt every 4 steps if optimizer_i == 1: if batch_nb % 4 == 0 : optimizer.step() optimizer.zero_grad() on_before_zero_grad Called in the training loop after taking an optimizer step and before zeroing grads. Good place to inspect weight information with weights updated. Called once per optimizer def on_before_zero_grad(self, optimizer): # do something with the optimizer or inspect it. on_after_backward Called in the training loop after model.backward() This is the ideal place to inspect or log gradient information def on_after_backward(self): # example to inspect gradient information in tensorboard if self.trainer.global_step % 25 == 0: # don't make the tf file huge params = self.state_dict() for k, v in params.items(): grads = v name = k self.experiment.add_histogram(tag=name, values=grads, global_step=self.trainer.global_step)","title":"Hooks"},{"location":"Trainer/hooks/#hooks","text":"[ Github Code ] There are cases when you might want to do something different at different parts of the training/validation loop. To enable a hook, simply override the method in your LightningModule and the trainer will call it at the correct time. Contributing If there's a hook you'd like to add, simply: 1. Fork PyTorchLightning. 2. Add the hook here . 3. Add the correct place in the Trainer where it should be called.","title":"Hooks"},{"location":"Trainer/hooks/#on_epoch_start","text":"Called in the training loop at the very beginning of the epoch. def on_epoch_start(self): # do something when the epoch starts","title":"on_epoch_start"},{"location":"Trainer/hooks/#on_epoch_end","text":"Called in the training loop at the very end of the epoch. def on_epoch_end(self): # do something when the epoch ends","title":"on_epoch_end"},{"location":"Trainer/hooks/#on_batch_start","text":"Called in the training loop before anything happens for that batch. def on_batch_start(self): # do something when the batch starts","title":"on_batch_start"},{"location":"Trainer/hooks/#on_batch_end","text":"Called in the training loop after the batch. def on_batch_end(self): # do something when the batch ends","title":"on_batch_end"},{"location":"Trainer/hooks/#on_pre_performance_check","text":"Called at the very beginning of the validation loop. def on_pre_performance_check(self): # do something before validation starts","title":"on_pre_performance_check"},{"location":"Trainer/hooks/#on_post_performance_check","text":"Called at the very end of the validation loop. def on_post_performance_check(self): # do something before validation end","title":"on_post_performance_check"},{"location":"Trainer/hooks/#on_tng_metrics","text":"Called in the training loop, right before metrics are logged. Although you can log at any time by using self.experiment, you can use this callback to modify what will be logged. def on_tng_metrics(self, metrics): # do something before validation end","title":"on_tng_metrics"},{"location":"Trainer/hooks/#optimizer_step","text":"Calls .step() and .zero_grad for each optimizer. You can override this method to adjust how you do the optimizer step for each optimizer Called once per optimizer # DEFAULT def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i): optimizer.step() optimizer.zero_grad() # Alternating schedule for optimizer steps (ie: GANs) def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i): # update generator opt every 2 steps if optimizer_i == 0: if batch_nb % 2 == 0 : optimizer.step() optimizer.zero_grad() # update discriminator opt every 4 steps if optimizer_i == 1: if batch_nb % 4 == 0 : optimizer.step() optimizer.zero_grad()","title":"optimizer_step"},{"location":"Trainer/hooks/#on_before_zero_grad","text":"Called in the training loop after taking an optimizer step and before zeroing grads. Good place to inspect weight information with weights updated. Called once per optimizer def on_before_zero_grad(self, optimizer): # do something with the optimizer or inspect it.","title":"on_before_zero_grad"},{"location":"Trainer/hooks/#on_after_backward","text":"Called in the training loop after model.backward() This is the ideal place to inspect or log gradient information def on_after_backward(self): # example to inspect gradient information in tensorboard if self.trainer.global_step % 25 == 0: # don't make the tf file huge params = self.state_dict() for k, v in params.items(): grads = v name = k self.experiment.add_histogram(tag=name, values=grads, global_step=self.trainer.global_step)","title":"on_after_backward"},{"location":"examples/Examples/","text":"Template model definition In 99% of cases you want to just copy this template to start a new lightningModule and change the core of what your model is actually trying to do. # get a copy of the module template wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/examples/new_project_templates/lightning_module_template.py Trainer Example __main__ function Normally, we want to let the __main__ function start the training. Inside the main we parse training arguments with whatever hyperparameters we want. Your LightningModule will have a chance to add hyperparameters. from test_tube import HyperOptArgumentParser if __name__ == '__main__': # use default args given by lightning root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0] parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False) add_default_args(parent_parser, root_dir) # allow model to overwrite or extend args parser = ExampleModel.add_model_specific_args(parent_parser) hyperparams = parser.parse_args() # train model main(hyperparams) Main Function The main function is your entry into the program. This is where you init your model, checkpoint directory, and launch the training. The main function should have 3 arguments: - hparams: a configuration of hyperparameters. - slurm_manager: Slurm cluster manager object (can be None) - dict: for you to return any values you want (useful in meta-learning, otherwise set to _) def main(hparams, cluster, results_dict): \"\"\" Main training routine specific for this project :param hparams: :return: \"\"\" # init experiment log_dir = os.path.dirname(os.path.realpath(__file__)) exp = Experiment( name='test_tube_exp', debug=True, save_dir=log_dir, version=0, autosave=False, description='test demo' ) # set the hparams for the experiment exp.argparse(hparams) exp.save() # build model model = MyLightningModule(hparams) # callbacks early_stop = EarlyStopping( monitor=hparams.early_stop_metric, patience=hparams.early_stop_patience, verbose=True, mode=hparams.early_stop_mode ) model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) checkpoint = ModelCheckpoint( filepath=model_save_path, save_function=None, save_best_only=True, verbose=True, monitor=hparams.model_save_monitor_value, mode=hparams.model_save_monitor_mode ) # configure trainer trainer = Trainer( experiment=exp, cluster=cluster, checkpoint_callback=checkpoint, early_stop_callback=early_stop, ) # train model trainer.fit(model) The main function will start training on your main function. If you use the HyperParameterOptimizer in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple argument parser you get the default arguments in the argument parser. So, calling main(hyperparams) runs the model with the default argparse arguments. main(hyperparams) CPU hyperparameter search # run a grid search over 20 hyperparameter combinations. hyperparams.optimize_parallel_cpu( main_local, nb_trials=20, nb_workers=1 ) Hyperparameter search on a single or multiple GPUs # run a grid search over 20 hyperparameter combinations. hyperparams.optimize_parallel_gpu( main_local, nb_trials=20, nb_workers=1, gpus=[0,1,2,3] ) Hyperparameter search on a SLURM HPC cluster def optimize_on_cluster(hyperparams): # enable cluster training cluster = SlurmCluster( hyperparam_optimizer=hyperparams, log_path=hyperparams.tt_save_path, test_tube_exp_name=hyperparams.tt_name ) # email for cluster coms cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True) # configure cluster cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus cluster.job_time = '48:00:00' cluster.gpu_type = '1080ti' cluster.memory_mb_per_node = 48000 # any modules for code to run in env cluster.add_command('source activate pytorch_lightning') # name of exp job_display_name = hyperparams.tt_name.split('_')[0] job_display_name = job_display_name[0:3] # run hopt print('submitting jobs...') cluster.optimize_parallel_cluster_gpu( main, nb_trials=hyperparams.nb_hopt_trials, job_name=job_display_name ) # run cluster hyperparameter search optimize_on_cluster(hyperparams)","title":"Examples"},{"location":"examples/Examples/#template-model-definition","text":"In 99% of cases you want to just copy this template to start a new lightningModule and change the core of what your model is actually trying to do. # get a copy of the module template wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/examples/new_project_templates/lightning_module_template.py","title":"Template model definition"},{"location":"examples/Examples/#trainer-example","text":"__main__ function Normally, we want to let the __main__ function start the training. Inside the main we parse training arguments with whatever hyperparameters we want. Your LightningModule will have a chance to add hyperparameters. from test_tube import HyperOptArgumentParser if __name__ == '__main__': # use default args given by lightning root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0] parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False) add_default_args(parent_parser, root_dir) # allow model to overwrite or extend args parser = ExampleModel.add_model_specific_args(parent_parser) hyperparams = parser.parse_args() # train model main(hyperparams) Main Function The main function is your entry into the program. This is where you init your model, checkpoint directory, and launch the training. The main function should have 3 arguments: - hparams: a configuration of hyperparameters. - slurm_manager: Slurm cluster manager object (can be None) - dict: for you to return any values you want (useful in meta-learning, otherwise set to _) def main(hparams, cluster, results_dict): \"\"\" Main training routine specific for this project :param hparams: :return: \"\"\" # init experiment log_dir = os.path.dirname(os.path.realpath(__file__)) exp = Experiment( name='test_tube_exp', debug=True, save_dir=log_dir, version=0, autosave=False, description='test demo' ) # set the hparams for the experiment exp.argparse(hparams) exp.save() # build model model = MyLightningModule(hparams) # callbacks early_stop = EarlyStopping( monitor=hparams.early_stop_metric, patience=hparams.early_stop_patience, verbose=True, mode=hparams.early_stop_mode ) model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) checkpoint = ModelCheckpoint( filepath=model_save_path, save_function=None, save_best_only=True, verbose=True, monitor=hparams.model_save_monitor_value, mode=hparams.model_save_monitor_mode ) # configure trainer trainer = Trainer( experiment=exp, cluster=cluster, checkpoint_callback=checkpoint, early_stop_callback=early_stop, ) # train model trainer.fit(model) The main function will start training on your main function. If you use the HyperParameterOptimizer in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple argument parser you get the default arguments in the argument parser. So, calling main(hyperparams) runs the model with the default argparse arguments. main(hyperparams)","title":"Trainer Example"},{"location":"examples/Examples/#cpu-hyperparameter-search","text":"# run a grid search over 20 hyperparameter combinations. hyperparams.optimize_parallel_cpu( main_local, nb_trials=20, nb_workers=1 )","title":"CPU hyperparameter search"},{"location":"examples/Examples/#hyperparameter-search-on-a-single-or-multiple-gpus","text":"# run a grid search over 20 hyperparameter combinations. hyperparams.optimize_parallel_gpu( main_local, nb_trials=20, nb_workers=1, gpus=[0,1,2,3] )","title":"Hyperparameter search on a single or multiple GPUs"},{"location":"examples/Examples/#hyperparameter-search-on-a-slurm-hpc-cluster","text":"def optimize_on_cluster(hyperparams): # enable cluster training cluster = SlurmCluster( hyperparam_optimizer=hyperparams, log_path=hyperparams.tt_save_path, test_tube_exp_name=hyperparams.tt_name ) # email for cluster coms cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True) # configure cluster cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus cluster.job_time = '48:00:00' cluster.gpu_type = '1080ti' cluster.memory_mb_per_node = 48000 # any modules for code to run in env cluster.add_command('source activate pytorch_lightning') # name of exp job_display_name = hyperparams.tt_name.split('_')[0] job_display_name = job_display_name[0:3] # run hopt print('submitting jobs...') cluster.optimize_parallel_cluster_gpu( main, nb_trials=hyperparams.nb_hopt_trials, job_name=job_display_name ) # run cluster hyperparameter search optimize_on_cluster(hyperparams)","title":"Hyperparameter search on a SLURM HPC cluster"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"New project Quick Start To start a new project you define two files, a LightningModule and a Trainer file. A separate trainer file allows to run many LightningModules. Each LightningModule has the core logic to a particular research project. For example, one lightningModule could be an image classifier, the other one could be a seq-2-seq model, both (optionally) ran by the same trainer file. MNIST LightningModule Trainer Basic CPU Trainer Template Multi-GPU Trainer Template GPU cluster Trainer Template Docs shortcuts LightningModule Trainer Quick start examples CPU example Hyperparameter search on single GPU Hyperparameter search on multiple GPUs on same node Hyperparameter search on a SLURM HPC cluster Checkpointing Model saving Model loading Restoring training session Computing cluster (SLURM) Running grid search on a cluster Walltime auto-resubmit Debugging Fast dev run Inspect gradient norms Log GPU usage Make model overfit on subset of data Print the parameter count by layer Pring which gradients are nan Print input and output size of every module in system Distributed training 16-bit mixed precision Multi-GPU Multi-node Single GPU Self-balancing architecture Experiment Logging Display metrics in progress bar Log metric row every k batches Process position Tensorboard support Save a snapshot of all hyperparameters Snapshot code for a training run Write logs file to csv every k batches Training loop Accumulate gradients Force training for min or max epochs Force disable early stop Gradient Clipping Hooks Learning rate scheduling Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%) Step optimizers at arbitrary intervals Validation loop Check validation every n epochs Hooks Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps","title":"Home"},{"location":"#new-project-quick-start","text":"To start a new project you define two files, a LightningModule and a Trainer file. A separate trainer file allows to run many LightningModules. Each LightningModule has the core logic to a particular research project. For example, one lightningModule could be an image classifier, the other one could be a seq-2-seq model, both (optionally) ran by the same trainer file. MNIST LightningModule Trainer Basic CPU Trainer Template Multi-GPU Trainer Template GPU cluster Trainer Template","title":"New project Quick Start"},{"location":"#docs-shortcuts","text":"LightningModule Trainer","title":"Docs shortcuts"},{"location":"#quick-start-examples","text":"CPU example Hyperparameter search on single GPU Hyperparameter search on multiple GPUs on same node Hyperparameter search on a SLURM HPC cluster","title":"Quick start examples"},{"location":"#checkpointing","text":"Model saving Model loading Restoring training session","title":"Checkpointing"},{"location":"#computing-cluster-slurm","text":"Running grid search on a cluster Walltime auto-resubmit","title":"Computing cluster (SLURM)"},{"location":"#debugging","text":"Fast dev run Inspect gradient norms Log GPU usage Make model overfit on subset of data Print the parameter count by layer Pring which gradients are nan Print input and output size of every module in system","title":"Debugging"},{"location":"#distributed-training","text":"16-bit mixed precision Multi-GPU Multi-node Single GPU Self-balancing architecture","title":"Distributed training"},{"location":"#experiment-logging","text":"Display metrics in progress bar Log metric row every k batches Process position Tensorboard support Save a snapshot of all hyperparameters Snapshot code for a training run Write logs file to csv every k batches","title":"Experiment Logging"},{"location":"#training-loop","text":"Accumulate gradients Force training for min or max epochs Force disable early stop Gradient Clipping Hooks Learning rate scheduling Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%) Step optimizers at arbitrary intervals","title":"Training loop"},{"location":"#validation-loop","text":"Check validation every n epochs Hooks Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps","title":"Validation loop"},{"location":"LightningModule/RequiredTrainerInterface/","text":"Lightning Module interface [ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. The easiest thing to do is copy the minimal example below and modify accordingly. Otherwise, to Define a Lightning Module, implement the following methods: Required : training_step tng_dataloader configure_optimizers Optional : - validation_step - validation_end - val_dataloader - test_dataloader on_save_checkpoint on_load_checkpoint update_tng_log_metrics add_model_specific_args Minimal example import os import torch from torch.nn import functional as F from torch.utils.data import DataLoader from torchvision.datasets import MNIST import torchvision.transforms as transforms import pytorch_lightning as pl class CoolModel(pl.LightningModule): def __init__(self): super(CoolModel, self).__init__() # not the best model... self.l1 = torch.nn.Linear(28 * 28, 10) def forward(self, x): return torch.relu(self.l1(x.view(x.size(0), -1))) def training_step(self, batch, batch_nb): # REQUIRED x, y = batch y_hat = self.forward(x) return {'loss': F.cross_entropy(y_hat, y)(y_hat, y)} def validation_step(self, batch, batch_nb): # OPTIONAL x, y = batch y_hat = self.forward(x) return {'val_loss': F.cross_entropy(y_hat, y)(y_hat, y)} def validation_end(self, outputs): # OPTIONAL avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() return {'avg_val_loss': avg_loss} def configure_optimizers(self): # REQUIRED return [torch.optim.Adam(self.parameters(), lr=0.02)] @pl.data_loader def tng_dataloader(self): return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) @pl.data_loader def val_dataloader(self): # OPTIONAL # can also return a list of val dataloaders return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) @pl.data_loader def test_dataloader(self): # OPTIONAL return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) How do these methods fit into the broader training? The LightningModule interface is on the right. Each method corresponds to a part of a research project. Lightning automates everything not in blue. Required Methods training_step def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output tng_dataloader @pl.data_loader def tng_dataloader(self) Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. Return PyTorch DataLoader Example @pl.data_loader def tng_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader configure_optimizers def configure_optimizers(self) Set up as many optimizers and (optionally) learning rate schedulers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one in every epoch. If you use 16 bit precision it will also handle that. Return List or Tuple - List of optimizers with an optional second list of learning-rate schedulers Example # most cases def configure_optimizers(self): opt = Adam(self.parameters(), lr=0.01) return [opt] # gan example, with scheduler for discriminator def configure_optimizers(self): generator_opt = Adam(self.model_gen.parameters(), lr=0.01) disriminator_opt = Adam(self.model_disc.parameters(), lr=0.02) discriminator_sched = CosineAnnealing(discriminator_opt, T_max=10) return [generator_opt, disriminator_opt], [discriminator_sched] If you need to control how often those optimizers step or override the default .step() schedule, override the optimizer_step hook. Optional Methods validation_step def validation_step(self, data_batch, batch_nb) # if have multiple val dataloaders: def validation_step(self, data_batch, batch_nb, dataloader_idx) OPTIONAL If you don't need to validate you don't need to implement this method. In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes, calculate accuracy, or save example outputs (using self.experiment or whatever you want). Really, anything you want. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is dataloader_i Integer displaying which dataloader this is (only if multiple val datasets used) Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example # CASE 1: A single validation dataset def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output If you pass in multiple validation datasets, validation_step will have an additional argument. # CASE 2: multiple validation datasets def validation_step(self, data_batch, batch_nb, dataset_idx): # dataset_idx tells you which dataset this is. The dataset_idx corresponds to the order of datasets returned in val_dataloader . validation_end def validation_end(self, outputs) If you didn't define a validation_step, this won't be called. Called at the end of the validation loop with the output of each validation_step. Called once per validation dataset. The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic on_save_checkpoint def on_save_checkpoint(self, checkpoint) Called by lightning to checkpoint your model. Lightning saves the training state (current epoch, global_step, etc) and also saves the model state_dict. If you want to save anything else, use this method to add your own key-value pair. Return Nothing Example def on_save_checkpoint(self, checkpoint): # 99% of use cases you don't need to implement this method checkpoint['something_cool_i_want_to_save'] = my_cool_pickable_object on_load_checkpoint def on_load_checkpoint(self, checkpoint) Called by lightning to restore your model. Lighting auto-restores global step, epoch, etc... It also restores the model state_dict. If you saved something with on_save_checkpoint this is your chance to restore this. Return Nothing Example def on_load_checkpoint(self, checkpoint): # 99% of the time you don't need to implement this method self.something_cool_i_want_to_save = checkpoint['something_cool_i_want_to_save'] val_dataloader @pl.data_loader def tng_dataloader(self) OPTIONAL If you don't need a validation dataset and a validation_step, you don't need to implement this method. Called by lightning during validation loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. Return PyTorch DataLoader or list of PyTorch Dataloaders. Example @pl.data_loader def val_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader # can also return multiple dataloaders @pl.data_loader def val_dataloader(self): return [loader_a, loader_b, ..., loader_n] In the case where you return multiple val_dataloaders, the validation_step will have an arguement dataset_idx which matches the order here. test_dataloader @pl.data_loader def test_dataloader(self) OPTIONAL If you don't need a test dataset and a test_step, you don't need to implement this method. Called by lightning during test loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. Return PyTorch DataLoader Example @pl.data_loader def test_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader update_tng_log_metrics def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged. Return Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs add_model_specific_args @staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The hyperparameter argument parser is available anywhere in your model by calling self.hparams. Return An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Lightning Module interface"},{"location":"LightningModule/RequiredTrainerInterface/#lightning-module-interface","text":"[ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. The easiest thing to do is copy the minimal example below and modify accordingly. Otherwise, to Define a Lightning Module, implement the following methods: Required : training_step tng_dataloader configure_optimizers Optional : - validation_step - validation_end - val_dataloader - test_dataloader on_save_checkpoint on_load_checkpoint update_tng_log_metrics add_model_specific_args","title":"Lightning Module interface"},{"location":"LightningModule/RequiredTrainerInterface/#minimal-example","text":"import os import torch from torch.nn import functional as F from torch.utils.data import DataLoader from torchvision.datasets import MNIST import torchvision.transforms as transforms import pytorch_lightning as pl class CoolModel(pl.LightningModule): def __init__(self): super(CoolModel, self).__init__() # not the best model... self.l1 = torch.nn.Linear(28 * 28, 10) def forward(self, x): return torch.relu(self.l1(x.view(x.size(0), -1))) def training_step(self, batch, batch_nb): # REQUIRED x, y = batch y_hat = self.forward(x) return {'loss': F.cross_entropy(y_hat, y)(y_hat, y)} def validation_step(self, batch, batch_nb): # OPTIONAL x, y = batch y_hat = self.forward(x) return {'val_loss': F.cross_entropy(y_hat, y)(y_hat, y)} def validation_end(self, outputs): # OPTIONAL avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean() return {'avg_val_loss': avg_loss} def configure_optimizers(self): # REQUIRED return [torch.optim.Adam(self.parameters(), lr=0.02)] @pl.data_loader def tng_dataloader(self): return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) @pl.data_loader def val_dataloader(self): # OPTIONAL # can also return a list of val dataloaders return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32) @pl.data_loader def test_dataloader(self): # OPTIONAL return DataLoader(MNIST(os.getcwd(), train=True, download=True, transform=transforms.ToTensor()), batch_size=32)","title":"Minimal example"},{"location":"LightningModule/RequiredTrainerInterface/#how-do-these-methods-fit-into-the-broader-training","text":"The LightningModule interface is on the right. Each method corresponds to a part of a research project. Lightning automates everything not in blue.","title":"How do these methods fit into the broader training?"},{"location":"LightningModule/RequiredTrainerInterface/#required-methods","text":"","title":"Required Methods"},{"location":"LightningModule/RequiredTrainerInterface/#training_step","text":"def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output","title":"training_step"},{"location":"LightningModule/RequiredTrainerInterface/#tng_dataloader","text":"@pl.data_loader def tng_dataloader(self) Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.","title":"tng_dataloader"},{"location":"LightningModule/RequiredTrainerInterface/#return","text":"PyTorch DataLoader Example @pl.data_loader def tng_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#configure_optimizers","text":"def configure_optimizers(self) Set up as many optimizers and (optionally) learning rate schedulers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one in every epoch. If you use 16 bit precision it will also handle that.","title":"configure_optimizers"},{"location":"LightningModule/RequiredTrainerInterface/#return_1","text":"List or Tuple - List of optimizers with an optional second list of learning-rate schedulers Example # most cases def configure_optimizers(self): opt = Adam(self.parameters(), lr=0.01) return [opt] # gan example, with scheduler for discriminator def configure_optimizers(self): generator_opt = Adam(self.model_gen.parameters(), lr=0.01) disriminator_opt = Adam(self.model_disc.parameters(), lr=0.02) discriminator_sched = CosineAnnealing(discriminator_opt, T_max=10) return [generator_opt, disriminator_opt], [discriminator_sched] If you need to control how often those optimizers step or override the default .step() schedule, override the optimizer_step hook.","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#optional-methods","text":"","title":"Optional Methods"},{"location":"LightningModule/RequiredTrainerInterface/#validation_step","text":"def validation_step(self, data_batch, batch_nb) # if have multiple val dataloaders: def validation_step(self, data_batch, batch_nb, dataloader_idx) OPTIONAL If you don't need to validate you don't need to implement this method. In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes, calculate accuracy, or save example outputs (using self.experiment or whatever you want). Really, anything you want. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is dataloader_i Integer displaying which dataloader this is (only if multiple val datasets used) Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example # CASE 1: A single validation dataset def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output If you pass in multiple validation datasets, validation_step will have an additional argument. # CASE 2: multiple validation datasets def validation_step(self, data_batch, batch_nb, dataset_idx): # dataset_idx tells you which dataset this is. The dataset_idx corresponds to the order of datasets returned in val_dataloader .","title":"validation_step"},{"location":"LightningModule/RequiredTrainerInterface/#validation_end","text":"def validation_end(self, outputs) If you didn't define a validation_step, this won't be called. Called at the end of the validation loop with the output of each validation_step. Called once per validation dataset. The outputs here are strictly for the progress bar. If you don't need to display anything, don't return anything. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic","title":"validation_end"},{"location":"LightningModule/RequiredTrainerInterface/#on_save_checkpoint","text":"def on_save_checkpoint(self, checkpoint) Called by lightning to checkpoint your model. Lightning saves the training state (current epoch, global_step, etc) and also saves the model state_dict. If you want to save anything else, use this method to add your own key-value pair.","title":"on_save_checkpoint"},{"location":"LightningModule/RequiredTrainerInterface/#return_2","text":"Nothing Example def on_save_checkpoint(self, checkpoint): # 99% of use cases you don't need to implement this method checkpoint['something_cool_i_want_to_save'] = my_cool_pickable_object","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#on_load_checkpoint","text":"def on_load_checkpoint(self, checkpoint) Called by lightning to restore your model. Lighting auto-restores global step, epoch, etc... It also restores the model state_dict. If you saved something with on_save_checkpoint this is your chance to restore this.","title":"on_load_checkpoint"},{"location":"LightningModule/RequiredTrainerInterface/#return_3","text":"Nothing Example def on_load_checkpoint(self, checkpoint): # 99% of the time you don't need to implement this method self.something_cool_i_want_to_save = checkpoint['something_cool_i_want_to_save']","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#val_dataloader","text":"@pl.data_loader def tng_dataloader(self) OPTIONAL If you don't need a validation dataset and a validation_step, you don't need to implement this method. Called by lightning during validation loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.","title":"val_dataloader"},{"location":"LightningModule/RequiredTrainerInterface/#return_4","text":"PyTorch DataLoader or list of PyTorch Dataloaders. Example @pl.data_loader def val_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader # can also return multiple dataloaders @pl.data_loader def val_dataloader(self): return [loader_a, loader_b, ..., loader_n] In the case where you return multiple val_dataloaders, the validation_step will have an arguement dataset_idx which matches the order here.","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#test_dataloader","text":"@pl.data_loader def test_dataloader(self) OPTIONAL If you don't need a test dataset and a test_step, you don't need to implement this method. Called by lightning during test loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed.","title":"test_dataloader"},{"location":"LightningModule/RequiredTrainerInterface/#return_5","text":"PyTorch DataLoader Example @pl.data_loader def test_dataloader(self): transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) return loader","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#update_tng_log_metrics","text":"def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged.","title":"update_tng_log_metrics"},{"location":"LightningModule/RequiredTrainerInterface/#return_6","text":"Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs","title":"Return"},{"location":"LightningModule/RequiredTrainerInterface/#add_model_specific_args","text":"@staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The hyperparameter argument parser is available anywhere in your model by calling self.hparams.","title":"add_model_specific_args"},{"location":"LightningModule/RequiredTrainerInterface/#return_7","text":"An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Return"},{"location":"LightningModule/methods/","text":"Lightning modules are strict superclasses of torch.nn.Module. A LightningModule offers the following in addition to that API. freeze Freeze all params for inference model = MyLightningModule(...) model.freeze() load_from_metrics This is the easiest/fastest way which uses the meta_tags.csv file from test-tube to rebuild the model. The meta_tags.csv file can be found in the test-tube experiment save_dir. pretrained_model = MyLightningModule.load_from_metrics( weights_path='/path/to/pytorch_checkpoint.ckpt', tags_csv='/path/to/test_tube/experiment/version/meta_tags.csv', on_gpu=True, map_location=None ) # predict pretrained_model.eval() pretrained_model.freeze() y_hat = pretrained_model(x) Params Param description weights_path Path to a PyTorch checkpoint tags_csv Path to meta_tags.csv file generated by the test-tube Experiment on_gpu if True, puts model on GPU. Make sure to use transforms option if model devices have changed map_location A dictionary mapping saved weight GPU devices to new GPU devices Returns LightningModule - The pretrained LightningModule unfreeze Unfreeze all params for inference model = MyLightningModule(...) model.unfreeze()","title":"Methods"},{"location":"LightningModule/methods/#freeze","text":"Freeze all params for inference model = MyLightningModule(...) model.freeze()","title":"freeze"},{"location":"LightningModule/methods/#load_from_metrics","text":"This is the easiest/fastest way which uses the meta_tags.csv file from test-tube to rebuild the model. The meta_tags.csv file can be found in the test-tube experiment save_dir. pretrained_model = MyLightningModule.load_from_metrics( weights_path='/path/to/pytorch_checkpoint.ckpt', tags_csv='/path/to/test_tube/experiment/version/meta_tags.csv', on_gpu=True, map_location=None ) # predict pretrained_model.eval() pretrained_model.freeze() y_hat = pretrained_model(x) Params Param description weights_path Path to a PyTorch checkpoint tags_csv Path to meta_tags.csv file generated by the test-tube Experiment on_gpu if True, puts model on GPU. Make sure to use transforms option if model devices have changed map_location A dictionary mapping saved weight GPU devices to new GPU devices Returns LightningModule - The pretrained LightningModule","title":"load_from_metrics"},{"location":"LightningModule/methods/#unfreeze","text":"Unfreeze all params for inference model = MyLightningModule(...) model.unfreeze()","title":"unfreeze"},{"location":"LightningModule/properties/","text":"A LightningModule has the following properties which you can access at any time current_epoch The current epoch dtype Current dtype experiment An instance of test-tube Experiment which you can use to log anything for tensorboarX. self.experiment.add_embedding(...) self.experiment.log({'val_loss': 0.9}) self.experiment.add_scalars(...) global_step Total training batches seen across all epochs gradient_clip The current gradient clip value on_gpu True if your model is currently running on GPUs. Useful to set flags around the LightningModule for different CPU vs GPU behavior. trainer Last resort access to any state the trainer has. Changing certain properties here could affect your training run. self.trainer.optimizers self.trainer.current_epoch ... Debugging The LightningModule also offers these tricks to help debug. example_input_array In the LightningModule init, you can set a dummy tensor for this property to get a print out of sizes coming into and out of every layer. def __init__(self): # put the dimensions of the first input to your system self.example_input_array = torch.rand(5, 28 * 28)","title":"Properties"},{"location":"LightningModule/properties/#current_epoch","text":"The current epoch","title":"current_epoch"},{"location":"LightningModule/properties/#dtype","text":"Current dtype","title":"dtype"},{"location":"LightningModule/properties/#experiment","text":"An instance of test-tube Experiment which you can use to log anything for tensorboarX. self.experiment.add_embedding(...) self.experiment.log({'val_loss': 0.9}) self.experiment.add_scalars(...)","title":"experiment"},{"location":"LightningModule/properties/#global_step","text":"Total training batches seen across all epochs","title":"global_step"},{"location":"LightningModule/properties/#gradient_clip","text":"The current gradient clip value","title":"gradient_clip"},{"location":"LightningModule/properties/#on_gpu","text":"True if your model is currently running on GPUs. Useful to set flags around the LightningModule for different CPU vs GPU behavior.","title":"on_gpu"},{"location":"LightningModule/properties/#trainer","text":"Last resort access to any state the trainer has. Changing certain properties here could affect your training run. self.trainer.optimizers self.trainer.current_epoch ...","title":"trainer"},{"location":"LightningModule/properties/#debugging","text":"The LightningModule also offers these tricks to help debug.","title":"Debugging"},{"location":"LightningModule/properties/#example_input_array","text":"In the LightningModule init, you can set a dummy tensor for this property to get a print out of sizes coming into and out of every layer. def __init__(self): # put the dimensions of the first input to your system self.example_input_array = torch.rand(5, 28 * 28)","title":"example_input_array"},{"location":"Trainer/","text":"Trainer [ Github Code ] The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. This is the basic use of the trainer: from pytorch_lightning import Trainer model = LightningTemplate() trainer = Trainer() trainer.fit(model) But of course the fun is in all the advanced things it can do: Checkpointing Model saving Model loading Restoring training session Computing cluster (SLURM) Running grid search on a cluster Walltime auto-resubmit Debugging Fast dev run Inspect gradient norms Log GPU usage Make model overfit on subset of data Print the parameter count by layer Pring which gradients are nan Print input and output size of every module in system Distributed training 16-bit mixed precision Multi-GPU Multi-node Single GPU Self-balancing architecture Experiment Logging Display metrics in progress bar Log metric row every k batches Process position Tensorboard support Save a snapshot of all hyperparameters Snapshot code for a training run Write logs file to csv every k batches Training loop Accumulate gradients Force training for min or max epochs Force disable early stop Gradient Clipping Hooks Learning rate scheduling Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%) Step optimizers at arbitrary intervals Validation loop Check validation every n epochs Hooks Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps","title":"Trainer"},{"location":"Trainer/#trainer","text":"[ Github Code ] The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. This is the basic use of the trainer: from pytorch_lightning import Trainer model = LightningTemplate() trainer = Trainer() trainer.fit(model) But of course the fun is in all the advanced things it can do: Checkpointing Model saving Model loading Restoring training session Computing cluster (SLURM) Running grid search on a cluster Walltime auto-resubmit Debugging Fast dev run Inspect gradient norms Log GPU usage Make model overfit on subset of data Print the parameter count by layer Pring which gradients are nan Print input and output size of every module in system Distributed training 16-bit mixed precision Multi-GPU Multi-node Single GPU Self-balancing architecture Experiment Logging Display metrics in progress bar Log metric row every k batches Process position Tensorboard support Save a snapshot of all hyperparameters Snapshot code for a training run Write logs file to csv every k batches Training loop Accumulate gradients Force training for min or max epochs Force disable early stop Gradient Clipping Hooks Learning rate scheduling Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%) Step optimizers at arbitrary intervals Validation loop Check validation every n epochs Hooks Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps","title":"Trainer"},{"location":"Trainer/Checkpointing/","text":"i Lightning can automate saving and loading checkpoints. Model saving To enable checkpointing, define the checkpoint callback and give it to the trainer. from pytorch_lightning.callbacks import ModelCheckpoint checkpoint_callback = ModelCheckpoint( filepath='/path/to/store/weights.ckpt', save_best_only=True, verbose=True, monitor='val_loss', mode='min' ) trainer = Trainer(checkpoint_callback=checkpoint_callback) Restoring training session You might want to not only load a model but also continue training it. Use this method to restore the trainer state as well. This will continue from the epoch and global step you last left off. However, the dataloaders will start from the first batch again (if you shuffled it shouldn't matter). Lightning will restore the session if you pass an experiment with the same version and there's a saved checkpoint. from test_tube import Experiment exp = Experiment(version=a_previous_version_with_a_saved_checkpoint) trainer = Trainer(experiment=exp) # this fit call loads model weights and trainer state # the trainer continues seamlessly from where you left off # without having to do anything else. trainer.fit(model) The trainer restores: - global_step - current_epoch - All optimizers - All lr_schedulers - Model weights You can even change the logic of your model as long as the weights and \"architecture\" of the system isn't different. If you add a layer, for instance, it might not work. At a rough level, here's what happens inside Trainer : self.global_step = checkpoint['global_step'] self.current_epoch = checkpoint['epoch'] # restore the optimizers optimizer_states = checkpoint['optimizer_states'] for optimizer, opt_state in zip(self.optimizers, optimizer_states): optimizer.load_state_dict(opt_state) # restore the lr schedulers lr_schedulers = checkpoint['lr_schedulers'] for scheduler, lrs_state in zip(self.lr_schedulers, lr_schedulers): scheduler.load_state_dict(lrs_state) # uses the model you passed into trainer model.load_state_dict(checkpoint['state_dict'])","title":"Checkpointing"},{"location":"Trainer/Checkpointing/#model-saving","text":"To enable checkpointing, define the checkpoint callback and give it to the trainer. from pytorch_lightning.callbacks import ModelCheckpoint checkpoint_callback = ModelCheckpoint( filepath='/path/to/store/weights.ckpt', save_best_only=True, verbose=True, monitor='val_loss', mode='min' ) trainer = Trainer(checkpoint_callback=checkpoint_callback)","title":"Model saving"},{"location":"Trainer/Checkpointing/#restoring-training-session","text":"You might want to not only load a model but also continue training it. Use this method to restore the trainer state as well. This will continue from the epoch and global step you last left off. However, the dataloaders will start from the first batch again (if you shuffled it shouldn't matter). Lightning will restore the session if you pass an experiment with the same version and there's a saved checkpoint. from test_tube import Experiment exp = Experiment(version=a_previous_version_with_a_saved_checkpoint) trainer = Trainer(experiment=exp) # this fit call loads model weights and trainer state # the trainer continues seamlessly from where you left off # without having to do anything else. trainer.fit(model) The trainer restores: - global_step - current_epoch - All optimizers - All lr_schedulers - Model weights You can even change the logic of your model as long as the weights and \"architecture\" of the system isn't different. If you add a layer, for instance, it might not work. At a rough level, here's what happens inside Trainer : self.global_step = checkpoint['global_step'] self.current_epoch = checkpoint['epoch'] # restore the optimizers optimizer_states = checkpoint['optimizer_states'] for optimizer, opt_state in zip(self.optimizers, optimizer_states): optimizer.load_state_dict(opt_state) # restore the lr schedulers lr_schedulers = checkpoint['lr_schedulers'] for scheduler, lrs_state in zip(self.lr_schedulers, lr_schedulers): scheduler.load_state_dict(lrs_state) # uses the model you passed into trainer model.load_state_dict(checkpoint['state_dict'])","title":"Restoring training session"},{"location":"Trainer/Distributed training/","text":"Lightning makes multi-gpu training and 16 bit training trivial. Note: None of the flags below require changing anything about your lightningModel definition. Choosing a backend Lightning supports two backends. DataParallel and DistributedDataParallel. Both can be used for single-node multi-GPU training. For multi-node training you must use DistributedDataParallel. You can toggle between each mode by setting this flag. # DEFAULT uses DataParallel trainer = Trainer(distributed_backend='dp') # change to distributed data parallel trainer = Trainer(distributed_backend='ddp') If you request multiple nodes, the back-end will auto-switch to ddp. We recommend you use DistributedDataparallel even for single-node multi-GPU training. It is MUCH faster than DP but may have configuration issues depending on your cluster. For a deeper understanding of what lightning is doing, feel free to read this guide . Distributed and 16-bit precision. Due to an issue with apex and DistributedDataParallel (PyTorch and NVIDIA issue), Lightning does not allow 16-bit and DP training. We tried to get this to work, but it's an issue on their end. Below are the possible configurations we support. 1 GPU 1+ GPUs DP DDP 16-bit command Y Trainer(gpus=[0]) Y Y Trainer(gpus=[0], use_amp=True) Y Y Trainer(gpus=[0, ...]) Y Y Trainer(gpus=[0, ...], distributed_backend='ddp') Y Y Y Trainer(gpus=[0, ...], distributed_backend='ddp', use_amp=True) CUDA flags CUDA flags make certain GPUs visible to your script. Lightning sets these for you automatically, there's NO NEED to do this yourself. # lightning will set according to what you give the trainer # os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\" # os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\" 16-bit mixed precision 16 bit precision can cut your memory footprint by half. If using volta architecture GPUs it can give a dramatic training speed-up as well. First, install apex (if install fails, look here ): $ git clone https://github.com/NVIDIA/apex $ cd apex $ pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./ then set this use_amp to True. # DEFAULT trainer = Trainer(amp_level='O2', use_amp=False) Single-gpu Make sure you're on a GPU machine. # DEFAULT trainer = Trainer(gpus=[0]) multi-gpu Make sure you're on a GPU machine. You can set as many GPUs as you want. In this setting, the model will run on all 8 GPUs at once using DataParallel under the hood. # to use DataParallel (default) trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='dp') # RECOMMENDED use DistributedDataParallel trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='ddp') Multi-node Multi-node training is easily done by specifying these flags. # train on 12*8 GPUs trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], nb_gpu_nodes=12) In addition, make sure to set up your SLURM job correctly via the SlurmClusterObject . In particular, specify the number of tasks per node correctly. cluster = SlurmCluster( hyperparam_optimizer=test_tube.HyperOptArgumentParser(), log_path='/some/path/to/save', ) # OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT # which interface your nodes use for communication cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo') # see output of the NCCL connection process # NCCL is how the nodes talk to each other cluster.add_command('export NCCL_DEBUG=INFO') # setting a master port here is a good idea. cluster.add_command('export MASTER_PORT=%r' % PORT) # good to load the latest NCCL version cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0']) # configure cluster cluster.per_experiment_nb_nodes = 12 cluster.per_experiment_nb_gpus = 8 cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu') Finally, make sure to add a distributed sampler to your dataset. The distributed sampler copies a portion of your dataset onto each GPU. (World_size = gpus_per_node * nb_nodes). # ie: this: dataset = myDataset() dataloader = Dataloader(dataset) # becomes: dataset = myDataset() dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) dataloader = Dataloader(dataset, sampler=dist_sampler) Self-balancing architecture Here lightning distributes parts of your module across available GPUs to optimize for speed and memory. COMING SOON.","title":"Distributed training"},{"location":"Trainer/Distributed training/#choosing-a-backend","text":"Lightning supports two backends. DataParallel and DistributedDataParallel. Both can be used for single-node multi-GPU training. For multi-node training you must use DistributedDataParallel. You can toggle between each mode by setting this flag. # DEFAULT uses DataParallel trainer = Trainer(distributed_backend='dp') # change to distributed data parallel trainer = Trainer(distributed_backend='ddp') If you request multiple nodes, the back-end will auto-switch to ddp. We recommend you use DistributedDataparallel even for single-node multi-GPU training. It is MUCH faster than DP but may have configuration issues depending on your cluster. For a deeper understanding of what lightning is doing, feel free to read this guide .","title":"Choosing a backend"},{"location":"Trainer/Distributed training/#distributed-and-16-bit-precision","text":"Due to an issue with apex and DistributedDataParallel (PyTorch and NVIDIA issue), Lightning does not allow 16-bit and DP training. We tried to get this to work, but it's an issue on their end. Below are the possible configurations we support. 1 GPU 1+ GPUs DP DDP 16-bit command Y Trainer(gpus=[0]) Y Y Trainer(gpus=[0], use_amp=True) Y Y Trainer(gpus=[0, ...]) Y Y Trainer(gpus=[0, ...], distributed_backend='ddp') Y Y Y Trainer(gpus=[0, ...], distributed_backend='ddp', use_amp=True)","title":"Distributed and 16-bit precision."},{"location":"Trainer/Distributed training/#cuda-flags","text":"CUDA flags make certain GPUs visible to your script. Lightning sets these for you automatically, there's NO NEED to do this yourself. # lightning will set according to what you give the trainer # os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\" # os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"","title":"CUDA flags"},{"location":"Trainer/Distributed training/#16-bit-mixed-precision","text":"16 bit precision can cut your memory footprint by half. If using volta architecture GPUs it can give a dramatic training speed-up as well. First, install apex (if install fails, look here ): $ git clone https://github.com/NVIDIA/apex $ cd apex $ pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ./ then set this use_amp to True. # DEFAULT trainer = Trainer(amp_level='O2', use_amp=False)","title":"16-bit mixed precision"},{"location":"Trainer/Distributed training/#single-gpu","text":"Make sure you're on a GPU machine. # DEFAULT trainer = Trainer(gpus=[0])","title":"Single-gpu"},{"location":"Trainer/Distributed training/#multi-gpu","text":"Make sure you're on a GPU machine. You can set as many GPUs as you want. In this setting, the model will run on all 8 GPUs at once using DataParallel under the hood. # to use DataParallel (default) trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='dp') # RECOMMENDED use DistributedDataParallel trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='ddp')","title":"multi-gpu"},{"location":"Trainer/Distributed training/#multi-node","text":"Multi-node training is easily done by specifying these flags. # train on 12*8 GPUs trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], nb_gpu_nodes=12) In addition, make sure to set up your SLURM job correctly via the SlurmClusterObject . In particular, specify the number of tasks per node correctly. cluster = SlurmCluster( hyperparam_optimizer=test_tube.HyperOptArgumentParser(), log_path='/some/path/to/save', ) # OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT # which interface your nodes use for communication cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo') # see output of the NCCL connection process # NCCL is how the nodes talk to each other cluster.add_command('export NCCL_DEBUG=INFO') # setting a master port here is a good idea. cluster.add_command('export MASTER_PORT=%r' % PORT) # good to load the latest NCCL version cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0']) # configure cluster cluster.per_experiment_nb_nodes = 12 cluster.per_experiment_nb_gpus = 8 cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu') Finally, make sure to add a distributed sampler to your dataset. The distributed sampler copies a portion of your dataset onto each GPU. (World_size = gpus_per_node * nb_nodes). # ie: this: dataset = myDataset() dataloader = Dataloader(dataset) # becomes: dataset = myDataset() dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset) dataloader = Dataloader(dataset, sampler=dist_sampler)","title":"Multi-node"},{"location":"Trainer/Distributed training/#self-balancing-architecture","text":"Here lightning distributes parts of your module across available GPUs to optimize for speed and memory. COMING SOON.","title":"Self-balancing architecture"},{"location":"Trainer/Logging/","text":"Lighting offers a few options for logging information about model, gpu usage, etc (via test-tube). It also offers printing options for training monitoring. Display metrics in progress bar # DEFAULT trainer = Trainer(progress_bar=True) Log metric row every k batches Every k batches lightning will make an entry in the metrics log # DEFAULT (ie: save a .csv log file every 10 batches) trainer = Trainer(add_log_row_interval=10) Process position When running multiple models on the same machine we want to decide which progress bar to use. Lightning will stack progress bars according to this value. # DEFAULT trainer = Trainer(process_position=0) # if this is the second model on the node, show the second progress bar below trainer = Trainer(process_position=1) Save a snapshot of all hyperparameters Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use. Give lightning a test-tube Experiment object to automate this for you. from test_tube import Experiment exp = Experiment(...) Trainer(experiment=exp) Snapshot code for a training run Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag. Give lightning a test-tube Experiment object to automate this for you. from test_tube import Experiment exp = Experiment(create_git_tag=True) Trainer(experiment=exp) Tensorboard support In the LightningModule you can access the experiment logger by doing: self.experiment # add image # Look at PyTorch SummaryWriter docs for what you can do. self.experiment.add_image(...) The experiment object is a strict subclass of PyTorch SummaryWriter. However, this class also snapshots every detail about the experiment (data folder paths, code, hyperparams), and allows you to visualize it using tensorboard. from test_tube import Experiment, HyperOptArgumentParser # exp hyperparams args = HyperOptArgumentParser() hparams = args.parse_args() # this is a summaryWriter with nicer logging structure exp = Experiment(save_dir='/some/path', create_git_tag=True) # track experiment details (must be ArgumentParser or HyperOptArgumentParser). # each option in the parser is tracked exp.argparse(hparams) exp.tag({'description': 'running demo'}) # trainer uses the exp object to log exp data trainer = Trainer(experiment=exp) trainer.fit(model) # view logs at: # tensorboard --logdir /some/path Write logs file to csv every k batches Every k batches, lightning will write the new logs to disk # DEFAULT (ie: save a .csv log file every 100 batches) trainer = Trainer(log_save_interval=100)","title":"Logging"},{"location":"Trainer/Logging/#display-metrics-in-progress-bar","text":"# DEFAULT trainer = Trainer(progress_bar=True)","title":"Display metrics in progress bar"},{"location":"Trainer/Logging/#log-metric-row-every-k-batches","text":"Every k batches lightning will make an entry in the metrics log # DEFAULT (ie: save a .csv log file every 10 batches) trainer = Trainer(add_log_row_interval=10)","title":"Log metric row every k batches"},{"location":"Trainer/Logging/#process-position","text":"When running multiple models on the same machine we want to decide which progress bar to use. Lightning will stack progress bars according to this value. # DEFAULT trainer = Trainer(process_position=0) # if this is the second model on the node, show the second progress bar below trainer = Trainer(process_position=1)","title":"Process position"},{"location":"Trainer/Logging/#save-a-snapshot-of-all-hyperparameters","text":"Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use. Give lightning a test-tube Experiment object to automate this for you. from test_tube import Experiment exp = Experiment(...) Trainer(experiment=exp)","title":"Save a snapshot of all hyperparameters"},{"location":"Trainer/Logging/#snapshot-code-for-a-training-run","text":"Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag. Give lightning a test-tube Experiment object to automate this for you. from test_tube import Experiment exp = Experiment(create_git_tag=True) Trainer(experiment=exp)","title":"Snapshot code for a training run"},{"location":"Trainer/Logging/#tensorboard-support","text":"In the LightningModule you can access the experiment logger by doing: self.experiment # add image # Look at PyTorch SummaryWriter docs for what you can do. self.experiment.add_image(...) The experiment object is a strict subclass of PyTorch SummaryWriter. However, this class also snapshots every detail about the experiment (data folder paths, code, hyperparams), and allows you to visualize it using tensorboard. from test_tube import Experiment, HyperOptArgumentParser # exp hyperparams args = HyperOptArgumentParser() hparams = args.parse_args() # this is a summaryWriter with nicer logging structure exp = Experiment(save_dir='/some/path', create_git_tag=True) # track experiment details (must be ArgumentParser or HyperOptArgumentParser). # each option in the parser is tracked exp.argparse(hparams) exp.tag({'description': 'running demo'}) # trainer uses the exp object to log exp data trainer = Trainer(experiment=exp) trainer.fit(model) # view logs at: # tensorboard --logdir /some/path","title":"Tensorboard support"},{"location":"Trainer/Logging/#write-logs-file-to-csv-every-k-batches","text":"Every k batches, lightning will write the new logs to disk # DEFAULT (ie: save a .csv log file every 100 batches) trainer = Trainer(log_save_interval=100)","title":"Write logs file to csv every k batches"},{"location":"Trainer/SLURM Managed Cluster/","text":"Lightning supports model training on a cluster managed by SLURM in the following cases: Training on single or multi-cpus only. Training on single or multi-gpus on the same node. Coming SOON: Training across multiple nodes. Running grid search on a cluster To use lightning to run a hyperparameter search (grid-search or random-search) on a cluster do 4 things: (1). Define the parameters for the grid search from test_tube import HyperOptArgumentParser # subclass of argparse parser = HyperOptArgumentParser(strategy='random_search') parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate') # let's enable optimizing over the number of layers in the network parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8]) hparams = parser.parse_args() (2). Define the cluster options in the SlurmCluster object (over 5 nodes and 8 gpus) from test_tube.hpc import SlurmCluster # hyperparameters is a test-tube hyper params object # see https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/ hyperparams = args.parse() # init cluster cluster = SlurmCluster( hyperparam_optimizer=hyperparams, log_path='/path/to/log/results/to', python_cmd='python3' ) # let the cluster know where to email for a change in job status (ie: complete, fail, etc...) cluster.notify_job_status(email='some@email.com', on_done=True, on_fail=True) # set the job options. In this instance, we'll run 20 different models # each with its own set of hyperparameters giving each one 1 GPU (ie: taking up 20 GPUs) cluster.per_experiment_nb_gpus = 8 cluster.per_experiment_nb_nodes = 5 # we'll request 10GB of memory per node cluster.memory_mb_per_node = 10000 # set a walltime of 10 minues cluster.job_time = '10:00' (3). Give trainer the cluster_manager in your main function: from pytorch_lightning import Trainer def train_fx(trial_hparams, cluster_manager, _): # hparams has a specific set of hyperparams my_model = MyLightningModel() # give the trainer the cluster object trainer = Trainer(cluster=cluster_manager) trainer.fit(my_model) (4). Start the grid search # run the models on the cluster cluster.optimize_parallel_cluster_gpu( train_fx, nb_trials=20, job_name='my_grid_search_exp_name', job_display_name='my_exp') That's it! The SlurmCluster object will automatically checkpoint the lightning model and resubmit if it runs into the walltime! Walltime auto-resubmit Lightning automatically resubmits jobs when they reach the walltime. You get this behavior for free if you give lightning a slurm cluster object. def my_main_fx(hparams, slurm_manager, _): trainer = Trainer(cluster=slurm_manager) (See the grid search example above for cluster configuration). With this feature lightning will: automatically checkpoint the model checkpoint the trainer session resubmit a continuation job. load the checkpoint and trainer session in the new model","title":"SLURM Managed Cluster"},{"location":"Trainer/SLURM Managed Cluster/#running-grid-search-on-a-cluster","text":"To use lightning to run a hyperparameter search (grid-search or random-search) on a cluster do 4 things: (1). Define the parameters for the grid search from test_tube import HyperOptArgumentParser # subclass of argparse parser = HyperOptArgumentParser(strategy='random_search') parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate') # let's enable optimizing over the number of layers in the network parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8]) hparams = parser.parse_args() (2). Define the cluster options in the SlurmCluster object (over 5 nodes and 8 gpus) from test_tube.hpc import SlurmCluster # hyperparameters is a test-tube hyper params object # see https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/ hyperparams = args.parse() # init cluster cluster = SlurmCluster( hyperparam_optimizer=hyperparams, log_path='/path/to/log/results/to', python_cmd='python3' ) # let the cluster know where to email for a change in job status (ie: complete, fail, etc...) cluster.notify_job_status(email='some@email.com', on_done=True, on_fail=True) # set the job options. In this instance, we'll run 20 different models # each with its own set of hyperparameters giving each one 1 GPU (ie: taking up 20 GPUs) cluster.per_experiment_nb_gpus = 8 cluster.per_experiment_nb_nodes = 5 # we'll request 10GB of memory per node cluster.memory_mb_per_node = 10000 # set a walltime of 10 minues cluster.job_time = '10:00' (3). Give trainer the cluster_manager in your main function: from pytorch_lightning import Trainer def train_fx(trial_hparams, cluster_manager, _): # hparams has a specific set of hyperparams my_model = MyLightningModel() # give the trainer the cluster object trainer = Trainer(cluster=cluster_manager) trainer.fit(my_model) (4). Start the grid search # run the models on the cluster cluster.optimize_parallel_cluster_gpu( train_fx, nb_trials=20, job_name='my_grid_search_exp_name', job_display_name='my_exp') That's it! The SlurmCluster object will automatically checkpoint the lightning model and resubmit if it runs into the walltime!","title":"Running grid search on a cluster"},{"location":"Trainer/SLURM Managed Cluster/#walltime-auto-resubmit","text":"Lightning automatically resubmits jobs when they reach the walltime. You get this behavior for free if you give lightning a slurm cluster object. def my_main_fx(hparams, slurm_manager, _): trainer = Trainer(cluster=slurm_manager) (See the grid search example above for cluster configuration). With this feature lightning will: automatically checkpoint the model checkpoint the trainer session resubmit a continuation job. load the checkpoint and trainer session in the new model","title":"Walltime auto-resubmit"},{"location":"Trainer/Training Loop/","text":"The lightning training loop handles everything except the actual computations of your model. To decide what will happen in your training loop, define the training_step function . Below are all the things lightning automates for you in the training loop. Accumulated gradients Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. # DEFAULT (ie: no accumulated grads) trainer = Trainer(accumulate_grad_batches=1) Force training for min or max epochs It can be useful to force training for a minimum number of epochs or limit to a max number # DEFAULT trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000) Force disable early stop Use this to turn off early stopping and run training to the max_epoch # DEFAULT trainer = Trainer(enable_early_stop=True) Gradient Clipping Gradient clipping may be enabled to avoid exploding gradients. Specifically, this will clip the gradient norm computed over all model parameters together . # DEFAULT (ie: don't clip) trainer = Trainer(gradient_clip=0) # clip gradients with norm above 0.5 trainer = Trainer(gradient_clip=0.5) Inspect gradient norms Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2) Set how much of the training set to check If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1)","title":"Training Loop"},{"location":"Trainer/Training Loop/#accumulated-gradients","text":"Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. # DEFAULT (ie: no accumulated grads) trainer = Trainer(accumulate_grad_batches=1)","title":"Accumulated gradients"},{"location":"Trainer/Training Loop/#force-training-for-min-or-max-epochs","text":"It can be useful to force training for a minimum number of epochs or limit to a max number # DEFAULT trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)","title":"Force training for min or max epochs"},{"location":"Trainer/Training Loop/#force-disable-early-stop","text":"Use this to turn off early stopping and run training to the max_epoch # DEFAULT trainer = Trainer(enable_early_stop=True)","title":"Force disable early stop"},{"location":"Trainer/Training Loop/#gradient-clipping","text":"Gradient clipping may be enabled to avoid exploding gradients. Specifically, this will clip the gradient norm computed over all model parameters together . # DEFAULT (ie: don't clip) trainer = Trainer(gradient_clip=0) # clip gradients with norm above 0.5 trainer = Trainer(gradient_clip=0.5)","title":"Gradient Clipping"},{"location":"Trainer/Training Loop/#inspect-gradient-norms","text":"Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2)","title":"Inspect gradient norms"},{"location":"Trainer/Training Loop/#set-how-much-of-the-training-set-to-check","text":"If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1)","title":"Set how much of the training set to check"},{"location":"Trainer/Validation loop/","text":"The lightning validation loop handles everything except the actual computations of your model. To decide what will happen in your validation loop, define the validation_step function . Below are all the things lightning automates for you in the validation loop. Note Lightning will run 5 steps of validation in the beginning of training as a sanity check so you don't have to wait until a full epoch to catch possible validation issues. Check validation every n epochs If you have a small dataset you might want to check validation every n epochs # DEFAULT trainer = Trainer(check_val_every_n_epoch=1) Set how much of the validation set to check If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(val_percent_check=1.0) # check 10% only trainer = Trainer(val_percent_check=0.1) Set how much of the test set to check If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(test_percent_check=1.0) # check 10% only trainer = Trainer(test_percent_check=0.1) Set validation check frequency within 1 training epoch For large datasets it's often desirable to check validation multiple times within a training loop # DEFAULT trainer = Trainer(val_check_interval=0.95) # check every .25 of an epoch trainer = Trainer(val_check_interval=0.25) Set the number of validation sanity steps Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop. # DEFAULT trainer = Trainer(nb_sanity_val_steps=5)","title":"Validation loop"},{"location":"Trainer/Validation loop/#check-validation-every-n-epochs","text":"If you have a small dataset you might want to check validation every n epochs # DEFAULT trainer = Trainer(check_val_every_n_epoch=1)","title":"Check validation every n epochs"},{"location":"Trainer/Validation loop/#set-how-much-of-the-validation-set-to-check","text":"If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(val_percent_check=1.0) # check 10% only trainer = Trainer(val_percent_check=0.1)","title":"Set how much of the validation set to check"},{"location":"Trainer/Validation loop/#set-how-much-of-the-test-set-to-check","text":"If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(test_percent_check=1.0) # check 10% only trainer = Trainer(test_percent_check=0.1)","title":"Set how much of the test set to check"},{"location":"Trainer/Validation loop/#set-validation-check-frequency-within-1-training-epoch","text":"For large datasets it's often desirable to check validation multiple times within a training loop # DEFAULT trainer = Trainer(val_check_interval=0.95) # check every .25 of an epoch trainer = Trainer(val_check_interval=0.25)","title":"Set validation check frequency within 1 training epoch"},{"location":"Trainer/Validation loop/#set-the-number-of-validation-sanity-steps","text":"Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop. # DEFAULT trainer = Trainer(nb_sanity_val_steps=5)","title":"Set the number of validation sanity steps"},{"location":"Trainer/debugging/","text":"These flags are useful to help debug a model. Fast dev run This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch. Use this to debug a full run of your program quickly # DEFAULT trainer = Trainer(fast_dev_run=False) Inspect gradient norms Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2) Make model overfit on subset of data A useful debugging trick is to make your model overfit a tiny fraction of the data. # DEFAULT don't overfit (ie: normal training) trainer = Trainer(overfit_pct=0.0) # overfit on 1% of data trainer = Trainer(overfit_pct=0.01) Print the parameter count by layer By default lightning prints a list of parameters and submodules when it starts training. Print which gradients are nan This option prints a list of tensors with nan gradients. # DEFAULT trainer = Trainer(print_nan_grads=False) Log GPU usage Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.","title":"Debugging"},{"location":"Trainer/debugging/#fast-dev-run","text":"This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch. Use this to debug a full run of your program quickly # DEFAULT trainer = Trainer(fast_dev_run=False)","title":"Fast dev run"},{"location":"Trainer/debugging/#inspect-gradient-norms","text":"Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2)","title":"Inspect gradient norms"},{"location":"Trainer/debugging/#make-model-overfit-on-subset-of-data","text":"A useful debugging trick is to make your model overfit a tiny fraction of the data. # DEFAULT don't overfit (ie: normal training) trainer = Trainer(overfit_pct=0.0) # overfit on 1% of data trainer = Trainer(overfit_pct=0.01)","title":"Make model overfit on subset of data"},{"location":"Trainer/debugging/#print-the-parameter-count-by-layer","text":"By default lightning prints a list of parameters and submodules when it starts training.","title":"Print the parameter count by layer"},{"location":"Trainer/debugging/#print-which-gradients-are-nan","text":"This option prints a list of tensors with nan gradients. # DEFAULT trainer = Trainer(print_nan_grads=False)","title":"Print which gradients are nan"},{"location":"Trainer/debugging/#log-gpu-usage","text":"Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.","title":"Log GPU usage"},{"location":"Trainer/hooks/","text":"Hooks [ Github Code ] There are cases when you might want to do something different at different parts of the training/validation loop. To enable a hook, simply override the method in your LightningModule and the trainer will call it at the correct time. Contributing If there's a hook you'd like to add, simply: 1. Fork PyTorchLightning. 2. Add the hook here . 3. Add the correct place in the Trainer where it should be called. on_epoch_start Called in the training loop at the very beginning of the epoch. def on_epoch_start(self): # do something when the epoch starts on_epoch_end Called in the training loop at the very end of the epoch. def on_epoch_end(self): # do something when the epoch ends on_batch_start Called in the training loop before anything happens for that batch. def on_batch_start(self): # do something when the batch starts on_batch_end Called in the training loop after the batch. def on_batch_end(self): # do something when the batch ends on_pre_performance_check Called at the very beginning of the validation loop. def on_pre_performance_check(self): # do something before validation starts on_post_performance_check Called at the very end of the validation loop. def on_post_performance_check(self): # do something before validation end on_tng_metrics Called in the training loop, right before metrics are logged. Although you can log at any time by using self.experiment, you can use this callback to modify what will be logged. def on_tng_metrics(self, metrics): # do something before validation end optimizer_step Calls .step() and .zero_grad for each optimizer. You can override this method to adjust how you do the optimizer step for each optimizer Called once per optimizer # DEFAULT def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i): optimizer.step() optimizer.zero_grad() # Alternating schedule for optimizer steps (ie: GANs) def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i): # update generator opt every 2 steps if optimizer_i == 0: if batch_nb % 2 == 0 : optimizer.step() optimizer.zero_grad() # update discriminator opt every 4 steps if optimizer_i == 1: if batch_nb % 4 == 0 : optimizer.step() optimizer.zero_grad() on_before_zero_grad Called in the training loop after taking an optimizer step and before zeroing grads. Good place to inspect weight information with weights updated. Called once per optimizer def on_before_zero_grad(self, optimizer): # do something with the optimizer or inspect it. on_after_backward Called in the training loop after model.backward() This is the ideal place to inspect or log gradient information def on_after_backward(self): # example to inspect gradient information in tensorboard if self.trainer.global_step % 25 == 0: # don't make the tf file huge params = self.state_dict() for k, v in params.items(): grads = v name = k self.experiment.add_histogram(tag=name, values=grads, global_step=self.trainer.global_step)","title":"Hooks"},{"location":"Trainer/hooks/#hooks","text":"[ Github Code ] There are cases when you might want to do something different at different parts of the training/validation loop. To enable a hook, simply override the method in your LightningModule and the trainer will call it at the correct time. Contributing If there's a hook you'd like to add, simply: 1. Fork PyTorchLightning. 2. Add the hook here . 3. Add the correct place in the Trainer where it should be called.","title":"Hooks"},{"location":"Trainer/hooks/#on_epoch_start","text":"Called in the training loop at the very beginning of the epoch. def on_epoch_start(self): # do something when the epoch starts","title":"on_epoch_start"},{"location":"Trainer/hooks/#on_epoch_end","text":"Called in the training loop at the very end of the epoch. def on_epoch_end(self): # do something when the epoch ends","title":"on_epoch_end"},{"location":"Trainer/hooks/#on_batch_start","text":"Called in the training loop before anything happens for that batch. def on_batch_start(self): # do something when the batch starts","title":"on_batch_start"},{"location":"Trainer/hooks/#on_batch_end","text":"Called in the training loop after the batch. def on_batch_end(self): # do something when the batch ends","title":"on_batch_end"},{"location":"Trainer/hooks/#on_pre_performance_check","text":"Called at the very beginning of the validation loop. def on_pre_performance_check(self): # do something before validation starts","title":"on_pre_performance_check"},{"location":"Trainer/hooks/#on_post_performance_check","text":"Called at the very end of the validation loop. def on_post_performance_check(self): # do something before validation end","title":"on_post_performance_check"},{"location":"Trainer/hooks/#on_tng_metrics","text":"Called in the training loop, right before metrics are logged. Although you can log at any time by using self.experiment, you can use this callback to modify what will be logged. def on_tng_metrics(self, metrics): # do something before validation end","title":"on_tng_metrics"},{"location":"Trainer/hooks/#optimizer_step","text":"Calls .step() and .zero_grad for each optimizer. You can override this method to adjust how you do the optimizer step for each optimizer Called once per optimizer # DEFAULT def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i): optimizer.step() optimizer.zero_grad() # Alternating schedule for optimizer steps (ie: GANs) def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i): # update generator opt every 2 steps if optimizer_i == 0: if batch_nb % 2 == 0 : optimizer.step() optimizer.zero_grad() # update discriminator opt every 4 steps if optimizer_i == 1: if batch_nb % 4 == 0 : optimizer.step() optimizer.zero_grad()","title":"optimizer_step"},{"location":"Trainer/hooks/#on_before_zero_grad","text":"Called in the training loop after taking an optimizer step and before zeroing grads. Good place to inspect weight information with weights updated. Called once per optimizer def on_before_zero_grad(self, optimizer): # do something with the optimizer or inspect it.","title":"on_before_zero_grad"},{"location":"Trainer/hooks/#on_after_backward","text":"Called in the training loop after model.backward() This is the ideal place to inspect or log gradient information def on_after_backward(self): # example to inspect gradient information in tensorboard if self.trainer.global_step % 25 == 0: # don't make the tf file huge params = self.state_dict() for k, v in params.items(): grads = v name = k self.experiment.add_histogram(tag=name, values=grads, global_step=self.trainer.global_step)","title":"on_after_backward"},{"location":"examples/Examples/","text":"Template model definition In 99% of cases you want to just copy this template to start a new lightningModule and change the core of what your model is actually trying to do. # get a copy of the module template wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/examples/new_project_templates/lightning_module_template.py Trainer Example __main__ function Normally, we want to let the __main__ function start the training. Inside the main we parse training arguments with whatever hyperparameters we want. Your LightningModule will have a chance to add hyperparameters. from test_tube import HyperOptArgumentParser if __name__ == '__main__': # use default args given by lightning root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0] parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False) add_default_args(parent_parser, root_dir) # allow model to overwrite or extend args parser = ExampleModel.add_model_specific_args(parent_parser) hyperparams = parser.parse_args() # train model main(hyperparams) Main Function The main function is your entry into the program. This is where you init your model, checkpoint directory, and launch the training. The main function should have 3 arguments: - hparams: a configuration of hyperparameters. - slurm_manager: Slurm cluster manager object (can be None) - dict: for you to return any values you want (useful in meta-learning, otherwise set to _) def main(hparams, cluster, results_dict): \"\"\" Main training routine specific for this project :param hparams: :return: \"\"\" # init experiment log_dir = os.path.dirname(os.path.realpath(__file__)) exp = Experiment( name='test_tube_exp', debug=True, save_dir=log_dir, version=0, autosave=False, description='test demo' ) # set the hparams for the experiment exp.argparse(hparams) exp.save() # build model model = MyLightningModule(hparams) # callbacks early_stop = EarlyStopping( monitor=hparams.early_stop_metric, patience=hparams.early_stop_patience, verbose=True, mode=hparams.early_stop_mode ) model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) checkpoint = ModelCheckpoint( filepath=model_save_path, save_function=None, save_best_only=True, verbose=True, monitor=hparams.model_save_monitor_value, mode=hparams.model_save_monitor_mode ) # configure trainer trainer = Trainer( experiment=exp, cluster=cluster, checkpoint_callback=checkpoint, early_stop_callback=early_stop, ) # train model trainer.fit(model) The main function will start training on your main function. If you use the HyperParameterOptimizer in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple argument parser you get the default arguments in the argument parser. So, calling main(hyperparams) runs the model with the default argparse arguments. main(hyperparams) CPU hyperparameter search # run a grid search over 20 hyperparameter combinations. hyperparams.optimize_parallel_cpu( main_local, nb_trials=20, nb_workers=1 ) Hyperparameter search on a single or multiple GPUs # run a grid search over 20 hyperparameter combinations. hyperparams.optimize_parallel_gpu( main_local, nb_trials=20, nb_workers=1, gpus=[0,1,2,3] ) Hyperparameter search on a SLURM HPC cluster def optimize_on_cluster(hyperparams): # enable cluster training cluster = SlurmCluster( hyperparam_optimizer=hyperparams, log_path=hyperparams.tt_save_path, test_tube_exp_name=hyperparams.tt_name ) # email for cluster coms cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True) # configure cluster cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus cluster.job_time = '48:00:00' cluster.gpu_type = '1080ti' cluster.memory_mb_per_node = 48000 # any modules for code to run in env cluster.add_command('source activate pytorch_lightning') # name of exp job_display_name = hyperparams.tt_name.split('_')[0] job_display_name = job_display_name[0:3] # run hopt print('submitting jobs...') cluster.optimize_parallel_cluster_gpu( main, nb_trials=hyperparams.nb_hopt_trials, job_name=job_display_name ) # run cluster hyperparameter search optimize_on_cluster(hyperparams)","title":"Examples"},{"location":"examples/Examples/#template-model-definition","text":"In 99% of cases you want to just copy this template to start a new lightningModule and change the core of what your model is actually trying to do. # get a copy of the module template wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/examples/new_project_templates/lightning_module_template.py","title":"Template model definition"},{"location":"examples/Examples/#trainer-example","text":"__main__ function Normally, we want to let the __main__ function start the training. Inside the main we parse training arguments with whatever hyperparameters we want. Your LightningModule will have a chance to add hyperparameters. from test_tube import HyperOptArgumentParser if __name__ == '__main__': # use default args given by lightning root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0] parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False) add_default_args(parent_parser, root_dir) # allow model to overwrite or extend args parser = ExampleModel.add_model_specific_args(parent_parser) hyperparams = parser.parse_args() # train model main(hyperparams) Main Function The main function is your entry into the program. This is where you init your model, checkpoint directory, and launch the training. The main function should have 3 arguments: - hparams: a configuration of hyperparameters. - slurm_manager: Slurm cluster manager object (can be None) - dict: for you to return any values you want (useful in meta-learning, otherwise set to _) def main(hparams, cluster, results_dict): \"\"\" Main training routine specific for this project :param hparams: :return: \"\"\" # init experiment log_dir = os.path.dirname(os.path.realpath(__file__)) exp = Experiment( name='test_tube_exp', debug=True, save_dir=log_dir, version=0, autosave=False, description='test demo' ) # set the hparams for the experiment exp.argparse(hparams) exp.save() # build model model = MyLightningModule(hparams) # callbacks early_stop = EarlyStopping( monitor=hparams.early_stop_metric, patience=hparams.early_stop_patience, verbose=True, mode=hparams.early_stop_mode ) model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version) checkpoint = ModelCheckpoint( filepath=model_save_path, save_function=None, save_best_only=True, verbose=True, monitor=hparams.model_save_monitor_value, mode=hparams.model_save_monitor_mode ) # configure trainer trainer = Trainer( experiment=exp, cluster=cluster, checkpoint_callback=checkpoint, early_stop_callback=early_stop, ) # train model trainer.fit(model) The main function will start training on your main function. If you use the HyperParameterOptimizer in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple argument parser you get the default arguments in the argument parser. So, calling main(hyperparams) runs the model with the default argparse arguments. main(hyperparams)","title":"Trainer Example"},{"location":"examples/Examples/#cpu-hyperparameter-search","text":"# run a grid search over 20 hyperparameter combinations. hyperparams.optimize_parallel_cpu( main_local, nb_trials=20, nb_workers=1 )","title":"CPU hyperparameter search"},{"location":"examples/Examples/#hyperparameter-search-on-a-single-or-multiple-gpus","text":"# run a grid search over 20 hyperparameter combinations. hyperparams.optimize_parallel_gpu( main_local, nb_trials=20, nb_workers=1, gpus=[0,1,2,3] )","title":"Hyperparameter search on a single or multiple GPUs"},{"location":"examples/Examples/#hyperparameter-search-on-a-slurm-hpc-cluster","text":"def optimize_on_cluster(hyperparams): # enable cluster training cluster = SlurmCluster( hyperparam_optimizer=hyperparams, log_path=hyperparams.tt_save_path, test_tube_exp_name=hyperparams.tt_name ) # email for cluster coms cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True) # configure cluster cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus cluster.job_time = '48:00:00' cluster.gpu_type = '1080ti' cluster.memory_mb_per_node = 48000 # any modules for code to run in env cluster.add_command('source activate pytorch_lightning') # name of exp job_display_name = hyperparams.tt_name.split('_')[0] job_display_name = job_display_name[0:3] # run hopt print('submitting jobs...') cluster.optimize_parallel_cluster_gpu( main, nb_trials=hyperparams.nb_hopt_trials, job_name=job_display_name ) # run cluster hyperparameter search optimize_on_cluster(hyperparams)","title":"Hyperparameter search on a SLURM HPC cluster"}]}
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 40f79450..ee15f132 100644
Binary files a/sitemap.xml.gz and b/sitemap.xml.gz differ