From f324ebf0ad97956410fcf0f5f352efae2301e409 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 27 Jun 2019 12:29:15 -0500 Subject: [PATCH] Deployed db29488 with MkDocs version: 1.0.4 --- 404.html | 6 +- Pytorch-Lightning/LightningModule/index.html | 6 +- Trainer/Checkpointing/index.html | 6 +- Trainer/Distributed training/index.html | 6 +- Trainer/SLURM Managed Cluster/index.html | 6 +- Trainer/Training Loop/index.html | 60 +++- Trainer/Validation loop/index.html | 256 ++++++++++++++++++ Trainer/{Vaildation loop => hooks}/index.html | 20 +- Trainer/index.html | 21 +- index.html | 8 +- search.html | 6 +- search/search_index.json | 2 +- sitemap.xml | 5 + sitemap.xml.gz | Bin 201 -> 202 bytes 14 files changed, 369 insertions(+), 39 deletions(-) create mode 100644 Trainer/Validation loop/index.html rename Trainer/{Vaildation loop => hooks}/index.html (89%) diff --git a/404.html b/404.html index ebc26eaf..31bc72d4 100644 --- a/404.html +++ b/404.html @@ -83,7 +83,11 @@
  • - Vaildation loop + Validation loop +
    • +
  • + + Hooks
    • diff --git a/Pytorch-Lightning/LightningModule/index.html b/Pytorch-Lightning/LightningModule/index.html index e6dd4193..6c806263 100644 --- a/Pytorch-Lightning/LightningModule/index.html +++ b/Pytorch-Lightning/LightningModule/index.html @@ -122,7 +122,11 @@
  • - Vaildation loop + Validation loop +
    • +
  • + + Hooks
    • diff --git a/Trainer/Checkpointing/index.html b/Trainer/Checkpointing/index.html index a071b5ef..59dc7699 100644 --- a/Trainer/Checkpointing/index.html +++ b/Trainer/Checkpointing/index.html @@ -93,7 +93,11 @@
  • - Vaildation loop + Validation loop +
    • +
  • + + Hooks
    • diff --git a/Trainer/Distributed training/index.html b/Trainer/Distributed training/index.html index f648fc1c..4701f9cf 100644 --- a/Trainer/Distributed training/index.html +++ b/Trainer/Distributed training/index.html @@ -93,7 +93,11 @@
  • - Vaildation loop + Validation loop +
    • +
  • + + Hooks
    • diff --git a/Trainer/SLURM Managed Cluster/index.html b/Trainer/SLURM Managed Cluster/index.html index 8915ca67..f8848604 100644 --- a/Trainer/SLURM Managed Cluster/index.html +++ b/Trainer/SLURM Managed Cluster/index.html @@ -93,7 +93,11 @@
  • - Vaildation loop + Validation loop +
    • +
  • + + Hooks
    • diff --git a/Trainer/Training Loop/index.html b/Trainer/Training Loop/index.html index 2150d68a..a57e3408 100644 --- a/Trainer/Training Loop/index.html +++ b/Trainer/Training Loop/index.html @@ -101,24 +101,30 @@
  • Check which gradients are nan
    • -
  • Check validation every n epochs
    • - -
  • Display metrics in progress bar
  • Display the parameter count by layer
    • +
  • Fast dev run
    • + +
  • Force training for min or max epochs
    • +
  • Force disable early stop
    • + +
  • Inspect gradient norms
  • Make model overfit on subset of data
    • +
  • Process position
    • + +
  • Set how much of the training set to check
    • @@ -126,7 +132,11 @@
  • - Vaildation loop + Validation loop +
    • +
  • + + Hooks
    • @@ -170,7 +180,8 @@
    -

    The asdf

    +

    The lightning training loop handles everything except the actual computations of your model. To decide what will happen in your training loop, define the training_step function.

    +

    Below are all the things lightning automates for you in the training loop.

    Accumulated gradients

    Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN.

    @@ -198,13 +209,6 @@ trainer = Trainer(lr_scheduler_milestones=[100, 200, 300]) trainer = Trainer(print_nan_grads=False) -
    -

    Check validation every n epochs

    -

    If you have a small dataset you might want to check validation every n epochs

    -
    # DEFAULT
-trainer = Trainer(check_val_every_n_epoch=1)
-
    -

    Display metrics in progress bar

    # DEFAULT
@@ -214,6 +218,14 @@ trainer = Trainer(progress_bar=True)
 
    
 
    Display the parameter count by layer
    
 
    By default lightning prints a list of parameters and submodules when it starts training.
    
+
    
+
    Fast dev run
    
+
    This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch.
+Use this to debug a full run of your program quickly
    
+
    # DEFAULT
+trainer = Trainer(fast_dev_run=False)
+
    
+
 
    
 
    Force training for min or max epochs
    
 
    It can be useful to force training for a minimum number of epochs or limit to a max number
    
@@ -221,6 +233,13 @@ trainer = Trainer(progress_bar=True)
 trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)
    +
    +

    Force disable early stop

    +

    Use this to turn off early stopping and run training to the max_epoch

    +
    # DEFAULT
+trainer = Trainer(enable_early_stop=True)
+
    +

    Inspect gradient norms

    Looking at grad norms can help you figure out where training might be going wrong.

    @@ -241,9 +260,20 @@ trainer = Trainer(overfit_pct=0.0) trainer = Trainer(overfit_pct=0.01) +
    +

    Process position

    +

    When running multiple models on the same machine we want to decide which progress bar to use. +Lightning will stack progress bars according to this value.

    +
    # DEFAULT
+trainer = Trainer(process_position=0)
+
+# if this is the second model on the node, show the second progress bar below
+trainer = Trainer(process_position=1)
+
    +

    Set how much of the training set to check

    -

    If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag

    +

    If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag

    # DEFAULT
 trainer = Trainer(train_percent_check=1.0)
 
@@ -257,7 +287,7 @@ trainer = Trainer(train_percent_check=0.1)
   
     
diff --git a/Trainer/Validation loop/index.html b/Trainer/Validation loop/index.html
new file mode 100644
index 00000000..98c5042d
--- /dev/null
+++ b/Trainer/Validation loop/index.html	
@@ -0,0 +1,256 @@
+
+
+  
+
+  
+  
+  
+  
+  
+  
+  Validation loop - Pytorch lightning Documentation
+  
+
+  
+  
+  
+  
+  
+  
+  
+  
+  
+   
+  
+
+
+
+
+  
    
+
+    
+    
+
+    
    
+
+      
+      
+
+      
+      
    
+        
    
+          
    
+  
+  
    
+
    
+          
    
+            
    
+              
+                
    The lightning validation loop handles everything except the actual computations of your model. To decide what will happen in your validation loop, define the validation_step function.
+Below are all the things lightning automates for you in the validation loop.
    
+
    Note 
    
+Lightning will run 5 steps of validation in the beginning of training as a sanity check so you don't have to wait until a full epoch to catch possible validation issues.
    
+
    
+
    Check validation every n epochs
    
+
    If you have a small dataset you might want to check validation every n epochs
    
+
    # DEFAULT
+trainer = Trainer(check_val_every_n_epoch=1)
+
    
+
+
    
+
    Set how much of the validation set to check
    
+
    If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag
    
+
    # DEFAULT
+trainer = Trainer(val_percent_check=1.0)
+
+# check 10% only
+trainer = Trainer(val_percent_check=0.1)
+
    
+
+
    
+
    Set how much of the test set to check
    
+
    If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag
    
+
    # DEFAULT
+trainer = Trainer(test_percent_check=1.0)
+
+# check 10% only
+trainer = Trainer(test_percent_check=0.1)
+
    
+
+
    
+
    Set validation check frequency within 1 training epoch
    
+
    For large datasets it's often desirable to check validation multiple times within a training loop
    
+
    # DEFAULT
+trainer = Trainer(val_check_interval=0.95)
+
+# check every .25 of an epoch 
+trainer = Trainer(val_check_interval=0.25)
+
    
+
+
    
+
    Set the number of validation sanity steps
    
+
    Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop.
    
+
    # DEFAULT
+trainer = Trainer(nb_sanity_val_steps=5)
+
    
+              
+            
    
+          
    
+          
+      
+        
    
+      
    
+
+    
    
+
+  
    
+
+  
    
+    
+      
+           GitHub
+      
+      
+        « Previous
+      
+      
+        Next »
+      
+    
+
    
+    
+    
+      
+
+
+
diff --git a/Trainer/Vaildation loop/index.html b/Trainer/hooks/index.html
similarity index 89%
rename from Trainer/Vaildation loop/index.html
rename to Trainer/hooks/index.html
index 6d2be8a2..69192cd1 100644
--- a/Trainer/Vaildation loop/index.html	
+++ b/Trainer/hooks/index.html
@@ -8,7 +8,7 @@
   
   
   
-  Vaildation loop - Pytorch lightning Documentation
+  Hooks - Pytorch lightning Documentation
   
 
   
@@ -17,8 +17,8 @@
   
   
   
@@ -87,10 +87,14 @@
                 
  • 
                     
     Training Loop
+                
    • 
+                
  • 
+                    
+    Validation loop
                 
    • 
                 
  • 
                     
-    Vaildation loop
+    Hooks
     
      
             
     
    
@@ -124,10 +128,10 @@
         
       
     
-    
  • Vaildation loop
    • 
+    
  • Hooks
    • 
     
  • 
       
-         Edit on GitHub
       
     
    • 
@@ -146,7 +150,7 @@
     
   
@@ -174,7 +178,7 @@
            GitHub
       
       
-        « Previous
+        « Previous
       
       
     
diff --git a/Trainer/index.html b/Trainer/index.html
index 99d07029..dbd4d6c0 100644
--- a/Trainer/index.html
+++ b/Trainer/index.html
@@ -96,7 +96,11 @@
                 
                 
  • 
                     
-    Vaildation loop
+    Validation loop
+                
    • 
+                
  • 
+                    
+    Hooks
                 
    • 
     
 	    
@@ -159,22 +163,25 @@ trainer.fit(model)
 
  • Anneal Learning rate
    • 
 
  • Check GPU usage
    • 
 
  • Check which gradients are nan
    • 
-
  • Check validation every n epochs
    • 
 
  • Display metrics in progress bar
    • 
 
  • Display the parameter count by layer
    • 
+
  • Fast dev run
    • 
 
  • Force training for min or max epochs
    • 
+
  • Force disable early stop
    • 
 
  • Inspect gradient norms
    • 
 
  • Make model overfit on subset of data
    • 
 
  • Use multiple optimizers (like GANs)
    • 
+
  • Process position
    • 
 
  • Set how much of the training set to check (1-100%)
    • 
 
 
    Validation loop    
    
 
 
    Distributed training    
    
 
      
diff --git a/index.html b/index.html
index f447036b..4caed35c 100644
--- a/index.html
+++ b/index.html
@@ -114,7 +114,11 @@
                 
                 
    • 
                     
-    Vaildation loop
+    Validation loop
+                
      • 
+                
    • 
+                    
+    Hooks
                 
      • 
     
    
 	    
@@ -261,5 +265,5 @@
 
 
diff --git a/search.html b/search.html
index 7d73030f..5c53cbed 100644
--- a/search.html
+++ b/search.html
@@ -83,7 +83,11 @@
                 
                 
  • 
                     
-    Vaildation loop
+    Validation loop
+                
    • 
+                
  • 
+                    
+    Hooks
                 
    • 
     
 	    
diff --git a/search/search_index.json b/search/search_index.json
index d3ddb95c..50bb7182 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"PYTORCH-LIGHTNING DOCUMENTATION Quick start Define a LightningModule Set up the trainer Quick start examples CPU example Single GPU example Multi-gpu example SLURM cluster grid search example Training loop Accumulate gradients Check GPU usage Check which gradients are nan Check validation every n epochs Display metrics in progress bar Force training for min or max epochs Inspect gradient norms Hooks Learning rate annealing Make model overfit on subset of data Multiple optimizers (like GANs) Set how much of the training set to check (1-100%) training_step function Validation loop Display metrics in progress bar hooks Set how much of the validation set to check (1-100%) Set validation check frequency within 1 training epoch (1-100%) validation_step function Why does validation run first for 5 steps? Distributed training Single-gpu Multi-gpu Multi-node 16-bit mixed precision Checkpointing Model saving Model loading Computing cluster (SLURM) Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"PYTORCH-LIGHTNING DOCUMENTATION"},{"location":"#pytorch-lightning-documentation","text":"","title":"PYTORCH-LIGHTNING DOCUMENTATION"},{"location":"#quick-start","text":"Define a LightningModule Set up the trainer","title":"Quick start"},{"location":"#quick-start-examples","text":"CPU example Single GPU example Multi-gpu example SLURM cluster grid search example","title":"Quick start examples"},{"location":"#training-loop","text":"Accumulate gradients Check GPU usage Check which gradients are nan Check validation every n epochs Display metrics in progress bar Force training for min or max epochs Inspect gradient norms Hooks Learning rate annealing Make model overfit on subset of data Multiple optimizers (like GANs) Set how much of the training set to check (1-100%) training_step function","title":"Training loop"},{"location":"#validation-loop","text":"Display metrics in progress bar hooks Set how much of the validation set to check (1-100%) Set validation check frequency within 1 training epoch (1-100%) validation_step function Why does validation run first for 5 steps?","title":"Validation loop"},{"location":"#distributed-training","text":"Single-gpu Multi-gpu Multi-node 16-bit mixed precision","title":"Distributed training"},{"location":"#checkpointing","text":"Model saving Model loading","title":"Checkpointing"},{"location":"#computing-cluster-slurm","text":"Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"Computing cluster (SLURM)"},{"location":"Pytorch-Lightning/LightningModule/","text":"Lightning module [ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. The easiest thing to do is copy this template and modify accordingly. Otherwise, to Define a Lightning Module, implement the following methods: Required : training_step validation_step validation_end configure_optimizers get_save_dict load_model_specific tng_dataloader tng_dataloader test_dataloader Optional : update_tng_log_metrics add_model_specific_args training_step def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output validation_step def validation_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output validation_end def validation_end(self, outputs) Called at the end of the validation loop with the output of each validation_step. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic configure_optimizers def configure_optimizers(self) Set up as many optimizers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one. If you use 16 bit precision it will also handle that. Return List - List of optimizers Example # most cases def configure_optimizers(self): opt = Adam(lr=0.01) return [opt] # gan example def configure_optimizers(self): generator_opt = Adam(lr=0.01) disriminator_opt = Adam(lr=0.02) return [generator_opt, disriminator_opt] get_save_dict def get_save_dict(self) Called by lightning to checkpoint your model. Lightning saves current epoch, current batch nb, etc... All you have to return is what specifically about your lightning model you want to checkpoint. Return Dictionary - No required keys. Most of the time as described in this example. Example def get_save_dict(self): # 99% of use cases this is all you need to return checkpoint = {'state_dict': self.state_dict()} return checkpoint load_model_specific def load_model_specific(self, checkpoint) Called by lightning to restore your model. This is your chance to restore your model using the keys you added in get_save_dict. Lightning will automatically restore current epoch, batch nb, etc. Return Nothing Example def load_model_specific(self, checkpoint): # you defined 'state_dict' in get_save_dict() self.load_state_dict(checkpoint['state_dict']) tng_dataloader @property def tng_dataloader(self) Called by lightning during training loop. Define it as a property. Return Pytorch DataLoader Example @property def tng_dataloader(self): if self._tng_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._tng_dataloader = loader except Exception as e: raise e return self._tng_dataloader val_dataloader @property def tng_dataloader(self) Called by lightning during validation loop. Define it as a property. Return Pytorch DataLoader Example @property def val_dataloader(self): if self._val_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._val_dataloader = loader except Exception as e: raise e return self._val_dataloader test_dataloader @property def test_dataloader(self) Called by lightning during test loop. Define it as a property. Return Pytorch DataLoader Example @property def test_dataloader(self): if self._test_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._test_dataloader = loader except Exception as e: raise e return self._test_dataloader update_tng_log_metrics def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged. Return Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs add_model_specific_args @staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The argument parser is available anywhere in your model by calling self.hparams Return An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Lightning module"},{"location":"Pytorch-Lightning/LightningModule/#lightning-module","text":"[ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. The easiest thing to do is copy this template and modify accordingly. Otherwise, to Define a Lightning Module, implement the following methods: Required : training_step validation_step validation_end configure_optimizers get_save_dict load_model_specific tng_dataloader tng_dataloader test_dataloader Optional : update_tng_log_metrics add_model_specific_args","title":"Lightning module"},{"location":"Pytorch-Lightning/LightningModule/#training_step","text":"def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output","title":"training_step"},{"location":"Pytorch-Lightning/LightningModule/#validation_step","text":"def validation_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output","title":"validation_step"},{"location":"Pytorch-Lightning/LightningModule/#validation_end","text":"def validation_end(self, outputs) Called at the end of the validation loop with the output of each validation_step. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic","title":"validation_end"},{"location":"Pytorch-Lightning/LightningModule/#configure_optimizers","text":"def configure_optimizers(self) Set up as many optimizers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one. If you use 16 bit precision it will also handle that.","title":"configure_optimizers"},{"location":"Pytorch-Lightning/LightningModule/#return","text":"List - List of optimizers Example # most cases def configure_optimizers(self): opt = Adam(lr=0.01) return [opt] # gan example def configure_optimizers(self): generator_opt = Adam(lr=0.01) disriminator_opt = Adam(lr=0.02) return [generator_opt, disriminator_opt]","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#get_save_dict","text":"def get_save_dict(self) Called by lightning to checkpoint your model. Lightning saves current epoch, current batch nb, etc... All you have to return is what specifically about your lightning model you want to checkpoint.","title":"get_save_dict"},{"location":"Pytorch-Lightning/LightningModule/#return_1","text":"Dictionary - No required keys. Most of the time as described in this example. Example def get_save_dict(self): # 99% of use cases this is all you need to return checkpoint = {'state_dict': self.state_dict()} return checkpoint","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#load_model_specific","text":"def load_model_specific(self, checkpoint) Called by lightning to restore your model. This is your chance to restore your model using the keys you added in get_save_dict. Lightning will automatically restore current epoch, batch nb, etc.","title":"load_model_specific"},{"location":"Pytorch-Lightning/LightningModule/#return_2","text":"Nothing Example def load_model_specific(self, checkpoint): # you defined 'state_dict' in get_save_dict() self.load_state_dict(checkpoint['state_dict'])","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#tng_dataloader","text":"@property def tng_dataloader(self) Called by lightning during training loop. Define it as a property.","title":"tng_dataloader"},{"location":"Pytorch-Lightning/LightningModule/#return_3","text":"Pytorch DataLoader Example @property def tng_dataloader(self): if self._tng_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._tng_dataloader = loader except Exception as e: raise e return self._tng_dataloader","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#val_dataloader","text":"@property def tng_dataloader(self) Called by lightning during validation loop. Define it as a property.","title":"val_dataloader"},{"location":"Pytorch-Lightning/LightningModule/#return_4","text":"Pytorch DataLoader Example @property def val_dataloader(self): if self._val_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._val_dataloader = loader except Exception as e: raise e return self._val_dataloader","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#test_dataloader","text":"@property def test_dataloader(self) Called by lightning during test loop. Define it as a property.","title":"test_dataloader"},{"location":"Pytorch-Lightning/LightningModule/#return_5","text":"Pytorch DataLoader Example @property def test_dataloader(self): if self._test_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._test_dataloader = loader except Exception as e: raise e return self._test_dataloader","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#update_tng_log_metrics","text":"def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged.","title":"update_tng_log_metrics"},{"location":"Pytorch-Lightning/LightningModule/#return_6","text":"Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#add_model_specific_args","text":"@staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The argument parser is available anywhere in your model by calling self.hparams","title":"add_model_specific_args"},{"location":"Pytorch-Lightning/LightningModule/#return_7","text":"An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Return"},{"location":"Trainer/","text":"Trainer [ Github Code ] The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. This is the basic use of the trainer: from pytorch_lightning import Trainer model = LightningTemplate() trainer = Trainer() trainer.fit(model) But of course the fun is in all the advanced things it can do: Training loop Accumulate gradients Anneal Learning rate Check GPU usage Check which gradients are nan Check validation every n epochs Display metrics in progress bar Display the parameter count by layer Force training for min or max epochs Inspect gradient norms Make model overfit on subset of data Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%) Validation loop Display metrics in progress bar Set how much of the validation set to check (1-100%) Set validation check frequency within 1 training epoch (1-100%) validation_step function Why does validation run first for 5 steps? Distributed training Single-gpu Multi-gpu Multi-node 16-bit mixed precision Checkpointing Model saving Model loading Computing cluster (SLURM) Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"Trainer"},{"location":"Trainer/#trainer","text":"[ Github Code ] The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. This is the basic use of the trainer: from pytorch_lightning import Trainer model = LightningTemplate() trainer = Trainer() trainer.fit(model) But of course the fun is in all the advanced things it can do: Training loop Accumulate gradients Anneal Learning rate Check GPU usage Check which gradients are nan Check validation every n epochs Display metrics in progress bar Display the parameter count by layer Force training for min or max epochs Inspect gradient norms Make model overfit on subset of data Use multiple optimizers (like GANs) Set how much of the training set to check (1-100%) Validation loop Display metrics in progress bar Set how much of the validation set to check (1-100%) Set validation check frequency within 1 training epoch (1-100%) validation_step function Why does validation run first for 5 steps? Distributed training Single-gpu Multi-gpu Multi-node 16-bit mixed precision Checkpointing Model saving Model loading Computing cluster (SLURM) Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"Trainer"},{"location":"Trainer/Checkpointing/","text":"","title":"Checkpointing"},{"location":"Trainer/Distributed training/","text":"","title":"Distributed training"},{"location":"Trainer/SLURM Managed Cluster/","text":"","title":"SLURM Managed Cluster"},{"location":"Trainer/Training Loop/","text":"The asdf Accumulated gradients Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. # DEFAULT (ie: no accumulated grads) trainer = Trainer(accumulate_grad_batches=1) Anneal Learning rate Cut the learning rate by 10 at every epoch listed in this list. # DEFAULT (don't anneal) trainer = Trainer(lr_scheduler_milestones=None) # cut LR by 10 at 100, 200, and 300 epochs trainer = Trainer(lr_scheduler_milestones=[100, 200, 300]) Check GPU usage Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training. Check which gradients are nan This option prints a list of tensors with nan gradients. # DEFAULT trainer = Trainer(print_nan_grads=False) Check validation every n epochs If you have a small dataset you might want to check validation every n epochs # DEFAULT trainer = Trainer(check_val_every_n_epoch=1) Display metrics in progress bar # DEFAULT trainer = Trainer(progress_bar=True) Display the parameter count by layer By default lightning prints a list of parameters and submodules when it starts training. Force training for min or max epochs It can be useful to force training for a minimum number of epochs or limit to a max number # DEFAULT trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000) Inspect gradient norms Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2) Make model overfit on subset of data A useful debugging trick is to make your model overfit a tiny fraction of the data. # DEFAULT don't overfit (ie: normal training) trainer = Trainer(overfit_pct=0.0) # overfit on 1% of data trainer = Trainer(overfit_pct=0.01) Set how much of the training set to check If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1)","title":"Training Loop"},{"location":"Trainer/Training Loop/#accumulated-gradients","text":"Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. # DEFAULT (ie: no accumulated grads) trainer = Trainer(accumulate_grad_batches=1)","title":"Accumulated gradients"},{"location":"Trainer/Training Loop/#anneal-learning-rate","text":"Cut the learning rate by 10 at every epoch listed in this list. # DEFAULT (don't anneal) trainer = Trainer(lr_scheduler_milestones=None) # cut LR by 10 at 100, 200, and 300 epochs trainer = Trainer(lr_scheduler_milestones=[100, 200, 300])","title":"Anneal Learning rate"},{"location":"Trainer/Training Loop/#check-gpu-usage","text":"Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.","title":"Check GPU usage"},{"location":"Trainer/Training Loop/#check-which-gradients-are-nan","text":"This option prints a list of tensors with nan gradients. # DEFAULT trainer = Trainer(print_nan_grads=False)","title":"Check which gradients are nan"},{"location":"Trainer/Training Loop/#check-validation-every-n-epochs","text":"If you have a small dataset you might want to check validation every n epochs # DEFAULT trainer = Trainer(check_val_every_n_epoch=1)","title":"Check validation every n epochs"},{"location":"Trainer/Training Loop/#display-metrics-in-progress-bar","text":"# DEFAULT trainer = Trainer(progress_bar=True)","title":"Display metrics in progress bar"},{"location":"Trainer/Training Loop/#display-the-parameter-count-by-layer","text":"By default lightning prints a list of parameters and submodules when it starts training.","title":"Display the parameter count by layer"},{"location":"Trainer/Training Loop/#force-training-for-min-or-max-epochs","text":"It can be useful to force training for a minimum number of epochs or limit to a max number # DEFAULT trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)","title":"Force training for min or max epochs"},{"location":"Trainer/Training Loop/#inspect-gradient-norms","text":"Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2)","title":"Inspect gradient norms"},{"location":"Trainer/Training Loop/#make-model-overfit-on-subset-of-data","text":"A useful debugging trick is to make your model overfit a tiny fraction of the data. # DEFAULT don't overfit (ie: normal training) trainer = Trainer(overfit_pct=0.0) # overfit on 1% of data trainer = Trainer(overfit_pct=0.01)","title":"Make model overfit on subset of data"},{"location":"Trainer/Training Loop/#set-how-much-of-the-training-set-to-check","text":"If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1)","title":"Set how much of the training set to check"},{"location":"Trainer/Vaildation loop/","text":"","title":"Vaildation loop"}]}
\ No newline at end of file
+{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"PYTORCH-LIGHTNING DOCUMENTATION Quick start Define a LightningModule Set up the trainer Quick start examples CPU example Single GPU example Multi-gpu example SLURM cluster grid search example Training loop Accumulate gradients Check GPU usage Check which gradients are nan Check validation every n epochs Display metrics in progress bar Force training for min or max epochs Inspect gradient norms Hooks Learning rate annealing Make model overfit on subset of data Multiple optimizers (like GANs) Set how much of the training set to check (1-100%) training_step function Validation loop Display metrics in progress bar hooks Set how much of the validation set to check (1-100%) Set validation check frequency within 1 training epoch (1-100%) validation_step function Why does validation run first for 5 steps? Distributed training Single-gpu Multi-gpu Multi-node 16-bit mixed precision Checkpointing Model saving Model loading Computing cluster (SLURM) Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"PYTORCH-LIGHTNING DOCUMENTATION"},{"location":"#pytorch-lightning-documentation","text":"","title":"PYTORCH-LIGHTNING DOCUMENTATION"},{"location":"#quick-start","text":"Define a LightningModule Set up the trainer","title":"Quick start"},{"location":"#quick-start-examples","text":"CPU example Single GPU example Multi-gpu example SLURM cluster grid search example","title":"Quick start examples"},{"location":"#training-loop","text":"Accumulate gradients Check GPU usage Check which gradients are nan Check validation every n epochs Display metrics in progress bar Force training for min or max epochs Inspect gradient norms Hooks Learning rate annealing Make model overfit on subset of data Multiple optimizers (like GANs) Set how much of the training set to check (1-100%) training_step function","title":"Training loop"},{"location":"#validation-loop","text":"Display metrics in progress bar hooks Set how much of the validation set to check (1-100%) Set validation check frequency within 1 training epoch (1-100%) validation_step function Why does validation run first for 5 steps?","title":"Validation loop"},{"location":"#distributed-training","text":"Single-gpu Multi-gpu Multi-node 16-bit mixed precision","title":"Distributed training"},{"location":"#checkpointing","text":"Model saving Model loading","title":"Checkpointing"},{"location":"#computing-cluster-slurm","text":"Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"Computing cluster (SLURM)"},{"location":"Pytorch-Lightning/LightningModule/","text":"Lightning module [ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. The easiest thing to do is copy this template and modify accordingly. Otherwise, to Define a Lightning Module, implement the following methods: Required : training_step validation_step validation_end configure_optimizers get_save_dict load_model_specific tng_dataloader tng_dataloader test_dataloader Optional : update_tng_log_metrics add_model_specific_args training_step def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output validation_step def validation_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output validation_end def validation_end(self, outputs) Called at the end of the validation loop with the output of each validation_step. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic configure_optimizers def configure_optimizers(self) Set up as many optimizers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one. If you use 16 bit precision it will also handle that. Return List - List of optimizers Example # most cases def configure_optimizers(self): opt = Adam(lr=0.01) return [opt] # gan example def configure_optimizers(self): generator_opt = Adam(lr=0.01) disriminator_opt = Adam(lr=0.02) return [generator_opt, disriminator_opt] get_save_dict def get_save_dict(self) Called by lightning to checkpoint your model. Lightning saves current epoch, current batch nb, etc... All you have to return is what specifically about your lightning model you want to checkpoint. Return Dictionary - No required keys. Most of the time as described in this example. Example def get_save_dict(self): # 99% of use cases this is all you need to return checkpoint = {'state_dict': self.state_dict()} return checkpoint load_model_specific def load_model_specific(self, checkpoint) Called by lightning to restore your model. This is your chance to restore your model using the keys you added in get_save_dict. Lightning will automatically restore current epoch, batch nb, etc. Return Nothing Example def load_model_specific(self, checkpoint): # you defined 'state_dict' in get_save_dict() self.load_state_dict(checkpoint['state_dict']) tng_dataloader @property def tng_dataloader(self) Called by lightning during training loop. Define it as a property. Return Pytorch DataLoader Example @property def tng_dataloader(self): if self._tng_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._tng_dataloader = loader except Exception as e: raise e return self._tng_dataloader val_dataloader @property def tng_dataloader(self) Called by lightning during validation loop. Define it as a property. Return Pytorch DataLoader Example @property def val_dataloader(self): if self._val_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._val_dataloader = loader except Exception as e: raise e return self._val_dataloader test_dataloader @property def test_dataloader(self) Called by lightning during test loop. Define it as a property. Return Pytorch DataLoader Example @property def test_dataloader(self): if self._test_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._test_dataloader = loader except Exception as e: raise e return self._test_dataloader update_tng_log_metrics def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged. Return Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs add_model_specific_args @staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The argument parser is available anywhere in your model by calling self.hparams Return An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Lightning module"},{"location":"Pytorch-Lightning/LightningModule/#lightning-module","text":"[ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. The easiest thing to do is copy this template and modify accordingly. Otherwise, to Define a Lightning Module, implement the following methods: Required : training_step validation_step validation_end configure_optimizers get_save_dict load_model_specific tng_dataloader tng_dataloader test_dataloader Optional : update_tng_log_metrics add_model_specific_args","title":"Lightning module"},{"location":"Pytorch-Lightning/LightningModule/#training_step","text":"def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output","title":"training_step"},{"location":"Pytorch-Lightning/LightningModule/#validation_step","text":"def validation_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output","title":"validation_step"},{"location":"Pytorch-Lightning/LightningModule/#validation_end","text":"def validation_end(self, outputs) Called at the end of the validation loop with the output of each validation_step. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic","title":"validation_end"},{"location":"Pytorch-Lightning/LightningModule/#configure_optimizers","text":"def configure_optimizers(self) Set up as many optimizers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one. If you use 16 bit precision it will also handle that.","title":"configure_optimizers"},{"location":"Pytorch-Lightning/LightningModule/#return","text":"List - List of optimizers Example # most cases def configure_optimizers(self): opt = Adam(lr=0.01) return [opt] # gan example def configure_optimizers(self): generator_opt = Adam(lr=0.01) disriminator_opt = Adam(lr=0.02) return [generator_opt, disriminator_opt]","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#get_save_dict","text":"def get_save_dict(self) Called by lightning to checkpoint your model. Lightning saves current epoch, current batch nb, etc... All you have to return is what specifically about your lightning model you want to checkpoint.","title":"get_save_dict"},{"location":"Pytorch-Lightning/LightningModule/#return_1","text":"Dictionary - No required keys. Most of the time as described in this example. Example def get_save_dict(self): # 99% of use cases this is all you need to return checkpoint = {'state_dict': self.state_dict()} return checkpoint","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#load_model_specific","text":"def load_model_specific(self, checkpoint) Called by lightning to restore your model. This is your chance to restore your model using the keys you added in get_save_dict. Lightning will automatically restore current epoch, batch nb, etc.","title":"load_model_specific"},{"location":"Pytorch-Lightning/LightningModule/#return_2","text":"Nothing Example def load_model_specific(self, checkpoint): # you defined 'state_dict' in get_save_dict() self.load_state_dict(checkpoint['state_dict'])","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#tng_dataloader","text":"@property def tng_dataloader(self) Called by lightning during training loop. Define it as a property.","title":"tng_dataloader"},{"location":"Pytorch-Lightning/LightningModule/#return_3","text":"Pytorch DataLoader Example @property def tng_dataloader(self): if self._tng_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._tng_dataloader = loader except Exception as e: raise e return self._tng_dataloader","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#val_dataloader","text":"@property def tng_dataloader(self) Called by lightning during validation loop. Define it as a property.","title":"val_dataloader"},{"location":"Pytorch-Lightning/LightningModule/#return_4","text":"Pytorch DataLoader Example @property def val_dataloader(self): if self._val_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._val_dataloader = loader except Exception as e: raise e return self._val_dataloader","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#test_dataloader","text":"@property def test_dataloader(self) Called by lightning during test loop. Define it as a property.","title":"test_dataloader"},{"location":"Pytorch-Lightning/LightningModule/#return_5","text":"Pytorch DataLoader Example @property def test_dataloader(self): if self._test_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._test_dataloader = loader except Exception as e: raise e return self._test_dataloader","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#update_tng_log_metrics","text":"def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged.","title":"update_tng_log_metrics"},{"location":"Pytorch-Lightning/LightningModule/#return_6","text":"Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#add_model_specific_args","text":"@staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The argument parser is available anywhere in your model by calling self.hparams","title":"add_model_specific_args"},{"location":"Pytorch-Lightning/LightningModule/#return_7","text":"An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Return"},{"location":"Trainer/","text":"Trainer [ Github Code ] The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. This is the basic use of the trainer: from pytorch_lightning import Trainer model = LightningTemplate() trainer = Trainer() trainer.fit(model) But of course the fun is in all the advanced things it can do: Training loop Accumulate gradients Anneal Learning rate Check GPU usage Check which gradients are nan Display metrics in progress bar Display the parameter count by layer Fast dev run Force training for min or max epochs Force disable early stop Inspect gradient norms Make model overfit on subset of data Use multiple optimizers (like GANs) Process position Set how much of the training set to check (1-100%) Validation loop Check validation every n epochs Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps Check validation every n epochs Distributed training Single-gpu Multi-gpu Multi-node 16-bit mixed precision Checkpointing Model saving Model loading Computing cluster (SLURM) Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"Trainer"},{"location":"Trainer/#trainer","text":"[ Github Code ] The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research. This is the basic use of the trainer: from pytorch_lightning import Trainer model = LightningTemplate() trainer = Trainer() trainer.fit(model) But of course the fun is in all the advanced things it can do: Training loop Accumulate gradients Anneal Learning rate Check GPU usage Check which gradients are nan Display metrics in progress bar Display the parameter count by layer Fast dev run Force training for min or max epochs Force disable early stop Inspect gradient norms Make model overfit on subset of data Use multiple optimizers (like GANs) Process position Set how much of the training set to check (1-100%) Validation loop Check validation every n epochs Set how much of the validation set to check Set how much of the test set to check Set validation check frequency within 1 training epoch Set the number of validation sanity steps Check validation every n epochs Distributed training Single-gpu Multi-gpu Multi-node 16-bit mixed precision Checkpointing Model saving Model loading Computing cluster (SLURM) Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"Trainer"},{"location":"Trainer/Checkpointing/","text":"","title":"Checkpointing"},{"location":"Trainer/Distributed training/","text":"","title":"Distributed training"},{"location":"Trainer/SLURM Managed Cluster/","text":"","title":"SLURM Managed Cluster"},{"location":"Trainer/Training Loop/","text":"The lightning training loop handles everything except the actual computations of your model. To decide what will happen in your training loop, define the training_step function . Below are all the things lightning automates for you in the training loop. Accumulated gradients Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. # DEFAULT (ie: no accumulated grads) trainer = Trainer(accumulate_grad_batches=1) Anneal Learning rate Cut the learning rate by 10 at every epoch listed in this list. # DEFAULT (don't anneal) trainer = Trainer(lr_scheduler_milestones=None) # cut LR by 10 at 100, 200, and 300 epochs trainer = Trainer(lr_scheduler_milestones=[100, 200, 300]) Check GPU usage Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training. Check which gradients are nan This option prints a list of tensors with nan gradients. # DEFAULT trainer = Trainer(print_nan_grads=False) Display metrics in progress bar # DEFAULT trainer = Trainer(progress_bar=True) Display the parameter count by layer By default lightning prints a list of parameters and submodules when it starts training. Fast dev run This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch. Use this to debug a full run of your program quickly # DEFAULT trainer = Trainer(fast_dev_run=False) Force training for min or max epochs It can be useful to force training for a minimum number of epochs or limit to a max number # DEFAULT trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000) Force disable early stop Use this to turn off early stopping and run training to the max_epoch # DEFAULT trainer = Trainer(enable_early_stop=True) Inspect gradient norms Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2) Make model overfit on subset of data A useful debugging trick is to make your model overfit a tiny fraction of the data. # DEFAULT don't overfit (ie: normal training) trainer = Trainer(overfit_pct=0.0) # overfit on 1% of data trainer = Trainer(overfit_pct=0.01) Process position When running multiple models on the same machine we want to decide which progress bar to use. Lightning will stack progress bars according to this value. # DEFAULT trainer = Trainer(process_position=0) # if this is the second model on the node, show the second progress bar below trainer = Trainer(process_position=1) Set how much of the training set to check If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1)","title":"Training Loop"},{"location":"Trainer/Training Loop/#accumulated-gradients","text":"Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. # DEFAULT (ie: no accumulated grads) trainer = Trainer(accumulate_grad_batches=1)","title":"Accumulated gradients"},{"location":"Trainer/Training Loop/#anneal-learning-rate","text":"Cut the learning rate by 10 at every epoch listed in this list. # DEFAULT (don't anneal) trainer = Trainer(lr_scheduler_milestones=None) # cut LR by 10 at 100, 200, and 300 epochs trainer = Trainer(lr_scheduler_milestones=[100, 200, 300])","title":"Anneal Learning rate"},{"location":"Trainer/Training Loop/#check-gpu-usage","text":"Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.","title":"Check GPU usage"},{"location":"Trainer/Training Loop/#check-which-gradients-are-nan","text":"This option prints a list of tensors with nan gradients. # DEFAULT trainer = Trainer(print_nan_grads=False)","title":"Check which gradients are nan"},{"location":"Trainer/Training Loop/#display-metrics-in-progress-bar","text":"# DEFAULT trainer = Trainer(progress_bar=True)","title":"Display metrics in progress bar"},{"location":"Trainer/Training Loop/#display-the-parameter-count-by-layer","text":"By default lightning prints a list of parameters and submodules when it starts training.","title":"Display the parameter count by layer"},{"location":"Trainer/Training Loop/#fast-dev-run","text":"This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch. Use this to debug a full run of your program quickly # DEFAULT trainer = Trainer(fast_dev_run=False)","title":"Fast dev run"},{"location":"Trainer/Training Loop/#force-training-for-min-or-max-epochs","text":"It can be useful to force training for a minimum number of epochs or limit to a max number # DEFAULT trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)","title":"Force training for min or max epochs"},{"location":"Trainer/Training Loop/#force-disable-early-stop","text":"Use this to turn off early stopping and run training to the max_epoch # DEFAULT trainer = Trainer(enable_early_stop=True)","title":"Force disable early stop"},{"location":"Trainer/Training Loop/#inspect-gradient-norms","text":"Looking at grad norms can help you figure out where training might be going wrong. # DEFAULT (-1 doesn't track norms) trainer = Trainer(track_grad_norm=-1) # track the LP norm (P=2 here) trainer = Trainer(track_grad_norm=2)","title":"Inspect gradient norms"},{"location":"Trainer/Training Loop/#make-model-overfit-on-subset-of-data","text":"A useful debugging trick is to make your model overfit a tiny fraction of the data. # DEFAULT don't overfit (ie: normal training) trainer = Trainer(overfit_pct=0.0) # overfit on 1% of data trainer = Trainer(overfit_pct=0.01)","title":"Make model overfit on subset of data"},{"location":"Trainer/Training Loop/#process-position","text":"When running multiple models on the same machine we want to decide which progress bar to use. Lightning will stack progress bars according to this value. # DEFAULT trainer = Trainer(process_position=0) # if this is the second model on the node, show the second progress bar below trainer = Trainer(process_position=1)","title":"Process position"},{"location":"Trainer/Training Loop/#set-how-much-of-the-training-set-to-check","text":"If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(train_percent_check=1.0) # check 10% only trainer = Trainer(train_percent_check=0.1)","title":"Set how much of the training set to check"},{"location":"Trainer/Validation loop/","text":"The lightning validation loop handles everything except the actual computations of your model. To decide what will happen in your validation loop, define the validation_step function . Below are all the things lightning automates for you in the validation loop. Note Lightning will run 5 steps of validation in the beginning of training as a sanity check so you don't have to wait until a full epoch to catch possible validation issues. Check validation every n epochs If you have a small dataset you might want to check validation every n epochs # DEFAULT trainer = Trainer(check_val_every_n_epoch=1) Set how much of the validation set to check If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(val_percent_check=1.0) # check 10% only trainer = Trainer(val_percent_check=0.1) Set how much of the test set to check If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(test_percent_check=1.0) # check 10% only trainer = Trainer(test_percent_check=0.1) Set validation check frequency within 1 training epoch For large datasets it's often desirable to check validation multiple times within a training loop # DEFAULT trainer = Trainer(val_check_interval=0.95) # check every .25 of an epoch trainer = Trainer(val_check_interval=0.25) Set the number of validation sanity steps Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop. # DEFAULT trainer = Trainer(nb_sanity_val_steps=5)","title":"Validation loop"},{"location":"Trainer/Validation loop/#check-validation-every-n-epochs","text":"If you have a small dataset you might want to check validation every n epochs # DEFAULT trainer = Trainer(check_val_every_n_epoch=1)","title":"Check validation every n epochs"},{"location":"Trainer/Validation loop/#set-how-much-of-the-validation-set-to-check","text":"If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(val_percent_check=1.0) # check 10% only trainer = Trainer(val_percent_check=0.1)","title":"Set how much of the validation set to check"},{"location":"Trainer/Validation loop/#set-how-much-of-the-test-set-to-check","text":"If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag # DEFAULT trainer = Trainer(test_percent_check=1.0) # check 10% only trainer = Trainer(test_percent_check=0.1)","title":"Set how much of the test set to check"},{"location":"Trainer/Validation loop/#set-validation-check-frequency-within-1-training-epoch","text":"For large datasets it's often desirable to check validation multiple times within a training loop # DEFAULT trainer = Trainer(val_check_interval=0.95) # check every .25 of an epoch trainer = Trainer(val_check_interval=0.25)","title":"Set validation check frequency within 1 training epoch"},{"location":"Trainer/Validation loop/#set-the-number-of-validation-sanity-steps","text":"Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop. # DEFAULT trainer = Trainer(nb_sanity_val_steps=5)","title":"Set the number of validation sanity steps"},{"location":"Trainer/hooks/","text":"","title":"Hooks"}]}
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 4ef71170..5684891e 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -40,4 +40,9 @@
      2019-06-27
      daily
     
+    
+     None
+     2019-06-27
+     daily
+    
 
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index ec79a08e2d2c6ffabefdba45c7df61efa16acd3b..96f8e55f6482d78d54a6f54adc777da2333ea809 100644
GIT binary patch
delta 179
zcmV;k08IbM0m=awABzYG>--d50{?SqbY*Q}a4vXlYyj<)kr*L=-oS=!_V3?6`?_V-
zqwmmNIj^lLaYiUc%1B#l)0FspI;1;X@r2!6=T$&4YXC*&cqZmx;V+tR7Tl1V|MM^hmzK!d}TOzU*rcoECuDlky
hdouWOW(s^UD{uu8uE75S-z~lk{sN&$ycT8z005lAQBnW^

delta 178
zcmV;j08RhO0m%UvABzYG>gN<)0{?SqbY*Q}a4vXlYyj<(kr*L=dIz@IzkmPitCm%d
zzC(BAytbyq8KD>{BWc!xRbq#pg
z#=*m40wo3J35qoxm=$83mHcRpVlb(XDSYT{&2yR+Dcz*`Hm(=Nb
g$>7JCDe%Rtz!kU=0{?*T7QdLk0N(_eTlE6~0Pdex)&Kwi