From 49a2a87c4d65df46da167f0c23a586c60cd35cd3 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 27 Jun 2019 09:05:57 -0500 Subject: [PATCH] Deployed 3b7c7c6 with MkDocs version: 1.0.4 --- 404.html | 15 + Pytorch-Lightning/LightningModule/index.html | 596 +++++++++++++++++++ Pytorch-Lightning/Trainer/index.html | 163 +++++ index.html | 83 ++- search.html | 15 + search/search_index.json | 2 +- sitemap.xml | 12 +- sitemap.xml.gz | Bin 190 -> 195 bytes source/examples/example_model.py | 8 +- 9 files changed, 863 insertions(+), 31 deletions(-) create mode 100644 Pytorch-Lightning/LightningModule/index.html create mode 100644 Pytorch-Lightning/Trainer/index.html diff --git a/404.html b/404.html index 06a9fd80..02970fa1 100644 --- a/404.html +++ b/404.html @@ -46,6 +46,21 @@ PYTORCH-LIGHTNING DOCUMENTATION +
  • + + Pytorch Lightning + +
    • +   diff --git a/Pytorch-Lightning/LightningModule/index.html b/Pytorch-Lightning/LightningModule/index.html new file mode 100644 index 00000000..bda91e12 --- /dev/null +++ b/Pytorch-Lightning/LightningModule/index.html @@ -0,0 +1,596 @@ + + + + + + + + + + + Lightning module - Pytorch lightning Documentation + + + + + + + + + + + + + + + + + +
    + + + + +
    + + + + + +
    +
    +
    + +
    +
    +
    +
    + +

    Lightning module

    +

    [Github Code]

    +

    A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model.

    +

    To Define a Lightning Module, implement the following methods:

    +

    Required:

    + +

    Optional:

    + +
    +

    training_step

    +
    def training_step(self, data_batch, batch_nb)
+
    + +

    In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model.

    +

    Params

    + + + + + + + + + + + + + + + + + +
    Paramdescription
    data_batchThe output of your dataloader. A tensor, tuple or list
    batch_nbInteger displaying which batch this is
    +

    Return

    +

    Dictionary or OrderedDict

    + + + + + + + + + + + + + + + + + + + + +
    keyvalueis required
    losstensor scalarY
    progDict for progress bar display. Must have only tensorsN
    +

    Example

    +
    def training_step(self, data_batch, batch_nb):
+    x, y, z = data_batch
+
+    # implement your own
+    out = self.forward(x)
+    loss = self.loss(out, x)
+
+    output = {
+        'loss': loss, # required
+        'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional
+    }
+
+    # return a dict
+    return output
+
    + +
    +

    validation_step

    +
    def validation_step(self, data_batch, batch_nb)
+
    + +

    In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. +This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation.

    +

    Params

    + + + + + + + + + + + + + + + + + +
    Paramdescription
    data_batchThe output of your dataloader. A tensor, tuple or list
    batch_nbInteger displaying which batch this is
    +

    Return

    + + + + + + + + + + + + + + + +
    Returndescriptionoptional
    dictDict of OrderedDict with metrics to display in progress bar. All keys must be tensors.Y
    +

    Example

    +
    def validation_step(self, data_batch, batch_nb):
+    x, y, z = data_batch
+
+    # implement your own
+    out = self.forward(x)
+    loss = self.loss(out, x)
+
+    # calculate acc
+    labels_hat = torch.argmax(out, dim=1)
+    val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
+
+    # all optional...
+    # return whatever you need for the collation function validation_end
+    output = OrderedDict({
+        'val_loss': loss_val,
+        'val_acc': torch.tensor(val_acc), # everything must be a tensor
+    })
+
+    # return an optional dict
+    return output
+
    + +
    +

    validation_end

    +
    def validation_end(self, outputs)
+
    + +

    Called at the end of the validation loop with the output of each validation_step.

    +

    Params

    + + + + + + + + + + + + + +
    Paramdescription
    outputsList of outputs you defined in validation_step
    +

    Return

    + + + + + + + + + + + + + + + +
    Returndescriptionoptional
    dictDict of OrderedDict with metrics to display in progress barY
    +

    Example

    +
    def validation_end(self, outputs):
+    """
+    Called at the end of validation to aggregate outputs
+    :param outputs: list of individual outputs of each validation step
+    :return:
+    """
+    val_loss_mean = 0
+    val_acc_mean = 0
+    for output in outputs:
+        val_loss_mean += output['val_loss']
+        val_acc_mean += output['val_acc']
+
+    val_loss_mean /= len(outputs)
+    val_acc_mean /= len(outputs)
+    tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
+    return tqdm_dic
+
    + +
    +

    configure_optimizers

    +
    def configure_optimizers(self)
+
    + +

    Set up as many optimizers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. +Lightning will call .backward() and .step() on each one. If you use 16 bit precision it will also handle that.

    +
    Return
    +

    List - List of optimizers

    +

    Example

    +
    # most cases
+def configure_optimizers(self):
+    opt = Adam(lr=0.01)
+    return [opt]
+
+# gan example
+def configure_optimizers(self):
+    generator_opt = Adam(lr=0.01)
+    disriminator_opt = Adam(lr=0.02)
+    return [generator_opt, disriminator_opt] 
+
    + +
    +

    get_save_dict

    +
    def get_save_dict(self)
+
    + +

    Called by lightning to checkpoint your model. Lightning saves current epoch, current batch nb, etc... +All you have to return is what specifically about your lightning model you want to checkpoint.

    +
    Return
    +

    Dictionary - No required keys. Most of the time as described in this example.

    +

    Example

    +
    def get_save_dict(self):
+    # 99% of use cases this is all you need to return
+    checkpoint = {'state_dict': self.state_dict()}
+    return checkpoint
+
    + +
    +

    load_model_specific

    +
    def load_model_specific(self, checkpoint)
+
    + +

    Called by lightning to restore your model. This is your chance to restore your model using the keys you added in get_save_dict. +Lightning will automatically restore current epoch, batch nb, etc.

    +
    Return
    +

    Nothing

    +

    Example

    +
    def load_model_specific(self, checkpoint):
+    # you defined 'state_dict' in get_save_dict()
+    self.load_state_dict(checkpoint['state_dict'])
+
    + +
    +

    tng_dataloader

    +
    @property
+def tng_dataloader(self)
+
    + +

    Called by lightning during training loop. Define it as a property.

    +
    Return
    +

    Pytorch DataLoader

    +

    Example

    +
    @property
+def tng_dataloader(self):
+    if self._tng_dataloader is None:
+        try:
+            transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+            dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True)
+            loader = torch.utils.data.DataLoader(
+                dataset=dataset,
+                batch_size=self.hparams.batch_size,
+                shuffle=True
+            )
+            self._tng_dataloader = loader
+        except Exception as e:
+            raise e
+
+    return self._tng_dataloader
+
    + +
    +

    val_dataloader

    +
    @property
+def tng_dataloader(self)
+
    + +

    Called by lightning during validation loop. Define it as a property.

    +
    Return
    +

    Pytorch DataLoader

    +

    Example

    +
    @property
+def val_dataloader(self):
+    if self._val_dataloader is None:
+        try:
+            transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+            dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True)
+            loader = torch.utils.data.DataLoader(
+                dataset=dataset,
+                batch_size=self.hparams.batch_size,
+                shuffle=True
+            )
+            self._val_dataloader = loader
+        except Exception as e:
+            raise e
+
+    return self._val_dataloader
+
    + +
    +

    test_dataloader

    +
    @property
+def test_dataloader(self)
+
    + +

    Called by lightning during test loop. Define it as a property.

    +
    Return
    +

    Pytorch DataLoader

    +

    Example

    +
    @property
+def test_dataloader(self):
+    if self._test_dataloader is None:
+        try:
+            transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+            dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True)
+            loader = torch.utils.data.DataLoader(
+                dataset=dataset,
+                batch_size=self.hparams.batch_size,
+                shuffle=True
+            )
+            self._test_dataloader = loader
+        except Exception as e:
+            raise e
+
+    return self._test_dataloader
+
    + +
    +

    update_tng_log_metrics

    +
    def update_tng_log_metrics(self, logs)
+
    + +

    Called by lightning right before it logs metrics for this batch. +This is a chance to ammend or add to the metrics about to be logged.

    +
    Return
    +

    Dict

    +

    Example

    +
    def update_tng_log_metrics(self, logs):
+    # modify or add to logs
+    return logs
+
    + +
    +

    add_model_specific_args

    +
    @staticmethod
+def add_model_specific_args(parent_parser, root_dir)
+
    + +

    Lightning has a list of default argparse commands. +This method is your chance to add or modify commands specific to your model. +The argument parser is available anywhere in your model by calling self.hparams

    +
    Return
    +

    An argument parser

    +

    Example

    +
    @staticmethod
+def add_model_specific_args(parent_parser, root_dir):
+    parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+
+    # param overwrites
+    # parser.set_defaults(gradient_clip=5.0)
+
+    # network params
+    parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
+    parser.add_argument('--in_features', default=28*28)
+    parser.add_argument('--out_features', default=10)
+    parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference
+
+    # data
+    parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
+
+    # training params (opt)
+    parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
+                    tunable=False)
+    parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False)
+    parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
+    return parser
+
    + +
    +
    + + +
    +
    + +
    + +
    + +
    + + + GitHub + + + « Previous + + + Next » + + +
    + + + + + + diff --git a/Pytorch-Lightning/Trainer/index.html b/Pytorch-Lightning/Trainer/index.html new file mode 100644 index 00000000..e97589f6 --- /dev/null +++ b/Pytorch-Lightning/Trainer/index.html @@ -0,0 +1,163 @@ + + + + + + + + + + + Trainer - Pytorch lightning Documentation + + + + + + + + + + + + + + + + + +
    + + + + +
    + + + + + +
    +
    +
    + +
    +
    +
    +
    + +

    Trainer

    + +
    +
    + + +
    +
    + +
    + +
    + +
    + + + GitHub + + + « Previous + + + +
    + + + + + + diff --git a/index.html b/index.html index 19928f21..12d67d80 100644 --- a/index.html +++ b/index.html @@ -5,7 +5,7 @@ - + PYTORCH-LIGHTNING DOCUMENTATION - Pytorch lightning Documentation @@ -61,17 +61,34 @@
  • Quick start examples
    • +
  • Training loop
    • + +
  • Validation loop
    • +
  • Distributed training
  • Checkpointing
  • Computing cluster (SLURM)
    • -
  • Common training use cases
    • - + + + +
  • + + Pytorch Lightning +
    • @@ -113,7 +130,7 @@

    PYTORCH-LIGHTNING DOCUMENTATION

    Quick start
    Quick start examples
    @@ -121,13 +138,39 @@
  • CPU example
  • Single GPU example
  • Multi-gpu example
    • -
  • SLURM cluster example
    • +
  • SLURM cluster grid search example
    • + +
    Training loop
    + +
    Validation loop
    +
    Distributed training
    Checkpointing
      diff --git a/search/search_index.json b/search/search_index.json index 077ab1a1..3ea7627b 100644 --- a/search/search_index.json +++ b/search/search_index.json @@ -1 +1 @@ -{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"PYTORCH-LIGHTNING DOCUMENTATION Quick start Define a lightning model Set up the trainer Quick start examples CPU example Single GPU example Multi-gpu example SLURM cluster example Distributed training Single-gpu Multi-gpu Multi-node Checkpointing Model saving Model loading Computing cluster (SLURM) Automatic checkpointing Automatic saving, loading Walltime auto-resubmit Common training use cases 16-bit mixed precision Accumulate gradients Check val many times during 1 training epoch Check GPU usage Check validation every n epochs Check which gradients are nan Inspect gradient norms Learning rate annealing Make model overfit on subset of data Min, max epochs Multiple optimizers (like GANs) Run a sanity check of model val and tng step Set how much of the tng, val, test sets to check (1-100%)","title":"PYTORCH-LIGHTNING DOCUMENTATION"},{"location":"#pytorch-lightning-documentation","text":"","title":"PYTORCH-LIGHTNING DOCUMENTATION"},{"location":"#quick-start","text":"Define a lightning model Set up the trainer","title":"Quick start"},{"location":"#quick-start-examples","text":"CPU example Single GPU example Multi-gpu example SLURM cluster example","title":"Quick start examples"},{"location":"#distributed-training","text":"Single-gpu Multi-gpu Multi-node","title":"Distributed training"},{"location":"#checkpointing","text":"Model saving Model loading","title":"Checkpointing"},{"location":"#computing-cluster-slurm","text":"Automatic checkpointing Automatic saving, loading Walltime auto-resubmit","title":"Computing cluster (SLURM)"},{"location":"#common-training-use-cases","text":"16-bit mixed precision Accumulate gradients Check val many times during 1 training epoch Check GPU usage Check validation every n epochs Check which gradients are nan Inspect gradient norms Learning rate annealing Make model overfit on subset of data Min, max epochs Multiple optimizers (like GANs) Run a sanity check of model val and tng step Set how much of the tng, val, test sets to check (1-100%)","title":"Common training use cases"}]} \ No newline at end of file +{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"PYTORCH-LIGHTNING DOCUMENTATION Quick start Define a LightningModule Set up the trainer Quick start examples CPU example Single GPU example Multi-gpu example SLURM cluster grid search example Training loop Accumulate gradients Check GPU usage Check which gradients are nan Check validation every n epochs Display metrics in progress bar Force training for min or max epochs Inspect gradient norms Hooks Learning rate annealing Make model overfit on subset of data Multiple optimizers (like GANs) Set how much of the training set to check (1-100%) training_step function Validation loop Display metrics in progress bar hooks Set how much of the validation set to check (1-100%) Set validation check frequency within 1 training epoch (1-100%) validation_step function Why does validation run first for 5 steps? Distributed training Single-gpu Multi-gpu Multi-node 16-bit mixed precision Checkpointing Model saving Model loading Computing cluster (SLURM) Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"PYTORCH-LIGHTNING DOCUMENTATION"},{"location":"#pytorch-lightning-documentation","text":"","title":"PYTORCH-LIGHTNING DOCUMENTATION"},{"location":"#quick-start","text":"Define a LightningModule Set up the trainer","title":"Quick start"},{"location":"#quick-start-examples","text":"CPU example Single GPU example Multi-gpu example SLURM cluster grid search example","title":"Quick start examples"},{"location":"#training-loop","text":"Accumulate gradients Check GPU usage Check which gradients are nan Check validation every n epochs Display metrics in progress bar Force training for min or max epochs Inspect gradient norms Hooks Learning rate annealing Make model overfit on subset of data Multiple optimizers (like GANs) Set how much of the training set to check (1-100%) training_step function","title":"Training loop"},{"location":"#validation-loop","text":"Display metrics in progress bar hooks Set how much of the validation set to check (1-100%) Set validation check frequency within 1 training epoch (1-100%) validation_step function Why does validation run first for 5 steps?","title":"Validation loop"},{"location":"#distributed-training","text":"Single-gpu Multi-gpu Multi-node 16-bit mixed precision","title":"Distributed training"},{"location":"#checkpointing","text":"Model saving Model loading","title":"Checkpointing"},{"location":"#computing-cluster-slurm","text":"Automatic checkpointing Automatic saving, loading Running grid search on a cluster Walltime auto-resubmit","title":"Computing cluster (SLURM)"},{"location":"Pytorch-Lightning/LightningModule/","text":"Lightning module [ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. To Define a Lightning Module, implement the following methods: Required : training_step validation_step validation_end configure_optimizers get_save_dict load_model_specific tng_dataloader tng_dataloader test_dataloader Optional : update_tng_log_metrics add_model_specific_args training_step def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output validation_step def validation_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output validation_end def validation_end(self, outputs) Called at the end of the validation loop with the output of each validation_step. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic configure_optimizers def configure_optimizers(self) Set up as many optimizers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one. If you use 16 bit precision it will also handle that. Return List - List of optimizers Example # most cases def configure_optimizers(self): opt = Adam(lr=0.01) return [opt] # gan example def configure_optimizers(self): generator_opt = Adam(lr=0.01) disriminator_opt = Adam(lr=0.02) return [generator_opt, disriminator_opt] get_save_dict def get_save_dict(self) Called by lightning to checkpoint your model. Lightning saves current epoch, current batch nb, etc... All you have to return is what specifically about your lightning model you want to checkpoint. Return Dictionary - No required keys. Most of the time as described in this example. Example def get_save_dict(self): # 99% of use cases this is all you need to return checkpoint = {'state_dict': self.state_dict()} return checkpoint load_model_specific def load_model_specific(self, checkpoint) Called by lightning to restore your model. This is your chance to restore your model using the keys you added in get_save_dict. Lightning will automatically restore current epoch, batch nb, etc. Return Nothing Example def load_model_specific(self, checkpoint): # you defined 'state_dict' in get_save_dict() self.load_state_dict(checkpoint['state_dict']) tng_dataloader @property def tng_dataloader(self) Called by lightning during training loop. Define it as a property. Return Pytorch DataLoader Example @property def tng_dataloader(self): if self._tng_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._tng_dataloader = loader except Exception as e: raise e return self._tng_dataloader val_dataloader @property def tng_dataloader(self) Called by lightning during validation loop. Define it as a property. Return Pytorch DataLoader Example @property def val_dataloader(self): if self._val_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._val_dataloader = loader except Exception as e: raise e return self._val_dataloader test_dataloader @property def test_dataloader(self) Called by lightning during test loop. Define it as a property. Return Pytorch DataLoader Example @property def test_dataloader(self): if self._test_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._test_dataloader = loader except Exception as e: raise e return self._test_dataloader update_tng_log_metrics def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged. Return Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs add_model_specific_args @staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The argument parser is available anywhere in your model by calling self.hparams Return An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Lightning module"},{"location":"Pytorch-Lightning/LightningModule/#lightning-module","text":"[ Github Code ] A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model. To Define a Lightning Module, implement the following methods: Required : training_step validation_step validation_end configure_optimizers get_save_dict load_model_specific tng_dataloader tng_dataloader test_dataloader Optional : update_tng_log_metrics add_model_specific_args","title":"Lightning module"},{"location":"Pytorch-Lightning/LightningModule/#training_step","text":"def training_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Dictionary or OrderedDict key value is required loss tensor scalar Y prog Dict for progress bar display. Must have only tensors N Example def training_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) output = { 'loss': loss, # required 'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional } # return a dict return output","title":"training_step"},{"location":"Pytorch-Lightning/LightningModule/#validation_step","text":"def validation_step(self, data_batch, batch_nb) In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model. This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation. Params Param description data_batch The output of your dataloader. A tensor, tuple or list batch_nb Integer displaying which batch this is Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. Y Example def validation_step(self, data_batch, batch_nb): x, y, z = data_batch # implement your own out = self.forward(x) loss = self.loss(out, x) # calculate acc labels_hat = torch.argmax(out, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) # all optional... # return whatever you need for the collation function validation_end output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc), # everything must be a tensor }) # return an optional dict return output","title":"validation_step"},{"location":"Pytorch-Lightning/LightningModule/#validation_end","text":"def validation_end(self, outputs) Called at the end of the validation loop with the output of each validation_step. Params Param description outputs List of outputs you defined in validation_step Return Return description optional dict Dict of OrderedDict with metrics to display in progress bar Y Example def validation_end(self, outputs): \"\"\" Called at the end of validation to aggregate outputs :param outputs: list of individual outputs of each validation step :return: \"\"\" val_loss_mean = 0 val_acc_mean = 0 for output in outputs: val_loss_mean += output['val_loss'] val_acc_mean += output['val_acc'] val_loss_mean /= len(outputs) val_acc_mean /= len(outputs) tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()} return tqdm_dic","title":"validation_end"},{"location":"Pytorch-Lightning/LightningModule/#configure_optimizers","text":"def configure_optimizers(self) Set up as many optimizers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple. Lightning will call .backward() and .step() on each one. If you use 16 bit precision it will also handle that.","title":"configure_optimizers"},{"location":"Pytorch-Lightning/LightningModule/#return","text":"List - List of optimizers Example # most cases def configure_optimizers(self): opt = Adam(lr=0.01) return [opt] # gan example def configure_optimizers(self): generator_opt = Adam(lr=0.01) disriminator_opt = Adam(lr=0.02) return [generator_opt, disriminator_opt]","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#get_save_dict","text":"def get_save_dict(self) Called by lightning to checkpoint your model. Lightning saves current epoch, current batch nb, etc... All you have to return is what specifically about your lightning model you want to checkpoint.","title":"get_save_dict"},{"location":"Pytorch-Lightning/LightningModule/#return_1","text":"Dictionary - No required keys. Most of the time as described in this example. Example def get_save_dict(self): # 99% of use cases this is all you need to return checkpoint = {'state_dict': self.state_dict()} return checkpoint","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#load_model_specific","text":"def load_model_specific(self, checkpoint) Called by lightning to restore your model. This is your chance to restore your model using the keys you added in get_save_dict. Lightning will automatically restore current epoch, batch nb, etc.","title":"load_model_specific"},{"location":"Pytorch-Lightning/LightningModule/#return_2","text":"Nothing Example def load_model_specific(self, checkpoint): # you defined 'state_dict' in get_save_dict() self.load_state_dict(checkpoint['state_dict'])","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#tng_dataloader","text":"@property def tng_dataloader(self) Called by lightning during training loop. Define it as a property.","title":"tng_dataloader"},{"location":"Pytorch-Lightning/LightningModule/#return_3","text":"Pytorch DataLoader Example @property def tng_dataloader(self): if self._tng_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._tng_dataloader = loader except Exception as e: raise e return self._tng_dataloader","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#val_dataloader","text":"@property def tng_dataloader(self) Called by lightning during validation loop. Define it as a property.","title":"val_dataloader"},{"location":"Pytorch-Lightning/LightningModule/#return_4","text":"Pytorch DataLoader Example @property def val_dataloader(self): if self._val_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._val_dataloader = loader except Exception as e: raise e return self._val_dataloader","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#test_dataloader","text":"@property def test_dataloader(self) Called by lightning during test loop. Define it as a property.","title":"test_dataloader"},{"location":"Pytorch-Lightning/LightningModule/#return_5","text":"Pytorch DataLoader Example @property def test_dataloader(self): if self._test_dataloader is None: try: transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))]) dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True) loader = torch.utils.data.DataLoader( dataset=dataset, batch_size=self.hparams.batch_size, shuffle=True ) self._test_dataloader = loader except Exception as e: raise e return self._test_dataloader","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#update_tng_log_metrics","text":"def update_tng_log_metrics(self, logs) Called by lightning right before it logs metrics for this batch. This is a chance to ammend or add to the metrics about to be logged.","title":"update_tng_log_metrics"},{"location":"Pytorch-Lightning/LightningModule/#return_6","text":"Dict Example def update_tng_log_metrics(self, logs): # modify or add to logs return logs","title":"Return"},{"location":"Pytorch-Lightning/LightningModule/#add_model_specific_args","text":"@staticmethod def add_model_specific_args(parent_parser, root_dir) Lightning has a list of default argparse commands. This method is your chance to add or modify commands specific to your model. The argument parser is available anywhere in your model by calling self.hparams","title":"add_model_specific_args"},{"location":"Pytorch-Lightning/LightningModule/#return_7","text":"An argument parser Example @staticmethod def add_model_specific_args(parent_parser, root_dir): parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser]) # param overwrites # parser.set_defaults(gradient_clip=5.0) # network params parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False) parser.add_argument('--in_features', default=28*28) parser.add_argument('--out_features', default=10) parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference # data parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str) # training params (opt) parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005], tunable=False) parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False) parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False) return parser","title":"Return"},{"location":"Pytorch-Lightning/Trainer/","text":"Trainer","title":"Trainer"},{"location":"Pytorch-Lightning/Trainer/#trainer","text":"","title":"Trainer"}]} \ No newline at end of file diff --git a/sitemap.xml b/sitemap.xml index 642b465c..014f5456 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -2,7 +2,17 @@ None - 2019-06-26 + 2019-06-27 + daily + + + None + 2019-06-27 + daily + + + None + 2019-06-27 daily \ No newline at end of file diff --git a/sitemap.xml.gz b/sitemap.xml.gz index 27b57bf421e098ac619fc32f17b8380cb146c778..7e0704ceff9c2e95c53f63cfaaf5c0f7602716e3 100644 GIT binary patch literal 195 zcmV;!06hO6iwFos%@kb%|8r?{Wo=<_E_iKh0Ns%>62c%5Mf;qBp$|(^sBsACEUoka z48epM0uj*Y?Ik8Up22PQ@7q87i)GDYFzBv)(9YJlAQU5IrE9hAYJ5H&@*Qq?#%|sO z6;U$VP~$$vaSt#}6A3!1L5zVo-$9Ul8Um~kD5)^dP^$UBq7XAq@}skg!K^+|ggCgC xm$ayIy2;CJTCZ%i$X?h^UA(&TR%r8N@I#gq_{RKq;Jc-V@D~`QHJ`u%006liT;~7) literal 190 zcmV;v073sBiwFpx3KU%e|8r?{Wo=<_E_iKh08NfP4#F@Dg?CO7WpCVes3=O)jgbis z5T&7wlqMw(6mC!YgM=49e}3{aZ{MROFCEZ$*U*CHlxU}XXI$UV+x46sX)6x=87%5Z zlG$NHhZv_5z&y_kP1K{DkonL9sv!*sJA|CoR2)dE<-n#0w_b^hcbdbZeo$f@eJ9Gi ssI$Dv$||jQwp-L7U9ay!KSU>u{c!jrTMDc(YfJBnFPzWwHQoRK08lAa?*IS* diff --git a/source/examples/example_model.py b/source/examples/example_model.py index f141911e..70b8eb17 100644 --- a/source/examples/example_model.py +++ b/source/examples/example_model.py @@ -1,6 +1,6 @@ import torch.nn as nn import numpy as np -from pytorch_lightning.root_module.root_module import RootModule +from pytorch_lightning.root_module.root_module import LightningModule from test_tube import HyperOptArgumentParser from torchvision.datasets import MNIST import torchvision.transforms as transforms @@ -10,7 +10,7 @@ import os, pdb from collections import OrderedDict -class ExampleModel(RootModule): +class ExampleModel(LightningModule): """ Sample model to show how to define a template """ @@ -71,9 +71,6 @@ class ExampleModel(RootModule): # calculate loss loss_val = self.loss(y, y_hat) - # tqdm_dic = {'tng_loss': loss_val.item()} - # return loss_val, tqdm_dic - output = OrderedDict({ 'loss': loss_val, 'tqdm_metrics': {} @@ -96,7 +93,6 @@ class ExampleModel(RootModule): labels_hat = torch.argmax(y_hat, dim=1) val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0) - # output = {'y_hat': y_hat, 'val_loss': loss_val.item(), 'val_acc': val_acc} output = OrderedDict({ 'val_loss': loss_val, 'val_acc': torch.tensor(val_acc),