mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-09 11:32:07 +08:00
Deployed ed31417 with MkDocs version: 1.0.4
This commit is contained in:
William Falcon
@@ -54,9 +54,36 @@
|
||||
|
||||
<a class="" href="/Pytorch-Lightning/LightningModule/">Lightning module</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="/Pytorch-Lightning/Trainer/">Trainer</a>
|
||||
<a class="" href="/Trainer/">Trainer</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="/Trainer/Checkpointing/">Checkpointing</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="/Trainer/Distributed training/">Distributed training</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="/Trainer/SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="/Trainer/Training Loop/">Training Loop</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="/Trainer/Vaildation loop/">Vaildation loop</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
@@ -93,9 +93,36 @@
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Trainer/">Trainer</a>
|
||||
<a class="" href="../../Trainer/">Trainer</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../../Trainer/Checkpointing/">Checkpointing</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../../Trainer/Distributed training/">Distributed training</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../../Trainer/SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../../Trainer/Training Loop/">Training Loop</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../../Trainer/Vaildation loop/">Vaildation loop</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
@@ -142,7 +169,8 @@
|
||||
<h1 id="lightning-module">Lightning module</h1>
|
||||
<p>[<a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/root_module/root_module.py">Github Code</a>]</p>
|
||||
<p>A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model.</p>
|
||||
<p>To Define a Lightning Module, implement the following methods:</p>
|
||||
<p>The easiest thing to do is copy <a href="../lightning_module_template.py">this template</a> and modify accordingly. </p>
|
||||
<p>Otherwise, to Define a Lightning Module, implement the following methods:</p>
|
||||
<p><strong>Required</strong>: </p>
|
||||
<ul>
|
||||
<li><a href="./#training_step">training_step</a> </li>
|
||||
@@ -550,7 +578,7 @@ def add_model_specific_args(parent_parser, root_dir):
|
||||
|
||||
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
|
||||
|
||||
<a href="../Trainer/" class="btn btn-neutral float-right" title="Trainer">Next <span class="icon icon-circle-arrow-right"></span></a>
|
||||
<a href="../../Trainer/" class="btn btn-neutral float-right" title="Trainer">Next <span class="icon icon-circle-arrow-right"></span></a>
|
||||
|
||||
|
||||
<a href="../.." class="btn btn-neutral" title="PYTORCH-LIGHTNING DOCUMENTATION"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
||||
@@ -584,7 +612,7 @@ def add_model_specific_args(parent_parser, root_dir):
|
||||
<span><a href="../.." style="color: #fcfcfc;">« Previous</a></span>
|
||||
|
||||
|
||||
<span style="margin-left: 15px"><a href="../Trainer/" style="color: #fcfcfc">Next »</a></span>
|
||||
<span style="margin-left: 15px"><a href="../../Trainer/" style="color: #fcfcfc">Next »</a></span>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
import torch.nn as nn
|
||||
from torchvision.datasets import MNIST
|
||||
import torchvision.transforms as transforms
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from test_tube import HyperOptArgumentParser
|
||||
from torch import optim
|
||||
|
||||
from pytorch_lightning.root_module.root_module import LightningModule
|
||||
|
||||
|
||||
class LightningTemplateModel(LightningModule):
|
||||
"""
|
||||
Sample model to show how to define a template
|
||||
"""
|
||||
|
||||
def __init__(self, hparams):
|
||||
"""
|
||||
Pass in parsed HyperOptArgumentParser to the model
|
||||
:param hparams:
|
||||
"""
|
||||
# init superclass
|
||||
super(LightningTemplateModel, self).__init__(hparams)
|
||||
|
||||
self.batch_size = hparams.batch_size
|
||||
|
||||
# build model
|
||||
self.__build_model()
|
||||
|
||||
# ---------------------
|
||||
# MODEL SETUP
|
||||
# ---------------------
|
||||
def __build_model(self):
|
||||
"""
|
||||
Layout model
|
||||
:return:
|
||||
"""
|
||||
self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
|
||||
self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
|
||||
self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
|
||||
|
||||
self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)
|
||||
|
||||
# ---------------------
|
||||
# TRAINING
|
||||
# ---------------------
|
||||
def forward(self, x):
|
||||
"""
|
||||
No special modification required for lightning, define as you normally would
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
|
||||
x = self.c_d1(x)
|
||||
x = torch.tanh(x)
|
||||
x = self.c_d1_bn(x)
|
||||
x = self.c_d1_drop(x)
|
||||
|
||||
x = self.c_d2(x)
|
||||
logits = F.log_softmax(x, dim=1)
|
||||
|
||||
return logits
|
||||
|
||||
def loss(self, labels, logits):
|
||||
nll = F.nll_loss(logits, labels)
|
||||
return nll
|
||||
|
||||
def training_step(self, data_batch, batch_i):
|
||||
"""
|
||||
Lightning calls this inside the training loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
# forward pass
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
# calculate loss
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
output = OrderedDict({
|
||||
'loss': loss_val,
|
||||
'tqdm_metrics': {}
|
||||
})
|
||||
return output
|
||||
|
||||
def validation_step(self, data_batch, batch_i):
|
||||
"""
|
||||
Lightning calls this inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': torch.tensor(val_acc),
|
||||
})
|
||||
return output
|
||||
|
||||
def validation_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
val_loss_mean = 0
|
||||
val_acc_mean = 0
|
||||
for output in outputs:
|
||||
val_loss_mean += output['val_loss']
|
||||
val_acc_mean += output['val_acc']
|
||||
|
||||
val_loss_mean /= len(outputs)
|
||||
val_acc_mean /= len(outputs)
|
||||
tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
|
||||
def update_tng_log_metrics(self, logs):
|
||||
return logs
|
||||
|
||||
# ---------------------
|
||||
# MODEL SAVING
|
||||
# ---------------------
|
||||
def get_save_dict(self):
|
||||
checkpoint = {'state_dict': self.state_dict()}
|
||||
return checkpoint
|
||||
|
||||
def load_model_specific(self, checkpoint):
|
||||
self.load_state_dict(checkpoint['state_dict'])
|
||||
pass
|
||||
|
||||
# ---------------------
|
||||
# TRAINING SETUP
|
||||
# ---------------------
|
||||
def configure_optimizers(self):
|
||||
"""
|
||||
return whatever optimizers we want here
|
||||
:return: list of optimizers
|
||||
"""
|
||||
optimizer = optim.Adam(lr=self.hparams.learning_rate)
|
||||
return [optimizer]
|
||||
|
||||
def __dataloader(self, train):
|
||||
# init data generators
|
||||
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
|
||||
|
||||
dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)
|
||||
|
||||
loader = torch.utils.data.DataLoader(
|
||||
dataset=dataset,
|
||||
batch_size=self.hparams.batch_size,
|
||||
shuffle=True
|
||||
)
|
||||
|
||||
return loader
|
||||
|
||||
@property
|
||||
def tng_dataloader(self):
|
||||
if self._tng_dataloader is None:
|
||||
try:
|
||||
self._tng_dataloader = self.__dataloader(train=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise e
|
||||
return self._tng_dataloader
|
||||
|
||||
@property
|
||||
def val_dataloader(self):
|
||||
if self._val_dataloader is None:
|
||||
try:
|
||||
self._val_dataloader = self.__dataloader(train=False)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise e
|
||||
return self._val_dataloader
|
||||
|
||||
@property
|
||||
def test_dataloader(self):
|
||||
if self._test_dataloader is None:
|
||||
try:
|
||||
self._test_dataloader = self.__dataloader(train=False)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise e
|
||||
return self._test_dataloader
|
||||
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser, root_dir):
|
||||
"""
|
||||
Parameters you define here will be available to your model through self.hparams
|
||||
:param parent_parser:
|
||||
:param root_dir:
|
||||
:return:
|
||||
"""
|
||||
parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
|
||||
|
||||
# param overwrites
|
||||
# parser.set_defaults(gradient_clip=5.0)
|
||||
|
||||
# network params
|
||||
parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
|
||||
parser.add_argument('--in_features', default=28*28)
|
||||
parser.add_argument('--out_features', default=10)
|
||||
parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference
|
||||
|
||||
# data
|
||||
parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
|
||||
|
||||
# training params (opt)
|
||||
parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
|
||||
tunable=False)
|
||||
parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False)
|
||||
parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
|
||||
return parser
|
||||
@@ -0,0 +1,191 @@
|
||||
<!DOCTYPE html>
|
||||
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
||||
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
|
||||
<link rel="shortcut icon" href="../../img/favicon.ico">
|
||||
<title>Checkpointing - Pytorch lightning Documentation</title>
|
||||
<link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
|
||||
|
||||
<link rel="stylesheet" href="../../css/theme.css" type="text/css" />
|
||||
<link rel="stylesheet" href="../../css/theme_extra.css" type="text/css" />
|
||||
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
|
||||
|
||||
<script>
|
||||
// Current page data
|
||||
var mkdocs_page_name = "Checkpointing";
|
||||
var mkdocs_page_input_path = "Trainer/Checkpointing.md";
|
||||
var mkdocs_page_url = null;
|
||||
</script>
|
||||
|
||||
<script src="../../js/jquery-2.1.1.min.js" defer></script>
|
||||
<script src="../../js/modernizr-2.8.3.min.js" defer></script>
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
|
||||
<script>hljs.initHighlightingOnLoad();</script>
|
||||
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav" role="document">
|
||||
|
||||
<div class="wy-grid-for-nav">
|
||||
|
||||
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
|
||||
<div class="wy-side-nav-search">
|
||||
<a href="../.." class="icon icon-home"> Pytorch lightning Documentation</a>
|
||||
<div role="search">
|
||||
<form id ="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" title="Type search term here" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
||||
<ul class="current">
|
||||
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<a class="" href="../..">PYTORCH-LIGHTNING DOCUMENTATION</a>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Pytorch Lightning</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../../Pytorch-Lightning/LightningModule/">Lightning module</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../">Trainer</a>
|
||||
</li>
|
||||
<li class=" current">
|
||||
|
||||
<a class="current" href="./">Checkpointing</a>
|
||||
<ul class="subnav">
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Distributed training/">Distributed training</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Training Loop/">Training Loop</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Vaildation loop/">Vaildation loop</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
||||
|
||||
|
||||
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="../..">Pytorch lightning Documentation</a>
|
||||
</nav>
|
||||
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="breadcrumbs navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="../..">Docs</a> »</li>
|
||||
|
||||
|
||||
|
||||
<li>Trainer »</li>
|
||||
|
||||
|
||||
|
||||
<li>Checkpointing</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Checkpointing.md"
|
||||
class="icon icon-github"> Edit on GitHub</a>
|
||||
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main">
|
||||
<div class="section">
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
|
||||
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
|
||||
|
||||
<a href="../Distributed training/" class="btn btn-neutral float-right" title="Distributed training">Next <span class="icon icon-circle-arrow-right"></span></a>
|
||||
|
||||
|
||||
<a href="../" class="btn btn-neutral" title="Trainer"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<!-- Copyright etc -->
|
||||
|
||||
</div>
|
||||
|
||||
Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="rst-versions" role="note" style="cursor: pointer">
|
||||
<span class="rst-current-version" data-toggle="rst-current-version">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" class="fa fa-github" style="float: left; color: #fcfcfc"> GitHub</a>
|
||||
|
||||
|
||||
<span><a href="../" style="color: #fcfcfc;">« Previous</a></span>
|
||||
|
||||
|
||||
<span style="margin-left: 15px"><a href="../Distributed training/" style="color: #fcfcfc">Next »</a></span>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
<script>var base_url = '../..';</script>
|
||||
<script src="../../js/theme.js" defer></script>
|
||||
<script src="../../search/main.js" defer></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,191 @@
|
||||
<!DOCTYPE html>
|
||||
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
||||
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
|
||||
<link rel="shortcut icon" href="../../img/favicon.ico">
|
||||
<title>Distributed training - Pytorch lightning Documentation</title>
|
||||
<link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
|
||||
|
||||
<link rel="stylesheet" href="../../css/theme.css" type="text/css" />
|
||||
<link rel="stylesheet" href="../../css/theme_extra.css" type="text/css" />
|
||||
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
|
||||
|
||||
<script>
|
||||
// Current page data
|
||||
var mkdocs_page_name = "Distributed training";
|
||||
var mkdocs_page_input_path = "Trainer/Distributed training.md";
|
||||
var mkdocs_page_url = null;
|
||||
</script>
|
||||
|
||||
<script src="../../js/jquery-2.1.1.min.js" defer></script>
|
||||
<script src="../../js/modernizr-2.8.3.min.js" defer></script>
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
|
||||
<script>hljs.initHighlightingOnLoad();</script>
|
||||
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav" role="document">
|
||||
|
||||
<div class="wy-grid-for-nav">
|
||||
|
||||
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
|
||||
<div class="wy-side-nav-search">
|
||||
<a href="../.." class="icon icon-home"> Pytorch lightning Documentation</a>
|
||||
<div role="search">
|
||||
<form id ="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" title="Type search term here" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
||||
<ul class="current">
|
||||
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<a class="" href="../..">PYTORCH-LIGHTNING DOCUMENTATION</a>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Pytorch Lightning</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../../Pytorch-Lightning/LightningModule/">Lightning module</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../">Trainer</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Checkpointing/">Checkpointing</a>
|
||||
</li>
|
||||
<li class=" current">
|
||||
|
||||
<a class="current" href="./">Distributed training</a>
|
||||
<ul class="subnav">
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Training Loop/">Training Loop</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Vaildation loop/">Vaildation loop</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
||||
|
||||
|
||||
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="../..">Pytorch lightning Documentation</a>
|
||||
</nav>
|
||||
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="breadcrumbs navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="../..">Docs</a> »</li>
|
||||
|
||||
|
||||
|
||||
<li>Trainer »</li>
|
||||
|
||||
|
||||
|
||||
<li>Distributed training</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Distributed training.md"
|
||||
class="icon icon-github"> Edit on GitHub</a>
|
||||
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main">
|
||||
<div class="section">
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
|
||||
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
|
||||
|
||||
<a href="../SLURM Managed Cluster/" class="btn btn-neutral float-right" title="SLURM Managed Cluster">Next <span class="icon icon-circle-arrow-right"></span></a>
|
||||
|
||||
|
||||
<a href="../Checkpointing/" class="btn btn-neutral" title="Checkpointing"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<!-- Copyright etc -->
|
||||
|
||||
</div>
|
||||
|
||||
Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="rst-versions" role="note" style="cursor: pointer">
|
||||
<span class="rst-current-version" data-toggle="rst-current-version">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" class="fa fa-github" style="float: left; color: #fcfcfc"> GitHub</a>
|
||||
|
||||
|
||||
<span><a href="../Checkpointing/" style="color: #fcfcfc;">« Previous</a></span>
|
||||
|
||||
|
||||
<span style="margin-left: 15px"><a href="../SLURM Managed Cluster/" style="color: #fcfcfc">Next »</a></span>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
<script>var base_url = '../..';</script>
|
||||
<script src="../../js/theme.js" defer></script>
|
||||
<script src="../../search/main.js" defer></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,191 @@
|
||||
<!DOCTYPE html>
|
||||
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
||||
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
|
||||
<link rel="shortcut icon" href="../../img/favicon.ico">
|
||||
<title>SLURM Managed Cluster - Pytorch lightning Documentation</title>
|
||||
<link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
|
||||
|
||||
<link rel="stylesheet" href="../../css/theme.css" type="text/css" />
|
||||
<link rel="stylesheet" href="../../css/theme_extra.css" type="text/css" />
|
||||
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
|
||||
|
||||
<script>
|
||||
// Current page data
|
||||
var mkdocs_page_name = "SLURM Managed Cluster";
|
||||
var mkdocs_page_input_path = "Trainer/SLURM Managed Cluster.md";
|
||||
var mkdocs_page_url = null;
|
||||
</script>
|
||||
|
||||
<script src="../../js/jquery-2.1.1.min.js" defer></script>
|
||||
<script src="../../js/modernizr-2.8.3.min.js" defer></script>
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
|
||||
<script>hljs.initHighlightingOnLoad();</script>
|
||||
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav" role="document">
|
||||
|
||||
<div class="wy-grid-for-nav">
|
||||
|
||||
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
|
||||
<div class="wy-side-nav-search">
|
||||
<a href="../.." class="icon icon-home"> Pytorch lightning Documentation</a>
|
||||
<div role="search">
|
||||
<form id ="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" title="Type search term here" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
||||
<ul class="current">
|
||||
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<a class="" href="../..">PYTORCH-LIGHTNING DOCUMENTATION</a>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Pytorch Lightning</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../../Pytorch-Lightning/LightningModule/">Lightning module</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../">Trainer</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Checkpointing/">Checkpointing</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Distributed training/">Distributed training</a>
|
||||
</li>
|
||||
<li class=" current">
|
||||
|
||||
<a class="current" href="./">SLURM Managed Cluster</a>
|
||||
<ul class="subnav">
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Training Loop/">Training Loop</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Vaildation loop/">Vaildation loop</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
||||
|
||||
|
||||
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="../..">Pytorch lightning Documentation</a>
|
||||
</nav>
|
||||
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="breadcrumbs navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="../..">Docs</a> »</li>
|
||||
|
||||
|
||||
|
||||
<li>Trainer »</li>
|
||||
|
||||
|
||||
|
||||
<li>SLURM Managed Cluster</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/SLURM Managed Cluster.md"
|
||||
class="icon icon-github"> Edit on GitHub</a>
|
||||
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main">
|
||||
<div class="section">
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
|
||||
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
|
||||
|
||||
<a href="../Training Loop/" class="btn btn-neutral float-right" title="Training Loop">Next <span class="icon icon-circle-arrow-right"></span></a>
|
||||
|
||||
|
||||
<a href="../Distributed training/" class="btn btn-neutral" title="Distributed training"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<!-- Copyright etc -->
|
||||
|
||||
</div>
|
||||
|
||||
Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="rst-versions" role="note" style="cursor: pointer">
|
||||
<span class="rst-current-version" data-toggle="rst-current-version">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" class="fa fa-github" style="float: left; color: #fcfcfc"> GitHub</a>
|
||||
|
||||
|
||||
<span><a href="../Distributed training/" style="color: #fcfcfc;">« Previous</a></span>
|
||||
|
||||
|
||||
<span style="margin-left: 15px"><a href="../Training Loop/" style="color: #fcfcfc">Next »</a></span>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
<script>var base_url = '../..';</script>
|
||||
<script src="../../js/theme.js" defer></script>
|
||||
<script src="../../search/main.js" defer></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,303 @@
|
||||
<!DOCTYPE html>
|
||||
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
||||
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
|
||||
<link rel="shortcut icon" href="../../img/favicon.ico">
|
||||
<title>Training Loop - Pytorch lightning Documentation</title>
|
||||
<link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
|
||||
|
||||
<link rel="stylesheet" href="../../css/theme.css" type="text/css" />
|
||||
<link rel="stylesheet" href="../../css/theme_extra.css" type="text/css" />
|
||||
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
|
||||
|
||||
<script>
|
||||
// Current page data
|
||||
var mkdocs_page_name = "Training Loop";
|
||||
var mkdocs_page_input_path = "Trainer/Training Loop.md";
|
||||
var mkdocs_page_url = null;
|
||||
</script>
|
||||
|
||||
<script src="../../js/jquery-2.1.1.min.js" defer></script>
|
||||
<script src="../../js/modernizr-2.8.3.min.js" defer></script>
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
|
||||
<script>hljs.initHighlightingOnLoad();</script>
|
||||
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav" role="document">
|
||||
|
||||
<div class="wy-grid-for-nav">
|
||||
|
||||
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
|
||||
<div class="wy-side-nav-search">
|
||||
<a href="../.." class="icon icon-home"> Pytorch lightning Documentation</a>
|
||||
<div role="search">
|
||||
<form id ="rtd-search-form" class="wy-form" action="../../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" title="Type search term here" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
||||
<ul class="current">
|
||||
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<a class="" href="../..">PYTORCH-LIGHTNING DOCUMENTATION</a>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Pytorch Lightning</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../../Pytorch-Lightning/LightningModule/">Lightning module</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../">Trainer</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Checkpointing/">Checkpointing</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Distributed training/">Distributed training</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
||||
</li>
|
||||
<li class=" current">
|
||||
|
||||
<a class="current" href="./">Training Loop</a>
|
||||
<ul class="subnav">
|
||||
|
||||
<li class="toctree-l3"><a href="#accumulated-gradients">Accumulated gradients</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#anneal-learning-rate">Anneal Learning rate</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#check-gpu-usage">Check GPU usage</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#check-which-gradients-are-nan">Check which gradients are nan</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#check-validation-every-n-epochs">Check validation every n epochs</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#display-metrics-in-progress-bar">Display metrics in progress bar</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#display-the-parameter-count-by-layer">Display the parameter count by layer</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#force-training-for-min-or-max-epochs">Force training for min or max epochs</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#inspect-gradient-norms">Inspect gradient norms</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#make-model-overfit-on-subset-of-data">Make model overfit on subset of data</a></li>
|
||||
|
||||
|
||||
<li class="toctree-l3"><a href="#set-how-much-of-the-training-set-to-check">Set how much of the training set to check</a></li>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Vaildation loop/">Vaildation loop</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
||||
|
||||
|
||||
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="../..">Pytorch lightning Documentation</a>
|
||||
</nav>
|
||||
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="breadcrumbs navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="../..">Docs</a> »</li>
|
||||
|
||||
|
||||
|
||||
<li>Trainer »</li>
|
||||
|
||||
|
||||
|
||||
<li>Training Loop</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Training Loop.md"
|
||||
class="icon icon-github"> Edit on GitHub</a>
|
||||
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main">
|
||||
<div class="section">
|
||||
|
||||
<p>The asdf</p>
|
||||
<hr />
|
||||
<h4 id="accumulated-gradients">Accumulated gradients</h4>
|
||||
<p>Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. </p>
|
||||
<pre><code class="python"># DEFAULT (ie: no accumulated grads)
|
||||
trainer = Trainer(accumulate_grad_batches=1)
|
||||
</code></pre>
|
||||
|
||||
<hr />
|
||||
<h4 id="anneal-learning-rate">Anneal Learning rate</h4>
|
||||
<p>Cut the learning rate by 10 at every epoch listed in this list.</p>
|
||||
<pre><code class="python"># DEFAULT (don't anneal)
|
||||
trainer = Trainer(lr_scheduler_milestones=None)
|
||||
|
||||
# cut LR by 10 at 100, 200, and 300 epochs
|
||||
trainer = Trainer(lr_scheduler_milestones=[100, 200, 300])
|
||||
</code></pre>
|
||||
|
||||
<hr />
|
||||
<h4 id="check-gpu-usage">Check GPU usage</h4>
|
||||
<p>Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.</p>
|
||||
<hr />
|
||||
<h4 id="check-which-gradients-are-nan">Check which gradients are nan</h4>
|
||||
<p>This option prints a list of tensors with nan gradients.</p>
|
||||
<pre><code class="python"># DEFAULT
|
||||
trainer = Trainer(print_nan_grads=False)
|
||||
</code></pre>
|
||||
|
||||
<hr />
|
||||
<h4 id="check-validation-every-n-epochs">Check validation every n epochs</h4>
|
||||
<p>If you have a small dataset you might want to check validation every n epochs</p>
|
||||
<pre><code class="python"># DEFAULT
|
||||
trainer = Trainer(check_val_every_n_epoch=1)
|
||||
</code></pre>
|
||||
|
||||
<hr />
|
||||
<h4 id="display-metrics-in-progress-bar">Display metrics in progress bar</h4>
|
||||
<pre><code class="python"># DEFAULT
|
||||
trainer = Trainer(progress_bar=True)
|
||||
</code></pre>
|
||||
|
||||
<hr />
|
||||
<h4 id="display-the-parameter-count-by-layer">Display the parameter count by layer</h4>
|
||||
<p>By default lightning prints a list of parameters <em>and submodules</em> when it starts training.</p>
|
||||
<hr />
|
||||
<h4 id="force-training-for-min-or-max-epochs">Force training for min or max epochs</h4>
|
||||
<p>It can be useful to force training for a minimum number of epochs or limit to a max number</p>
|
||||
<pre><code class="python"># DEFAULT
|
||||
trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)
|
||||
</code></pre>
|
||||
|
||||
<hr />
|
||||
<h4 id="inspect-gradient-norms">Inspect gradient norms</h4>
|
||||
<p>Looking at grad norms can help you figure out where training might be going wrong.</p>
|
||||
<pre><code class="python"># DEFAULT (-1 doesn't track norms)
|
||||
trainer = Trainer(track_grad_norm=-1)
|
||||
|
||||
# track the LP norm (P=2 here)
|
||||
trainer = Trainer(track_grad_norm=2)
|
||||
</code></pre>
|
||||
|
||||
<hr />
|
||||
<h4 id="make-model-overfit-on-subset-of-data">Make model overfit on subset of data</h4>
|
||||
<p>A useful debugging trick is to make your model overfit a tiny fraction of the data.</p>
|
||||
<pre><code class="python"># DEFAULT don't overfit (ie: normal training)
|
||||
trainer = Trainer(overfit_pct=0.0)
|
||||
|
||||
# overfit on 1% of data
|
||||
trainer = Trainer(overfit_pct=0.01)
|
||||
</code></pre>
|
||||
|
||||
<hr />
|
||||
<h4 id="set-how-much-of-the-training-set-to-check">Set how much of the training set to check</h4>
|
||||
<p>If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag</p>
|
||||
<pre><code class="python"># DEFAULT
|
||||
trainer = Trainer(train_percent_check=1.0)
|
||||
|
||||
# check 10% only
|
||||
trainer = Trainer(train_percent_check=0.1)
|
||||
</code></pre>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
|
||||
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
|
||||
|
||||
<a href="../Vaildation loop/" class="btn btn-neutral float-right" title="Vaildation loop">Next <span class="icon icon-circle-arrow-right"></span></a>
|
||||
|
||||
|
||||
<a href="../SLURM Managed Cluster/" class="btn btn-neutral" title="SLURM Managed Cluster"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<!-- Copyright etc -->
|
||||
|
||||
</div>
|
||||
|
||||
Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="rst-versions" role="note" style="cursor: pointer">
|
||||
<span class="rst-current-version" data-toggle="rst-current-version">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" class="fa fa-github" style="float: left; color: #fcfcfc"> GitHub</a>
|
||||
|
||||
|
||||
<span><a href="../SLURM Managed Cluster/" style="color: #fcfcfc;">« Previous</a></span>
|
||||
|
||||
|
||||
<span style="margin-left: 15px"><a href="../Vaildation loop/" style="color: #fcfcfc">Next »</a></span>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
<script>var base_url = '../..';</script>
|
||||
<script src="../../js/theme.js" defer></script>
|
||||
<script src="../../search/main.js" defer></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
|
||||
<link rel="shortcut icon" href="../../img/favicon.ico">
|
||||
<title>Trainer - Pytorch lightning Documentation</title>
|
||||
<title>Vaildation loop - Pytorch lightning Documentation</title>
|
||||
<link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
|
||||
|
||||
<link rel="stylesheet" href="../../css/theme.css" type="text/css" />
|
||||
@@ -17,8 +17,8 @@
|
||||
|
||||
<script>
|
||||
// Current page data
|
||||
var mkdocs_page_name = "Trainer";
|
||||
var mkdocs_page_input_path = "Pytorch-Lightning/Trainer.md";
|
||||
var mkdocs_page_name = "Vaildation loop";
|
||||
var mkdocs_page_input_path = "Trainer/Vaildation loop.md";
|
||||
var mkdocs_page_url = null;
|
||||
</script>
|
||||
|
||||
@@ -59,16 +59,40 @@
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../LightningModule/">Lightning module</a>
|
||||
<a class="" href="../../Pytorch-Lightning/LightningModule/">Lightning module</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../">Trainer</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Checkpointing/">Checkpointing</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Distributed training/">Distributed training</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Training Loop/">Training Loop</a>
|
||||
</li>
|
||||
<li class=" current">
|
||||
|
||||
<a class="current" href="./">Trainer</a>
|
||||
<a class="current" href="./">Vaildation loop</a>
|
||||
<ul class="subnav">
|
||||
|
||||
<li class="toctree-l3"><a href="#trainer">Trainer</a></li>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
</ul>
|
||||
@@ -96,14 +120,14 @@
|
||||
|
||||
|
||||
|
||||
<li>Pytorch Lightning »</li>
|
||||
<li>Trainer »</li>
|
||||
|
||||
|
||||
|
||||
<li>Trainer</li>
|
||||
<li>Vaildation loop</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Pytorch-Lightning/Trainer.md"
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Vaildation loop.md"
|
||||
class="icon icon-github"> Edit on GitHub</a>
|
||||
|
||||
</li>
|
||||
@@ -113,7 +137,7 @@
|
||||
<div role="main">
|
||||
<div class="section">
|
||||
|
||||
<h1 id="trainer">Trainer</h1>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
@@ -122,7 +146,7 @@
|
||||
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
|
||||
|
||||
|
||||
<a href="../LightningModule/" class="btn btn-neutral" title="Lightning module"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
||||
<a href="../Training Loop/" class="btn btn-neutral" title="Training Loop"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
||||
|
||||
</div>
|
||||
|
||||
@@ -150,7 +174,7 @@
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" class="fa fa-github" style="float: left; color: #fcfcfc"> GitHub</a>
|
||||
|
||||
|
||||
<span><a href="../LightningModule/" style="color: #fcfcfc;">« Previous</a></span>
|
||||
<span><a href="../Training Loop/" style="color: #fcfcfc;">« Previous</a></span>
|
||||
|
||||
|
||||
</span>
|
||||
@@ -0,0 +1,248 @@
|
||||
<!DOCTYPE html>
|
||||
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
||||
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
|
||||
<link rel="shortcut icon" href="../img/favicon.ico">
|
||||
<title>Trainer - Pytorch lightning Documentation</title>
|
||||
<link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
|
||||
|
||||
<link rel="stylesheet" href="../css/theme.css" type="text/css" />
|
||||
<link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
|
||||
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
|
||||
|
||||
<script>
|
||||
// Current page data
|
||||
var mkdocs_page_name = "Trainer";
|
||||
var mkdocs_page_input_path = "Trainer/index.md";
|
||||
var mkdocs_page_url = null;
|
||||
</script>
|
||||
|
||||
<script src="../js/jquery-2.1.1.min.js" defer></script>
|
||||
<script src="../js/modernizr-2.8.3.min.js" defer></script>
|
||||
<script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
|
||||
<script>hljs.initHighlightingOnLoad();</script>
|
||||
|
||||
</head>
|
||||
|
||||
<body class="wy-body-for-nav" role="document">
|
||||
|
||||
<div class="wy-grid-for-nav">
|
||||
|
||||
|
||||
<nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
|
||||
<div class="wy-side-nav-search">
|
||||
<a href=".." class="icon icon-home"> Pytorch lightning Documentation</a>
|
||||
<div role="search">
|
||||
<form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
||||
<input type="text" name="q" placeholder="Search docs" title="Type search term here" />
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
||||
<ul class="current">
|
||||
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<a class="" href="..">PYTORCH-LIGHTNING DOCUMENTATION</a>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Pytorch Lightning</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="../Pytorch-Lightning/LightningModule/">Lightning module</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class=" current">
|
||||
|
||||
<a class="current" href="./">Trainer</a>
|
||||
<ul class="subnav">
|
||||
|
||||
<li class="toctree-l3"><a href="#trainer">Trainer</a></li>
|
||||
|
||||
|
||||
</ul>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Checkpointing/">Checkpointing</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Distributed training/">Distributed training</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Training Loop/">Training Loop</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Vaildation loop/">Vaildation loop</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
</nav>
|
||||
|
||||
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
||||
|
||||
|
||||
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
||||
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
||||
<a href="..">Pytorch lightning Documentation</a>
|
||||
</nav>
|
||||
|
||||
|
||||
<div class="wy-nav-content">
|
||||
<div class="rst-content">
|
||||
<div role="navigation" aria-label="breadcrumbs navigation">
|
||||
<ul class="wy-breadcrumbs">
|
||||
<li><a href="..">Docs</a> »</li>
|
||||
|
||||
|
||||
|
||||
<li>Trainer »</li>
|
||||
|
||||
|
||||
|
||||
<li>Trainer</li>
|
||||
<li class="wy-breadcrumbs-aside">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/index.md"
|
||||
class="icon icon-github"> Edit on GitHub</a>
|
||||
|
||||
</li>
|
||||
</ul>
|
||||
<hr/>
|
||||
</div>
|
||||
<div role="main">
|
||||
<div class="section">
|
||||
|
||||
<h1 id="trainer">Trainer</h1>
|
||||
<p>[<a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/models/trainer.py">Github Code</a>]</p>
|
||||
<p>The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research.</p>
|
||||
<p>This is the basic use of the trainer:</p>
|
||||
<pre><code class="python">from pytorch_lightning import Trainer
|
||||
|
||||
model = LightningTemplate()
|
||||
|
||||
trainer = Trainer()
|
||||
trainer.fit(model)
|
||||
</code></pre>
|
||||
|
||||
<p>But of course the fun is in all the advanced things it can do:</p>
|
||||
<p><strong>Training loop</strong> </p>
|
||||
<ul>
|
||||
<li><a href="Training%20Loop/#accumulated-gradients">Accumulate gradients</a></li>
|
||||
<li><a href="Training%20Loop/#anneal-learning-rate">Anneal Learning rate</a></li>
|
||||
<li><a href="Training%20Loop/#Check-gpu-usage">Check GPU usage</a></li>
|
||||
<li><a href="Training%20Loop/#check-which-gradients-are-nan">Check which gradients are nan</a></li>
|
||||
<li><a href="Training%20Loop/#check-validation-every-n-epochs">Check validation every n epochs</a></li>
|
||||
<li><a href="Training%20Loop/#display-metrics-in-progress-bar">Display metrics in progress bar</a></li>
|
||||
<li><a href="Training%20Loop/#display-the-parameter-count-by-layer">Display the parameter count by layer</a></li>
|
||||
<li><a href="Training%20Loop/#force-training-for-min-or-max-epochs">Force training for min or max epochs</a></li>
|
||||
<li><a href="Training%20Loop/#inspect-gradient-norms">Inspect gradient norms</a></li>
|
||||
<li><a href="Training%20Loop/#make-model-overfit-on-subset-of-data">Make model overfit on subset of data</a></li>
|
||||
<li><a href="../Pytorch-lightning/LightningModule/#configure_optimizers">Use multiple optimizers (like GANs)</a></li>
|
||||
<li><a href="Training%20Loop/#set-how-much-of-the-training-set-to-check">Set how much of the training set to check (1-100%)</a></li>
|
||||
</ul>
|
||||
<p><strong>Validation loop</strong> </p>
|
||||
<ul>
|
||||
<li>Display metrics in progress bar</li>
|
||||
<li>Set how much of the validation set to check (1-100%)</li>
|
||||
<li>Set validation check frequency within 1 training epoch (1-100%)</li>
|
||||
<li>validation_step function</li>
|
||||
<li>Why does validation run first for 5 steps?</li>
|
||||
</ul>
|
||||
<p><strong>Distributed training</strong> </p>
|
||||
<ul>
|
||||
<li>Single-gpu </li>
|
||||
<li>Multi-gpu </li>
|
||||
<li>Multi-node </li>
|
||||
<li>16-bit mixed precision</li>
|
||||
</ul>
|
||||
<p><strong>Checkpointing</strong> </p>
|
||||
<ul>
|
||||
<li>Model saving</li>
|
||||
<li>Model loading </li>
|
||||
</ul>
|
||||
<p><strong>Computing cluster (SLURM)</strong> </p>
|
||||
<ul>
|
||||
<li>Automatic checkpointing </li>
|
||||
<li>Automatic saving, loading </li>
|
||||
<li>Running grid search on a cluster </li>
|
||||
<li>Walltime auto-resubmit </li>
|
||||
</ul>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<footer>
|
||||
|
||||
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
|
||||
|
||||
<a href="Checkpointing/" class="btn btn-neutral float-right" title="Checkpointing">Next <span class="icon icon-circle-arrow-right"></span></a>
|
||||
|
||||
|
||||
<a href="../Pytorch-Lightning/LightningModule/" class="btn btn-neutral" title="Lightning module"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
<hr/>
|
||||
|
||||
<div role="contentinfo">
|
||||
<!-- Copyright etc -->
|
||||
|
||||
</div>
|
||||
|
||||
Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</section>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="rst-versions" role="note" style="cursor: pointer">
|
||||
<span class="rst-current-version" data-toggle="rst-current-version">
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" class="fa fa-github" style="float: left; color: #fcfcfc"> GitHub</a>
|
||||
|
||||
|
||||
<span><a href="../Pytorch-Lightning/LightningModule/" style="color: #fcfcfc;">« Previous</a></span>
|
||||
|
||||
|
||||
<span style="margin-left: 15px"><a href="Checkpointing/" style="color: #fcfcfc">Next »</a></span>
|
||||
|
||||
</span>
|
||||
</div>
|
||||
<script>var base_url = '..';</script>
|
||||
<script src="../js/theme.js" defer></script>
|
||||
<script src="../search/main.js" defer></script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
+29
-2
@@ -85,9 +85,36 @@
|
||||
|
||||
<a class="" href="Pytorch-Lightning/LightningModule/">Lightning module</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Pytorch-Lightning/Trainer/">Trainer</a>
|
||||
<a class="" href="Trainer/">Trainer</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Trainer/Checkpointing/">Checkpointing</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Trainer/Distributed training/">Distributed training</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Trainer/SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Trainer/Training Loop/">Training Loop</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="Trainer/Vaildation loop/">Vaildation loop</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
@@ -234,5 +261,5 @@
|
||||
|
||||
<!--
|
||||
MkDocs version : 1.0.4
|
||||
Build Date UTC : 2019-06-27 14:05:57
|
||||
Build Date UTC : 2019-06-27 15:59:38
|
||||
-->
|
||||
|
||||
+28
-1
@@ -54,9 +54,36 @@
|
||||
|
||||
<a class="" href="./Pytorch-Lightning/LightningModule/">Lightning module</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
<li class="toctree-l1">
|
||||
|
||||
<span class="caption-text">Trainer</span>
|
||||
<ul class="subnav">
|
||||
<li class="">
|
||||
|
||||
<a class="" href="./Pytorch-Lightning/Trainer/">Trainer</a>
|
||||
<a class="" href="./Trainer/">Trainer</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="./Trainer/Checkpointing/">Checkpointing</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="./Trainer/Distributed training/">Distributed training</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="./Trainer/SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="./Trainer/Training Loop/">Training Loop</a>
|
||||
</li>
|
||||
<li class="">
|
||||
|
||||
<a class="" href="./Trainer/Vaildation loop/">Vaildation loop</a>
|
||||
</li>
|
||||
</ul>
|
||||
</li>
|
||||
|
||||
File diff suppressed because one or more lines are too long
+25
@@ -15,4 +15,29 @@
|
||||
<lastmod>2019-06-27</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2019-06-27</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2019-06-27</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2019-06-27</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2019-06-27</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
<url>
|
||||
<loc>None</loc>
|
||||
<lastmod>2019-06-27</lastmod>
|
||||
<changefreq>daily</changefreq>
|
||||
</url>
|
||||
</urlset>
|
||||
Binary file not shown.
@@ -4,7 +4,7 @@ import sys
|
||||
from test_tube import HyperOptArgumentParser, Experiment
|
||||
from pytorch_lightning.models.trainer import Trainer
|
||||
from pytorch_lightning.utils.arg_parse import add_default_args
|
||||
from pytorch_lightning.utils.pt_callbacks import EarlyStopping, ModelCheckpoint
|
||||
from pytorch_lightning.callbacks.pt_callbacks import EarlyStopping, ModelCheckpoint
|
||||
from docs.source.examples.example_model import ExampleModel
|
||||
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
|
||||
from pytorch_lightning.models.trainer import Trainer
|
||||
from pytorch_lightning.utils.arg_parse import add_default_args
|
||||
|
||||
from pytorch_lightning.utils.pt_callbacks import EarlyStopping, ModelCheckpoint
|
||||
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
|
||||
|
||||
SEED = 2334
|
||||
torch.manual_seed(SEED)
|
||||
|
||||
Reference in New Issue
Block a user