wassname/pytorch-lightning
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-lightning.git synced 2026-09-11 12:31:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

Deployed b35229d with MkDocs version: 1.0.4

Browse Source
This commit is contained in:
William Falcon
2019-11-06 15:08:12 -05:00
parent 65095b6cac
commit 2cc732ca16
46 changed files with 995 additions and 718 deletions
Download Patch File Download Diff File Expand all files Collapse all files
404.html
+4 -4
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="/assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="/assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="/assets/stylesheets/application.0284f74d.css">
@@ -167,7 +167,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -501,7 +501,7 @@
</div>
<script src="/assets/javascripts/application.ac79c3b0.js"></script>
<script src="/assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"/"}})</script>
LightningModule/RequiredTrainerInterface/index.html
+213 -51
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -276,21 +276,21 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#minimal-example" class="md-nav__link">
<a href="#minimal-example" title="Minimal example" class="md-nav__link">
Minimal example
</a>
</li>
<li class="md-nav__item">
<a href="#how-do-these-methods-fit-into-the-broader-training" class="md-nav__link">
<a href="#how-do-these-methods-fit-into-the-broader-training" title="How do these methods fit into the broader training?" class="md-nav__link">
How do these methods fit into the broader training?
</a>
</li>
<li class="md-nav__item">
<a href="#required-methods" class="md-nav__link">
<a href="#required-methods" title="Required Methods" class="md-nav__link">
Required Methods
</a>
@@ -298,14 +298,21 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#training_step" class="md-nav__link">
<a href="#training_step" title="training_step" class="md-nav__link">
training_step
</a>
</li>
<li class="md-nav__item">
<a href="#train_dataloader" class="md-nav__link">
<a href="#training_end" title="training_end" class="md-nav__link">
training_end
</a>
</li>
<li class="md-nav__item">
<a href="#train_dataloader" title="train_dataloader" class="md-nav__link">
train_dataloader
</a>
@@ -313,7 +320,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return" class="md-nav__link">
<a href="#return" title="Return" class="md-nav__link">
Return
</a>
@@ -325,7 +332,7 @@
</li>
<li class="md-nav__item">
<a href="#configure_optimizers" class="md-nav__link">
<a href="#configure_optimizers" title="configure_optimizers" class="md-nav__link">
configure_optimizers
</a>
@@ -333,7 +340,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_1" class="md-nav__link">
<a href="#return_1" title="Return" class="md-nav__link">
Return
</a>
@@ -350,7 +357,7 @@
</li>
<li class="md-nav__item">
<a href="#optional-methods" class="md-nav__link">
<a href="#optional-methods" title="Optional Methods" class="md-nav__link">
Optional Methods
</a>
@@ -358,35 +365,35 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#validation_step" class="md-nav__link">
<a href="#validation_step" title="validation_step" class="md-nav__link">
validation_step
</a>
</li>
<li class="md-nav__item">
<a href="#validation_end" class="md-nav__link">
<a href="#validation_end" title="validation_end" class="md-nav__link">
validation_end
</a>
</li>
<li class="md-nav__item">
<a href="#test_step" class="md-nav__link">
<a href="#test_step" title="test_step" class="md-nav__link">
test_step
</a>
</li>
<li class="md-nav__item">
<a href="#test_end" class="md-nav__link">
<a href="#test_end" title="test_end" class="md-nav__link">
test_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_save_checkpoint" class="md-nav__link">
<a href="#on_save_checkpoint" title="on_save_checkpoint" class="md-nav__link">
on_save_checkpoint
</a>
@@ -394,7 +401,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_2" class="md-nav__link">
<a href="#return_2" title="Return" class="md-nav__link">
Return
</a>
@@ -406,7 +413,7 @@
</li>
<li class="md-nav__item">
<a href="#on_load_checkpoint" class="md-nav__link">
<a href="#on_load_checkpoint" title="on_load_checkpoint" class="md-nav__link">
on_load_checkpoint
</a>
@@ -414,7 +421,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_3" class="md-nav__link">
<a href="#return_3" title="Return" class="md-nav__link">
Return
</a>
@@ -426,7 +433,7 @@
</li>
<li class="md-nav__item">
<a href="#val_dataloader" class="md-nav__link">
<a href="#val_dataloader" title="val_dataloader" class="md-nav__link">
val_dataloader
</a>
@@ -434,7 +441,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_4" class="md-nav__link">
<a href="#return_4" title="Return" class="md-nav__link">
Return
</a>
@@ -446,7 +453,7 @@
</li>
<li class="md-nav__item">
<a href="#test_dataloader" class="md-nav__link">
<a href="#test_dataloader" title="test_dataloader" class="md-nav__link">
test_dataloader
</a>
@@ -454,7 +461,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_5" class="md-nav__link">
<a href="#return_5" title="Return" class="md-nav__link">
Return
</a>
@@ -466,7 +473,7 @@
</li>
<li class="md-nav__item">
<a href="#add_model_specific_args" class="md-nav__link">
<a href="#add_model_specific_args" title="add_model_specific_args" class="md-nav__link">
add_model_specific_args
</a>
@@ -474,7 +481,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_6" class="md-nav__link">
<a href="#return_6" title="Return" class="md-nav__link">
Return
</a>
@@ -731,21 +738,21 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#minimal-example" class="md-nav__link">
<a href="#minimal-example" title="Minimal example" class="md-nav__link">
Minimal example
</a>
</li>
<li class="md-nav__item">
<a href="#how-do-these-methods-fit-into-the-broader-training" class="md-nav__link">
<a href="#how-do-these-methods-fit-into-the-broader-training" title="How do these methods fit into the broader training?" class="md-nav__link">
How do these methods fit into the broader training?
</a>
</li>
<li class="md-nav__item">
<a href="#required-methods" class="md-nav__link">
<a href="#required-methods" title="Required Methods" class="md-nav__link">
Required Methods
</a>
@@ -753,14 +760,21 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#training_step" class="md-nav__link">
<a href="#training_step" title="training_step" class="md-nav__link">
training_step
</a>
</li>
<li class="md-nav__item">
<a href="#train_dataloader" class="md-nav__link">
<a href="#training_end" title="training_end" class="md-nav__link">
training_end
</a>
</li>
<li class="md-nav__item">
<a href="#train_dataloader" title="train_dataloader" class="md-nav__link">
train_dataloader
</a>
@@ -768,7 +782,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return" class="md-nav__link">
<a href="#return" title="Return" class="md-nav__link">
Return
</a>
@@ -780,7 +794,7 @@
</li>
<li class="md-nav__item">
<a href="#configure_optimizers" class="md-nav__link">
<a href="#configure_optimizers" title="configure_optimizers" class="md-nav__link">
configure_optimizers
</a>
@@ -788,7 +802,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_1" class="md-nav__link">
<a href="#return_1" title="Return" class="md-nav__link">
Return
</a>
@@ -805,7 +819,7 @@
</li>
<li class="md-nav__item">
<a href="#optional-methods" class="md-nav__link">
<a href="#optional-methods" title="Optional Methods" class="md-nav__link">
Optional Methods
</a>
@@ -813,35 +827,35 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#validation_step" class="md-nav__link">
<a href="#validation_step" title="validation_step" class="md-nav__link">
validation_step
</a>
</li>
<li class="md-nav__item">
<a href="#validation_end" class="md-nav__link">
<a href="#validation_end" title="validation_end" class="md-nav__link">
validation_end
</a>
</li>
<li class="md-nav__item">
<a href="#test_step" class="md-nav__link">
<a href="#test_step" title="test_step" class="md-nav__link">
test_step
</a>
</li>
<li class="md-nav__item">
<a href="#test_end" class="md-nav__link">
<a href="#test_end" title="test_end" class="md-nav__link">
test_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_save_checkpoint" class="md-nav__link">
<a href="#on_save_checkpoint" title="on_save_checkpoint" class="md-nav__link">
on_save_checkpoint
</a>
@@ -849,7 +863,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_2" class="md-nav__link">
<a href="#return_2" title="Return" class="md-nav__link">
Return
</a>
@@ -861,7 +875,7 @@
</li>
<li class="md-nav__item">
<a href="#on_load_checkpoint" class="md-nav__link">
<a href="#on_load_checkpoint" title="on_load_checkpoint" class="md-nav__link">
on_load_checkpoint
</a>
@@ -869,7 +883,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_3" class="md-nav__link">
<a href="#return_3" title="Return" class="md-nav__link">
Return
</a>
@@ -881,7 +895,7 @@
</li>
<li class="md-nav__item">
<a href="#val_dataloader" class="md-nav__link">
<a href="#val_dataloader" title="val_dataloader" class="md-nav__link">
val_dataloader
</a>
@@ -889,7 +903,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_4" class="md-nav__link">
<a href="#return_4" title="Return" class="md-nav__link">
Return
</a>
@@ -901,7 +915,7 @@
</li>
<li class="md-nav__item">
<a href="#test_dataloader" class="md-nav__link">
<a href="#test_dataloader" title="test_dataloader" class="md-nav__link">
test_dataloader
</a>
@@ -909,7 +923,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_5" class="md-nav__link">
<a href="#return_5" title="Return" class="md-nav__link">
Return
</a>
@@ -921,7 +935,7 @@
</li>
<li class="md-nav__item">
<a href="#add_model_specific_args" class="md-nav__link">
<a href="#add_model_specific_args" title="add_model_specific_args" class="md-nav__link">
add_model_specific_args
</a>
@@ -929,7 +943,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_6" class="md-nav__link">
<a href="#return_6" title="Return" class="md-nav__link">
Return
</a>
@@ -977,6 +991,7 @@
</ul>
<p><strong>Optional</strong>: </p>
<ul>
<li><a href="./#training_end">training_end</a> </li>
<li><a href="./#validation_step">validation_step</a> </li>
<li><a href="./#validation_end">validation_end</a> </li>
<li><a href="./#test_step">test_step</a> </li>
@@ -1244,6 +1259,153 @@
</pre></div>
</td></tr></table>
<p>If you add truncated back propagation through time you will also get an additional argument with the hidden states of the previous step. </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># Truncated back-propagation through time </span>
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">,</span> <span class="n">hiddens</span><span class="p">):</span>
<span class="c1"># hiddens are the hiddens from the previous truncated backprop step</span>
</pre></div>
</td></tr></table>
<p>You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to
break out of the current training epoch early.</p>
<hr />
<h3 id="training_end">training_end</h3>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">training_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">train_step_outputs</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<p>In certain cases (dp, ddp2), you might want to use all outputs of every process to do something.
For instance, if using negative samples, you could run a batch via dp and use ALL the outputs
for a single softmax across the full batch (ie: the denominator would use the full batch).</p>
<p>In this case you should define training_end to perform those calculations.</p>
<p><strong>Params</strong> </p>
<table>
<thead>
<tr>
<th>Param</th>
<th>description</th>
</tr>
</thead>
<tbody>
<tr>
<td>outputs</td>
<td>What you return in training_step.</td>
</tr>
</tbody>
</table>
<p><strong>Return</strong> </p>
<p>Dictionary or OrderedDict </p>
<table>
<thead>
<tr>
<th>key</th>
<th>value</th>
<th>is required</th>
</tr>
</thead>
<tbody>
<tr>
<td>loss</td>
<td>tensor scalar</td>
<td>Y</td>
</tr>
<tr>
<td>progress_bar</td>
<td>Dict for progress bar display. Must have only tensors</td>
<td>N</td>
</tr>
<tr>
<td>log</td>
<td>Dict of metrics to add to logger. Must have only tensors (no images, etc)</td>
<td>N</td>
</tr>
</tbody>
</table>
<p><strong>Example</strong></p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># WITHOUT training_end</span>
<span class="c1"># if used in DP or DDP2, this batch is 1/nb_gpus large</span>
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
<span class="c1"># batch is 1/nb_gpus big</span>
<span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">batch</span>
<span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
<span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">out</span><span class="p">)</span>
<span class="n">loss</span> <span class="o">=</span> <span class="n">nce_loss</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
<span class="k">return</span> <span class="p">{</span><span class="s1">&#39;loss&#39;</span><span class="p">:</span> <span class="n">loss</span><span class="p">}</span>
<span class="c1"># --------------</span>
<span class="c1"># with training_end to do softmax over the full batch</span>
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
<span class="c1"># batch is 1/nb_gpus big</span>
<span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">batch</span>
<span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
<span class="k">return</span> <span class="p">{</span><span class="s1">&#39;out&#39;</span><span class="p">:</span> <span class="n">out</span><span class="p">}</span>
<span class="k">def</span> <span class="nf">training_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
<span class="c1"># this out is now the full size of the batch</span>
<span class="n">out</span> <span class="o">=</span> <span class="n">outputs</span><span class="p">[</span><span class="s1">&#39;out&#39;</span><span class="p">]</span>
<span class="c1"># this softmax now uses the full batch size</span>
<span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">out</span><span class="p">)</span>
<span class="n">loss</span> <span class="o">=</span> <span class="n">nce_loss</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
<span class="k">return</span> <span class="p">{</span><span class="s1">&#39;loss&#39;</span><span class="p">:</span> <span class="n">loss</span><span class="p">}</span>
</pre></div>
</td></tr></table>
<p>If you define multiple optimizers, this step will also be called with an additional <code>optimizer_idx</code> param. </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4
5
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># Multiple optimizers (ie: GANs) </span>
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">,</span> <span class="n">optimizer_idx</span><span class="p">):</span>
<span class="k">if</span> <span class="n">optimizer_idx</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
<span class="c1"># do training_step with encoder</span>
<span class="k">if</span> <span class="n">optimizer_idx</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
<span class="c1"># do training_step with decoder </span>
</pre></div>
</td></tr></table>
<p>If you add truncated back propagation through time you will also get an additional argument with the hidden states of the previous step. </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># Truncated back-propagation through time </span>
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">,</span> <span class="n">hiddens</span><span class="p">):</span>
<span class="c1"># hiddens are the hiddens from the previous truncated backprop step</span>
</pre></div>
</td></tr></table>
<p>You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to
break out of the current training epoch early.</p>
<hr />
@@ -1255,7 +1417,7 @@ break out of the current training epoch early.</p>
</td></tr></table>
<p>Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. <br />
If you want to change the data during every epoch DON'T use the data_loader decorator. </p>
If you want to change the data during every epoch DON'T use the data_loader decorator.</p>
<h5 id="return">Return</h5>
<p>PyTorch DataLoader</p>
<p><strong>Example</strong></p>
@@ -2124,7 +2286,7 @@ The <a href="https://williamfalcon.github.io/test-tube/hyperparameter_optimizati
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
LightningModule/methods/index.html
+49 -12
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -284,21 +284,28 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#freeze" class="md-nav__link">
<a href="#freeze" title="freeze" class="md-nav__link">
freeze
</a>
</li>
<li class="md-nav__item">
<a href="#load_from_metrics" class="md-nav__link">
<a href="#load_from_metrics" title="load_from_metrics" class="md-nav__link">
load_from_metrics
</a>
</li>
<li class="md-nav__item">
<a href="#unfreeze" class="md-nav__link">
<a href="#load_from_metrics_1" title="load_from_metrics" class="md-nav__link">
load_from_metrics
</a>
</li>
<li class="md-nav__item">
<a href="#unfreeze" title="unfreeze" class="md-nav__link">
unfreeze
</a>
@@ -531,21 +538,28 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#freeze" class="md-nav__link">
<a href="#freeze" title="freeze" class="md-nav__link">
freeze
</a>
</li>
<li class="md-nav__item">
<a href="#load_from_metrics" class="md-nav__link">
<a href="#load_from_metrics" title="load_from_metrics" class="md-nav__link">
load_from_metrics
</a>
</li>
<li class="md-nav__item">
<a href="#unfreeze" class="md-nav__link">
<a href="#load_from_metrics_1" title="load_from_metrics" class="md-nav__link">
load_from_metrics
</a>
</li>
<li class="md-nav__item">
<a href="#unfreeze" title="unfreeze" class="md-nav__link">
unfreeze
</a>
@@ -584,8 +598,31 @@
<hr />
<h3 id="load_from_metrics">load_from_metrics</h3>
<p>This is the easiest/fastest way which uses the meta_tags.csv file from test-tube to rebuild the model.
The meta_tags.csv file can be found in the test-tube experiment save_dir. </p>
<p>This is the easiest/fastest way which loads hyperparameters and weights from a checkpoint,
such as the one saved by the <code>ModelCheckpoint</code> callback</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4
5
6
7
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">pretrained_model</span> <span class="o">=</span> <span class="n">MyLightningModule</span><span class="o">.</span><span class="n">load_from_checkpoint</span><span class="p">(</span>
<span class="n">checkpoint_path</span><span class="o">=</span><span class="s1">&#39;/path/to/pytorch_checkpoint.ckpt&#39;</span>
<span class="p">)</span>
<span class="c1"># predict</span>
<span class="n">pretrained_model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
<span class="n">pretrained_model</span><span class="o">.</span><span class="n">freeze</span><span class="p">()</span>
<span class="n">y_hat</span> <span class="o">=</span> <span class="n">pretrained_model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<hr />
<h3 id="load_from_metrics_1">load_from_metrics</h3>
<p>If you're using test tube, there is an alternate method which uses the meta_tags.csv
file from test-tube to rebuild the model. The meta_tags.csv file can be found in the
test-tube experiment save_dir. </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
@@ -716,7 +753,7 @@ The meta_tags.csv file can be found in the test-tube experiment save_dir.
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
LightningModule/properties/index.html
+22 -22
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -296,56 +296,56 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#current_epoch" class="md-nav__link">
<a href="#current_epoch" title="current_epoch" class="md-nav__link">
current_epoch
</a>
</li>
<li class="md-nav__item">
<a href="#dtype" class="md-nav__link">
<a href="#dtype" title="dtype" class="md-nav__link">
dtype
</a>
</li>
<li class="md-nav__item">
<a href="#logger" class="md-nav__link">
<a href="#logger" title="logger" class="md-nav__link">
logger
</a>
</li>
<li class="md-nav__item">
<a href="#global_step" class="md-nav__link">
<a href="#global_step" title="global_step" class="md-nav__link">
global_step
</a>
</li>
<li class="md-nav__item">
<a href="#gradient_clip_val" class="md-nav__link">
<a href="#gradient_clip_val" title="gradient_clip_val" class="md-nav__link">
gradient_clip_val
</a>
</li>
<li class="md-nav__item">
<a href="#on_gpu" class="md-nav__link">
<a href="#on_gpu" title="on_gpu" class="md-nav__link">
on_gpu
</a>
</li>
<li class="md-nav__item">
<a href="#trainer" class="md-nav__link">
<a href="#trainer" title="trainer" class="md-nav__link">
trainer
</a>
</li>
<li class="md-nav__item">
<a href="#debugging" class="md-nav__link">
<a href="#debugging" title="Debugging" class="md-nav__link">
Debugging
</a>
@@ -353,7 +353,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#example_input_array" class="md-nav__link">
<a href="#example_input_array" title="example_input_array" class="md-nav__link">
example_input_array
</a>
@@ -579,56 +579,56 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#current_epoch" class="md-nav__link">
<a href="#current_epoch" title="current_epoch" class="md-nav__link">
current_epoch
</a>
</li>
<li class="md-nav__item">
<a href="#dtype" class="md-nav__link">
<a href="#dtype" title="dtype" class="md-nav__link">
dtype
</a>
</li>
<li class="md-nav__item">
<a href="#logger" class="md-nav__link">
<a href="#logger" title="logger" class="md-nav__link">
logger
</a>
</li>
<li class="md-nav__item">
<a href="#global_step" class="md-nav__link">
<a href="#global_step" title="global_step" class="md-nav__link">
global_step
</a>
</li>
<li class="md-nav__item">
<a href="#gradient_clip_val" class="md-nav__link">
<a href="#gradient_clip_val" title="gradient_clip_val" class="md-nav__link">
gradient_clip_val
</a>
</li>
<li class="md-nav__item">
<a href="#on_gpu" class="md-nav__link">
<a href="#on_gpu" title="on_gpu" class="md-nav__link">
on_gpu
</a>
</li>
<li class="md-nav__item">
<a href="#trainer" class="md-nav__link">
<a href="#trainer" title="trainer" class="md-nav__link">
trainer
</a>
</li>
<li class="md-nav__item">
<a href="#debugging" class="md-nav__link">
<a href="#debugging" title="Debugging" class="md-nav__link">
Debugging
</a>
@@ -636,7 +636,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#example_input_array" class="md-nav__link">
<a href="#example_input_array" title="example_input_array" class="md-nav__link">
example_input_array
</a>
@@ -798,7 +798,7 @@ to get a print out of sizes coming into and out of every layer. </p>
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Checkpointing/index.html
+34 -20
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -344,14 +344,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#model-saving" class="md-nav__link">
<a href="#model-saving" title="Model saving" class="md-nav__link">
Model saving
</a>
</li>
<li class="md-nav__item">
<a href="#restoring-training-session" class="md-nav__link">
<a href="#restoring-training-session" title="Restoring training session" class="md-nav__link">
Restoring training session
</a>
@@ -524,14 +524,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#model-saving" class="md-nav__link">
<a href="#model-saving" title="Model saving" class="md-nav__link">
Model saving
</a>
</li>
<li class="md-nav__item">
<a href="#restoring-training-session" class="md-nav__link">
<a href="#restoring-training-session" title="Restoring training session" class="md-nav__link">
Restoring training session
</a>
@@ -601,19 +601,33 @@ To change the checkpoint path pass in :</p>
<p>You might want to not only load a model but also continue training it. Use this method to
restore the trainer state as well. This will continue from the epoch and global step you last left off.<br />
However, the dataloaders will start from the first batch again (if you shuffled it shouldn't matter). </p>
<p>Lightning will restore the session if you pass an experiment with the same version and there's a saved checkpoint. </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4
5
6
7
8
9</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">test_tube</span> <span class="kn">import</span> <span class="n">Experiment</span>
<p>Lightning will restore the session if you pass a logger with the same version and there's a saved checkpoint. </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning</span> <span class="kn">import</span> <span class="n">Trainer</span>
<span class="kn">from</span> <span class="nn">pytorch_lightning.logging</span> <span class="kn">import</span> <span class="n">TestTubeLogger</span>
<span class="n">exp</span> <span class="o">=</span> <span class="n">Experiment</span><span class="p">(</span><span class="n">version</span><span class="o">=</span><span class="n">a_previous_version_with_a_saved_checkpoint</span><span class="p">)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">experiment</span><span class="o">=</span><span class="n">exp</span><span class="p">)</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">TestTubeLogger</span><span class="p">(</span>
<span class="n">save_dir</span><span class="o">=</span><span class="s1">&#39;./savepath&#39;</span><span class="p">,</span>
<span class="n">version</span><span class="o">=</span><span class="mi">1</span> <span class="c1"># An existing version with a saved checkpoint</span>
<span class="p">)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span>
<span class="n">logger</span><span class="o">=</span><span class="n">logger</span><span class="p">,</span>
<span class="n">default_save_path</span><span class="o">=</span><span class="s1">&#39;./savepath&#39;</span>
<span class="p">)</span>
<span class="c1"># this fit call loads model weights and trainer state</span>
<span class="c1"># the trainer continues seamlessly from where you left off</span>
@@ -733,7 +747,7 @@ the system isn't different. If you add a layer, for instance, it might not work.
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Distributed training/index.html
+51 -37
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -356,7 +356,7 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#choosing-a-backend" class="md-nav__link">
<a href="#choosing-a-backend" title="Choosing a backend" class="md-nav__link">
Choosing a backend
</a>
@@ -364,21 +364,21 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#dataparallel-dp" class="md-nav__link">
<a href="#dataparallel-dp" title="DataParallel (dp)" class="md-nav__link">
DataParallel (dp)
</a>
</li>
<li class="md-nav__item">
<a href="#distributeddataparallel-ddp" class="md-nav__link">
<a href="#distributeddataparallel-ddp" title="DistributedDataParallel (ddp)" class="md-nav__link">
DistributedDataParallel (ddp)
</a>
</li>
<li class="md-nav__item">
<a href="#distributeddataparallel-2-ddp2" class="md-nav__link">
<a href="#distributeddataparallel-2-ddp2" title="DistributedDataParallel-2 (ddp2)" class="md-nav__link">
DistributedDataParallel-2 (ddp2)
</a>
@@ -390,56 +390,56 @@
</li>
<li class="md-nav__item">
<a href="#distributed-and-16-bit-precision" class="md-nav__link">
<a href="#distributed-and-16-bit-precision" title="Distributed and 16-bit precision." class="md-nav__link">
Distributed and 16-bit precision.
</a>
</li>
<li class="md-nav__item">
<a href="#cuda-flags" class="md-nav__link">
<a href="#cuda-flags" title="CUDA flags" class="md-nav__link">
CUDA flags
</a>
</li>
<li class="md-nav__item">
<a href="#16-bit-mixed-precision" class="md-nav__link">
<a href="#16-bit-mixed-precision" title="16-bit mixed precision" class="md-nav__link">
16-bit mixed precision
</a>
</li>
<li class="md-nav__item">
<a href="#single-gpu" class="md-nav__link">
<a href="#single-gpu" title="Single-gpu" class="md-nav__link">
Single-gpu
</a>
</li>
<li class="md-nav__item">
<a href="#multi-gpu" class="md-nav__link">
<a href="#multi-gpu" title="multi-gpu" class="md-nav__link">
multi-gpu
</a>
</li>
<li class="md-nav__item">
<a href="#multi-node" class="md-nav__link">
<a href="#multi-node" title="Multi-node" class="md-nav__link">
Multi-node
</a>
</li>
<li class="md-nav__item">
<a href="#auto-slurm-job-submission" class="md-nav__link">
<a href="#auto-slurm-job-submission" title="Auto-slurm-job-submission" class="md-nav__link">
Auto-slurm-job-submission
</a>
</li>
<li class="md-nav__item">
<a href="#self-balancing-architecture" class="md-nav__link">
<a href="#self-balancing-architecture" title="Self-balancing architecture" class="md-nav__link">
Self-balancing architecture
</a>
@@ -600,7 +600,7 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#choosing-a-backend" class="md-nav__link">
<a href="#choosing-a-backend" title="Choosing a backend" class="md-nav__link">
Choosing a backend
</a>
@@ -608,21 +608,21 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#dataparallel-dp" class="md-nav__link">
<a href="#dataparallel-dp" title="DataParallel (dp)" class="md-nav__link">
DataParallel (dp)
</a>
</li>
<li class="md-nav__item">
<a href="#distributeddataparallel-ddp" class="md-nav__link">
<a href="#distributeddataparallel-ddp" title="DistributedDataParallel (ddp)" class="md-nav__link">
DistributedDataParallel (ddp)
</a>
</li>
<li class="md-nav__item">
<a href="#distributeddataparallel-2-ddp2" class="md-nav__link">
<a href="#distributeddataparallel-2-ddp2" title="DistributedDataParallel-2 (ddp2)" class="md-nav__link">
DistributedDataParallel-2 (ddp2)
</a>
@@ -634,56 +634,56 @@
</li>
<li class="md-nav__item">
<a href="#distributed-and-16-bit-precision" class="md-nav__link">
<a href="#distributed-and-16-bit-precision" title="Distributed and 16-bit precision." class="md-nav__link">
Distributed and 16-bit precision.
</a>
</li>
<li class="md-nav__item">
<a href="#cuda-flags" class="md-nav__link">
<a href="#cuda-flags" title="CUDA flags" class="md-nav__link">
CUDA flags
</a>
</li>
<li class="md-nav__item">
<a href="#16-bit-mixed-precision" class="md-nav__link">
<a href="#16-bit-mixed-precision" title="16-bit mixed precision" class="md-nav__link">
16-bit mixed precision
</a>
</li>
<li class="md-nav__item">
<a href="#single-gpu" class="md-nav__link">
<a href="#single-gpu" title="Single-gpu" class="md-nav__link">
Single-gpu
</a>
</li>
<li class="md-nav__item">
<a href="#multi-gpu" class="md-nav__link">
<a href="#multi-gpu" title="multi-gpu" class="md-nav__link">
multi-gpu
</a>
</li>
<li class="md-nav__item">
<a href="#multi-node" class="md-nav__link">
<a href="#multi-node" title="Multi-node" class="md-nav__link">
Multi-node
</a>
</li>
<li class="md-nav__item">
<a href="#auto-slurm-job-submission" class="md-nav__link">
<a href="#auto-slurm-job-submission" title="Auto-slurm-job-submission" class="md-nav__link">
Auto-slurm-job-submission
</a>
</li>
<li class="md-nav__item">
<a href="#self-balancing-architecture" class="md-nav__link">
<a href="#self-balancing-architecture" title="Self-balancing architecture" class="md-nav__link">
Self-balancing architecture
</a>
@@ -814,21 +814,35 @@ not allow 16-bit and DP training. We tried to get this to work, but it's an issu
</tbody>
</table>
<p>You also have the option of specifying which GPUs to use by passing a list: </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4
5
6
7
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (int)</span>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (int) specifies how many GPUs to use.</span>
<span class="n">Trainer</span><span class="p">(</span><span class="n">gpus</span><span class="o">=</span><span class="n">k</span><span class="p">)</span>
<span class="c1"># Above is equivalent to </span>
<span class="n">Trainer</span><span class="p">(</span><span class="n">gpus</span><span class="o">=</span><span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">k</span><span class="p">)))</span>
<span class="c1"># You specify which GPUs (don&#39;t use if running on cluster) </span>
<span class="n">Trainer</span><span class="p">(</span><span class="n">gpus</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
<span class="c1"># can also be a string</span>
<span class="n">Trainer</span><span class="p">(</span><span class="n">gpus</span><span class="o">=</span><span class="s1">&#39;0, 1&#39;</span><span class="p">)</span>
<span class="c1"># can also be -1 or &#39;-1&#39;, this uses all available GPUs</span>
<span class="c1"># this is equivalent to list(range(torch.cuda.available_devices()))</span>
<span class="n">Trainer</span><span class="p">(</span><span class="n">gpus</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -1025,7 +1039,7 @@ portion of your dataset onto each GPU. (World_size = gpus_per_node * nb_nodes).
<p>Instead of manually building SLURM scripts, you can use the <a href="https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/">SlurmCluster object</a> to
do this for you. The SlurmCluster can also run a grid search if you pass in a <a href="https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/">HyperOptArgumentParser</a>.</p>
<p>Here is an example where you run a grid search of 9 combinations of hyperparams.
<a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/new_project_templates/multi_node_examples">The full examples are here</a>.</p>
<a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/new_project_templates/multi_node_examples">The full examples are here</a>.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
@@ -1184,7 +1198,7 @@ do this for you. The SlurmCluster can also run a grid search if you pass in a <a
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Logging/index.html
+76 -28
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -368,14 +368,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#default_save_path" class="md-nav__link">
<a href="#default_save_path" title="default_save_path" class="md-nav__link">
default_save_path
</a>
</li>
<li class="md-nav__item">
<a href="#setting-up-logging" class="md-nav__link">
<a href="#setting-up-logging" title="Setting up logging" class="md-nav__link">
Setting up logging
</a>
@@ -383,70 +383,77 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#test-tube" class="md-nav__link">
<a href="#test-tube" title="Test tube" class="md-nav__link">
Test tube
</a>
</li>
<li class="md-nav__item">
<a href="#mlflow" class="md-nav__link">
<a href="#mlflow" title="MLFlow" class="md-nav__link">
MLFlow
</a>
</li>
<li class="md-nav__item">
<a href="#custom-logger" class="md-nav__link">
<a href="#cometml" title="Comet.ml" class="md-nav__link">
Comet.ml
</a>
</li>
<li class="md-nav__item">
<a href="#custom-logger" title="Custom logger" class="md-nav__link">
Custom logger
</a>
</li>
<li class="md-nav__item">
<a href="#using-loggers" class="md-nav__link">
<a href="#using-loggers" title="Using loggers" class="md-nav__link">
Using loggers
</a>
</li>
<li class="md-nav__item">
<a href="#display-metrics-in-progress-bar" class="md-nav__link">
<a href="#display-metrics-in-progress-bar" title="Display metrics in progress bar" class="md-nav__link">
Display metrics in progress bar
</a>
</li>
<li class="md-nav__item">
<a href="#log-metric-row-every-k-batches" class="md-nav__link">
<a href="#log-metric-row-every-k-batches" title="Log metric row every k batches" class="md-nav__link">
Log metric row every k batches
</a>
</li>
<li class="md-nav__item">
<a href="#log-gpu-memory" class="md-nav__link">
<a href="#log-gpu-memory" title="Log GPU memory" class="md-nav__link">
Log GPU memory
</a>
</li>
<li class="md-nav__item">
<a href="#process-position" class="md-nav__link">
<a href="#process-position" title="Process position" class="md-nav__link">
Process position
</a>
</li>
<li class="md-nav__item">
<a href="#save-a-snapshot-of-all-hyperparameters" class="md-nav__link">
<a href="#save-a-snapshot-of-all-hyperparameters" title="Save a snapshot of all hyperparameters" class="md-nav__link">
Save a snapshot of all hyperparameters
</a>
</li>
<li class="md-nav__item">
<a href="#write-logs-file-to-csv-every-k-batches" class="md-nav__link">
<a href="#write-logs-file-to-csv-every-k-batches" title="Write logs file to csv every k batches" class="md-nav__link">
Write logs file to csv every k batches
</a>
@@ -600,14 +607,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#default_save_path" class="md-nav__link">
<a href="#default_save_path" title="default_save_path" class="md-nav__link">
default_save_path
</a>
</li>
<li class="md-nav__item">
<a href="#setting-up-logging" class="md-nav__link">
<a href="#setting-up-logging" title="Setting up logging" class="md-nav__link">
Setting up logging
</a>
@@ -615,70 +622,77 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#test-tube" class="md-nav__link">
<a href="#test-tube" title="Test tube" class="md-nav__link">
Test tube
</a>
</li>
<li class="md-nav__item">
<a href="#mlflow" class="md-nav__link">
<a href="#mlflow" title="MLFlow" class="md-nav__link">
MLFlow
</a>
</li>
<li class="md-nav__item">
<a href="#custom-logger" class="md-nav__link">
<a href="#cometml" title="Comet.ml" class="md-nav__link">
Comet.ml
</a>
</li>
<li class="md-nav__item">
<a href="#custom-logger" title="Custom logger" class="md-nav__link">
Custom logger
</a>
</li>
<li class="md-nav__item">
<a href="#using-loggers" class="md-nav__link">
<a href="#using-loggers" title="Using loggers" class="md-nav__link">
Using loggers
</a>
</li>
<li class="md-nav__item">
<a href="#display-metrics-in-progress-bar" class="md-nav__link">
<a href="#display-metrics-in-progress-bar" title="Display metrics in progress bar" class="md-nav__link">
Display metrics in progress bar
</a>
</li>
<li class="md-nav__item">
<a href="#log-metric-row-every-k-batches" class="md-nav__link">
<a href="#log-metric-row-every-k-batches" title="Log metric row every k batches" class="md-nav__link">
Log metric row every k batches
</a>
</li>
<li class="md-nav__item">
<a href="#log-gpu-memory" class="md-nav__link">
<a href="#log-gpu-memory" title="Log GPU memory" class="md-nav__link">
Log GPU memory
</a>
</li>
<li class="md-nav__item">
<a href="#process-position" class="md-nav__link">
<a href="#process-position" title="Process position" class="md-nav__link">
Process position
</a>
</li>
<li class="md-nav__item">
<a href="#save-a-snapshot-of-all-hyperparameters" class="md-nav__link">
<a href="#save-a-snapshot-of-all-hyperparameters" title="Save a snapshot of all hyperparameters" class="md-nav__link">
Save a snapshot of all hyperparameters
</a>
</li>
<li class="md-nav__item">
<a href="#write-logs-file-to-csv-every-k-batches" class="md-nav__link">
<a href="#write-logs-file-to-csv-every-k-batches" title="Write logs file to csv every k batches" class="md-nav__link">
Write logs file to csv every k batches
</a>
@@ -806,6 +820,40 @@ to manage experiments and snapshots all hyperparameters you pass to a LightningM
</pre></div>
</td></tr></table>
<hr />
<h4 id="cometml">Comet.ml</h4>
<p>Log using <a href="https://www.comet.ml">comet</a></p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4
5
6
7</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.logging</span> <span class="kn">import</span> <span class="n">CometLogger</span>
<span class="c1"># arguments made to CometLogger are passed on to the comet_ml.Experiment class</span>
<span class="n">comet_logger</span> <span class="o">=</span> <span class="n">CometLogger</span><span class="p">(</span>
<span class="n">api_key</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">&quot;COMET_KEY&quot;</span><span class="p">],</span>
<span class="n">workspace</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">&quot;COMET_KEY&quot;</span><span class="p">],</span>
<span class="p">)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">comet_logger</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<p>Use the logger anywhere in you LightningModule as follows:</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4
5
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">train_step</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
<span class="c1"># example</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">whatever_comet_ml_supports</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="k">def</span> <span class="nf">any_lightning_module_function_or_hook</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">whatever_comet_ml_supports</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<hr />
<h4 id="custom-logger">Custom logger</h4>
<p>You can implement your own logger by writing a class that inherits from
@@ -1039,7 +1087,7 @@ Lightning will stack progress bars according to this value. </p>
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/SLURM Managed Cluster/index.html
+8 -8
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -380,14 +380,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#running-grid-search-on-a-cluster" class="md-nav__link">
<a href="#running-grid-search-on-a-cluster" title="Running grid search on a cluster" class="md-nav__link">
Running grid search on a cluster
</a>
</li>
<li class="md-nav__item">
<a href="#walltime-auto-resubmit" class="md-nav__link">
<a href="#walltime-auto-resubmit" title="Walltime auto-resubmit" class="md-nav__link">
Walltime auto-resubmit
</a>
@@ -524,14 +524,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#running-grid-search-on-a-cluster" class="md-nav__link">
<a href="#running-grid-search-on-a-cluster" title="Running grid search on a cluster" class="md-nav__link">
Running grid search on a cluster
</a>
</li>
<li class="md-nav__item">
<a href="#walltime-auto-resubmit" class="md-nav__link">
<a href="#walltime-auto-resubmit" title="Walltime auto-resubmit" class="md-nav__link">
Walltime auto-resubmit
</a>
@@ -777,7 +777,7 @@ your SLURM script. </p>
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Testing loop/index.html
+6 -6
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -392,7 +392,7 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#test" class="md-nav__link">
<a href="#test" title="test" class="md-nav__link">
test
</a>
@@ -517,7 +517,7 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#test" class="md-nav__link">
<a href="#test" title="test" class="md-nav__link">
test
</a>
@@ -658,7 +658,7 @@ First case is where you test right after a full training routine.</p>
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Training Loop/index.html
+93 -22
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -404,54 +404,68 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#accumulated-gradients" class="md-nav__link">
<a href="#accumulated-gradients" title="Accumulated gradients" class="md-nav__link">
Accumulated gradients
</a>
</li>
<li class="md-nav__item">
<a href="#force-training-for-min-or-max-epochs" class="md-nav__link">
<a href="#force-training-for-min-or-max-epochs" title="Force training for min or max epochs" class="md-nav__link">
Force training for min or max epochs
</a>
</li>
<li class="md-nav__item">
<a href="#early-stopping" class="md-nav__link">
<a href="#early-stopping" title="Early stopping" class="md-nav__link">
Early stopping
</a>
</li>
<li class="md-nav__item">
<a href="#force-disable-early-stop" class="md-nav__link">
<a href="#force-disable-early-stop" title="Force disable early stop" class="md-nav__link">
Force disable early stop
</a>
</li>
<li class="md-nav__item">
<a href="#gradient-clipping" class="md-nav__link">
<a href="#gradient-clipping" title="Gradient Clipping" class="md-nav__link">
Gradient Clipping
</a>
</li>
<li class="md-nav__item">
<a href="#inspect-gradient-norms" class="md-nav__link">
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
Inspect gradient norms
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-training-set-to-check" class="md-nav__link">
<a href="#set-how-much-of-the-training-set-to-check" title="Set how much of the training set to check" class="md-nav__link">
Set how much of the training set to check
</a>
</li>
<li class="md-nav__item">
<a href="#packed-sequences-as-inputs" title="Packed sequences as inputs" class="md-nav__link">
Packed sequences as inputs
</a>
</li>
<li class="md-nav__item">
<a href="#truncated-back-propagation-through-time" title="Truncated Back Propagation Through Time" class="md-nav__link">
Truncated Back Propagation Through Time
</a>
</li>
@@ -559,54 +573,68 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#accumulated-gradients" class="md-nav__link">
<a href="#accumulated-gradients" title="Accumulated gradients" class="md-nav__link">
Accumulated gradients
</a>
</li>
<li class="md-nav__item">
<a href="#force-training-for-min-or-max-epochs" class="md-nav__link">
<a href="#force-training-for-min-or-max-epochs" title="Force training for min or max epochs" class="md-nav__link">
Force training for min or max epochs
</a>
</li>
<li class="md-nav__item">
<a href="#early-stopping" class="md-nav__link">
<a href="#early-stopping" title="Early stopping" class="md-nav__link">
Early stopping
</a>
</li>
<li class="md-nav__item">
<a href="#force-disable-early-stop" class="md-nav__link">
<a href="#force-disable-early-stop" title="Force disable early stop" class="md-nav__link">
Force disable early stop
</a>
</li>
<li class="md-nav__item">
<a href="#gradient-clipping" class="md-nav__link">
<a href="#gradient-clipping" title="Gradient Clipping" class="md-nav__link">
Gradient Clipping
</a>
</li>
<li class="md-nav__item">
<a href="#inspect-gradient-norms" class="md-nav__link">
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
Inspect gradient norms
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-training-set-to-check" class="md-nav__link">
<a href="#set-how-much-of-the-training-set-to-check" title="Set how much of the training set to check" class="md-nav__link">
Set how much of the training set to check
</a>
</li>
<li class="md-nav__item">
<a href="#packed-sequences-as-inputs" title="Packed sequences as inputs" class="md-nav__link">
Packed sequences as inputs
</a>
</li>
<li class="md-nav__item">
<a href="#truncated-back-propagation-through-time" title="Truncated Back Propagation Through Time" class="md-nav__link">
Truncated Back Propagation Through Time
</a>
</li>
@@ -632,7 +660,7 @@
<p>Below are all the things lightning automates for you in the training loop.</p>
<hr />
<h4 id="accumulated-gradients">Accumulated gradients</h4>
<p>Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN. </p>
<p>Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (ie: no accumulated grads)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">accumulate_grad_batches</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -650,7 +678,7 @@
<hr />
<h4 id="early-stopping">Early stopping</h4>
<p>The trainer already sets up default early stopping for you.
<p>The trainer already sets up default early stopping for you.
To modify this behavior, pass in your own EarlyStopping callback.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
@@ -684,10 +712,10 @@ To modify this behavior, pass in your own EarlyStopping callback.</p>
<span class="c1"># without passing anything in, uses the default callback above</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">()</span>
<span class="c1"># pass in your own to override the default callback </span>
<span class="c1"># pass in your own to override the default callback</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">early_stop_callback</span><span class="o">=</span><span class="n">early_stop_callback</span><span class="p">)</span>
<span class="c1"># pass in None to disable it </span>
<span class="c1"># pass in None to disable it</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">early_stop_callback</span><span class="o">=</span><span class="bp">None</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -747,6 +775,49 @@ Specifically, this will <a href="https://pytorch.org/docs/stable/nn.html#torch.n
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">train_percent_check</span><span class="o">=</span><span class="mf">0.1</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<hr />
<h4 id="packed-sequences-as-inputs">Packed sequences as inputs</h4>
<p>When using PackedSequence, do 2 things:
1. return either a padded tensor in dataset or a list of variable length tensors in the dataloader collate_fn (example above shows the list implementation). <br />
2. Pack the sequence in forward or training and validation steps depending on use case.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># For use in dataloader</span>
<span class="k">def</span> <span class="nf">collate_fn</span><span class="p">(</span><span class="n">batch</span><span class="p">):</span>
<span class="n">x</span> <span class="o">=</span> <span class="p">[</span><span class="n">item</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">batch</span><span class="p">]</span>
<span class="n">y</span> <span class="o">=</span> <span class="p">[</span><span class="n">item</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">batch</span><span class="p">]</span>
<span class="k">return</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span>
<span class="c1"># In module</span>
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">pack_sequence</span><span class="p">(</span><span class="n">batch</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">enforce_sorted</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
<span class="n">y</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">pack_sequence</span><span class="p">(</span><span class="n">batch</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">enforce_sorted</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<hr />
<h4 id="truncated-back-propagation-through-time">Truncated Back Propagation Through Time</h4>
<p>There are times when multiple backwards passes are needed for each batch. For example, it may save memory to use Truncated Back Propagation Through Time when training RNNs.</p>
<p>When this flag is enabled each batch is split into sequences of size truncated_bptt_steps and passed to training_step(...) separately. A default splitting function is provided, however, you can override it for more flexibility. See <a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks#tbptt_split_batch">tbptt_split_batch</a>.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (single backwards pass per batch)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">truncated_bptt_steps</span><span class="o">=</span><span class="bp">None</span><span class="p">)</span>
<span class="c1"># (split batch into sequences of size 2)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">truncated_bptt_steps</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -816,7 +887,7 @@ Specifically, this will <a href="https://pytorch.org/docs/stable/nn.html#torch.n
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Validation loop/index.html
+25 -16
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -416,35 +416,35 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#check-validation-every-n-epochs" class="md-nav__link">
<a href="#check-validation-every-n-epochs" title="Check validation every n epochs" class="md-nav__link">
Check validation every n epochs
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-validation-set-to-check" class="md-nav__link">
<a href="#set-how-much-of-the-validation-set-to-check" title="Set how much of the validation set to check" class="md-nav__link">
Set how much of the validation set to check
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-test-set-to-check" class="md-nav__link">
<a href="#set-how-much-of-the-test-set-to-check" title="Set how much of the test set to check" class="md-nav__link">
Set how much of the test set to check
</a>
</li>
<li class="md-nav__item">
<a href="#set-validation-check-frequency-within-1-training-epoch" class="md-nav__link">
<a href="#set-validation-check-frequency-within-1-training-epoch" title="Set validation check frequency within 1 training epoch" class="md-nav__link">
Set validation check frequency within 1 training epoch
</a>
</li>
<li class="md-nav__item">
<a href="#set-the-number-of-validation-sanity-steps" class="md-nav__link">
<a href="#set-the-number-of-validation-sanity-steps" title="Set the number of validation sanity steps" class="md-nav__link">
Set the number of validation sanity steps
</a>
@@ -545,35 +545,35 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#check-validation-every-n-epochs" class="md-nav__link">
<a href="#check-validation-every-n-epochs" title="Check validation every n epochs" class="md-nav__link">
Check validation every n epochs
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-validation-set-to-check" class="md-nav__link">
<a href="#set-how-much-of-the-validation-set-to-check" title="Set how much of the validation set to check" class="md-nav__link">
Set how much of the validation set to check
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-test-set-to-check" class="md-nav__link">
<a href="#set-how-much-of-the-test-set-to-check" title="Set how much of the test set to check" class="md-nav__link">
Set how much of the test set to check
</a>
</li>
<li class="md-nav__item">
<a href="#set-validation-check-frequency-within-1-training-epoch" class="md-nav__link">
<a href="#set-validation-check-frequency-within-1-training-epoch" title="Set validation check frequency within 1 training epoch" class="md-nav__link">
Set validation check frequency within 1 training epoch
</a>
</li>
<li class="md-nav__item">
<a href="#set-the-number-of-validation-sanity-steps" class="md-nav__link">
<a href="#set-the-number-of-validation-sanity-steps" title="Set the number of validation sanity steps" class="md-nav__link">
Set the number of validation sanity steps
</a>
@@ -647,16 +647,25 @@ Lightning will run 5 steps of validation in the beginning of training as a sanit
<hr />
<h4 id="set-validation-check-frequency-within-1-training-epoch">Set validation check frequency within 1 training epoch</h4>
<p>For large datasets it's often desirable to check validation multiple times within a training loop</p>
<p>For large datasets it's often desirable to check validation multiple times within a training loop.
Pass in a float to check that often within 1 training epoch.
Pass in an int k to check every k training batches. Must use an int if using
an IterableDataset.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
5
6
7
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">val_check_interval</span><span class="o">=</span><span class="mf">0.95</span><span class="p">)</span>
<span class="c1"># check every .25 of an epoch </span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">val_check_interval</span><span class="o">=</span><span class="mf">0.25</span><span class="p">)</span>
<span class="c1"># check every 100 train batches (ie: for IterableDatasets or fixed frequency)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">val_check_interval</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -739,7 +748,7 @@ Lightning will run 5 steps of validation in the beginning of training as a sanit
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/debugging/index.html
+16 -16
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -428,42 +428,42 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#fast-dev-run" class="md-nav__link">
<a href="#fast-dev-run" title="Fast dev run" class="md-nav__link">
Fast dev run
</a>
</li>
<li class="md-nav__item">
<a href="#inspect-gradient-norms" class="md-nav__link">
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
Inspect gradient norms
</a>
</li>
<li class="md-nav__item">
<a href="#make-model-overfit-on-subset-of-data" class="md-nav__link">
<a href="#make-model-overfit-on-subset-of-data" title="Make model overfit on subset of data" class="md-nav__link">
Make model overfit on subset of data
</a>
</li>
<li class="md-nav__item">
<a href="#print-the-parameter-count-by-layer" class="md-nav__link">
<a href="#print-the-parameter-count-by-layer" title="Print the parameter count by layer" class="md-nav__link">
Print the parameter count by layer
</a>
</li>
<li class="md-nav__item">
<a href="#print-which-gradients-are-nan" class="md-nav__link">
<a href="#print-which-gradients-are-nan" title="Print which gradients are nan" class="md-nav__link">
Print which gradients are nan
</a>
</li>
<li class="md-nav__item">
<a href="#log-gpu-usage" class="md-nav__link">
<a href="#log-gpu-usage" title="Log GPU usage" class="md-nav__link">
Log GPU usage
</a>
@@ -552,42 +552,42 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#fast-dev-run" class="md-nav__link">
<a href="#fast-dev-run" title="Fast dev run" class="md-nav__link">
Fast dev run
</a>
</li>
<li class="md-nav__item">
<a href="#inspect-gradient-norms" class="md-nav__link">
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
Inspect gradient norms
</a>
</li>
<li class="md-nav__item">
<a href="#make-model-overfit-on-subset-of-data" class="md-nav__link">
<a href="#make-model-overfit-on-subset-of-data" title="Make model overfit on subset of data" class="md-nav__link">
Make model overfit on subset of data
</a>
</li>
<li class="md-nav__item">
<a href="#print-the-parameter-count-by-layer" class="md-nav__link">
<a href="#print-the-parameter-count-by-layer" title="Print the parameter count by layer" class="md-nav__link">
Print the parameter count by layer
</a>
</li>
<li class="md-nav__item">
<a href="#print-which-gradients-are-nan" class="md-nav__link">
<a href="#print-which-gradients-are-nan" title="Print which gradients are nan" class="md-nav__link">
Print which gradients are nan
</a>
</li>
<li class="md-nav__item">
<a href="#log-gpu-usage" class="md-nav__link">
<a href="#log-gpu-usage" title="Log GPU usage" class="md-nav__link">
Log GPU usage
</a>
@@ -752,7 +752,7 @@ Use this to debug a full run of your program quickly</p>
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/hooks/index.html
+311 -22
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -444,68 +444,103 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#on_epoch_start" class="md-nav__link">
<a href="#on_epoch_start" title="on_epoch_start" class="md-nav__link">
on_epoch_start
</a>
</li>
<li class="md-nav__item">
<a href="#on_epoch_end" class="md-nav__link">
<a href="#on_epoch_end" title="on_epoch_end" class="md-nav__link">
on_epoch_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_batch_start" class="md-nav__link">
<a href="#on_batch_start" title="on_batch_start" class="md-nav__link">
on_batch_start
</a>
</li>
<li class="md-nav__item">
<a href="#on_batch_end" class="md-nav__link">
<a href="#on_batch_end" title="on_batch_end" class="md-nav__link">
on_batch_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_pre_performance_check" class="md-nav__link">
<a href="#on_pre_performance_check" title="on_pre_performance_check" class="md-nav__link">
on_pre_performance_check
</a>
</li>
<li class="md-nav__item">
<a href="#on_post_performance_check" class="md-nav__link">
<a href="#on_post_performance_check" title="on_post_performance_check" class="md-nav__link">
on_post_performance_check
</a>
</li>
<li class="md-nav__item">
<a href="#optimizer_step" class="md-nav__link">
<a href="#optimizer_step" title="optimizer_step" class="md-nav__link">
optimizer_step
</a>
</li>
<li class="md-nav__item">
<a href="#on_before_zero_grad" class="md-nav__link">
<a href="#on_before_zero_grad" title="on_before_zero_grad" class="md-nav__link">
on_before_zero_grad
</a>
</li>
<li class="md-nav__item">
<a href="#on_after_backward" class="md-nav__link">
<a href="#backward" title="backward" class="md-nav__link">
backward
</a>
</li>
<li class="md-nav__item">
<a href="#on_after_backward" title="on_after_backward" class="md-nav__link">
on_after_backward
</a>
</li>
<li class="md-nav__item">
<a href="#tbptt_split_batch" title="tbptt_split_batch" class="md-nav__link">
tbptt_split_batch
</a>
</li>
<li class="md-nav__item">
<a href="#configure_apex" title="configure_apex" class="md-nav__link">
configure_apex
</a>
</li>
<li class="md-nav__item">
<a href="#configure_ddp" title="configure_ddp" class="md-nav__link">
configure_ddp
</a>
</li>
<li class="md-nav__item">
<a href="#init_ddp_connection" title="init_ddp_connection" class="md-nav__link">
init_ddp_connection
</a>
</li>
@@ -579,68 +614,103 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#on_epoch_start" class="md-nav__link">
<a href="#on_epoch_start" title="on_epoch_start" class="md-nav__link">
on_epoch_start
</a>
</li>
<li class="md-nav__item">
<a href="#on_epoch_end" class="md-nav__link">
<a href="#on_epoch_end" title="on_epoch_end" class="md-nav__link">
on_epoch_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_batch_start" class="md-nav__link">
<a href="#on_batch_start" title="on_batch_start" class="md-nav__link">
on_batch_start
</a>
</li>
<li class="md-nav__item">
<a href="#on_batch_end" class="md-nav__link">
<a href="#on_batch_end" title="on_batch_end" class="md-nav__link">
on_batch_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_pre_performance_check" class="md-nav__link">
<a href="#on_pre_performance_check" title="on_pre_performance_check" class="md-nav__link">
on_pre_performance_check
</a>
</li>
<li class="md-nav__item">
<a href="#on_post_performance_check" class="md-nav__link">
<a href="#on_post_performance_check" title="on_post_performance_check" class="md-nav__link">
on_post_performance_check
</a>
</li>
<li class="md-nav__item">
<a href="#optimizer_step" class="md-nav__link">
<a href="#optimizer_step" title="optimizer_step" class="md-nav__link">
optimizer_step
</a>
</li>
<li class="md-nav__item">
<a href="#on_before_zero_grad" class="md-nav__link">
<a href="#on_before_zero_grad" title="on_before_zero_grad" class="md-nav__link">
on_before_zero_grad
</a>
</li>
<li class="md-nav__item">
<a href="#on_after_backward" class="md-nav__link">
<a href="#backward" title="backward" class="md-nav__link">
backward
</a>
</li>
<li class="md-nav__item">
<a href="#on_after_backward" title="on_after_backward" class="md-nav__link">
on_after_backward
</a>
</li>
<li class="md-nav__item">
<a href="#tbptt_split_batch" title="tbptt_split_batch" class="md-nav__link">
tbptt_split_batch
</a>
</li>
<li class="md-nav__item">
<a href="#configure_apex" title="configure_apex" class="md-nav__link">
configure_apex
</a>
</li>
<li class="md-nav__item">
<a href="#configure_ddp" title="configure_ddp" class="md-nav__link">
configure_ddp
</a>
</li>
<li class="md-nav__item">
<a href="#init_ddp_connection" title="init_ddp_connection" class="md-nav__link">
init_ddp_connection
</a>
</li>
@@ -807,6 +877,39 @@ Good place to inspect weight information with weights updated.</p>
</pre></div>
</td></tr></table>
<hr />
<h4 id="backward">backward</h4>
<p>Called to perform backward step.
Feel free to override as needed.</p>
<p>The loss passed in has already been scaled for accumulated gradients if requested.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11
12
13</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">backward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">use_amp</span><span class="p">,</span> <span class="n">loss</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Override backward with your own implementation if you need to</span>
<span class="sd"> :param use_amp: Whether amp was requested or not</span>
<span class="sd"> :param loss: Loss is already scaled by accumulated grads</span>
<span class="sd"> :param optimizer: Current optimizer being used</span>
<span class="sd"> :return:</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="k">if</span> <span class="n">use_amp</span><span class="p">:</span>
<span class="k">with</span> <span class="n">amp</span><span class="o">.</span><span class="n">scale_loss</span><span class="p">(</span><span class="n">loss</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">)</span> <span class="k">as</span> <span class="n">scaled_loss</span><span class="p">:</span>
<span class="n">scaled_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="k">else</span><span class="p">:</span>
<span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
</pre></div>
</td></tr></table>
<hr />
<h4 id="on_after_backward">on_after_backward</h4>
<p>Called in the training loop after model.backward()
@@ -828,6 +931,192 @@ This is the ideal place to inspect or log gradient information </p>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_histogram</span><span class="p">(</span><span class="n">tag</span><span class="o">=</span><span class="n">name</span><span class="p">,</span> <span class="n">values</span><span class="o">=</span><span class="n">grads</span><span class="p">,</span> <span class="n">global_step</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">global_step</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<hr />
<h4 id="tbptt_split_batch">tbptt_split_batch</h4>
<p>Called in the training loop after on_batch_start if <code>truncated_bptt_steps &gt; 0</code>. Each returned batch split is passed separately to training_step(...).</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">tbptt_split_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">split_size</span><span class="p">):</span>
<span class="n">splits</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">time_dims</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">split_size</span><span class="p">):</span>
<span class="n">batch_split</span> <span class="o">=</span> <span class="p">[]</span>
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">batch</span><span class="p">):</span>
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
<span class="n">split_x</span> <span class="o">=</span> <span class="n">x</span><span class="p">[:,</span> <span class="n">t</span><span class="p">:</span><span class="n">t</span> <span class="o">+</span> <span class="n">split_size</span><span class="p">]</span>
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">collections</span><span class="o">.</span><span class="n">Sequence</span><span class="p">):</span>
<span class="n">split_x</span> <span class="o">=</span> <span class="p">[</span><span class="bp">None</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
<span class="k">for</span> <span class="n">batch_idx</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)):</span>
<span class="n">split_x</span><span class="p">[</span><span class="n">batch_idx</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="p">[</span><span class="n">batch_idx</span><span class="p">][</span><span class="n">t</span><span class="p">:</span><span class="n">t</span> <span class="o">+</span> <span class="n">split_size</span><span class="p">]</span>
<span class="n">batch_split</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">split_x</span><span class="p">)</span>
<span class="n">splits</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">batch_split</span><span class="p">)</span>
<span class="k">return</span> <span class="n">splits</span>
</pre></div>
</td></tr></table>
<hr />
<h4 id="configure_apex">configure_apex</h4>
<p>Overwrite to define your own Apex implementation init.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">configure_apex</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">amp</span><span class="p">,</span> <span class="n">model</span><span class="p">,</span> <span class="n">optimizers</span><span class="p">,</span> <span class="n">amp_level</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Override to init AMP your own way</span>
<span class="sd"> Must return a model and list of optimizers</span>
<span class="sd"> :param amp:</span>
<span class="sd"> :param model:</span>
<span class="sd"> :param optimizers:</span>
<span class="sd"> :param amp_level:</span>
<span class="sd"> :return: Apex wrapped model and optimizers</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="n">model</span><span class="p">,</span> <span class="n">optimizers</span> <span class="o">=</span> <span class="n">amp</span><span class="o">.</span><span class="n">initialize</span><span class="p">(</span>
<span class="n">model</span><span class="p">,</span> <span class="n">optimizers</span><span class="p">,</span> <span class="n">opt_level</span><span class="o">=</span><span class="n">amp_level</span><span class="p">,</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">model</span><span class="p">,</span> <span class="n">optimizers</span>
</pre></div>
</td></tr></table>
<hr />
<h4 id="configure_ddp">configure_ddp</h4>
<p>Overwrite to define your own DDP implementation init.
The only requirement is that:
1. On a validation batch the call goes to model.validation_step. <br />
2. On a training batch the call goes to model.training_step. <br />
3. On a testing batch, the call goes to model.test_step</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">configure_ddp</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">model</span><span class="p">,</span> <span class="n">device_ids</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Override to init DDP in a different way or use your own wrapper.</span>
<span class="sd"> Must return model.</span>
<span class="sd"> :param model:</span>
<span class="sd"> :param device_ids:</span>
<span class="sd"> :return: DDP wrapped model</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># Lightning DDP simply routes to test_step, val_step, etc...</span>
<span class="n">model</span> <span class="o">=</span> <span class="n">LightningDistributedDataParallel</span><span class="p">(</span>
<span class="n">model</span><span class="p">,</span>
<span class="n">device_ids</span><span class="o">=</span><span class="n">device_ids</span><span class="p">,</span>
<span class="n">find_unused_parameters</span><span class="o">=</span><span class="bp">True</span>
<span class="p">)</span>
<span class="k">return</span> <span class="n">model</span>
</pre></div>
</td></tr></table>
<hr />
<h4 id="init_ddp_connection">init_ddp_connection</h4>
<p>Override to init DDP in your own way. </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">init_ddp_connection</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Connect all procs in the world using the env:// init</span>
<span class="sd"> Use the first node as the root address</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># use slurm job id for the port number</span>
<span class="c1"># guarantees unique ports across jobs from same grid search</span>
<span class="k">try</span><span class="p">:</span>
<span class="c1"># use the last 4 numbers in the job id as the id</span>
<span class="n">default_port</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">&#39;SLURM_JOB_ID&#39;</span><span class="p">]</span>
<span class="n">default_port</span> <span class="o">=</span> <span class="n">default_port</span><span class="p">[</span><span class="o">-</span><span class="mi">4</span><span class="p">:]</span>
<span class="c1"># all ports should be in the 10k+ range</span>
<span class="n">default_port</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">default_port</span><span class="p">)</span> <span class="o">+</span> <span class="mi">15000</span>
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<span class="n">default_port</span> <span class="o">=</span> <span class="mi">12910</span>
<span class="c1"># if user gave a port number, use that one instead</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">default_port</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">&#39;MASTER_PORT&#39;</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">&#39;MASTER_PORT&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">default_port</span><span class="p">)</span>
<span class="c1"># figure out the root node addr</span>
<span class="k">try</span><span class="p">:</span>
<span class="n">root_node</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">&#39;SLURM_NODELIST&#39;</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">&#39; &#39;</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<span class="n">root_node</span> <span class="o">=</span> <span class="s1">&#39;127.0.0.2&#39;</span>
<span class="n">root_node</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">resolve_root_node_address</span><span class="p">(</span><span class="n">root_node</span><span class="p">)</span>
<span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">&#39;MASTER_ADDR&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">root_node</span>
<span class="n">dist</span><span class="o">.</span><span class="n">init_process_group</span><span class="p">(</span><span class="s1">&#39;nccl&#39;</span><span class="p">,</span> <span class="n">rank</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">proc_rank</span><span class="p">,</span> <span class="n">world_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">world_size</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -897,7 +1186,7 @@ This is the ideal place to inspect or log gradient information </p>
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/index.html
+7 -4
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -546,6 +546,7 @@
</ul>
<p><strong>Distributed training</strong> </p>
<ul>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#init_ddp_connection">Implement Your Own Distributed (DDP) training</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision">16-bit mixed precision</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU">Multi-GPU</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node">Multi-node</a></li>
@@ -574,6 +575,8 @@
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers">Use multiple optimizers (like GANs)</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check">Set how much of the training set to check (1-100%)</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step">Step optimizers at arbitrary intervals</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#packed-sequences-as-inputs">Packed sequences</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning//Training%20Loop/#truncated-back-propation-through-time">Truncated Back Propagation Through Time</a></li>
</ul>
<p><strong>Validation loop</strong> </p>
<ul>
@@ -657,7 +660,7 @@
</div>
<script src="../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:".."}})</script>
assets/javascripts/application.245445c6.js
+6
View File
File diff suppressed because one or more lines are too long
assets/javascripts/application.ac79c3b0.js
-60
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.ar.js
-20
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.da.js
-16
View File
@@ -1,17 +1 @@
/*!
* Lunr languages, `Danish` language
* https://github.com/MihaiValentin/lunr-languages
*
* Copyright 2014, Mihai Valentin
* http://www.mozilla.org/MPL/
*/
/*!
* based on
* Snowball JavaScript Library v0.3
* http://code.google.com/p/urim/
* http://snowball.tartarus.org/
*
* Copyright 2010, Oleg Mazko
* http://www.mozilla.org/MPL/
*/
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r,m,i;e.da=function(){this.pipeline.reset(),this.pipeline.add(e.da.trimmer,e.da.stopWordFilter,e.da.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.da.stemmer))},e.da.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stＡ-Ｚａ-ｚ",e.da.trimmer=e.trimmerSupport.generateTrimmer(e.da.wordCharacters),e.Pipeline.registerFunction(e.da.trimmer,"trimmer-da"),e.da.stemmer=(r=e.stemmerSupport.Among,m=e.stemmerSupport.SnowballProgram,i=new function(){var i,t,n,s=[new r("hed",-1,1),new r("ethed",0,1),new r("ered",-1,1),new r("e",-1,1),new r("erede",3,1),new r("ende",3,1),new r("erende",5,1),new r("ene",3,1),new r("erne",3,1),new r("ere",3,1),new r("en",-1,1),new r("heden",10,1),new r("eren",10,1),new r("er",-1,1),new r("heder",13,1),new r("erer",13,1),new r("s",-1,2),new r("heds",16,1),new r("es",16,1),new r("endes",18,1),new r("erendes",19,1),new r("enes",18,1),new r("ernes",18,1),new r("eres",18,1),new r("ens",16,1),new r("hedens",24,1),new r("erens",24,1),new r("ers",16,1),new r("ets",16,1),new r("erets",28,1),new r("et",-1,1),new r("eret",30,1)],o=[new r("gd",-1,-1),new r("dt",-1,-1),new r("gt",-1,-1),new r("kt",-1,-1)],a=[new r("ig",-1,1),new r("lig",0,1),new r("elig",1,1),new r("els",-1,1),new r("løst",-1,2)],d=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,48,0,128],u=[239,254,42,3,0,0,0,0,0,0,0,0,0,0,0,0,16],c=new m;function l(){var e,r=c.limit-c.cursor;c.cursor>=t&&(e=c.limit_backward,c.limit_backward=t,c.ket=c.cursor,c.find_among_b(o,4)?(c.bra=c.cursor,c.limit_backward=e,c.cursor=c.limit-r,c.cursor>c.limit_backward&&(c.cursor--,c.bra=c.cursor,c.slice_del())):c.limit_backward=e)}this.setCurrent=function(e){c.setCurrent(e)},this.getCurrent=function(){return c.getCurrent()},this.stem=function(){var e,r=c.cursor;return function(){var e,r=c.cursor+3;if(t=c.limit,0<=r&&r<=c.limit){for(i=r;;){if(e=c.cursor,c.in_grouping(d,97,248)){c.cursor=e;break}if((c.cursor=e)>=c.limit)return;c.cursor++}for(;!c.out_grouping(d,97,248);){if(c.cursor>=c.limit)return;c.cursor++}(t=c.cursor)<i&&(t=i)}}(),c.limit_backward=r,c.cursor=c.limit,function(){var e,r;if(c.cursor>=t&&(r=c.limit_backward,c.limit_backward=t,c.ket=c.cursor,e=c.find_among_b(s,32),c.limit_backward=r,e))switch(c.bra=c.cursor,e){case 1:c.slice_del();break;case 2:c.in_grouping_b(u,97,229)&&c.slice_del()}}(),c.cursor=c.limit,l(),c.cursor=c.limit,function(){var e,r,i,n=c.limit-c.cursor;if(c.ket=c.cursor,c.eq_s_b(2,"st")&&(c.bra=c.cursor,c.eq_s_b(2,"ig")&&c.slice_del()),c.cursor=c.limit-n,c.cursor>=t&&(r=c.limit_backward,c.limit_backward=t,c.ket=c.cursor,e=c.find_among_b(a,5),c.limit_backward=r,e))switch(c.bra=c.cursor,e){case 1:c.slice_del(),i=c.limit-c.cursor,l(),c.cursor=c.limit-i;break;case 2:c.slice_from("løs")}}(),c.cursor=c.limit,c.cursor>=t&&(e=c.limit_backward,c.limit_backward=t,c.ket=c.cursor,c.out_grouping_b(d,97,248)?(c.bra=c.cursor,n=c.slice_to(n),c.limit_backward=e,c.eq_v_b(n)&&c.slice_del()):c.limit_backward=e),!0}},function(e){return"function"==typeof e.update?e.update(function(e){return i.setCurrent(e),i.stem(),i.getCurrent()}):(i.setCurrent(e),i.stem(),i.getCurrent())}),e.Pipeline.registerFunction(e.da.stemmer,"stemmer-da"),e.da.stopWordFilter=e.generateStopWordFilter("ad af alle alt anden at blev blive bliver da de dem den denne der deres det dette dig din disse dog du efter eller en end er et for fra ham han hans har havde have hende hendes her hos hun hvad hvis hvor i ikke ind jeg jer jo kunne man mange med meget men mig min mine mit mod ned noget nogle nu når og også om op os over på selv sig sin sine sit skal skulle som sådan thi til ud under var vi vil ville vor være været".split(" ")),e.Pipeline.registerFunction(e.da.stopWordFilter,"stopWordFilter-da")}});
assets/javascripts/lunr/lunr.de.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.du.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.es.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.fi.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.fr.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.hu.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.it.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.ja.js
-16
View File
@@ -1,17 +1 @@
/*!
* Lunr languages, `Japanese` language
* https://github.com/MihaiValentin/lunr-languages
*
* Copyright 2014, Chad Liu
* http://www.mozilla.org/MPL/
*/
/*!
* based on
* Snowball JavaScript Library v0.3
* http://code.google.com/p/urim/
* http://snowball.tartarus.org/
*
* Copyright 2010, Oleg Mazko
* http://www.mozilla.org/MPL/
*/
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(m){if(void 0===m)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===m.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var l="2"==m.version[0];m.ja=function(){this.pipeline.reset(),this.pipeline.add(m.ja.trimmer,m.ja.stopWordFilter,m.ja.stemmer),l?this.tokenizer=m.ja.tokenizer:(m.tokenizer&&(m.tokenizer=m.ja.tokenizer),this.tokenizerFn&&(this.tokenizerFn=m.ja.tokenizer))};var j=new m.TinySegmenter;m.ja.tokenizer=function(e){var r,t,i,n,o,s,p,a,u;if(!arguments.length||null==e||null==e)return[];if(Array.isArray(e))return e.map(function(e){return l?new m.Token(e.toLowerCase()):e.toLowerCase()});for(r=(t=e.toString().toLowerCase().replace(/^\s+/,"")).length-1;0<=r;r--)if(/\S/.test(t.charAt(r))){t=t.substring(0,r+1);break}for(o=[],i=t.length,p=a=0;a<=i;a++)if(s=a-p,t.charAt(a).match(/\s/)||a==i){if(0<s)for(n=j.segment(t.slice(p,a)).filter(function(e){return!!e}),u=p,r=0;r<n.length;r++)l?o.push(new m.Token(n[r],{position:[u,n[r].length],index:o.length})):o.push(n[r]),u+=n[r].length;p=a+1}return o},m.ja.stemmer=function(e){return e},m.Pipeline.registerFunction(m.ja.stemmer,"stemmer-ja"),m.ja.wordCharacters="一二三四五六七八九十百千万億兆一-龠々〆ヵヶぁ-んァ-ヴーｱ-ﾝﾞa-zA-Zａ-ｚＡ-Ｚ0-9０-９",m.ja.trimmer=m.trimmerSupport.generateTrimmer(m.ja.wordCharacters),m.Pipeline.registerFunction(m.ja.trimmer,"trimmer-ja"),m.ja.stopWordFilter=m.generateStopWordFilter("これ それ あれ この その あの ここ そこ あそこ こちら どこ だれ なに なん 何 私 貴方 貴方方 我々 私達 あの人 あのかた 彼女 彼 です あります おります います は が の に を で え から まで より も どの と し それで しかし".split(" ")),m.Pipeline.registerFunction(m.ja.stopWordFilter,"stopWordFilter-ja"),m.jp=m.ja,m.Pipeline.registerFunction(m.jp.stemmer,"stemmer-jp"),m.Pipeline.registerFunction(m.jp.trimmer,"trimmer-jp"),m.Pipeline.registerFunction(m.jp.stopWordFilter,"stopWordFilter-jp")}});
assets/javascripts/lunr/lunr.multi.js
+1 -1
View File
@@ -1 +1 @@
!function(e,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(e.lunr)}(this,function(){return function(o){o.multiLanguage=function(){for(var e=Array.prototype.slice.call(arguments),t=e.join("-"),i="",r=[],n=[],s=0;s<e.length;++s)"en"==e[s]?(i+="\\w",r.unshift(o.stopWordFilter),r.push(o.stemmer),n.push(o.stemmer)):(i+=o[e[s]].wordCharacters,o[e[s]].stopWordFilter&&r.unshift(o[e[s]].stopWordFilter),o[e[s]].stemmer&&(r.push(o[e[s]].stemmer),n.push(o[e[s]].stemmer)));var p=o.trimmerSupport.generateTrimmer(i);return o.Pipeline.registerFunction(p,"lunr-multi-trimmer-"+t),r.unshift(p),function(){this.pipeline.reset(),this.pipeline.add.apply(this.pipeline,r),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add.apply(this.searchPipeline,n))}}}});
!function(e,i){"function"==typeof define&&define.amd?define(i):"object"==typeof exports?module.exports=i():i()(e.lunr)}(this,function(){return function(o){o.multiLanguage=function(){for(var e=Array.prototype.slice.call(arguments),i=e.join("-"),t="",r=[],n=[],s=0;s<e.length;++s)"en"==e[s]?(t+="\\w",r.unshift(o.stopWordFilter),r.push(o.stemmer),n.push(o.stemmer)):(t+=o[e[s]].wordCharacters,r.unshift(o[e[s]].stopWordFilter),r.push(o[e[s]].stemmer),n.push(o[e[s]].stemmer));var p=o.trimmerSupport.generateTrimmer(t);return o.Pipeline.registerFunction(p,"lunr-multi-trimmer-"+i),r.unshift(p),function(){this.pipeline.reset(),this.pipeline.add.apply(this.pipeline,r),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add.apply(this.searchPipeline,n))}}}});
assets/javascripts/lunr/lunr.nl.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.no.js
-16
View File
@@ -1,17 +1 @@
/*!
* Lunr languages, `Norwegian` language
* https://github.com/MihaiValentin/lunr-languages
*
* Copyright 2014, Mihai Valentin
* http://www.mozilla.org/MPL/
*/
/*!
* based on
* Snowball JavaScript Library v0.3
* http://code.google.com/p/urim/
* http://snowball.tartarus.org/
*
* Copyright 2010, Oleg Mazko
* http://www.mozilla.org/MPL/
*/
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r,n,i;e.no=function(){this.pipeline.reset(),this.pipeline.add(e.no.trimmer,e.no.stopWordFilter,e.no.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.no.stemmer))},e.no.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stＡ-Ｚａ-ｚ",e.no.trimmer=e.trimmerSupport.generateTrimmer(e.no.wordCharacters),e.Pipeline.registerFunction(e.no.trimmer,"trimmer-no"),e.no.stemmer=(r=e.stemmerSupport.Among,n=e.stemmerSupport.SnowballProgram,i=new function(){var o,s,a=[new r("a",-1,1),new r("e",-1,1),new r("ede",1,1),new r("ande",1,1),new r("ende",1,1),new r("ane",1,1),new r("ene",1,1),new r("hetene",6,1),new r("erte",1,3),new r("en",-1,1),new r("heten",9,1),new r("ar",-1,1),new r("er",-1,1),new r("heter",12,1),new r("s",-1,2),new r("as",14,1),new r("es",14,1),new r("edes",16,1),new r("endes",16,1),new r("enes",16,1),new r("hetenes",19,1),new r("ens",14,1),new r("hetens",21,1),new r("ers",14,1),new r("ets",14,1),new r("et",-1,1),new r("het",25,1),new r("ert",-1,3),new r("ast",-1,1)],m=[new r("dt",-1,-1),new r("vt",-1,-1)],l=[new r("leg",-1,1),new r("eleg",0,1),new r("ig",-1,1),new r("eig",2,1),new r("lig",2,1),new r("elig",4,1),new r("els",-1,1),new r("lov",-1,1),new r("elov",7,1),new r("slov",7,1),new r("hetslov",9,1)],u=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,48,0,128],d=[119,125,149,1],c=new n;this.setCurrent=function(e){c.setCurrent(e)},this.getCurrent=function(){return c.getCurrent()},this.stem=function(){var e,r,n,i,t=c.cursor;return function(){var e,r=c.cursor+3;if(s=c.limit,0<=r||r<=c.limit){for(o=r;;){if(e=c.cursor,c.in_grouping(u,97,248)){c.cursor=e;break}if(e>=c.limit)return;c.cursor=e+1}for(;!c.out_grouping(u,97,248);){if(c.cursor>=c.limit)return;c.cursor++}(s=c.cursor)<o&&(s=o)}}(),c.limit_backward=t,c.cursor=c.limit,function(){var e,r,n;if(c.cursor>=s&&(r=c.limit_backward,c.limit_backward=s,c.ket=c.cursor,e=c.find_among_b(a,29),c.limit_backward=r,e))switch(c.bra=c.cursor,e){case 1:c.slice_del();break;case 2:n=c.limit-c.cursor,c.in_grouping_b(d,98,122)?c.slice_del():(c.cursor=c.limit-n,c.eq_s_b(1,"k")&&c.out_grouping_b(u,97,248)&&c.slice_del());break;case 3:c.slice_from("er")}}(),c.cursor=c.limit,r=c.limit-c.cursor,c.cursor>=s&&(e=c.limit_backward,c.limit_backward=s,c.ket=c.cursor,c.find_among_b(m,2)?(c.bra=c.cursor,c.limit_backward=e,c.cursor=c.limit-r,c.cursor>c.limit_backward&&(c.cursor--,c.bra=c.cursor,c.slice_del())):c.limit_backward=e),c.cursor=c.limit,c.cursor>=s&&(i=c.limit_backward,c.limit_backward=s,c.ket=c.cursor,(n=c.find_among_b(l,11))?(c.bra=c.cursor,c.limit_backward=i,1==n&&c.slice_del()):c.limit_backward=i),!0}},function(e){return"function"==typeof e.update?e.update(function(e){return i.setCurrent(e),i.stem(),i.getCurrent()}):(i.setCurrent(e),i.stem(),i.getCurrent())}),e.Pipeline.registerFunction(e.no.stemmer,"stemmer-no"),e.no.stopWordFilter=e.generateStopWordFilter("alle at av bare begge ble blei bli blir blitt både båe da de deg dei deim deira deires dem den denne der dere deres det dette di din disse ditt du dykk dykkar då eg ein eit eitt eller elles en enn er et ett etter for fordi fra før ha hadde han hans har hennar henne hennes her hjå ho hoe honom hoss hossen hun hva hvem hver hvilke hvilken hvis hvor hvordan hvorfor i ikke ikkje ikkje ingen ingi inkje inn inni ja jeg kan kom korleis korso kun kunne kva kvar kvarhelst kven kvi kvifor man mange me med medan meg meget mellom men mi min mine mitt mot mykje ned no noe noen noka noko nokon nokor nokre nå når og også om opp oss over på samme seg selv si si sia sidan siden sin sine sitt sjøl skal skulle slik so som som somme somt så sånn til um upp ut uten var vart varte ved vere verte vi vil ville vore vors vort vår være være vært å".split(" ")),e.Pipeline.registerFunction(e.no.stopWordFilter,"stopWordFilter-no")}});
assets/javascripts/lunr/lunr.pt.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.ro.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.ru.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.stemmer.support.js
-8
View File
@@ -1,9 +1 @@
/*!
* Snowball JavaScript Library v0.3
* http://code.google.com/p/urim/
* http://snowball.tartarus.org/
*
* Copyright 2010, Oleg Mazko
* http://www.mozilla.org/MPL/
*/
!function(r,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(r.lunr)}(this,function(){return function(r){r.stemmerSupport={Among:function(r,t,i,s){if(this.toCharArray=function(r){for(var t=r.length,i=new Array(t),s=0;s<t;s++)i[s]=r.charCodeAt(s);return i},!r&&""!=r||!t&&0!=t||!i)throw"Bad Among initialisation: s:"+r+", substring_i: "+t+", result: "+i;this.s_size=r.length,this.s=this.toCharArray(r),this.substring_i=t,this.result=i,this.method=s},SnowballProgram:function(){var b;return{bra:0,ket:0,limit:0,cursor:0,limit_backward:0,setCurrent:function(r){b=r,this.cursor=0,this.limit=r.length,this.limit_backward=0,this.bra=this.cursor,this.ket=this.limit},getCurrent:function(){var r=b;return b=null,r},in_grouping:function(r,t,i){if(this.cursor<this.limit){var s=b.charCodeAt(this.cursor);if(s<=i&&t<=s&&r[(s-=t)>>3]&1<<(7&s))return this.cursor++,!0}return!1},in_grouping_b:function(r,t,i){if(this.cursor>this.limit_backward){var s=b.charCodeAt(this.cursor-1);if(s<=i&&t<=s&&r[(s-=t)>>3]&1<<(7&s))return this.cursor--,!0}return!1},out_grouping:function(r,t,i){if(this.cursor<this.limit){var s=b.charCodeAt(this.cursor);if(i<s||s<t)return this.cursor++,!0;if(!(r[(s-=t)>>3]&1<<(7&s)))return this.cursor++,!0}return!1},out_grouping_b:function(r,t,i){if(this.cursor>this.limit_backward){var s=b.charCodeAt(this.cursor-1);if(i<s||s<t)return this.cursor--,!0;if(!(r[(s-=t)>>3]&1<<(7&s)))return this.cursor--,!0}return!1},eq_s:function(r,t){if(this.limit-this.cursor<r)return!1;for(var i=0;i<r;i++)if(b.charCodeAt(this.cursor+i)!=t.charCodeAt(i))return!1;return this.cursor+=r,!0},eq_s_b:function(r,t){if(this.cursor-this.limit_backward<r)return!1;for(var i=0;i<r;i++)if(b.charCodeAt(this.cursor-r+i)!=t.charCodeAt(i))return!1;return this.cursor-=r,!0},find_among:function(r,t){for(var i=0,s=t,e=this.cursor,n=this.limit,u=0,o=0,h=!1;;){for(var c=i+(s-i>>1),a=0,f=u<o?u:o,l=r[c],_=f;_<l.s_size;_++){if(e+f==n){a=-1;break}if(a=b.charCodeAt(e+f)-l.s[_])break;f++}if(a<0?(s=c,o=f):(i=c,u=f),s-i<=1){if(0<i||s==i||h)break;h=!0}}for(;;){if(u>=(l=r[i]).s_size){if(this.cursor=e+l.s_size,!l.method)return l.result;var m=l.method();if(this.cursor=e+l.s_size,m)return l.result}if((i=l.substring_i)<0)return 0}},find_among_b:function(r,t){for(var i=0,s=t,e=this.cursor,n=this.limit_backward,u=0,o=0,h=!1;;){for(var c=i+(s-i>>1),a=0,f=u<o?u:o,l=(_=r[c]).s_size-1-f;0<=l;l--){if(e-f==n){a=-1;break}if(a=b.charCodeAt(e-1-f)-_.s[l])break;f++}if(a<0?(s=c,o=f):(i=c,u=f),s-i<=1){if(0<i||s==i||h)break;h=!0}}for(;;){var _;if(u>=(_=r[i]).s_size){if(this.cursor=e-_.s_size,!_.method)return _.result;var m=_.method();if(this.cursor=e-_.s_size,m)return _.result}if((i=_.substring_i)<0)return 0}},replace_s:function(r,t,i){var s=i.length-(t-r);return b=b.substring(0,r)+i+b.substring(t),this.limit+=s,this.cursor>=t?this.cursor+=s:this.cursor>r&&(this.cursor=r),s},slice_check:function(){if(this.bra<0||this.bra>this.ket||this.ket>this.limit||this.limit>b.length)throw"faulty slice operation"},slice_from:function(r){this.slice_check(),this.replace_s(this.bra,this.ket,r)},slice_del:function(){this.slice_from("")},insert:function(r,t,i){var s=this.replace_s(r,t,i);r<=this.bra&&(this.bra+=s),r<=this.ket&&(this.ket+=s)},slice_to:function(){return this.slice_check(),b.substring(this.bra,this.ket)},eq_v_b:function(r){return this.eq_s_b(r.length,r)}}}},r.trimmerSupport={generateTrimmer:function(r){var t=new RegExp("^[^"+r+"]+"),i=new RegExp("[^"+r+"]+$");return function(r){return"function"==typeof r.update?r.update(function(r){return r.replace(t,"").replace(i,"")}):r.replace(t,"").replace(i,"")}}}}});
assets/javascripts/lunr/lunr.sv.js
-16
View File
@@ -1,17 +1 @@
/*!
* Lunr languages, `Swedish` language
* https://github.com/MihaiValentin/lunr-languages
*
* Copyright 2014, Mihai Valentin
* http://www.mozilla.org/MPL/
*/
/*!
* based on
* Snowball JavaScript Library v0.3
* http://code.google.com/p/urim/
* http://snowball.tartarus.org/
*
* Copyright 2010, Oleg Mazko
* http://www.mozilla.org/MPL/
*/
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r,l,n;e.sv=function(){this.pipeline.reset(),this.pipeline.add(e.sv.trimmer,e.sv.stopWordFilter,e.sv.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.sv.stemmer))},e.sv.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stＡ-Ｚａ-ｚ",e.sv.trimmer=e.trimmerSupport.generateTrimmer(e.sv.wordCharacters),e.Pipeline.registerFunction(e.sv.trimmer,"trimmer-sv"),e.sv.stemmer=(r=e.stemmerSupport.Among,l=e.stemmerSupport.SnowballProgram,n=new function(){var n,t,i=[new r("a",-1,1),new r("arna",0,1),new r("erna",0,1),new r("heterna",2,1),new r("orna",0,1),new r("ad",-1,1),new r("e",-1,1),new r("ade",6,1),new r("ande",6,1),new r("arne",6,1),new r("are",6,1),new r("aste",6,1),new r("en",-1,1),new r("anden",12,1),new r("aren",12,1),new r("heten",12,1),new r("ern",-1,1),new r("ar",-1,1),new r("er",-1,1),new r("heter",18,1),new r("or",-1,1),new r("s",-1,2),new r("as",21,1),new r("arnas",22,1),new r("ernas",22,1),new r("ornas",22,1),new r("es",21,1),new r("ades",26,1),new r("andes",26,1),new r("ens",21,1),new r("arens",29,1),new r("hetens",29,1),new r("erns",21,1),new r("at",-1,1),new r("andet",-1,1),new r("het",-1,1),new r("ast",-1,1)],s=[new r("dd",-1,-1),new r("gd",-1,-1),new r("nn",-1,-1),new r("dt",-1,-1),new r("gt",-1,-1),new r("kt",-1,-1),new r("tt",-1,-1)],a=[new r("ig",-1,1),new r("lig",0,1),new r("els",-1,1),new r("fullt",-1,3),new r("löst",-1,2)],o=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,24,0,32],u=[119,127,149],m=new l;this.setCurrent=function(e){m.setCurrent(e)},this.getCurrent=function(){return m.getCurrent()},this.stem=function(){var e,r=m.cursor;return function(){var e,r=m.cursor+3;if(t=m.limit,0<=r||r<=m.limit){for(n=r;;){if(e=m.cursor,m.in_grouping(o,97,246)){m.cursor=e;break}if(m.cursor=e,m.cursor>=m.limit)return;m.cursor++}for(;!m.out_grouping(o,97,246);){if(m.cursor>=m.limit)return;m.cursor++}(t=m.cursor)<n&&(t=n)}}(),m.limit_backward=r,m.cursor=m.limit,function(){var e,r=m.limit_backward;if(m.cursor>=t&&(m.limit_backward=t,m.cursor=m.limit,m.ket=m.cursor,e=m.find_among_b(i,37),m.limit_backward=r,e))switch(m.bra=m.cursor,e){case 1:m.slice_del();break;case 2:m.in_grouping_b(u,98,121)&&m.slice_del()}}(),m.cursor=m.limit,e=m.limit_backward,m.cursor>=t&&(m.limit_backward=t,m.cursor=m.limit,m.find_among_b(s,7)&&(m.cursor=m.limit,m.ket=m.cursor,m.cursor>m.limit_backward&&(m.bra=--m.cursor,m.slice_del())),m.limit_backward=e),m.cursor=m.limit,function(){var e,r;if(m.cursor>=t){if(r=m.limit_backward,m.limit_backward=t,m.cursor=m.limit,m.ket=m.cursor,e=m.find_among_b(a,5))switch(m.bra=m.cursor,e){case 1:m.slice_del();break;case 2:m.slice_from("lös");break;case 3:m.slice_from("full")}m.limit_backward=r}}(),!0}},function(e){return"function"==typeof e.update?e.update(function(e){return n.setCurrent(e),n.stem(),n.getCurrent()}):(n.setCurrent(e),n.stem(),n.getCurrent())}),e.Pipeline.registerFunction(e.sv.stemmer,"stemmer-sv"),e.sv.stopWordFilter=e.generateStopWordFilter("alla allt att av blev bli blir blivit de dem den denna deras dess dessa det detta dig din dina ditt du där då efter ej eller en er era ert ett från för ha hade han hans har henne hennes hon honom hur här i icke ingen inom inte jag ju kan kunde man med mellan men mig min mina mitt mot mycket ni nu när någon något några och om oss på samma sedan sig sin sina sitta själv skulle som så sådan sådana sådant till under upp ut utan vad var vara varför varit varje vars vart vem vi vid vilka vilkas vilken vilket vår våra vårt än är åt över".split(" ")),e.Pipeline.registerFunction(e.sv.stopWordFilter,"stopWordFilter-sv")}});
assets/javascripts/lunr/lunr.th.js
-16
View File
@@ -1,17 +1 @@
/*!
* Lunr languages, `Thai` language
* https://github.com/MihaiValentin/lunr-languages
*
* Copyright 2017, Keerati Thiwanruk
* http://www.mozilla.org/MPL/
*/
/*!
* based on
* Snowball JavaScript Library v0.3
* http://code.google.com/p/urim/
* http://snowball.tartarus.org/
*
* Copyright 2010, Oleg Mazko
* http://www.mozilla.org/MPL/
*/
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(t){if(void 0===t)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===t.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var i="2"==t.version[0];t.th=function(){this.pipeline.reset(),this.pipeline.add(t.th.trimmer),i?this.tokenizer=t.th.tokenizer:(t.tokenizer&&(t.tokenizer=t.th.tokenizer),this.tokenizerFn&&(this.tokenizerFn=t.th.tokenizer))},t.th.wordCharacters="[฀-๿]",t.th.trimmer=t.trimmerSupport.generateTrimmer(t.th.wordCharacters),t.Pipeline.registerFunction(t.th.trimmer,"trimmer-th");var n=t.wordcut;n.init(),t.th.tokenizer=function(e){if(!arguments.length||null==e||null==e)return[];if(Array.isArray(e))return e.map(function(e){return i?new t.Token(e):e});var r=e.toString().replace(/^\s+/,"");return n.cut(r).split("|")}}});
assets/javascripts/lunr/lunr.tr.js
-16
View File
File diff suppressed because one or more lines are too long
assets/javascripts/lunr/lunr.vi.js
-17
View File
@@ -1,17 +0,0 @@
/*!
* Lunr languages, `Vietnamese` language
* https://github.com/MihaiValentin/lunr-languages
*
* Copyright 2017, Keerati Thiwanruk
* http://www.mozilla.org/MPL/
*/
/*!
* based on
* Snowball JavaScript Library v0.3
* http://code.google.com/p/urim/
* http://snowball.tartarus.org/
*
* Copyright 2010, Oleg Mazko
* http://www.mozilla.org/MPL/
*/
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");e.vi=function(){this.pipeline.reset(),this.pipeline.add(e.vi.stopWordFilter,e.vi.trimmer)},e.vi.wordCharacters="[A-Za-ẓ̀͐́͑̉̃̓ÂâÊêÔôĂ-ăĐ-đƠ-ơƯ-ư]",e.vi.trimmer=e.trimmerSupport.generateTrimmer(e.vi.wordCharacters),e.Pipeline.registerFunction(e.vi.trimmer,"trimmer-vi"),e.vi.stopWordFilter=e.generateStopWordFilter("là cái nhưng mà".split(" "))}});
assets/javascripts/lunr/tinyseg.js
+1 -1
View File
File diff suppressed because one or more lines are too long
assets/stylesheets/application-palette.a8b3c06d.css → assets/stylesheets/application-palette.01803549.css
+1 -1
View File
File diff suppressed because one or more lines are too long
assets/stylesheets/application.0284f74d.css
+1
View File
File diff suppressed because one or more lines are too long
assets/stylesheets/application.30686662.css
-1
View File
File diff suppressed because one or more lines are too long
examples/Examples/index.html
+17 -17
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -476,14 +476,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#template-model-definition" class="md-nav__link">
<a href="#template-model-definition" title="Template model definition" class="md-nav__link">
Template model definition
</a>
</li>
<li class="md-nav__item">
<a href="#trainer-example" class="md-nav__link">
<a href="#trainer-example" title="Trainer Example" class="md-nav__link">
Trainer Example
</a>
@@ -491,21 +491,21 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#cpu-hyperparameter-search" class="md-nav__link">
<a href="#cpu-hyperparameter-search" title="CPU hyperparameter search" class="md-nav__link">
CPU hyperparameter search
</a>
</li>
<li class="md-nav__item">
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" class="md-nav__link">
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" title="Hyperparameter search on a single or multiple GPUs" class="md-nav__link">
Hyperparameter search on a single or multiple GPUs
</a>
</li>
<li class="md-nav__item">
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" class="md-nav__link">
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" title="Hyperparameter search on a SLURM HPC cluster" class="md-nav__link">
Hyperparameter search on a SLURM HPC cluster
</a>
@@ -551,14 +551,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#template-model-definition" class="md-nav__link">
<a href="#template-model-definition" title="Template model definition" class="md-nav__link">
Template model definition
</a>
</li>
<li class="md-nav__item">
<a href="#trainer-example" class="md-nav__link">
<a href="#trainer-example" title="Trainer Example" class="md-nav__link">
Trainer Example
</a>
@@ -566,21 +566,21 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#cpu-hyperparameter-search" class="md-nav__link">
<a href="#cpu-hyperparameter-search" title="CPU hyperparameter search" class="md-nav__link">
CPU hyperparameter search
</a>
</li>
<li class="md-nav__item">
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" class="md-nav__link">
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" title="Hyperparameter search on a single or multiple GPUs" class="md-nav__link">
Hyperparameter search on a single or multiple GPUs
</a>
</li>
<li class="md-nav__item">
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" class="md-nav__link">
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" title="Hyperparameter search on a SLURM HPC cluster" class="md-nav__link">
Hyperparameter search on a SLURM HPC cluster
</a>
@@ -613,10 +613,10 @@
<h1>Examples</h1>
<h3 id="template-model-definition">Template model definition</h3>
<p>In 99% of cases you want to just copy <a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/examples">one of the examples</a> to start a new lightningModule and change the core of what your model is actually trying to do.</p>
<p>In 99% of cases you want to just copy <a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples">one of the examples</a> to start a new lightningModule and change the core of what your model is actually trying to do.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># get a copy of the module template</span>
wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/examples/new_project_templates/lightning_module_template.py
wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/pl_examples/new_project_templates/lightning_module_template.py
</pre></div>
</td></tr></table>
@@ -796,7 +796,7 @@ argument parser you get the default arguments in the argument parser.</p>
<span class="n">job_display_name</span> <span class="o">=</span> <span class="n">job_display_name</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="mi">3</span><span class="p">]</span>
<span class="c1"># run hopt</span>
<span class="k">print</span><span class="p">(</span><span class="s1">&#39;submitting jobs...&#39;</span><span class="p">)</span>
<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;submitting jobs...&#39;</span><span class="p">)</span>
<span class="n">cluster</span><span class="o">.</span><span class="n">optimize_parallel_cluster_gpu</span><span class="p">(</span>
<span class="n">main</span><span class="p">,</span>
<span class="n">nb_trials</span><span class="o">=</span><span class="n">hyperparams</span><span class="o">.</span><span class="n">nb_hopt_trials</span><span class="p">,</span>
@@ -862,7 +862,7 @@ argument parser you get the default arguments in the argument parser.</p>
</div>
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
index.html
+37 -36
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="assets/stylesheets/application.30686662.css">
<link rel="stylesheet" href="assets/stylesheets/application.0284f74d.css">
@@ -171,7 +171,7 @@
<main class="md-main" role="main">
<main class="md-main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -239,105 +239,105 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#new-project-quick-start" class="md-nav__link">
<a href="#new-project-quick-start" title="New project Quick Start" class="md-nav__link">
New project Quick Start
</a>
</li>
<li class="md-nav__item">
<a href="#case-1-bert" class="md-nav__link">
<a href="#case-1-bert" title="Case 1: BERT" class="md-nav__link">
Case 1: BERT
</a>
</li>
<li class="md-nav__item">
<a href="#case-2-cooler-not-bert" class="md-nav__link">
<a href="#case-2-cooler-not-bert" title="Case 2: COOLER NOT BERT" class="md-nav__link">
Case 2: COOLER NOT BERT
</a>
</li>
<li class="md-nav__item">
<a href="#rapid-research-flow" class="md-nav__link">
<a href="#rapid-research-flow" title="Rapid research flow" class="md-nav__link">
Rapid research flow
</a>
</li>
<li class="md-nav__item">
<a href="#templates" class="md-nav__link">
<a href="#templates" title="Templates" class="md-nav__link">
Templates
</a>
</li>
<li class="md-nav__item">
<a href="#docs-shortcuts" class="md-nav__link">
<a href="#docs-shortcuts" title="Docs shortcuts" class="md-nav__link">
Docs shortcuts
</a>
</li>
<li class="md-nav__item">
<a href="#quick-start-examples" class="md-nav__link">
<a href="#quick-start-examples" title="Quick start examples" class="md-nav__link">
Quick start examples
</a>
</li>
<li class="md-nav__item">
<a href="#checkpointing" class="md-nav__link">
<a href="#checkpointing" title="Checkpointing" class="md-nav__link">
Checkpointing
</a>
</li>
<li class="md-nav__item">
<a href="#computing-cluster-slurm" class="md-nav__link">
<a href="#computing-cluster-slurm" title="Computing cluster (SLURM)" class="md-nav__link">
Computing cluster (SLURM)
</a>
</li>
<li class="md-nav__item">
<a href="#debugging" class="md-nav__link">
<a href="#debugging" title="Debugging" class="md-nav__link">
Debugging
</a>
</li>
<li class="md-nav__item">
<a href="#distributed-training" class="md-nav__link">
<a href="#distributed-training" title="Distributed training" class="md-nav__link">
Distributed training
</a>
</li>
<li class="md-nav__item">
<a href="#experiment-logging" class="md-nav__link">
<a href="#experiment-logging" title="Experiment Logging" class="md-nav__link">
Experiment Logging
</a>
</li>
<li class="md-nav__item">
<a href="#training-loop" class="md-nav__link">
<a href="#training-loop" title="Training loop" class="md-nav__link">
Training loop
</a>
</li>
<li class="md-nav__item">
<a href="#validation-loop" class="md-nav__link">
<a href="#validation-loop" title="Validation loop" class="md-nav__link">
Validation loop
</a>
</li>
<li class="md-nav__item">
<a href="#testing-loop" class="md-nav__link">
<a href="#testing-loop" title="Testing loop" class="md-nav__link">
Testing loop
</a>
@@ -613,105 +613,105 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#new-project-quick-start" class="md-nav__link">
<a href="#new-project-quick-start" title="New project Quick Start" class="md-nav__link">
New project Quick Start
</a>
</li>
<li class="md-nav__item">
<a href="#case-1-bert" class="md-nav__link">
<a href="#case-1-bert" title="Case 1: BERT" class="md-nav__link">
Case 1: BERT
</a>
</li>
<li class="md-nav__item">
<a href="#case-2-cooler-not-bert" class="md-nav__link">
<a href="#case-2-cooler-not-bert" title="Case 2: COOLER NOT BERT" class="md-nav__link">
Case 2: COOLER NOT BERT
</a>
</li>
<li class="md-nav__item">
<a href="#rapid-research-flow" class="md-nav__link">
<a href="#rapid-research-flow" title="Rapid research flow" class="md-nav__link">
Rapid research flow
</a>
</li>
<li class="md-nav__item">
<a href="#templates" class="md-nav__link">
<a href="#templates" title="Templates" class="md-nav__link">
Templates
</a>
</li>
<li class="md-nav__item">
<a href="#docs-shortcuts" class="md-nav__link">
<a href="#docs-shortcuts" title="Docs shortcuts" class="md-nav__link">
Docs shortcuts
</a>
</li>
<li class="md-nav__item">
<a href="#quick-start-examples" class="md-nav__link">
<a href="#quick-start-examples" title="Quick start examples" class="md-nav__link">
Quick start examples
</a>
</li>
<li class="md-nav__item">
<a href="#checkpointing" class="md-nav__link">
<a href="#checkpointing" title="Checkpointing" class="md-nav__link">
Checkpointing
</a>
</li>
<li class="md-nav__item">
<a href="#computing-cluster-slurm" class="md-nav__link">
<a href="#computing-cluster-slurm" title="Computing cluster (SLURM)" class="md-nav__link">
Computing cluster (SLURM)
</a>
</li>
<li class="md-nav__item">
<a href="#debugging" class="md-nav__link">
<a href="#debugging" title="Debugging" class="md-nav__link">
Debugging
</a>
</li>
<li class="md-nav__item">
<a href="#distributed-training" class="md-nav__link">
<a href="#distributed-training" title="Distributed training" class="md-nav__link">
Distributed training
</a>
</li>
<li class="md-nav__item">
<a href="#experiment-logging" class="md-nav__link">
<a href="#experiment-logging" title="Experiment Logging" class="md-nav__link">
Experiment Logging
</a>
</li>
<li class="md-nav__item">
<a href="#training-loop" class="md-nav__link">
<a href="#training-loop" title="Training loop" class="md-nav__link">
Training loop
</a>
</li>
<li class="md-nav__item">
<a href="#validation-loop" class="md-nav__link">
<a href="#validation-loop" title="Validation loop" class="md-nav__link">
Validation loop
</a>
</li>
<li class="md-nav__item">
<a href="#testing-loop" class="md-nav__link">
<a href="#testing-loop" title="Testing loop" class="md-nav__link">
Testing loop
</a>
@@ -825,8 +825,8 @@ You would define a single LightningModule and use flags to switch between your d
<ol>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#minimal-example">MNIST LightningModule</a> </li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/">Trainer</a><ul>
<li><a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/basic_examples">Basic CPU, GPU Trainer Template</a> </li>
<li><a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/examples/multi_node_examples">GPU cluster Trainer Template</a></li>
<li><a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/basic_examples">Basic CPU, GPU Trainer Template</a></li>
<li><a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/multi_node_examples">GPU cluster Trainer Template</a></li>
</ul>
</li>
</ol>
@@ -866,6 +866,7 @@ You would define a single LightningModule and use flags to switch between your d
</ul>
<h6 id="distributed-training">Distributed training</h6>
<ul>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#init_ddp_connection">Implement Your Own Distributed (DDP) training</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision">16-bit mixed precision</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU">Multi-GPU</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node">Multi-node</a></li>
@@ -963,7 +964,7 @@ You would define a single LightningModule and use flags to switch between your d
</div>
<script src="assets/javascripts/application.ac79c3b0.js"></script>
<script src="assets/javascripts/application.245445c6.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"."}})</script>
search/search_index.json
+1 -1
View File
File diff suppressed because one or more lines are too long
sitemap.xml
+15 -15
View File
@@ -2,77 +2,77 @@
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-19</lastmod>
<lastmod>2019-11-06</lastmod>
<changefreq>daily</changefreq>
</url>
</urlset>
sitemap.xml.gz
BIN
View File
Binary file not shown.