wassname/pytorch-lightning
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-lightning.git synced 2026-09-09 11:32:07 +08:00
Code Issues Packages Projects Releases Wiki Activity

Deployed 0eb6950 with MkDocs version: 1.0.4

Browse Source
This commit is contained in:
William Falcon
2019-10-05 16:10:29 -05:00
parent c5d8042012
commit 09017dfd67
23 changed files with 575 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
404.html
+4 -4
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="/assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="/assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="/assets/stylesheets/application.30686662.css">
@@ -167,7 +167,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -501,7 +501,7 @@
</div>
<script src="/assets/javascripts/application.245445c6.js"></script>
<script src="/assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"/"}})</script>
LightningModule/RequiredTrainerInterface/index.html
+135 -71
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -276,21 +276,21 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#minimal-example" title="Minimal example" class="md-nav__link">
<a href="#minimal-example" class="md-nav__link">
Minimal example
</a>
</li>
<li class="md-nav__item">
<a href="#how-do-these-methods-fit-into-the-broader-training" title="How do these methods fit into the broader training?" class="md-nav__link">
<a href="#how-do-these-methods-fit-into-the-broader-training" class="md-nav__link">
How do these methods fit into the broader training?
</a>
</li>
<li class="md-nav__item">
<a href="#required-methods" title="Required Methods" class="md-nav__link">
<a href="#required-methods" class="md-nav__link">
Required Methods
</a>
@@ -298,14 +298,14 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#training_step" title="training_step" class="md-nav__link">
<a href="#training_step" class="md-nav__link">
training_step
</a>
</li>
<li class="md-nav__item">
<a href="#train_dataloader" title="train_dataloader" class="md-nav__link">
<a href="#train_dataloader" class="md-nav__link">
train_dataloader
</a>
@@ -313,7 +313,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return" title="Return" class="md-nav__link">
<a href="#return" class="md-nav__link">
Return
</a>
@@ -325,7 +325,7 @@
</li>
<li class="md-nav__item">
<a href="#configure_optimizers" title="configure_optimizers" class="md-nav__link">
<a href="#configure_optimizers" class="md-nav__link">
configure_optimizers
</a>
@@ -333,7 +333,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_1" title="Return" class="md-nav__link">
<a href="#return_1" class="md-nav__link">
Return
</a>
@@ -350,7 +350,7 @@
</li>
<li class="md-nav__item">
<a href="#optional-methods" title="Optional Methods" class="md-nav__link">
<a href="#optional-methods" class="md-nav__link">
Optional Methods
</a>
@@ -358,35 +358,35 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#validation_step" title="validation_step" class="md-nav__link">
<a href="#validation_step" class="md-nav__link">
validation_step
</a>
</li>
<li class="md-nav__item">
<a href="#validation_end" title="validation_end" class="md-nav__link">
<a href="#validation_end" class="md-nav__link">
validation_end
</a>
</li>
<li class="md-nav__item">
<a href="#test_step" title="test_step" class="md-nav__link">
<a href="#test_step" class="md-nav__link">
test_step
</a>
</li>
<li class="md-nav__item">
<a href="#test_end" title="test_end" class="md-nav__link">
<a href="#test_end" class="md-nav__link">
test_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_save_checkpoint" title="on_save_checkpoint" class="md-nav__link">
<a href="#on_save_checkpoint" class="md-nav__link">
on_save_checkpoint
</a>
@@ -394,7 +394,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_2" title="Return" class="md-nav__link">
<a href="#return_2" class="md-nav__link">
Return
</a>
@@ -406,7 +406,7 @@
</li>
<li class="md-nav__item">
<a href="#on_load_checkpoint" title="on_load_checkpoint" class="md-nav__link">
<a href="#on_load_checkpoint" class="md-nav__link">
on_load_checkpoint
</a>
@@ -414,7 +414,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_3" title="Return" class="md-nav__link">
<a href="#return_3" class="md-nav__link">
Return
</a>
@@ -426,7 +426,7 @@
</li>
<li class="md-nav__item">
<a href="#val_dataloader" title="val_dataloader" class="md-nav__link">
<a href="#val_dataloader" class="md-nav__link">
val_dataloader
</a>
@@ -434,7 +434,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_4" title="Return" class="md-nav__link">
<a href="#return_4" class="md-nav__link">
Return
</a>
@@ -446,7 +446,7 @@
</li>
<li class="md-nav__item">
<a href="#test_dataloader" title="test_dataloader" class="md-nav__link">
<a href="#test_dataloader" class="md-nav__link">
test_dataloader
</a>
@@ -454,7 +454,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_5" title="Return" class="md-nav__link">
<a href="#return_5" class="md-nav__link">
Return
</a>
@@ -466,7 +466,7 @@
</li>
<li class="md-nav__item">
<a href="#add_model_specific_args" title="add_model_specific_args" class="md-nav__link">
<a href="#add_model_specific_args" class="md-nav__link">
add_model_specific_args
</a>
@@ -474,7 +474,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_6" title="Return" class="md-nav__link">
<a href="#return_6" class="md-nav__link">
Return
</a>
@@ -731,21 +731,21 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#minimal-example" title="Minimal example" class="md-nav__link">
<a href="#minimal-example" class="md-nav__link">
Minimal example
</a>
</li>
<li class="md-nav__item">
<a href="#how-do-these-methods-fit-into-the-broader-training" title="How do these methods fit into the broader training?" class="md-nav__link">
<a href="#how-do-these-methods-fit-into-the-broader-training" class="md-nav__link">
How do these methods fit into the broader training?
</a>
</li>
<li class="md-nav__item">
<a href="#required-methods" title="Required Methods" class="md-nav__link">
<a href="#required-methods" class="md-nav__link">
Required Methods
</a>
@@ -753,14 +753,14 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#training_step" title="training_step" class="md-nav__link">
<a href="#training_step" class="md-nav__link">
training_step
</a>
</li>
<li class="md-nav__item">
<a href="#train_dataloader" title="train_dataloader" class="md-nav__link">
<a href="#train_dataloader" class="md-nav__link">
train_dataloader
</a>
@@ -768,7 +768,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return" title="Return" class="md-nav__link">
<a href="#return" class="md-nav__link">
Return
</a>
@@ -780,7 +780,7 @@
</li>
<li class="md-nav__item">
<a href="#configure_optimizers" title="configure_optimizers" class="md-nav__link">
<a href="#configure_optimizers" class="md-nav__link">
configure_optimizers
</a>
@@ -788,7 +788,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_1" title="Return" class="md-nav__link">
<a href="#return_1" class="md-nav__link">
Return
</a>
@@ -805,7 +805,7 @@
</li>
<li class="md-nav__item">
<a href="#optional-methods" title="Optional Methods" class="md-nav__link">
<a href="#optional-methods" class="md-nav__link">
Optional Methods
</a>
@@ -813,35 +813,35 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#validation_step" title="validation_step" class="md-nav__link">
<a href="#validation_step" class="md-nav__link">
validation_step
</a>
</li>
<li class="md-nav__item">
<a href="#validation_end" title="validation_end" class="md-nav__link">
<a href="#validation_end" class="md-nav__link">
validation_end
</a>
</li>
<li class="md-nav__item">
<a href="#test_step" title="test_step" class="md-nav__link">
<a href="#test_step" class="md-nav__link">
test_step
</a>
</li>
<li class="md-nav__item">
<a href="#test_end" title="test_end" class="md-nav__link">
<a href="#test_end" class="md-nav__link">
test_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_save_checkpoint" title="on_save_checkpoint" class="md-nav__link">
<a href="#on_save_checkpoint" class="md-nav__link">
on_save_checkpoint
</a>
@@ -849,7 +849,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_2" title="Return" class="md-nav__link">
<a href="#return_2" class="md-nav__link">
Return
</a>
@@ -861,7 +861,7 @@
</li>
<li class="md-nav__item">
<a href="#on_load_checkpoint" title="on_load_checkpoint" class="md-nav__link">
<a href="#on_load_checkpoint" class="md-nav__link">
on_load_checkpoint
</a>
@@ -869,7 +869,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_3" title="Return" class="md-nav__link">
<a href="#return_3" class="md-nav__link">
Return
</a>
@@ -881,7 +881,7 @@
</li>
<li class="md-nav__item">
<a href="#val_dataloader" title="val_dataloader" class="md-nav__link">
<a href="#val_dataloader" class="md-nav__link">
val_dataloader
</a>
@@ -889,7 +889,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_4" title="Return" class="md-nav__link">
<a href="#return_4" class="md-nav__link">
Return
</a>
@@ -901,7 +901,7 @@
</li>
<li class="md-nav__item">
<a href="#test_dataloader" title="test_dataloader" class="md-nav__link">
<a href="#test_dataloader" class="md-nav__link">
test_dataloader
</a>
@@ -909,7 +909,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_5" title="Return" class="md-nav__link">
<a href="#return_5" class="md-nav__link">
Return
</a>
@@ -921,7 +921,7 @@
</li>
<li class="md-nav__item">
<a href="#add_model_specific_args" title="add_model_specific_args" class="md-nav__link">
<a href="#add_model_specific_args" class="md-nav__link">
add_model_specific_args
</a>
@@ -929,7 +929,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#return_6" title="Return" class="md-nav__link">
<a href="#return_6" class="md-nav__link">
Return
</a>
@@ -1175,10 +1175,15 @@
<td>Y</td>
</tr>
<tr>
<td>progress</td>
<td>progress_bar</td>
<td>Dict for progress bar display. Must have only tensors</td>
<td>N</td>
</tr>
<tr>
<td>log</td>
<td>Dict of metrics to add to logger. Must have only tensors (no images, etc)</td>
<td>N</td>
</tr>
</tbody>
</table>
<p><strong>Example</strong></p>
@@ -1195,7 +1200,8 @@
11
12
13
14</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
14
15</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
<span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">z</span> <span class="o">=</span> <span class="n">batch</span>
<span class="c1"># implement your own</span>
@@ -1204,7 +1210,8 @@
<span class="n">output</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;loss&#39;</span><span class="p">:</span> <span class="n">loss</span><span class="p">,</span> <span class="c1"># required</span>
<span class="s1">&#39;progress&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;training_loss&#39;</span><span class="p">:</span> <span class="n">loss</span><span class="p">}</span> <span class="c1"># optional (MUST ALL BE TENSORS)</span>
<span class="s1">&#39;progress_bar&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;training_loss&#39;</span><span class="p">:</span> <span class="n">loss</span><span class="p">},</span> <span class="c1"># optional (MUST ALL BE TENSORS)</span>
<span class="s1">&#39;log&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;training_loss&#39;</span><span class="p">:</span> <span class="n">loss</span><span class="p">}</span> <span class="c1"># optional (MUST ALL BE TENSORS)</span>
<span class="p">}</span>
<span class="c1"># return a dict</span>
@@ -1227,6 +1234,8 @@
</pre></div>
</td></tr></table>
<p>You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to
break out of the current training epoch early.</p>
<hr />
<h3 id="train_dataloader">train_dataloader</h3>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
@@ -1270,7 +1279,8 @@ If you want to change the data during every epoch DON'T use the data_loader deco
<p>Set up as many optimizers and (optionally) learning rate schedulers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple.
Lightning will call .backward() and .step() on each one in every epoch. If you use 16 bit precision it will also handle that.</p>
<p><strong>Note:</strong> If you use multiple optimizers, training_step will have an additional <code>optimizer_idx</code> parameter. </p>
<p><strong>Note:</strong> If you use multiple optimizers, training_step will have an additional <code>optimizer_idx</code> parameter. <br />
<strong>Note 2:</strong> If you use LBFGS lightning handles the closure function automatically for you.</p>
<h5 id="return_1">Return</h5>
<p>Return any of these 3 options: <br />
Single optimizer <br />
@@ -1368,8 +1378,8 @@ If you don't need to validate you don't need to implement this method. In this s
<tbody>
<tr>
<td>dict</td>
<td>Dict or OrderedDict with metrics to display in progress bar. All keys must be tensors.</td>
<td>Y</td>
<td>Dict or OrderedDict - passed to the validation_end step</td>
<td>N</td>
</tr>
</tbody>
</table>
@@ -1412,7 +1422,7 @@ If you don't need to validate you don't need to implement this method. In this s
<span class="c1"># or generated text... or whatever</span>
<span class="n">sample_imgs</span> <span class="o">=</span> <span class="n">x</span><span class="p">[:</span><span class="mi">6</span><span class="p">]</span>
<span class="n">grid</span> <span class="o">=</span> <span class="n">torchvision</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">make_grid</span><span class="p">(</span><span class="n">sample_imgs</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_image</span><span class="p">(</span><span class="s1">&#39;example_images&#39;</span><span class="p">,</span> <span class="n">grid</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_image</span><span class="p">(</span><span class="s1">&#39;example_images&#39;</span><span class="p">,</span> <span class="n">grid</span><span class="p">,</span> <span class="mi">0</span><span class="p">)</span>
<span class="c1"># calculate acc</span>
<span class="n">labels_hat</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">argmax</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
@@ -1465,19 +1475,25 @@ If you don't need to validate you don't need to implement this method. In this s
</tbody>
</table>
<p><strong>Return</strong> </p>
<p>Dictionary or OrderedDict </p>
<table>
<thead>
<tr>
<th>Return</th>
<th>description</th>
<th>optional</th>
<th>key</th>
<th>value</th>
<th>is required</th>
</tr>
</thead>
<tbody>
<tr>
<td>dict</td>
<td>Dict of OrderedDict with metrics to display in progress bar</td>
<td>Y</td>
<td>progress_bar</td>
<td>Dict for progress bar display. Must have only tensors</td>
<td>N</td>
</tr>
<tr>
<td>log</td>
<td>Dict of metrics to add to logger. Must have only tensors (no images, etc)</td>
<td>N</td>
</tr>
</tbody>
</table>
@@ -1498,7 +1514,13 @@ If you don't need to validate you don't need to implement this method. In this s
13
14
15
16</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">validation_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
16
17
18
19
20
21
22</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">validation_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Called at the end of validation to aggregate outputs</span>
<span class="sd"> :param outputs: list of individual outputs of each validation step</span>
@@ -1513,7 +1535,13 @@ If you don't need to validate you don't need to implement this method. In this s
<span class="n">val_loss_mean</span> <span class="o">/=</span> <span class="nb">len</span><span class="p">(</span><span class="n">outputs</span><span class="p">)</span>
<span class="n">val_acc_mean</span> <span class="o">/=</span> <span class="nb">len</span><span class="p">(</span><span class="n">outputs</span><span class="p">)</span>
<span class="n">tqdm_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;val_loss&#39;</span><span class="p">:</span> <span class="n">val_loss_mean</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span> <span class="s1">&#39;val_acc&#39;</span><span class="p">:</span> <span class="n">val_acc_mean</span><span class="o">.</span><span class="n">item</span><span class="p">()}</span>
<span class="k">return</span> <span class="n">tqdm_dict</span>
<span class="c1"># show val_loss and val_acc in progress bar but only log val_loss</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;progress_bar&#39;</span><span class="p">:</span> <span class="n">tqdm_dict</span><span class="p">,</span>
<span class="s1">&#39;log&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;val_loss&#39;</span><span class="p">:</span> <span class="n">val_loss_mean</span><span class="o">.</span><span class="n">item</span><span class="p">()}</span>
<span class="p">}</span>
<span class="k">return</span> <span class="n">results</span>
</pre></div>
</td></tr></table>
@@ -1538,7 +1566,13 @@ each validation step for that dataloader.</p>
16
17
18
19</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">validation_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
19
20
21
22
23
24
25</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">validation_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Called at the end of validation to aggregate outputs</span>
<span class="sd"> :param outputs: list of list of individual outputs of each validation step</span>
@@ -1556,7 +1590,13 @@ each validation step for that dataloader.</p>
<span class="n">val_loss_mean</span> <span class="o">/=</span> <span class="n">i</span>
<span class="n">val_acc_mean</span> <span class="o">/=</span> <span class="n">i</span>
<span class="n">tqdm_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;val_loss&#39;</span><span class="p">:</span> <span class="n">val_loss_mean</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span> <span class="s1">&#39;val_acc&#39;</span><span class="p">:</span> <span class="n">val_acc_mean</span><span class="o">.</span><span class="n">item</span><span class="p">()}</span>
<span class="k">return</span> <span class="n">tqdm_dict</span>
<span class="c1"># show val_loss and val_acc in progress bar but only log val_loss</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;progress_bar&#39;</span><span class="p">:</span> <span class="n">tqdm_dict</span><span class="p">,</span>
<span class="s1">&#39;log&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;val_loss&#39;</span><span class="p">:</span> <span class="n">val_loss_mean</span><span class="o">.</span><span class="n">item</span><span class="p">()}</span>
<span class="p">}</span>
<span class="k">return</span> <span class="n">results</span>
</pre></div>
</td></tr></table>
@@ -1730,7 +1770,13 @@ If you don't need to test you don't need to implement this method. In this step
13
14
15
16</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">test_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
16
17
18
19
20
21
22</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">test_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Called at the end of test to aggregate outputs</span>
<span class="sd"> :param outputs: list of individual outputs of each test step</span>
@@ -1745,7 +1791,13 @@ If you don't need to test you don't need to implement this method. In this step
<span class="n">test_loss_mean</span> <span class="o">/=</span> <span class="nb">len</span><span class="p">(</span><span class="n">outputs</span><span class="p">)</span>
<span class="n">test_acc_mean</span> <span class="o">/=</span> <span class="nb">len</span><span class="p">(</span><span class="n">outputs</span><span class="p">)</span>
<span class="n">tqdm_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;test_loss&#39;</span><span class="p">:</span> <span class="n">test_loss_mean</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span> <span class="s1">&#39;test_acc&#39;</span><span class="p">:</span> <span class="n">test_acc_mean</span><span class="o">.</span><span class="n">item</span><span class="p">()}</span>
<span class="k">return</span> <span class="n">tqdm_dict</span>
<span class="c1"># show test_loss and test_acc in progress bar but only log test_loss</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;progress_bar&#39;</span><span class="p">:</span> <span class="n">tqdm_dict</span><span class="p">,</span>
<span class="s1">&#39;log&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;test_loss&#39;</span><span class="p">:</span> <span class="n">val_loss_mean</span><span class="o">.</span><span class="n">item</span><span class="p">()}</span>
<span class="p">}</span>
<span class="k">return</span> <span class="n">results</span>
</pre></div>
</td></tr></table>
@@ -1770,7 +1822,13 @@ each validation step for that dataloader.</p>
16
17
18
19</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">test_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
19
20
21
22
23
24
25</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">test_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Called at the end of test to aggregate outputs</span>
<span class="sd"> :param outputs: list of individual outputs of each test step</span>
@@ -1788,7 +1846,13 @@ each validation step for that dataloader.</p>
<span class="n">test_loss_mean</span> <span class="o">/=</span> <span class="n">i</span>
<span class="n">test_acc_mean</span> <span class="o">/=</span> <span class="n">i</span>
<span class="n">tqdm_dict</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;test_loss&#39;</span><span class="p">:</span> <span class="n">test_loss_mean</span><span class="o">.</span><span class="n">item</span><span class="p">(),</span> <span class="s1">&#39;test_acc&#39;</span><span class="p">:</span> <span class="n">test_acc_mean</span><span class="o">.</span><span class="n">item</span><span class="p">()}</span>
<span class="k">return</span> <span class="n">tqdm_dict</span>
<span class="c1"># show test_loss and test_acc in progress bar but only log test_loss</span>
<span class="n">results</span> <span class="o">=</span> <span class="p">{</span>
<span class="s1">&#39;progress_bar&#39;</span><span class="p">:</span> <span class="n">tqdm_dict</span><span class="p">,</span>
<span class="s1">&#39;log&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;test_loss&#39;</span><span class="p">:</span> <span class="n">val_loss_mean</span><span class="o">.</span><span class="n">item</span><span class="p">()}</span>
<span class="p">}</span>
<span class="k">return</span> <span class="n">results</span>
</pre></div>
</td></tr></table>
@@ -2050,7 +2114,7 @@ The <a href="https://williamfalcon.github.io/test-tube/hyperparameter_optimizati
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
LightningModule/methods/index.html
+10 -10
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -284,21 +284,21 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#freeze" title="freeze" class="md-nav__link">
<a href="#freeze" class="md-nav__link">
freeze
</a>
</li>
<li class="md-nav__item">
<a href="#load_from_metrics" title="load_from_metrics" class="md-nav__link">
<a href="#load_from_metrics" class="md-nav__link">
load_from_metrics
</a>
</li>
<li class="md-nav__item">
<a href="#unfreeze" title="unfreeze" class="md-nav__link">
<a href="#unfreeze" class="md-nav__link">
unfreeze
</a>
@@ -531,21 +531,21 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#freeze" title="freeze" class="md-nav__link">
<a href="#freeze" class="md-nav__link">
freeze
</a>
</li>
<li class="md-nav__item">
<a href="#load_from_metrics" title="load_from_metrics" class="md-nav__link">
<a href="#load_from_metrics" class="md-nav__link">
load_from_metrics
</a>
</li>
<li class="md-nav__item">
<a href="#unfreeze" title="unfreeze" class="md-nav__link">
<a href="#unfreeze" class="md-nav__link">
unfreeze
</a>
@@ -716,7 +716,7 @@ The meta_tags.csv file can be found in the test-tube experiment save_dir.
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
LightningModule/properties/index.html
+30 -28
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -296,56 +296,56 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#current_epoch" title="current_epoch" class="md-nav__link">
<a href="#current_epoch" class="md-nav__link">
current_epoch
</a>
</li>
<li class="md-nav__item">
<a href="#dtype" title="dtype" class="md-nav__link">
<a href="#dtype" class="md-nav__link">
dtype
</a>
</li>
<li class="md-nav__item">
<a href="#logger" title="logger" class="md-nav__link">
<a href="#logger" class="md-nav__link">
logger
</a>
</li>
<li class="md-nav__item">
<a href="#global_step" title="global_step" class="md-nav__link">
<a href="#global_step" class="md-nav__link">
global_step
</a>
</li>
<li class="md-nav__item">
<a href="#gradient_clip_val" title="gradient_clip_val" class="md-nav__link">
<a href="#gradient_clip_val" class="md-nav__link">
gradient_clip_val
</a>
</li>
<li class="md-nav__item">
<a href="#on_gpu" title="on_gpu" class="md-nav__link">
<a href="#on_gpu" class="md-nav__link">
on_gpu
</a>
</li>
<li class="md-nav__item">
<a href="#trainer" title="trainer" class="md-nav__link">
<a href="#trainer" class="md-nav__link">
trainer
</a>
</li>
<li class="md-nav__item">
<a href="#debugging" title="Debugging" class="md-nav__link">
<a href="#debugging" class="md-nav__link">
Debugging
</a>
@@ -353,7 +353,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#example_input_array" title="example_input_array" class="md-nav__link">
<a href="#example_input_array" class="md-nav__link">
example_input_array
</a>
@@ -579,56 +579,56 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#current_epoch" title="current_epoch" class="md-nav__link">
<a href="#current_epoch" class="md-nav__link">
current_epoch
</a>
</li>
<li class="md-nav__item">
<a href="#dtype" title="dtype" class="md-nav__link">
<a href="#dtype" class="md-nav__link">
dtype
</a>
</li>
<li class="md-nav__item">
<a href="#logger" title="logger" class="md-nav__link">
<a href="#logger" class="md-nav__link">
logger
</a>
</li>
<li class="md-nav__item">
<a href="#global_step" title="global_step" class="md-nav__link">
<a href="#global_step" class="md-nav__link">
global_step
</a>
</li>
<li class="md-nav__item">
<a href="#gradient_clip_val" title="gradient_clip_val" class="md-nav__link">
<a href="#gradient_clip_val" class="md-nav__link">
gradient_clip_val
</a>
</li>
<li class="md-nav__item">
<a href="#on_gpu" title="on_gpu" class="md-nav__link">
<a href="#on_gpu" class="md-nav__link">
on_gpu
</a>
</li>
<li class="md-nav__item">
<a href="#trainer" title="trainer" class="md-nav__link">
<a href="#trainer" class="md-nav__link">
trainer
</a>
</li>
<li class="md-nav__item">
<a href="#debugging" title="Debugging" class="md-nav__link">
<a href="#debugging" class="md-nav__link">
Debugging
</a>
@@ -636,7 +636,7 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#example_input_array" title="example_input_array" class="md-nav__link">
<a href="#example_input_array" class="md-nav__link">
example_input_array
</a>
@@ -677,20 +677,22 @@
<p>Current dtype </p>
<hr />
<h4 id="logger">logger</h4>
<p>A reference to the logger you passed into trainer. </p>
<p>A reference to the logger you passed into trainer.
Passing a logger is optional. If you don't pass one in, Lightning will create one for you automatically.
This logger saves logs to '''/os.getcwd()/lightning_logs'''</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">your_logger</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<p>Call it from anywhere in your LightningModule to add metrics, images, etc... whatever your logger supports. </p>
<p>Here is an example using the Test-tube logger (which is a wrapper on <a href="https://pytorch.org/docs/stable/tensorboard.html">PyTorch SummaryWriter</a> with versioned folder structure). </p>
<p>Here is an example using the TestTubeLogger (which is a wrapper on <a href="https://pytorch.org/docs/stable/tensorboard.html">PyTorch SummaryWriter</a> with versioned folder structure). </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># if logger is a tensorboard logger or test-tube experiment</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">add_embedding</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">log</span><span class="p">({</span><span class="s1">&#39;val_loss&#39;</span><span class="p">:</span> <span class="mf">0.9</span><span class="p">})</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">add_scalars</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
4</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># if logger is a tensorboard logger or TestTubeLogger</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_embedding</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">log</span><span class="p">({</span><span class="s1">&#39;val_loss&#39;</span><span class="p">:</span> <span class="mf">0.9</span><span class="p">})</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_scalars</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -796,7 +798,7 @@ to get a print out of sizes coming into and out of every layer. </p>
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Checkpointing/index.html
+19 -11
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -344,14 +344,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#model-saving" title="Model saving" class="md-nav__link">
<a href="#model-saving" class="md-nav__link">
Model saving
</a>
</li>
<li class="md-nav__item">
<a href="#restoring-training-session" title="Restoring training session" class="md-nav__link">
<a href="#restoring-training-session" class="md-nav__link">
Restoring training session
</a>
@@ -524,14 +524,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#model-saving" title="Model saving" class="md-nav__link">
<a href="#model-saving" class="md-nav__link">
Model saving
</a>
</li>
<li class="md-nav__item">
<a href="#restoring-training-session" title="Restoring training session" class="md-nav__link">
<a href="#restoring-training-session" class="md-nav__link">
Restoring training session
</a>
@@ -561,7 +561,13 @@
<p>Lightning can automate saving and loading checkpoints.</p>
<hr />
<h3 id="model-saving">Model saving</h3>
<p>To enable checkpointing, define the checkpoint callback and give it to the trainer.</p>
<p>Checkpointing is enabled by default to the current working directory.
To change the checkpoint path pass in :</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">Trainer</span><span class="p">(</span><span class="n">default_save_path</span><span class="o">=</span><span class="s1">&#39;/your/path/to/save/checkpoints&#39;</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<p>To modify the behavior of checkpointing pass in your own callback.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
@@ -573,10 +579,12 @@
9
10
11
12</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.callbacks</span> <span class="kn">import</span> <span class="n">ModelCheckpoint</span>
12
13</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.callbacks</span> <span class="kn">import</span> <span class="n">ModelCheckpoint</span>
<span class="c1"># DEFAULTS used by the Trainer</span>
<span class="n">checkpoint_callback</span> <span class="o">=</span> <span class="n">ModelCheckpoint</span><span class="p">(</span>
<span class="n">filepath</span><span class="o">=</span><span class="s1">&#39;/path/to/store/weights/&#39;</span><span class="p">,</span>
<span class="n">filepath</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">(),</span>
<span class="n">save_best_only</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
<span class="n">monitor</span><span class="o">=</span><span class="s1">&#39;val_loss&#39;</span><span class="p">,</span>
@@ -725,7 +733,7 @@ the system isn't different. If you add a layer, for instance, it might not work.
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Distributed training/index.html
+103 -38
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -356,63 +356,90 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#choosing-a-backend" title="Choosing a backend" class="md-nav__link">
<a href="#choosing-a-backend" class="md-nav__link">
Choosing a backend
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#dataparallel-dp" class="md-nav__link">
DataParallel (dp)
</a>
</li>
<li class="md-nav__item">
<a href="#distributeddataparallel-ddp" class="md-nav__link">
DistributedDataParallel (ddp)
</a>
</li>
<li class="md-nav__item">
<a href="#distributeddataparallel-2-ddp2" class="md-nav__link">
DistributedDataParallel-2 (ddp2)
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#distributed-and-16-bit-precision" title="Distributed and 16-bit precision." class="md-nav__link">
<a href="#distributed-and-16-bit-precision" class="md-nav__link">
Distributed and 16-bit precision.
</a>
</li>
<li class="md-nav__item">
<a href="#cuda-flags" title="CUDA flags" class="md-nav__link">
<a href="#cuda-flags" class="md-nav__link">
CUDA flags
</a>
</li>
<li class="md-nav__item">
<a href="#16-bit-mixed-precision" title="16-bit mixed precision" class="md-nav__link">
<a href="#16-bit-mixed-precision" class="md-nav__link">
16-bit mixed precision
</a>
</li>
<li class="md-nav__item">
<a href="#single-gpu" title="Single-gpu" class="md-nav__link">
<a href="#single-gpu" class="md-nav__link">
Single-gpu
</a>
</li>
<li class="md-nav__item">
<a href="#multi-gpu" title="multi-gpu" class="md-nav__link">
<a href="#multi-gpu" class="md-nav__link">
multi-gpu
</a>
</li>
<li class="md-nav__item">
<a href="#multi-node" title="Multi-node" class="md-nav__link">
<a href="#multi-node" class="md-nav__link">
Multi-node
</a>
</li>
<li class="md-nav__item">
<a href="#auto-slurm-job-submission" title="Auto-slurm-job-submission" class="md-nav__link">
<a href="#auto-slurm-job-submission" class="md-nav__link">
Auto-slurm-job-submission
</a>
</li>
<li class="md-nav__item">
<a href="#self-balancing-architecture" title="Self-balancing architecture" class="md-nav__link">
<a href="#self-balancing-architecture" class="md-nav__link">
Self-balancing architecture
</a>
@@ -573,63 +600,90 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#choosing-a-backend" title="Choosing a backend" class="md-nav__link">
<a href="#choosing-a-backend" class="md-nav__link">
Choosing a backend
</a>
<nav class="md-nav">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#dataparallel-dp" class="md-nav__link">
DataParallel (dp)
</a>
</li>
<li class="md-nav__item">
<a href="#distributeddataparallel-ddp" class="md-nav__link">
DistributedDataParallel (ddp)
</a>
</li>
<li class="md-nav__item">
<a href="#distributeddataparallel-2-ddp2" class="md-nav__link">
DistributedDataParallel-2 (ddp2)
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#distributed-and-16-bit-precision" title="Distributed and 16-bit precision." class="md-nav__link">
<a href="#distributed-and-16-bit-precision" class="md-nav__link">
Distributed and 16-bit precision.
</a>
</li>
<li class="md-nav__item">
<a href="#cuda-flags" title="CUDA flags" class="md-nav__link">
<a href="#cuda-flags" class="md-nav__link">
CUDA flags
</a>
</li>
<li class="md-nav__item">
<a href="#16-bit-mixed-precision" title="16-bit mixed precision" class="md-nav__link">
<a href="#16-bit-mixed-precision" class="md-nav__link">
16-bit mixed precision
</a>
</li>
<li class="md-nav__item">
<a href="#single-gpu" title="Single-gpu" class="md-nav__link">
<a href="#single-gpu" class="md-nav__link">
Single-gpu
</a>
</li>
<li class="md-nav__item">
<a href="#multi-gpu" title="multi-gpu" class="md-nav__link">
<a href="#multi-gpu" class="md-nav__link">
multi-gpu
</a>
</li>
<li class="md-nav__item">
<a href="#multi-node" title="Multi-node" class="md-nav__link">
<a href="#multi-node" class="md-nav__link">
Multi-node
</a>
</li>
<li class="md-nav__item">
<a href="#auto-slurm-job-submission" title="Auto-slurm-job-submission" class="md-nav__link">
<a href="#auto-slurm-job-submission" class="md-nav__link">
Auto-slurm-job-submission
</a>
</li>
<li class="md-nav__item">
<a href="#self-balancing-architecture" title="Self-balancing architecture" class="md-nav__link">
<a href="#self-balancing-architecture" class="md-nav__link">
Self-balancing architecture
</a>
@@ -663,16 +717,26 @@ None of the flags below require changing anything about your lightningModel defi
<h4 id="choosing-a-backend">Choosing a backend</h4>
<p>Lightning supports two backends. DataParallel and DistributedDataParallel. Both can be used for single-node multi-GPU training.
For multi-node training you must use DistributedDataParallel. </p>
<p><strong>Warning: Your cluster must have NCCL installed and you must load it when submitting your SLURM script</strong></p>
<h5 id="dataparallel-dp">DataParallel (dp)</h5>
<p>Splits a batch across multiple GPUs on the same node. Cannot be used for multi-node training. </p>
<h5 id="distributeddataparallel-ddp">DistributedDataParallel (ddp)</h5>
<p>Trains a copy of the model on each GPU and only syncs gradients. If used with DistributedSampler, each GPU trains
on a subset of the full dataset. </p>
<h5 id="distributeddataparallel-2-ddp2">DistributedDataParallel-2 (ddp2)</h5>
<p>Works like DDP, except each node trains a single copy of the model using ALL GPUs on that node.
Very useful when dealing with negative samples, etc...</p>
<p>You can toggle between each mode by setting this flag.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4
5
6
7
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (when using single GPU or no GPUs)</span>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (when using single GPU or no GPUs)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">distributed_backend</span><span class="o">=</span><span class="bp">None</span><span class="p">)</span>
<span class="c1"># Change to DataParallel (gpus &gt; 1)</span>
@@ -680,6 +744,9 @@ For multi-node training you must use DistributedDataParallel. </p>
<span class="c1"># change to distributed data parallel (gpus &gt; 1)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">distributed_backend</span><span class="o">=</span><span class="s1">&#39;ddp&#39;</span><span class="p">)</span>
<span class="c1"># change to distributed data parallel (gpus &gt; 1)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">distributed_backend</span><span class="o">=</span><span class="s1">&#39;ddp2&#39;</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -896,8 +963,7 @@ script for the above trainer configuration. </p>
30
31
32
33
34</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="ch">#!/bin/bash -l</span>
33</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="ch">#!/bin/bash -l</span>
<span class="c1"># SLURM SUBMIT SCRIPT</span>
<span class="c1">#SBATCH --nodes=12</span>
@@ -909,11 +975,6 @@ script for the above trainer configuration. </p>
<span class="c1"># activate conda env</span>
conda activate my_env
<span class="c1"># REQUIRED: Load the latest NCCL version</span>
<span class="c1"># the nccl version must match the cuda used to build your PyTorch distribution </span>
<span class="c1"># (ie: which instructions did you follow when installing PyTorch)</span>
<span class="c1"># module load NCCL/2.4.7-1-cuda.10.0</span>
<span class="c1"># -------------------------</span>
<span class="c1"># OPTIONAL</span>
<span class="c1"># -------------------------</span>
@@ -921,6 +982,10 @@ conda activate my_env
<span class="c1"># export NCCL_DEBUG=INFO</span>
<span class="c1"># export PYTHONFAULTHANDLER=1</span>
<span class="c1"># PyTorch comes with prebuilt NCCL support... but if you have issues with it</span>
<span class="c1"># you might need to load the latest version from your modules</span>
<span class="c1"># module load NCCL/2.4.7-1-cuda.10.0</span>
<span class="c1"># on your cluster you might need these:</span>
<span class="c1"># set the network interface</span>
<span class="c1"># export NCCL_SOCKET_IFNAME=^docker0,lo</span>
@@ -1119,7 +1184,7 @@ do this for you. The SlurmCluster can also run a grid search if you pass in a <a
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Logging/index.html
+109 -55
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -79,7 +79,7 @@
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
<a href="#setting-up-logging" tabindex="1" class="md-skip">
<a href="#default_save_path" tabindex="1" class="md-skip">
Skip to content
</a>
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -368,7 +368,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#setting-up-logging" title="Setting up logging" class="md-nav__link">
<a href="#default_save_path" class="md-nav__link">
default_save_path
</a>
</li>
<li class="md-nav__item">
<a href="#setting-up-logging" class="md-nav__link">
Setting up logging
</a>
@@ -376,76 +383,70 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#test-tube" title="Test tube" class="md-nav__link">
<a href="#test-tube" class="md-nav__link">
Test tube
</a>
</li>
<li class="md-nav__item">
<a href="#mlflow" title="MLFlow" class="md-nav__link">
<a href="#mlflow" class="md-nav__link">
MLFlow
</a>
</li>
<li class="md-nav__item">
<a href="#custom-logger" title="Custom logger" class="md-nav__link">
<a href="#custom-logger" class="md-nav__link">
Custom logger
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#using-loggers" title="Using loggers" class="md-nav__link">
<li class="md-nav__item">
<a href="#using-loggers" class="md-nav__link">
Using loggers
</a>
<nav class="md-nav">
<ul class="md-nav__list">
</li>
<li class="md-nav__item">
<a href="#display-metrics-in-progress-bar" title="Display metrics in progress bar" class="md-nav__link">
<a href="#display-metrics-in-progress-bar" class="md-nav__link">
Display metrics in progress bar
</a>
</li>
<li class="md-nav__item">
<a href="#log-metric-row-every-k-batches" title="Log metric row every k batches" class="md-nav__link">
<a href="#log-metric-row-every-k-batches" class="md-nav__link">
Log metric row every k batches
</a>
</li>
<li class="md-nav__item">
<a href="#log-gpu-memory" title="Log GPU memory" class="md-nav__link">
<a href="#log-gpu-memory" class="md-nav__link">
Log GPU memory
</a>
</li>
<li class="md-nav__item">
<a href="#process-position" title="Process position" class="md-nav__link">
<a href="#process-position" class="md-nav__link">
Process position
</a>
</li>
<li class="md-nav__item">
<a href="#save-a-snapshot-of-all-hyperparameters" title="Save a snapshot of all hyperparameters" class="md-nav__link">
<a href="#save-a-snapshot-of-all-hyperparameters" class="md-nav__link">
Save a snapshot of all hyperparameters
</a>
</li>
<li class="md-nav__item">
<a href="#write-logs-file-to-csv-every-k-batches" title="Write logs file to csv every k batches" class="md-nav__link">
<a href="#write-logs-file-to-csv-every-k-batches" class="md-nav__link">
Write logs file to csv every k batches
</a>
@@ -599,7 +600,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#setting-up-logging" title="Setting up logging" class="md-nav__link">
<a href="#default_save_path" class="md-nav__link">
default_save_path
</a>
</li>
<li class="md-nav__item">
<a href="#setting-up-logging" class="md-nav__link">
Setting up logging
</a>
@@ -607,76 +615,70 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#test-tube" title="Test tube" class="md-nav__link">
<a href="#test-tube" class="md-nav__link">
Test tube
</a>
</li>
<li class="md-nav__item">
<a href="#mlflow" title="MLFlow" class="md-nav__link">
<a href="#mlflow" class="md-nav__link">
MLFlow
</a>
</li>
<li class="md-nav__item">
<a href="#custom-logger" title="Custom logger" class="md-nav__link">
<a href="#custom-logger" class="md-nav__link">
Custom logger
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#using-loggers" title="Using loggers" class="md-nav__link">
<li class="md-nav__item">
<a href="#using-loggers" class="md-nav__link">
Using loggers
</a>
<nav class="md-nav">
<ul class="md-nav__list">
</li>
<li class="md-nav__item">
<a href="#display-metrics-in-progress-bar" title="Display metrics in progress bar" class="md-nav__link">
<a href="#display-metrics-in-progress-bar" class="md-nav__link">
Display metrics in progress bar
</a>
</li>
<li class="md-nav__item">
<a href="#log-metric-row-every-k-batches" title="Log metric row every k batches" class="md-nav__link">
<a href="#log-metric-row-every-k-batches" class="md-nav__link">
Log metric row every k batches
</a>
</li>
<li class="md-nav__item">
<a href="#log-gpu-memory" title="Log GPU memory" class="md-nav__link">
<a href="#log-gpu-memory" class="md-nav__link">
Log GPU memory
</a>
</li>
<li class="md-nav__item">
<a href="#process-position" title="Process position" class="md-nav__link">
<a href="#process-position" class="md-nav__link">
Process position
</a>
</li>
<li class="md-nav__item">
<a href="#save-a-snapshot-of-all-hyperparameters" title="Save a snapshot of all hyperparameters" class="md-nav__link">
<a href="#save-a-snapshot-of-all-hyperparameters" class="md-nav__link">
Save a snapshot of all hyperparameters
</a>
</li>
<li class="md-nav__item">
<a href="#write-logs-file-to-csv-every-k-batches" title="Write logs file to csv every k batches" class="md-nav__link">
<a href="#write-logs-file-to-csv-every-k-batches" class="md-nav__link">
Write logs file to csv every k batches
</a>
@@ -710,15 +712,28 @@
<p>Lighting offers options for logging information about model, gpu usage, etc, via several different logging frameworks. It also offers printing options for training monitoring.</p>
<hr />
<h3 id="default_save_path">default_save_path</h3>
<p>Lightning sets a default TestTubeLogger and CheckpointCallback for you which log to
<code>os.getcwd()</code> by default. To modify the logging path you can set:</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">Trainer</span><span class="p">(</span><span class="n">default_save_path</span><span class="o">=</span><span class="s1">&#39;/your/path/to/save/checkpoints&#39;</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<p>If you need more custom behavior (different paths for both, different metrics, etc...)
from the logger and the checkpointCallback, pass in your own instances as explained below.</p>
<hr />
<h3 id="setting-up-logging">Setting up logging</h3>
<p>Initialize your logger, which should inherit from <code>LightningBaseLogger</code>, and pass
it to <code>Trainer</code>.</p>
<p>The trainer inits a default logger for you (TestTubeLogger). All logs will
go to the current working directory under a folder named <code>`os.getcwd()/lightning_logs</code>. </p>
<p>If you want to modify the default logging behavior even more, pass in a logger
(which should inherit from <code>LightningBaseLogger</code>). </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">my_logger</span> <span class="o">=</span> <span class="n">MyLightningLogger</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">my_logger</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<p>The path in this logger will overwrite default_save_path.</p>
<p>Lightning supports several common experiment tracking frameworks out of the box</p>
<hr />
<h4 id="test-tube">Test tube</h4>
@@ -816,7 +831,18 @@ only the first process in DDP training logs data.</p>
<p>If you write a logger than may be useful to others, please send
a pull request to add it to Lighting!</p>
<hr />
<h3 id="using-loggers">Using loggers</h3>
<h4 id="using-loggers">Using loggers</h4>
<p>You can call the logger anywhere from your LightningModule by doing:</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="bp">self</span><span class="o">.</span><span class="n">logger</span>
<span class="c1"># add an image if using TestTubeLogger</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_image</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
</pre></div>
</td></tr></table>
<h4 id="display-metrics-in-progress-bar">Display metrics in progress bar</h4>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
@@ -837,8 +863,20 @@ a pull request to add it to Lighting!</p>
<h4 id="log-gpu-memory">Log GPU memory</h4>
<p>Logs GPU memory when metrics are logged. </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">log_gpu_memory</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
2
3
4
5
6
7
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">log_gpu_memory</span><span class="o">=</span><span class="bp">None</span><span class="p">)</span>
<span class="c1"># log only the min/max utilization</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">log_gpu_memory</span><span class="o">=</span><span class="s1">&#39;min_max&#39;</span><span class="p">)</span>
<span class="c1"># log all the GPU memory (if on DDP, logs only that node)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">log_gpu_memory</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -860,14 +898,30 @@ Lightning will stack progress bars according to this value. </p>
<hr />
<h4 id="save-a-snapshot-of-all-hyperparameters">Save a snapshot of all hyperparameters</h4>
<p>Log hyperparameters using the logger</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2
3
4</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">logger</span> <span class="o">=</span> <span class="n">TestTubeLogger</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="n">logger</span><span class="o">.</span><span class="n">log_hyperparams</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
<p>Automatically log hyperparameters stored in the <code>hparams</code> attribute as an <code>argparse.Namespace</code> </p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
4
5
6
7
8
9
10
11
12</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">class</span> <span class="nc">MyModel</span><span class="p">(</span><span class="n">pl</span><span class="o">.</span><span class="n">Lightning</span><span class="p">):</span>
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hparams</span><span class="p">):</span>
<span class="bp">self</span><span class="o">.</span><span class="n">hparams</span> <span class="o">=</span> <span class="n">hparams</span>
<span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">logger</span><span class="p">)</span>
<span class="o">...</span>
<span class="n">args</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_args</span><span class="p">()</span>
<span class="n">model</span> <span class="o">=</span> <span class="n">MyModel</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
<span class="n">logger</span> <span class="o">=</span> <span class="n">TestTubeLogger</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="n">t</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">logger</span><span class="p">)</span>
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -948,7 +1002,7 @@ Lightning will stack progress bars according to this value. </p>
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/SLURM Managed Cluster/index.html
+8 -8
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -380,14 +380,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#running-grid-search-on-a-cluster" title="Running grid search on a cluster" class="md-nav__link">
<a href="#running-grid-search-on-a-cluster" class="md-nav__link">
Running grid search on a cluster
</a>
</li>
<li class="md-nav__item">
<a href="#walltime-auto-resubmit" title="Walltime auto-resubmit" class="md-nav__link">
<a href="#walltime-auto-resubmit" class="md-nav__link">
Walltime auto-resubmit
</a>
@@ -524,14 +524,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#running-grid-search-on-a-cluster" title="Running grid search on a cluster" class="md-nav__link">
<a href="#running-grid-search-on-a-cluster" class="md-nav__link">
Running grid search on a cluster
</a>
</li>
<li class="md-nav__item">
<a href="#walltime-auto-resubmit" title="Walltime auto-resubmit" class="md-nav__link">
<a href="#walltime-auto-resubmit" class="md-nav__link">
Walltime auto-resubmit
</a>
@@ -777,7 +777,7 @@ your SLURM script. </p>
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Testing loop/index.html
+6 -6
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -392,7 +392,7 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#test" title="test" class="md-nav__link">
<a href="#test" class="md-nav__link">
test
</a>
@@ -517,7 +517,7 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#test" title="test" class="md-nav__link">
<a href="#test" class="md-nav__link">
test
</a>
@@ -658,7 +658,7 @@ First case is where you test right after a full training routine.</p>
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Training Loop/index.html
+23 -22
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -404,49 +404,49 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#accumulated-gradients" title="Accumulated gradients" class="md-nav__link">
<a href="#accumulated-gradients" class="md-nav__link">
Accumulated gradients
</a>
</li>
<li class="md-nav__item">
<a href="#force-training-for-min-or-max-epochs" title="Force training for min or max epochs" class="md-nav__link">
<a href="#force-training-for-min-or-max-epochs" class="md-nav__link">
Force training for min or max epochs
</a>
</li>
<li class="md-nav__item">
<a href="#early-stopping" title="Early stopping" class="md-nav__link">
<a href="#early-stopping" class="md-nav__link">
Early stopping
</a>
</li>
<li class="md-nav__item">
<a href="#force-disable-early-stop" title="Force disable early stop" class="md-nav__link">
<a href="#force-disable-early-stop" class="md-nav__link">
Force disable early stop
</a>
</li>
<li class="md-nav__item">
<a href="#gradient-clipping" title="Gradient Clipping" class="md-nav__link">
<a href="#gradient-clipping" class="md-nav__link">
Gradient Clipping
</a>
</li>
<li class="md-nav__item">
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
<a href="#inspect-gradient-norms" class="md-nav__link">
Inspect gradient norms
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-training-set-to-check" title="Set how much of the training set to check" class="md-nav__link">
<a href="#set-how-much-of-the-training-set-to-check" class="md-nav__link">
Set how much of the training set to check
</a>
@@ -559,49 +559,49 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#accumulated-gradients" title="Accumulated gradients" class="md-nav__link">
<a href="#accumulated-gradients" class="md-nav__link">
Accumulated gradients
</a>
</li>
<li class="md-nav__item">
<a href="#force-training-for-min-or-max-epochs" title="Force training for min or max epochs" class="md-nav__link">
<a href="#force-training-for-min-or-max-epochs" class="md-nav__link">
Force training for min or max epochs
</a>
</li>
<li class="md-nav__item">
<a href="#early-stopping" title="Early stopping" class="md-nav__link">
<a href="#early-stopping" class="md-nav__link">
Early stopping
</a>
</li>
<li class="md-nav__item">
<a href="#force-disable-early-stop" title="Force disable early stop" class="md-nav__link">
<a href="#force-disable-early-stop" class="md-nav__link">
Force disable early stop
</a>
</li>
<li class="md-nav__item">
<a href="#gradient-clipping" title="Gradient Clipping" class="md-nav__link">
<a href="#gradient-clipping" class="md-nav__link">
Gradient Clipping
</a>
</li>
<li class="md-nav__item">
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
<a href="#inspect-gradient-norms" class="md-nav__link">
Inspect gradient norms
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-training-set-to-check" title="Set how much of the training set to check" class="md-nav__link">
<a href="#set-how-much-of-the-training-set-to-check" class="md-nav__link">
Set how much of the training set to check
</a>
@@ -650,7 +650,8 @@
<hr />
<h4 id="early-stopping">Early stopping</h4>
<p>To enable early-stopping, define the callback and give it to the trainer.</p>
<p>The trainer already sets up default early stopping for you.
To modify this behavior, pass in your own EarlyStopping callback.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
2
3
@@ -664,13 +665,13 @@
11
12</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.callbacks</span> <span class="kn">import</span> <span class="n">EarlyStopping</span>
<span class="c1"># DEFAULTS</span>
<span class="c1"># DEFAULTS used by Trainer</span>
<span class="n">early_stop_callback</span> <span class="o">=</span> <span class="n">EarlyStopping</span><span class="p">(</span>
<span class="n">monitor</span><span class="o">=</span><span class="s1">&#39;val_loss&#39;</span><span class="p">,</span>
<span class="n">min_delta</span><span class="o">=</span><span class="mf">0.00</span><span class="p">,</span>
<span class="n">patience</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
<span class="n">patience</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
<span class="n">mode</span><span class="o">=</span><span class="s1">&#39;auto&#39;</span>
<span class="n">mode</span><span class="o">=</span><span class="s1">&#39;min&#39;</span>
<span class="p">)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">early_stop_callback</span><span class="o">=</span><span class="n">early_stop_callback</span><span class="p">)</span>
@@ -801,7 +802,7 @@ Specifically, this will <a href="https://pytorch.org/docs/stable/nn.html#torch.n
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/Validation loop/index.html
+14 -14
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -416,35 +416,35 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#check-validation-every-n-epochs" title="Check validation every n epochs" class="md-nav__link">
<a href="#check-validation-every-n-epochs" class="md-nav__link">
Check validation every n epochs
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-validation-set-to-check" title="Set how much of the validation set to check" class="md-nav__link">
<a href="#set-how-much-of-the-validation-set-to-check" class="md-nav__link">
Set how much of the validation set to check
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-test-set-to-check" title="Set how much of the test set to check" class="md-nav__link">
<a href="#set-how-much-of-the-test-set-to-check" class="md-nav__link">
Set how much of the test set to check
</a>
</li>
<li class="md-nav__item">
<a href="#set-validation-check-frequency-within-1-training-epoch" title="Set validation check frequency within 1 training epoch" class="md-nav__link">
<a href="#set-validation-check-frequency-within-1-training-epoch" class="md-nav__link">
Set validation check frequency within 1 training epoch
</a>
</li>
<li class="md-nav__item">
<a href="#set-the-number-of-validation-sanity-steps" title="Set the number of validation sanity steps" class="md-nav__link">
<a href="#set-the-number-of-validation-sanity-steps" class="md-nav__link">
Set the number of validation sanity steps
</a>
@@ -545,35 +545,35 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#check-validation-every-n-epochs" title="Check validation every n epochs" class="md-nav__link">
<a href="#check-validation-every-n-epochs" class="md-nav__link">
Check validation every n epochs
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-validation-set-to-check" title="Set how much of the validation set to check" class="md-nav__link">
<a href="#set-how-much-of-the-validation-set-to-check" class="md-nav__link">
Set how much of the validation set to check
</a>
</li>
<li class="md-nav__item">
<a href="#set-how-much-of-the-test-set-to-check" title="Set how much of the test set to check" class="md-nav__link">
<a href="#set-how-much-of-the-test-set-to-check" class="md-nav__link">
Set how much of the test set to check
</a>
</li>
<li class="md-nav__item">
<a href="#set-validation-check-frequency-within-1-training-epoch" title="Set validation check frequency within 1 training epoch" class="md-nav__link">
<a href="#set-validation-check-frequency-within-1-training-epoch" class="md-nav__link">
Set validation check frequency within 1 training epoch
</a>
</li>
<li class="md-nav__item">
<a href="#set-the-number-of-validation-sanity-steps" title="Set the number of validation sanity steps" class="md-nav__link">
<a href="#set-the-number-of-validation-sanity-steps" class="md-nav__link">
Set the number of validation sanity steps
</a>
@@ -739,7 +739,7 @@ Lightning will run 5 steps of validation in the beginning of training as a sanit
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/debugging/index.html
+16 -16
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -428,42 +428,42 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#fast-dev-run" title="Fast dev run" class="md-nav__link">
<a href="#fast-dev-run" class="md-nav__link">
Fast dev run
</a>
</li>
<li class="md-nav__item">
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
<a href="#inspect-gradient-norms" class="md-nav__link">
Inspect gradient norms
</a>
</li>
<li class="md-nav__item">
<a href="#make-model-overfit-on-subset-of-data" title="Make model overfit on subset of data" class="md-nav__link">
<a href="#make-model-overfit-on-subset-of-data" class="md-nav__link">
Make model overfit on subset of data
</a>
</li>
<li class="md-nav__item">
<a href="#print-the-parameter-count-by-layer" title="Print the parameter count by layer" class="md-nav__link">
<a href="#print-the-parameter-count-by-layer" class="md-nav__link">
Print the parameter count by layer
</a>
</li>
<li class="md-nav__item">
<a href="#print-which-gradients-are-nan" title="Print which gradients are nan" class="md-nav__link">
<a href="#print-which-gradients-are-nan" class="md-nav__link">
Print which gradients are nan
</a>
</li>
<li class="md-nav__item">
<a href="#log-gpu-usage" title="Log GPU usage" class="md-nav__link">
<a href="#log-gpu-usage" class="md-nav__link">
Log GPU usage
</a>
@@ -552,42 +552,42 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#fast-dev-run" title="Fast dev run" class="md-nav__link">
<a href="#fast-dev-run" class="md-nav__link">
Fast dev run
</a>
</li>
<li class="md-nav__item">
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
<a href="#inspect-gradient-norms" class="md-nav__link">
Inspect gradient norms
</a>
</li>
<li class="md-nav__item">
<a href="#make-model-overfit-on-subset-of-data" title="Make model overfit on subset of data" class="md-nav__link">
<a href="#make-model-overfit-on-subset-of-data" class="md-nav__link">
Make model overfit on subset of data
</a>
</li>
<li class="md-nav__item">
<a href="#print-the-parameter-count-by-layer" title="Print the parameter count by layer" class="md-nav__link">
<a href="#print-the-parameter-count-by-layer" class="md-nav__link">
Print the parameter count by layer
</a>
</li>
<li class="md-nav__item">
<a href="#print-which-gradients-are-nan" title="Print which gradients are nan" class="md-nav__link">
<a href="#print-which-gradients-are-nan" class="md-nav__link">
Print which gradients are nan
</a>
</li>
<li class="md-nav__item">
<a href="#log-gpu-usage" title="Log GPU usage" class="md-nav__link">
<a href="#log-gpu-usage" class="md-nav__link">
Log GPU usage
</a>
@@ -740,7 +740,7 @@ Use this to debug a full run of your program quickly</p>
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/hooks/index.html
+23 -48
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -444,70 +444,63 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#on_epoch_start" title="on_epoch_start" class="md-nav__link">
<a href="#on_epoch_start" class="md-nav__link">
on_epoch_start
</a>
</li>
<li class="md-nav__item">
<a href="#on_epoch_end" title="on_epoch_end" class="md-nav__link">
<a href="#on_epoch_end" class="md-nav__link">
on_epoch_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_batch_start" title="on_batch_start" class="md-nav__link">
<a href="#on_batch_start" class="md-nav__link">
on_batch_start
</a>
</li>
<li class="md-nav__item">
<a href="#on_batch_end" title="on_batch_end" class="md-nav__link">
<a href="#on_batch_end" class="md-nav__link">
on_batch_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_pre_performance_check" title="on_pre_performance_check" class="md-nav__link">
<a href="#on_pre_performance_check" class="md-nav__link">
on_pre_performance_check
</a>
</li>
<li class="md-nav__item">
<a href="#on_post_performance_check" title="on_post_performance_check" class="md-nav__link">
<a href="#on_post_performance_check" class="md-nav__link">
on_post_performance_check
</a>
</li>
<li class="md-nav__item">
<a href="#on_training_metrics" title="on_training_metrics" class="md-nav__link">
on_training_metrics
</a>
</li>
<li class="md-nav__item">
<a href="#optimizer_step" title="optimizer_step" class="md-nav__link">
<a href="#optimizer_step" class="md-nav__link">
optimizer_step
</a>
</li>
<li class="md-nav__item">
<a href="#on_before_zero_grad" title="on_before_zero_grad" class="md-nav__link">
<a href="#on_before_zero_grad" class="md-nav__link">
on_before_zero_grad
</a>
</li>
<li class="md-nav__item">
<a href="#on_after_backward" title="on_after_backward" class="md-nav__link">
<a href="#on_after_backward" class="md-nav__link">
on_after_backward
</a>
@@ -586,70 +579,63 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#on_epoch_start" title="on_epoch_start" class="md-nav__link">
<a href="#on_epoch_start" class="md-nav__link">
on_epoch_start
</a>
</li>
<li class="md-nav__item">
<a href="#on_epoch_end" title="on_epoch_end" class="md-nav__link">
<a href="#on_epoch_end" class="md-nav__link">
on_epoch_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_batch_start" title="on_batch_start" class="md-nav__link">
<a href="#on_batch_start" class="md-nav__link">
on_batch_start
</a>
</li>
<li class="md-nav__item">
<a href="#on_batch_end" title="on_batch_end" class="md-nav__link">
<a href="#on_batch_end" class="md-nav__link">
on_batch_end
</a>
</li>
<li class="md-nav__item">
<a href="#on_pre_performance_check" title="on_pre_performance_check" class="md-nav__link">
<a href="#on_pre_performance_check" class="md-nav__link">
on_pre_performance_check
</a>
</li>
<li class="md-nav__item">
<a href="#on_post_performance_check" title="on_post_performance_check" class="md-nav__link">
<a href="#on_post_performance_check" class="md-nav__link">
on_post_performance_check
</a>
</li>
<li class="md-nav__item">
<a href="#on_training_metrics" title="on_training_metrics" class="md-nav__link">
on_training_metrics
</a>
</li>
<li class="md-nav__item">
<a href="#optimizer_step" title="optimizer_step" class="md-nav__link">
<a href="#optimizer_step" class="md-nav__link">
optimizer_step
</a>
</li>
<li class="md-nav__item">
<a href="#on_before_zero_grad" title="on_before_zero_grad" class="md-nav__link">
<a href="#on_before_zero_grad" class="md-nav__link">
on_before_zero_grad
</a>
</li>
<li class="md-nav__item">
<a href="#on_after_backward" title="on_after_backward" class="md-nav__link">
<a href="#on_after_backward" class="md-nav__link">
on_after_backward
</a>
@@ -736,17 +722,6 @@ To enable a hook, simply override the method in your LightningModule and the tra
</pre></div>
</td></tr></table>
<hr />
<h4 id="on_training_metrics">on_training_metrics</h4>
<p>Called in the training loop, right before metrics are logged.
Although you can log at any time by using self.experiment, you can use
this callback to modify what will be logged.</p>
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_training_metrics</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">metrics</span><span class="p">):</span>
<span class="c1"># do something before validation end</span>
</pre></div>
</td></tr></table>
<hr />
<h4 id="optimizer_step">optimizer_step</h4>
<p>Calls .step() and .zero_grad for each optimizer.<br />
@@ -850,7 +825,7 @@ This is the ideal place to inspect or log gradient information </p>
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
<span class="n">grads</span> <span class="o">=</span> <span class="n">v</span>
<span class="n">name</span> <span class="o">=</span> <span class="n">k</span>
<span class="bp">self</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_histogram</span><span class="p">(</span><span class="n">tag</span><span class="o">=</span><span class="n">name</span><span class="p">,</span> <span class="n">values</span><span class="o">=</span><span class="n">grads</span><span class="p">,</span> <span class="n">global_step</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">global_step</span><span class="p">)</span>
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_histogram</span><span class="p">(</span><span class="n">tag</span><span class="o">=</span><span class="n">name</span><span class="p">,</span> <span class="n">values</span><span class="o">=</span><span class="n">grads</span><span class="p">,</span> <span class="n">global_step</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">global_step</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -922,7 +897,7 @@ This is the ideal place to inspect or log gradient information </p>
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
Trainer/index.html
+5 -5
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -541,7 +541,7 @@
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage">Log GPU usage</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data">Make model overfit on subset of data</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer">Print the parameter count by layer</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan">Pring which gradients are nan</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan">Print which gradients are nan</a></li>
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/properties/#example_input_array">Print input and output size of every module in system</a></li>
</ul>
<p><strong>Distributed training</strong> </p>
@@ -657,7 +657,7 @@
</div>
<script src="../assets/javascripts/application.245445c6.js"></script>
<script src="../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:".."}})</script>
assets/javascripts/application.245445c6.js → assets/javascripts/application.c648116f.js
+1 -1
View File
File diff suppressed because one or more lines are too long
assets/stylesheets/application-palette.01803549.css → assets/stylesheets/application-palette.a8b3c06d.css
+1 -1
View File
File diff suppressed because one or more lines are too long
assets/stylesheets/application.0284f74d.css
-1
View File
File diff suppressed because one or more lines are too long
assets/stylesheets/application.30686662.css
+1
View File
File diff suppressed because one or more lines are too long
examples/Examples/index.html
+16 -92
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="../../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -476,14 +476,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#template-model-definition" title="Template model definition" class="md-nav__link">
<a href="#template-model-definition" class="md-nav__link">
Template model definition
</a>
</li>
<li class="md-nav__item">
<a href="#trainer-example" title="Trainer Example" class="md-nav__link">
<a href="#trainer-example" class="md-nav__link">
Trainer Example
</a>
@@ -491,21 +491,21 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#cpu-hyperparameter-search" title="CPU hyperparameter search" class="md-nav__link">
<a href="#cpu-hyperparameter-search" class="md-nav__link">
CPU hyperparameter search
</a>
</li>
<li class="md-nav__item">
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" title="Hyperparameter search on a single or multiple GPUs" class="md-nav__link">
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" class="md-nav__link">
Hyperparameter search on a single or multiple GPUs
</a>
</li>
<li class="md-nav__item">
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" title="Hyperparameter search on a SLURM HPC cluster" class="md-nav__link">
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" class="md-nav__link">
Hyperparameter search on a SLURM HPC cluster
</a>
@@ -551,14 +551,14 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#template-model-definition" title="Template model definition" class="md-nav__link">
<a href="#template-model-definition" class="md-nav__link">
Template model definition
</a>
</li>
<li class="md-nav__item">
<a href="#trainer-example" title="Trainer Example" class="md-nav__link">
<a href="#trainer-example" class="md-nav__link">
Trainer Example
</a>
@@ -566,21 +566,21 @@
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#cpu-hyperparameter-search" title="CPU hyperparameter search" class="md-nav__link">
<a href="#cpu-hyperparameter-search" class="md-nav__link">
CPU hyperparameter search
</a>
</li>
<li class="md-nav__item">
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" title="Hyperparameter search on a single or multiple GPUs" class="md-nav__link">
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" class="md-nav__link">
Hyperparameter search on a single or multiple GPUs
</a>
</li>
<li class="md-nav__item">
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" title="Hyperparameter search on a SLURM HPC cluster" class="md-nav__link">
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" class="md-nav__link">
Hyperparameter search on a SLURM HPC cluster
</a>
@@ -677,93 +677,17 @@ The main function should have 3 arguments: <br />
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">main</span><span class="p">(</span><span class="n">hparams</span><span class="p">,</span> <span class="n">cluster</span><span class="p">,</span> <span class="n">results_dict</span><span class="p">):</span>
14</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">main</span><span class="p">(</span><span class="n">hparams</span><span class="p">,</span> <span class="n">cluster</span><span class="p">,</span> <span class="n">results_dict</span><span class="p">):</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd"> Main training routine specific for this project</span>
<span class="sd"> :param hparams:</span>
<span class="sd"> :return:</span>
<span class="sd"> &quot;&quot;&quot;</span>
<span class="c1"># init experiment</span>
<span class="n">log_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">realpath</span><span class="p">(</span><span class="vm">__file__</span><span class="p">))</span>
<span class="n">exp</span> <span class="o">=</span> <span class="n">Experiment</span><span class="p">(</span>
<span class="n">name</span><span class="o">=</span><span class="s1">&#39;test_tube_exp&#39;</span><span class="p">,</span>
<span class="n">debug</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
<span class="n">save_dir</span><span class="o">=</span><span class="n">log_dir</span><span class="p">,</span>
<span class="n">version</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
<span class="n">autosave</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
<span class="n">description</span><span class="o">=</span><span class="s1">&#39;test demo&#39;</span>
<span class="p">)</span>
<span class="c1"># set the hparams for the experiment</span>
<span class="n">exp</span><span class="o">.</span><span class="n">argparse</span><span class="p">(</span><span class="n">hparams</span><span class="p">)</span>
<span class="n">exp</span><span class="o">.</span><span class="n">save</span><span class="p">()</span>
<span class="c1"># build model</span>
<span class="n">model</span> <span class="o">=</span> <span class="n">MyLightningModule</span><span class="p">(</span><span class="n">hparams</span><span class="p">)</span>
<span class="c1"># callbacks</span>
<span class="n">early_stop</span> <span class="o">=</span> <span class="n">EarlyStopping</span><span class="p">(</span>
<span class="n">monitor</span><span class="o">=</span><span class="n">hparams</span><span class="o">.</span><span class="n">early_stop_metric</span><span class="p">,</span>
<span class="n">patience</span><span class="o">=</span><span class="n">hparams</span><span class="o">.</span><span class="n">early_stop_patience</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
<span class="n">mode</span><span class="o">=</span><span class="n">hparams</span><span class="o">.</span><span class="n">early_stop_mode</span>
<span class="p">)</span>
<span class="n">model_save_path</span> <span class="o">=</span> <span class="s1">&#39;{}/{}/{}&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">hparams</span><span class="o">.</span><span class="n">model_save_path</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">exp</span><span class="o">.</span><span class="n">version</span><span class="p">)</span>
<span class="n">checkpoint</span> <span class="o">=</span> <span class="n">ModelCheckpoint</span><span class="p">(</span>
<span class="n">filepath</span><span class="o">=</span><span class="n">model_save_path</span><span class="p">,</span>
<span class="n">save_function</span><span class="o">=</span><span class="bp">None</span><span class="p">,</span>
<span class="n">save_best_only</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
<span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
<span class="n">monitor</span><span class="o">=</span><span class="n">hparams</span><span class="o">.</span><span class="n">model_save_monitor_value</span><span class="p">,</span>
<span class="n">mode</span><span class="o">=</span><span class="n">hparams</span><span class="o">.</span><span class="n">model_save_monitor_mode</span>
<span class="p">)</span>
<span class="c1"># configure trainer</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span>
<span class="n">experiment</span><span class="o">=</span><span class="n">exp</span><span class="p">,</span>
<span class="n">cluster</span><span class="o">=</span><span class="n">cluster</span><span class="p">,</span>
<span class="n">checkpoint_callback</span><span class="o">=</span><span class="n">checkpoint</span><span class="p">,</span>
<span class="n">early_stop_callback</span><span class="o">=</span><span class="n">early_stop</span><span class="p">,</span>
<span class="p">)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">()</span>
<span class="c1"># train model</span>
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
@@ -938,7 +862,7 @@ argument parser you get the default arguments in the argument parser.</p>
</div>
<script src="../../assets/javascripts/application.245445c6.js"></script>
<script src="../../assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
index.html
+35 -35
View File
@@ -34,7 +34,7 @@
<meta name="lang:search.tokenizer" content="[\s\-]+">
<link rel="shortcut icon" href="assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.2">
@@ -42,7 +42,7 @@
<link rel="stylesheet" href="assets/stylesheets/application.0284f74d.css">
<link rel="stylesheet" href="assets/stylesheets/application.30686662.css">
@@ -171,7 +171,7 @@
<main class="md-main">
<main class="md-main" role="main">
<div class="md-main__inner md-grid" data-md-component="container">
@@ -239,105 +239,105 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#new-project-quick-start" title="New project Quick Start" class="md-nav__link">
<a href="#new-project-quick-start" class="md-nav__link">
New project Quick Start
</a>
</li>
<li class="md-nav__item">
<a href="#case-1-bert" title="Case 1: BERT" class="md-nav__link">
<a href="#case-1-bert" class="md-nav__link">
Case 1: BERT
</a>
</li>
<li class="md-nav__item">
<a href="#case-2-cooler-not-bert" title="Case 2: COOLER NOT BERT" class="md-nav__link">
<a href="#case-2-cooler-not-bert" class="md-nav__link">
Case 2: COOLER NOT BERT
</a>
</li>
<li class="md-nav__item">
<a href="#rapid-research-flow" title="Rapid research flow" class="md-nav__link">
<a href="#rapid-research-flow" class="md-nav__link">
Rapid research flow
</a>
</li>
<li class="md-nav__item">
<a href="#templates" title="Templates" class="md-nav__link">
<a href="#templates" class="md-nav__link">
Templates
</a>
</li>
<li class="md-nav__item">
<a href="#docs-shortcuts" title="Docs shortcuts" class="md-nav__link">
<a href="#docs-shortcuts" class="md-nav__link">
Docs shortcuts
</a>
</li>
<li class="md-nav__item">
<a href="#quick-start-examples" title="Quick start examples" class="md-nav__link">
<a href="#quick-start-examples" class="md-nav__link">
Quick start examples
</a>
</li>
<li class="md-nav__item">
<a href="#checkpointing" title="Checkpointing" class="md-nav__link">
<a href="#checkpointing" class="md-nav__link">
Checkpointing
</a>
</li>
<li class="md-nav__item">
<a href="#computing-cluster-slurm" title="Computing cluster (SLURM)" class="md-nav__link">
<a href="#computing-cluster-slurm" class="md-nav__link">
Computing cluster (SLURM)
</a>
</li>
<li class="md-nav__item">
<a href="#debugging" title="Debugging" class="md-nav__link">
<a href="#debugging" class="md-nav__link">
Debugging
</a>
</li>
<li class="md-nav__item">
<a href="#distributed-training" title="Distributed training" class="md-nav__link">
<a href="#distributed-training" class="md-nav__link">
Distributed training
</a>
</li>
<li class="md-nav__item">
<a href="#experiment-logging" title="Experiment Logging" class="md-nav__link">
<a href="#experiment-logging" class="md-nav__link">
Experiment Logging
</a>
</li>
<li class="md-nav__item">
<a href="#training-loop" title="Training loop" class="md-nav__link">
<a href="#training-loop" class="md-nav__link">
Training loop
</a>
</li>
<li class="md-nav__item">
<a href="#validation-loop" title="Validation loop" class="md-nav__link">
<a href="#validation-loop" class="md-nav__link">
Validation loop
</a>
</li>
<li class="md-nav__item">
<a href="#testing-loop" title="Testing loop" class="md-nav__link">
<a href="#testing-loop" class="md-nav__link">
Testing loop
</a>
@@ -613,105 +613,105 @@
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="#new-project-quick-start" title="New project Quick Start" class="md-nav__link">
<a href="#new-project-quick-start" class="md-nav__link">
New project Quick Start
</a>
</li>
<li class="md-nav__item">
<a href="#case-1-bert" title="Case 1: BERT" class="md-nav__link">
<a href="#case-1-bert" class="md-nav__link">
Case 1: BERT
</a>
</li>
<li class="md-nav__item">
<a href="#case-2-cooler-not-bert" title="Case 2: COOLER NOT BERT" class="md-nav__link">
<a href="#case-2-cooler-not-bert" class="md-nav__link">
Case 2: COOLER NOT BERT
</a>
</li>
<li class="md-nav__item">
<a href="#rapid-research-flow" title="Rapid research flow" class="md-nav__link">
<a href="#rapid-research-flow" class="md-nav__link">
Rapid research flow
</a>
</li>
<li class="md-nav__item">
<a href="#templates" title="Templates" class="md-nav__link">
<a href="#templates" class="md-nav__link">
Templates
</a>
</li>
<li class="md-nav__item">
<a href="#docs-shortcuts" title="Docs shortcuts" class="md-nav__link">
<a href="#docs-shortcuts" class="md-nav__link">
Docs shortcuts
</a>
</li>
<li class="md-nav__item">
<a href="#quick-start-examples" title="Quick start examples" class="md-nav__link">
<a href="#quick-start-examples" class="md-nav__link">
Quick start examples
</a>
</li>
<li class="md-nav__item">
<a href="#checkpointing" title="Checkpointing" class="md-nav__link">
<a href="#checkpointing" class="md-nav__link">
Checkpointing
</a>
</li>
<li class="md-nav__item">
<a href="#computing-cluster-slurm" title="Computing cluster (SLURM)" class="md-nav__link">
<a href="#computing-cluster-slurm" class="md-nav__link">
Computing cluster (SLURM)
</a>
</li>
<li class="md-nav__item">
<a href="#debugging" title="Debugging" class="md-nav__link">
<a href="#debugging" class="md-nav__link">
Debugging
</a>
</li>
<li class="md-nav__item">
<a href="#distributed-training" title="Distributed training" class="md-nav__link">
<a href="#distributed-training" class="md-nav__link">
Distributed training
</a>
</li>
<li class="md-nav__item">
<a href="#experiment-logging" title="Experiment Logging" class="md-nav__link">
<a href="#experiment-logging" class="md-nav__link">
Experiment Logging
</a>
</li>
<li class="md-nav__item">
<a href="#training-loop" title="Training loop" class="md-nav__link">
<a href="#training-loop" class="md-nav__link">
Training loop
</a>
</li>
<li class="md-nav__item">
<a href="#validation-loop" title="Validation loop" class="md-nav__link">
<a href="#validation-loop" class="md-nav__link">
Validation loop
</a>
</li>
<li class="md-nav__item">
<a href="#testing-loop" title="Testing loop" class="md-nav__link">
<a href="#testing-loop" class="md-nav__link">
Testing loop
</a>
@@ -811,7 +811,7 @@ You would define a single LightningModule and use flags to switch between your d
<span class="k">else</span><span class="p">:</span>
<span class="n">model</span> <span class="o">=</span> <span class="n">CoolerNotBERT</span><span class="p">()</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">gpus</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="n">use_amp</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">gpus</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">use_amp</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
</pre></div>
</td></tr></table>
@@ -964,7 +964,7 @@ You would define a single LightningModule and use flags to switch between your d
</div>
<script src="assets/javascripts/application.245445c6.js"></script>
<script src="assets/javascripts/application.c648116f.js"></script>
<script>app.initialize({version:"1.0.4",url:{base:"."}})</script>
search/search_index.json
+1 -1
View File
File diff suppressed because one or more lines are too long
sitemap.xml
+15 -15
View File
@@ -2,77 +2,77 @@
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
<lastmod>2019-10-02</lastmod>
<lastmod>2019-10-05</lastmod>
<changefreq>daily</changefreq>
</url>
</urlset>
sitemap.xml.gz
BIN
View File
Binary file not shown.