mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-09 11:32:07 +08:00
Deployed b35229d with MkDocs version: 1.0.4
This commit is contained in:
@@ -34,7 +34,7 @@
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@
|
||||
|
||||
|
||||
|
||||
<main class="md-main" role="main">
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
@@ -276,21 +276,21 @@
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#minimal-example" class="md-nav__link">
|
||||
<a href="#minimal-example" title="Minimal example" class="md-nav__link">
|
||||
Minimal example
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#how-do-these-methods-fit-into-the-broader-training" class="md-nav__link">
|
||||
<a href="#how-do-these-methods-fit-into-the-broader-training" title="How do these methods fit into the broader training?" class="md-nav__link">
|
||||
How do these methods fit into the broader training?
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#required-methods" class="md-nav__link">
|
||||
<a href="#required-methods" title="Required Methods" class="md-nav__link">
|
||||
Required Methods
|
||||
</a>
|
||||
|
||||
@@ -298,14 +298,21 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#training_step" class="md-nav__link">
|
||||
<a href="#training_step" title="training_step" class="md-nav__link">
|
||||
training_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#train_dataloader" class="md-nav__link">
|
||||
<a href="#training_end" title="training_end" class="md-nav__link">
|
||||
training_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#train_dataloader" title="train_dataloader" class="md-nav__link">
|
||||
train_dataloader
|
||||
</a>
|
||||
|
||||
@@ -313,7 +320,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return" class="md-nav__link">
|
||||
<a href="#return" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -325,7 +332,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#configure_optimizers" class="md-nav__link">
|
||||
<a href="#configure_optimizers" title="configure_optimizers" class="md-nav__link">
|
||||
configure_optimizers
|
||||
</a>
|
||||
|
||||
@@ -333,7 +340,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_1" class="md-nav__link">
|
||||
<a href="#return_1" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -350,7 +357,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#optional-methods" class="md-nav__link">
|
||||
<a href="#optional-methods" title="Optional Methods" class="md-nav__link">
|
||||
Optional Methods
|
||||
</a>
|
||||
|
||||
@@ -358,35 +365,35 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#validation_step" class="md-nav__link">
|
||||
<a href="#validation_step" title="validation_step" class="md-nav__link">
|
||||
validation_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#validation_end" class="md-nav__link">
|
||||
<a href="#validation_end" title="validation_end" class="md-nav__link">
|
||||
validation_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#test_step" class="md-nav__link">
|
||||
<a href="#test_step" title="test_step" class="md-nav__link">
|
||||
test_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#test_end" class="md-nav__link">
|
||||
<a href="#test_end" title="test_end" class="md-nav__link">
|
||||
test_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_save_checkpoint" class="md-nav__link">
|
||||
<a href="#on_save_checkpoint" title="on_save_checkpoint" class="md-nav__link">
|
||||
on_save_checkpoint
|
||||
</a>
|
||||
|
||||
@@ -394,7 +401,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_2" class="md-nav__link">
|
||||
<a href="#return_2" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -406,7 +413,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_load_checkpoint" class="md-nav__link">
|
||||
<a href="#on_load_checkpoint" title="on_load_checkpoint" class="md-nav__link">
|
||||
on_load_checkpoint
|
||||
</a>
|
||||
|
||||
@@ -414,7 +421,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_3" class="md-nav__link">
|
||||
<a href="#return_3" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -426,7 +433,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#val_dataloader" class="md-nav__link">
|
||||
<a href="#val_dataloader" title="val_dataloader" class="md-nav__link">
|
||||
val_dataloader
|
||||
</a>
|
||||
|
||||
@@ -434,7 +441,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_4" class="md-nav__link">
|
||||
<a href="#return_4" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -446,7 +453,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#test_dataloader" class="md-nav__link">
|
||||
<a href="#test_dataloader" title="test_dataloader" class="md-nav__link">
|
||||
test_dataloader
|
||||
</a>
|
||||
|
||||
@@ -454,7 +461,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_5" class="md-nav__link">
|
||||
<a href="#return_5" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -466,7 +473,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#add_model_specific_args" class="md-nav__link">
|
||||
<a href="#add_model_specific_args" title="add_model_specific_args" class="md-nav__link">
|
||||
add_model_specific_args
|
||||
</a>
|
||||
|
||||
@@ -474,7 +481,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_6" class="md-nav__link">
|
||||
<a href="#return_6" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -731,21 +738,21 @@
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#minimal-example" class="md-nav__link">
|
||||
<a href="#minimal-example" title="Minimal example" class="md-nav__link">
|
||||
Minimal example
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#how-do-these-methods-fit-into-the-broader-training" class="md-nav__link">
|
||||
<a href="#how-do-these-methods-fit-into-the-broader-training" title="How do these methods fit into the broader training?" class="md-nav__link">
|
||||
How do these methods fit into the broader training?
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#required-methods" class="md-nav__link">
|
||||
<a href="#required-methods" title="Required Methods" class="md-nav__link">
|
||||
Required Methods
|
||||
</a>
|
||||
|
||||
@@ -753,14 +760,21 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#training_step" class="md-nav__link">
|
||||
<a href="#training_step" title="training_step" class="md-nav__link">
|
||||
training_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#train_dataloader" class="md-nav__link">
|
||||
<a href="#training_end" title="training_end" class="md-nav__link">
|
||||
training_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#train_dataloader" title="train_dataloader" class="md-nav__link">
|
||||
train_dataloader
|
||||
</a>
|
||||
|
||||
@@ -768,7 +782,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return" class="md-nav__link">
|
||||
<a href="#return" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -780,7 +794,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#configure_optimizers" class="md-nav__link">
|
||||
<a href="#configure_optimizers" title="configure_optimizers" class="md-nav__link">
|
||||
configure_optimizers
|
||||
</a>
|
||||
|
||||
@@ -788,7 +802,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_1" class="md-nav__link">
|
||||
<a href="#return_1" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -805,7 +819,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#optional-methods" class="md-nav__link">
|
||||
<a href="#optional-methods" title="Optional Methods" class="md-nav__link">
|
||||
Optional Methods
|
||||
</a>
|
||||
|
||||
@@ -813,35 +827,35 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#validation_step" class="md-nav__link">
|
||||
<a href="#validation_step" title="validation_step" class="md-nav__link">
|
||||
validation_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#validation_end" class="md-nav__link">
|
||||
<a href="#validation_end" title="validation_end" class="md-nav__link">
|
||||
validation_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#test_step" class="md-nav__link">
|
||||
<a href="#test_step" title="test_step" class="md-nav__link">
|
||||
test_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#test_end" class="md-nav__link">
|
||||
<a href="#test_end" title="test_end" class="md-nav__link">
|
||||
test_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_save_checkpoint" class="md-nav__link">
|
||||
<a href="#on_save_checkpoint" title="on_save_checkpoint" class="md-nav__link">
|
||||
on_save_checkpoint
|
||||
</a>
|
||||
|
||||
@@ -849,7 +863,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_2" class="md-nav__link">
|
||||
<a href="#return_2" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -861,7 +875,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_load_checkpoint" class="md-nav__link">
|
||||
<a href="#on_load_checkpoint" title="on_load_checkpoint" class="md-nav__link">
|
||||
on_load_checkpoint
|
||||
</a>
|
||||
|
||||
@@ -869,7 +883,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_3" class="md-nav__link">
|
||||
<a href="#return_3" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -881,7 +895,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#val_dataloader" class="md-nav__link">
|
||||
<a href="#val_dataloader" title="val_dataloader" class="md-nav__link">
|
||||
val_dataloader
|
||||
</a>
|
||||
|
||||
@@ -889,7 +903,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_4" class="md-nav__link">
|
||||
<a href="#return_4" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -901,7 +915,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#test_dataloader" class="md-nav__link">
|
||||
<a href="#test_dataloader" title="test_dataloader" class="md-nav__link">
|
||||
test_dataloader
|
||||
</a>
|
||||
|
||||
@@ -909,7 +923,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_5" class="md-nav__link">
|
||||
<a href="#return_5" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -921,7 +935,7 @@
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#add_model_specific_args" class="md-nav__link">
|
||||
<a href="#add_model_specific_args" title="add_model_specific_args" class="md-nav__link">
|
||||
add_model_specific_args
|
||||
</a>
|
||||
|
||||
@@ -929,7 +943,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#return_6" class="md-nav__link">
|
||||
<a href="#return_6" title="Return" class="md-nav__link">
|
||||
Return
|
||||
</a>
|
||||
|
||||
@@ -977,6 +991,7 @@
|
||||
</ul>
|
||||
<p><strong>Optional</strong>: </p>
|
||||
<ul>
|
||||
<li><a href="./#training_end">training_end</a> </li>
|
||||
<li><a href="./#validation_step">validation_step</a> </li>
|
||||
<li><a href="./#validation_end">validation_end</a> </li>
|
||||
<li><a href="./#test_step">test_step</a> </li>
|
||||
@@ -1244,6 +1259,153 @@
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>If you add truncated back propagation through time you will also get an additional argument with the hidden states of the previous step. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># Truncated back-propagation through time </span>
|
||||
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">,</span> <span class="n">hiddens</span><span class="p">):</span>
|
||||
<span class="c1"># hiddens are the hiddens from the previous truncated backprop step</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to
|
||||
break out of the current training epoch early.</p>
|
||||
<hr />
|
||||
<h3 id="training_end">training_end</h3>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">training_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">train_step_outputs</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>In certain cases (dp, ddp2), you might want to use all outputs of every process to do something.
|
||||
For instance, if using negative samples, you could run a batch via dp and use ALL the outputs
|
||||
for a single softmax across the full batch (ie: the denominator would use the full batch).</p>
|
||||
<p>In this case you should define training_end to perform those calculations.</p>
|
||||
<p><strong>Params</strong> </p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Param</th>
|
||||
<th>description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>outputs</td>
|
||||
<td>What you return in training_step.</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p><strong>Return</strong> </p>
|
||||
<p>Dictionary or OrderedDict </p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>key</th>
|
||||
<th>value</th>
|
||||
<th>is required</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>loss</td>
|
||||
<td>tensor scalar</td>
|
||||
<td>Y</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>progress_bar</td>
|
||||
<td>Dict for progress bar display. Must have only tensors</td>
|
||||
<td>N</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>log</td>
|
||||
<td>Dict of metrics to add to logger. Must have only tensors (no images, etc)</td>
|
||||
<td>N</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p><strong>Example</strong></p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16
|
||||
17
|
||||
18
|
||||
19
|
||||
20
|
||||
21
|
||||
22
|
||||
23
|
||||
24
|
||||
25
|
||||
26
|
||||
27
|
||||
28</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># WITHOUT training_end</span>
|
||||
<span class="c1"># if used in DP or DDP2, this batch is 1/nb_gpus large</span>
|
||||
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
|
||||
<span class="c1"># batch is 1/nb_gpus big</span>
|
||||
<span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">batch</span>
|
||||
|
||||
<span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||||
<span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">out</span><span class="p">)</span>
|
||||
<span class="n">loss</span> <span class="o">=</span> <span class="n">nce_loss</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="p">{</span><span class="s1">'loss'</span><span class="p">:</span> <span class="n">loss</span><span class="p">}</span>
|
||||
|
||||
<span class="c1"># --------------</span>
|
||||
<span class="c1"># with training_end to do softmax over the full batch</span>
|
||||
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
|
||||
<span class="c1"># batch is 1/nb_gpus big</span>
|
||||
<span class="n">x</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">batch</span>
|
||||
|
||||
<span class="n">out</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">forward</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="p">{</span><span class="s1">'out'</span><span class="p">:</span> <span class="n">out</span><span class="p">}</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">training_end</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">):</span>
|
||||
<span class="c1"># this out is now the full size of the batch</span>
|
||||
<span class="n">out</span> <span class="o">=</span> <span class="n">outputs</span><span class="p">[</span><span class="s1">'out'</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># this softmax now uses the full batch size</span>
|
||||
<span class="n">loss</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">softmax</span><span class="p">(</span><span class="n">out</span><span class="p">)</span>
|
||||
<span class="n">loss</span> <span class="o">=</span> <span class="n">nce_loss</span><span class="p">(</span><span class="n">loss</span><span class="p">)</span>
|
||||
<span class="k">return</span> <span class="p">{</span><span class="s1">'loss'</span><span class="p">:</span> <span class="n">loss</span><span class="p">}</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>If you define multiple optimizers, this step will also be called with an additional <code>optimizer_idx</code> param. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># Multiple optimizers (ie: GANs) </span>
|
||||
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">,</span> <span class="n">optimizer_idx</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="n">optimizer_idx</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="c1"># do training_step with encoder</span>
|
||||
<span class="k">if</span> <span class="n">optimizer_idx</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="c1"># do training_step with decoder </span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>If you add truncated back propagation through time you will also get an additional argument with the hidden states of the previous step. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># Truncated back-propagation through time </span>
|
||||
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">,</span> <span class="n">hiddens</span><span class="p">):</span>
|
||||
<span class="c1"># hiddens are the hiddens from the previous truncated backprop step</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>You can also return a -1 instead of a dict to stop the current loop. This is useful if you want to
|
||||
break out of the current training epoch early.</p>
|
||||
<hr />
|
||||
@@ -1255,7 +1417,7 @@ break out of the current training epoch early.</p>
|
||||
</td></tr></table>
|
||||
|
||||
<p>Called by lightning during training loop. Make sure to use the @pl.data_loader decorator, this ensures not calling this function until the data are needed. <br />
|
||||
If you want to change the data during every epoch DON'T use the data_loader decorator. </p>
|
||||
If you want to change the data during every epoch DON'T use the data_loader decorator.</p>
|
||||
<h5 id="return">Return</h5>
|
||||
<p>PyTorch DataLoader</p>
|
||||
<p><strong>Example</strong></p>
|
||||
@@ -2124,7 +2286,7 @@ The <a href="https://williamfalcon.github.io/test-tube/hyperparameter_optimizati
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@
|
||||
|
||||
|
||||
|
||||
<main class="md-main" role="main">
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
@@ -284,21 +284,28 @@
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#freeze" class="md-nav__link">
|
||||
<a href="#freeze" title="freeze" class="md-nav__link">
|
||||
freeze
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#load_from_metrics" class="md-nav__link">
|
||||
<a href="#load_from_metrics" title="load_from_metrics" class="md-nav__link">
|
||||
load_from_metrics
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#unfreeze" class="md-nav__link">
|
||||
<a href="#load_from_metrics_1" title="load_from_metrics" class="md-nav__link">
|
||||
load_from_metrics
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#unfreeze" title="unfreeze" class="md-nav__link">
|
||||
unfreeze
|
||||
</a>
|
||||
|
||||
@@ -531,21 +538,28 @@
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#freeze" class="md-nav__link">
|
||||
<a href="#freeze" title="freeze" class="md-nav__link">
|
||||
freeze
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#load_from_metrics" class="md-nav__link">
|
||||
<a href="#load_from_metrics" title="load_from_metrics" class="md-nav__link">
|
||||
load_from_metrics
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#unfreeze" class="md-nav__link">
|
||||
<a href="#load_from_metrics_1" title="load_from_metrics" class="md-nav__link">
|
||||
load_from_metrics
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#unfreeze" title="unfreeze" class="md-nav__link">
|
||||
unfreeze
|
||||
</a>
|
||||
|
||||
@@ -584,8 +598,31 @@
|
||||
|
||||
<hr />
|
||||
<h3 id="load_from_metrics">load_from_metrics</h3>
|
||||
<p>This is the easiest/fastest way which uses the meta_tags.csv file from test-tube to rebuild the model.
|
||||
The meta_tags.csv file can be found in the test-tube experiment save_dir. </p>
|
||||
<p>This is the easiest/fastest way which loads hyperparameters and weights from a checkpoint,
|
||||
such as the one saved by the <code>ModelCheckpoint</code> callback</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">pretrained_model</span> <span class="o">=</span> <span class="n">MyLightningModule</span><span class="o">.</span><span class="n">load_from_checkpoint</span><span class="p">(</span>
|
||||
<span class="n">checkpoint_path</span><span class="o">=</span><span class="s1">'/path/to/pytorch_checkpoint.ckpt'</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># predict</span>
|
||||
<span class="n">pretrained_model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
|
||||
<span class="n">pretrained_model</span><span class="o">.</span><span class="n">freeze</span><span class="p">()</span>
|
||||
<span class="n">y_hat</span> <span class="o">=</span> <span class="n">pretrained_model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h3 id="load_from_metrics_1">load_from_metrics</h3>
|
||||
<p>If you're using test tube, there is an alternate method which uses the meta_tags.csv
|
||||
file from test-tube to rebuild the model. The meta_tags.csv file can be found in the
|
||||
test-tube experiment save_dir. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
@@ -716,7 +753,7 @@ The meta_tags.csv file can be found in the test-tube experiment save_dir.
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@
|
||||
|
||||
|
||||
|
||||
<main class="md-main" role="main">
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
@@ -296,56 +296,56 @@
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#current_epoch" class="md-nav__link">
|
||||
<a href="#current_epoch" title="current_epoch" class="md-nav__link">
|
||||
current_epoch
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#dtype" class="md-nav__link">
|
||||
<a href="#dtype" title="dtype" class="md-nav__link">
|
||||
dtype
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#logger" class="md-nav__link">
|
||||
<a href="#logger" title="logger" class="md-nav__link">
|
||||
logger
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#global_step" class="md-nav__link">
|
||||
<a href="#global_step" title="global_step" class="md-nav__link">
|
||||
global_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#gradient_clip_val" class="md-nav__link">
|
||||
<a href="#gradient_clip_val" title="gradient_clip_val" class="md-nav__link">
|
||||
gradient_clip_val
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_gpu" class="md-nav__link">
|
||||
<a href="#on_gpu" title="on_gpu" class="md-nav__link">
|
||||
on_gpu
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#trainer" class="md-nav__link">
|
||||
<a href="#trainer" title="trainer" class="md-nav__link">
|
||||
trainer
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#debugging" class="md-nav__link">
|
||||
<a href="#debugging" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
|
||||
@@ -353,7 +353,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#example_input_array" class="md-nav__link">
|
||||
<a href="#example_input_array" title="example_input_array" class="md-nav__link">
|
||||
example_input_array
|
||||
</a>
|
||||
|
||||
@@ -579,56 +579,56 @@
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#current_epoch" class="md-nav__link">
|
||||
<a href="#current_epoch" title="current_epoch" class="md-nav__link">
|
||||
current_epoch
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#dtype" class="md-nav__link">
|
||||
<a href="#dtype" title="dtype" class="md-nav__link">
|
||||
dtype
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#logger" class="md-nav__link">
|
||||
<a href="#logger" title="logger" class="md-nav__link">
|
||||
logger
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#global_step" class="md-nav__link">
|
||||
<a href="#global_step" title="global_step" class="md-nav__link">
|
||||
global_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#gradient_clip_val" class="md-nav__link">
|
||||
<a href="#gradient_clip_val" title="gradient_clip_val" class="md-nav__link">
|
||||
gradient_clip_val
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_gpu" class="md-nav__link">
|
||||
<a href="#on_gpu" title="on_gpu" class="md-nav__link">
|
||||
on_gpu
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#trainer" class="md-nav__link">
|
||||
<a href="#trainer" title="trainer" class="md-nav__link">
|
||||
trainer
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#debugging" class="md-nav__link">
|
||||
<a href="#debugging" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
|
||||
@@ -636,7 +636,7 @@
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#example_input_array" class="md-nav__link">
|
||||
<a href="#example_input_array" title="example_input_array" class="md-nav__link">
|
||||
example_input_array
|
||||
</a>
|
||||
|
||||
@@ -798,7 +798,7 @@ to get a print out of sizes coming into and out of every layer. </p>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user