mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-18 12:50:58 +08:00
Deployed f0af138 with MkDocs version: 1.0.4
This commit is contained in:
William Falcon
+97
-52
@@ -673,104 +673,149 @@ To enable a hook, simply override the method in your LightningModule and the tra
|
||||
<hr />
|
||||
<h4 id="on_epoch_start">on_epoch_start</h4>
|
||||
<p>Called in the training loop at the very beginning of the epoch. </p>
|
||||
<pre><code class="python">def on_epoch_start(self):
|
||||
# do something when the epoch starts
|
||||
</code></pre>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_epoch_start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># do something when the epoch starts</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="on_epoch_end">on_epoch_end</h4>
|
||||
<p>Called in the training loop at the very end of the epoch. </p>
|
||||
<pre><code class="python">def on_epoch_end(self):
|
||||
# do something when the epoch ends
|
||||
</code></pre>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_epoch_end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># do something when the epoch ends </span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="on_batch_start">on_batch_start</h4>
|
||||
<p>Called in the training loop before anything happens for that batch. </p>
|
||||
<pre><code class="python">def on_batch_start(self):
|
||||
# do something when the batch starts
|
||||
</code></pre>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_batch_start</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># do something when the batch starts</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="on_batch_end">on_batch_end</h4>
|
||||
<p>Called in the training loop after the batch. </p>
|
||||
<pre><code class="python">def on_batch_end(self):
|
||||
# do something when the batch ends
|
||||
</code></pre>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_batch_end</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># do something when the batch ends </span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="on_pre_performance_check">on_pre_performance_check</h4>
|
||||
<p>Called at the very beginning of the validation loop. </p>
|
||||
<pre><code class="python">def on_pre_performance_check(self):
|
||||
# do something before validation starts
|
||||
</code></pre>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_pre_performance_check</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># do something before validation starts </span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="on_post_performance_check">on_post_performance_check</h4>
|
||||
<p>Called at the very end of the validation loop. </p>
|
||||
<pre><code class="python">def on_post_performance_check(self):
|
||||
# do something before validation end
|
||||
</code></pre>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_post_performance_check</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># do something before validation end</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="on_tng_metrics">on_tng_metrics</h4>
|
||||
<p>Called in the training loop, right before metrics are logged.
|
||||
Although you can log at any time by using self.experiment, you can use
|
||||
this callback to modify what will be logged.</p>
|
||||
<pre><code class="python">def on_tng_metrics(self, metrics):
|
||||
# do something before validation end
|
||||
</code></pre>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_tng_metrics</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">metrics</span><span class="p">):</span>
|
||||
<span class="c1"># do something before validation end</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="optimizer_step">optimizer_step</h4>
|
||||
<p>Calls .step() and .zero_grad for each optimizer.<br />
|
||||
You can override this method to adjust how you do the optimizer step for each optimizer</p>
|
||||
<p>Called once per optimizer</p>
|
||||
<pre><code class="python"># DEFAULT
|
||||
def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16
|
||||
17
|
||||
18
|
||||
19
|
||||
20
|
||||
21</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="k">def</span> <span class="nf">optimizer_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">current_epoch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">optimizer_i</span><span class="p">):</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||
|
||||
# Alternating schedule for optimizer steps (ie: GANs)
|
||||
def optimizer_step(self, current_epoch, batch_nb, optimizer, optimizer_i):
|
||||
# update generator opt every 2 steps
|
||||
if optimizer_i == 0:
|
||||
if batch_nb % 2 == 0 :
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
<span class="c1"># Alternating schedule for optimizer steps (ie: GANs) </span>
|
||||
<span class="k">def</span> <span class="nf">optimizer_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">current_epoch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">optimizer_i</span><span class="p">):</span>
|
||||
<span class="c1"># update generator opt every 2 steps</span>
|
||||
<span class="k">if</span> <span class="n">optimizer_i</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">batch_nb</span> <span class="o">%</span> <span class="mi">2</span> <span class="o">==</span> <span class="mi">0</span> <span class="p">:</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||
|
||||
# update discriminator opt every 4 steps
|
||||
if optimizer_i == 1:
|
||||
if batch_nb % 4 == 0 :
|
||||
optimizer.step()
|
||||
optimizer.zero_grad()
|
||||
<span class="c1"># update discriminator opt every 4 steps</span>
|
||||
<span class="k">if</span> <span class="n">optimizer_i</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
|
||||
<span class="k">if</span> <span class="n">batch_nb</span> <span class="o">%</span> <span class="mi">4</span> <span class="o">==</span> <span class="mi">0</span> <span class="p">:</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
|
||||
|
||||
# ...
|
||||
# add as many optimizers as you want
|
||||
</code></pre>
|
||||
<span class="c1"># ...</span>
|
||||
<span class="c1"># add as many optimizers as you want </span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="on_before_zero_grad">on_before_zero_grad</h4>
|
||||
<p>Called in the training loop after taking an optimizer step and before zeroing grads.
|
||||
Good place to inspect weight information with weights updated.</p>
|
||||
<p>Called once per optimizer</p>
|
||||
<pre><code class="python">def on_before_zero_grad(self, optimizer):
|
||||
# do something with the optimizer or inspect it.
|
||||
</code></pre>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_before_zero_grad</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">):</span>
|
||||
<span class="c1"># do something with the optimizer or inspect it. </span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="on_after_backward">on_after_backward</h4>
|
||||
<p>Called in the training loop after model.backward()
|
||||
This is the ideal place to inspect or log gradient information </p>
|
||||
<pre><code class="python">def on_after_backward(self):
|
||||
# example to inspect gradient information in tensorboard
|
||||
if self.trainer.global_step % 25 == 0: # don't make the tf file huge
|
||||
params = self.state_dict()
|
||||
for k, v in params.items():
|
||||
grads = v
|
||||
name = k
|
||||
self.experiment.add_histogram(tag=name, values=grads, global_step=self.trainer.global_step)
|
||||
</code></pre>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">on_after_backward</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># example to inspect gradient information in tensorboard</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">global_step</span> <span class="o">%</span> <span class="mi">25</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span> <span class="c1"># don't make the tf file huge</span>
|
||||
<span class="n">params</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">state_dict</span><span class="p">()</span>
|
||||
<span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">params</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
|
||||
<span class="n">grads</span> <span class="o">=</span> <span class="n">v</span>
|
||||
<span class="n">name</span> <span class="o">=</span> <span class="n">k</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_histogram</span><span class="p">(</span><span class="n">tag</span><span class="o">=</span><span class="n">name</span><span class="p">,</span> <span class="n">values</span><span class="o">=</span><span class="n">grads</span><span class="p">,</span> <span class="n">global_step</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">global_step</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user