mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-12 12:40:20 +08:00
Deployed b35229d with MkDocs version: 1.0.4
This commit is contained in:
William Falcon
+311
-22
@@ -34,7 +34,7 @@
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.3">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.30686662.css">
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@
|
||||
|
||||
|
||||
|
||||
<main class="md-main" role="main">
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
@@ -444,68 +444,103 @@
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_epoch_start" class="md-nav__link">
|
||||
<a href="#on_epoch_start" title="on_epoch_start" class="md-nav__link">
|
||||
on_epoch_start
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_epoch_end" class="md-nav__link">
|
||||
<a href="#on_epoch_end" title="on_epoch_end" class="md-nav__link">
|
||||
on_epoch_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_batch_start" class="md-nav__link">
|
||||
<a href="#on_batch_start" title="on_batch_start" class="md-nav__link">
|
||||
on_batch_start
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_batch_end" class="md-nav__link">
|
||||
<a href="#on_batch_end" title="on_batch_end" class="md-nav__link">
|
||||
on_batch_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_pre_performance_check" class="md-nav__link">
|
||||
<a href="#on_pre_performance_check" title="on_pre_performance_check" class="md-nav__link">
|
||||
on_pre_performance_check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_post_performance_check" class="md-nav__link">
|
||||
<a href="#on_post_performance_check" title="on_post_performance_check" class="md-nav__link">
|
||||
on_post_performance_check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#optimizer_step" class="md-nav__link">
|
||||
<a href="#optimizer_step" title="optimizer_step" class="md-nav__link">
|
||||
optimizer_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_before_zero_grad" class="md-nav__link">
|
||||
<a href="#on_before_zero_grad" title="on_before_zero_grad" class="md-nav__link">
|
||||
on_before_zero_grad
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_after_backward" class="md-nav__link">
|
||||
<a href="#backward" title="backward" class="md-nav__link">
|
||||
backward
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_after_backward" title="on_after_backward" class="md-nav__link">
|
||||
on_after_backward
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#tbptt_split_batch" title="tbptt_split_batch" class="md-nav__link">
|
||||
tbptt_split_batch
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#configure_apex" title="configure_apex" class="md-nav__link">
|
||||
configure_apex
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#configure_ddp" title="configure_ddp" class="md-nav__link">
|
||||
configure_ddp
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#init_ddp_connection" title="init_ddp_connection" class="md-nav__link">
|
||||
init_ddp_connection
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -579,68 +614,103 @@
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_epoch_start" class="md-nav__link">
|
||||
<a href="#on_epoch_start" title="on_epoch_start" class="md-nav__link">
|
||||
on_epoch_start
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_epoch_end" class="md-nav__link">
|
||||
<a href="#on_epoch_end" title="on_epoch_end" class="md-nav__link">
|
||||
on_epoch_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_batch_start" class="md-nav__link">
|
||||
<a href="#on_batch_start" title="on_batch_start" class="md-nav__link">
|
||||
on_batch_start
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_batch_end" class="md-nav__link">
|
||||
<a href="#on_batch_end" title="on_batch_end" class="md-nav__link">
|
||||
on_batch_end
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_pre_performance_check" class="md-nav__link">
|
||||
<a href="#on_pre_performance_check" title="on_pre_performance_check" class="md-nav__link">
|
||||
on_pre_performance_check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_post_performance_check" class="md-nav__link">
|
||||
<a href="#on_post_performance_check" title="on_post_performance_check" class="md-nav__link">
|
||||
on_post_performance_check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#optimizer_step" class="md-nav__link">
|
||||
<a href="#optimizer_step" title="optimizer_step" class="md-nav__link">
|
||||
optimizer_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_before_zero_grad" class="md-nav__link">
|
||||
<a href="#on_before_zero_grad" title="on_before_zero_grad" class="md-nav__link">
|
||||
on_before_zero_grad
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_after_backward" class="md-nav__link">
|
||||
<a href="#backward" title="backward" class="md-nav__link">
|
||||
backward
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_after_backward" title="on_after_backward" class="md-nav__link">
|
||||
on_after_backward
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#tbptt_split_batch" title="tbptt_split_batch" class="md-nav__link">
|
||||
tbptt_split_batch
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#configure_apex" title="configure_apex" class="md-nav__link">
|
||||
configure_apex
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#configure_ddp" title="configure_ddp" class="md-nav__link">
|
||||
configure_ddp
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#init_ddp_connection" title="init_ddp_connection" class="md-nav__link">
|
||||
init_ddp_connection
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -807,6 +877,39 @@ Good place to inspect weight information with weights updated.</p>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="backward">backward</h4>
|
||||
<p>Called to perform backward step.
|
||||
Feel free to override as needed.</p>
|
||||
<p>The loss passed in has already been scaled for accumulated gradients if requested.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">backward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">use_amp</span><span class="p">,</span> <span class="n">loss</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">):</span>
|
||||
<span class="sd">"""</span>
|
||||
<span class="sd"> Override backward with your own implementation if you need to</span>
|
||||
<span class="sd"> :param use_amp: Whether amp was requested or not</span>
|
||||
<span class="sd"> :param loss: Loss is already scaled by accumulated grads</span>
|
||||
<span class="sd"> :param optimizer: Current optimizer being used</span>
|
||||
<span class="sd"> :return:</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="k">if</span> <span class="n">use_amp</span><span class="p">:</span>
|
||||
<span class="k">with</span> <span class="n">amp</span><span class="o">.</span><span class="n">scale_loss</span><span class="p">(</span><span class="n">loss</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">)</span> <span class="k">as</span> <span class="n">scaled_loss</span><span class="p">:</span>
|
||||
<span class="n">scaled_loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="on_after_backward">on_after_backward</h4>
|
||||
<p>Called in the training loop after model.backward()
|
||||
@@ -828,6 +931,192 @@ This is the ideal place to inspect or log gradient information </p>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_histogram</span><span class="p">(</span><span class="n">tag</span><span class="o">=</span><span class="n">name</span><span class="p">,</span> <span class="n">values</span><span class="o">=</span><span class="n">grads</span><span class="p">,</span> <span class="n">global_step</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">global_step</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="tbptt_split_batch">tbptt_split_batch</h4>
|
||||
<p>Called in the training loop after on_batch_start if <code>truncated_bptt_steps > 0</code>. Each returned batch split is passed separately to training_step(...).</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16
|
||||
17</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">tbptt_split_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">split_size</span><span class="p">):</span>
|
||||
<span class="n">splits</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="n">time_dims</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">split_size</span><span class="p">):</span>
|
||||
<span class="n">batch_split</span> <span class="o">=</span> <span class="p">[]</span>
|
||||
<span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">batch</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
|
||||
<span class="n">split_x</span> <span class="o">=</span> <span class="n">x</span><span class="p">[:,</span> <span class="n">t</span><span class="p">:</span><span class="n">t</span> <span class="o">+</span> <span class="n">split_size</span><span class="p">]</span>
|
||||
<span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">collections</span><span class="o">.</span><span class="n">Sequence</span><span class="p">):</span>
|
||||
<span class="n">split_x</span> <span class="o">=</span> <span class="p">[</span><span class="bp">None</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||||
<span class="k">for</span> <span class="n">batch_idx</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">x</span><span class="p">)):</span>
|
||||
<span class="n">split_x</span><span class="p">[</span><span class="n">batch_idx</span><span class="p">]</span> <span class="o">=</span> <span class="n">x</span><span class="p">[</span><span class="n">batch_idx</span><span class="p">][</span><span class="n">t</span><span class="p">:</span><span class="n">t</span> <span class="o">+</span> <span class="n">split_size</span><span class="p">]</span>
|
||||
|
||||
<span class="n">batch_split</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">split_x</span><span class="p">)</span>
|
||||
|
||||
<span class="n">splits</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">batch_split</span><span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">splits</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="configure_apex">configure_apex</h4>
|
||||
<p>Overwrite to define your own Apex implementation init.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">configure_apex</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">amp</span><span class="p">,</span> <span class="n">model</span><span class="p">,</span> <span class="n">optimizers</span><span class="p">,</span> <span class="n">amp_level</span><span class="p">):</span>
|
||||
<span class="sd">"""</span>
|
||||
<span class="sd"> Override to init AMP your own way</span>
|
||||
<span class="sd"> Must return a model and list of optimizers</span>
|
||||
<span class="sd"> :param amp:</span>
|
||||
<span class="sd"> :param model:</span>
|
||||
<span class="sd"> :param optimizers:</span>
|
||||
<span class="sd"> :param amp_level:</span>
|
||||
<span class="sd"> :return: Apex wrapped model and optimizers</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="n">model</span><span class="p">,</span> <span class="n">optimizers</span> <span class="o">=</span> <span class="n">amp</span><span class="o">.</span><span class="n">initialize</span><span class="p">(</span>
|
||||
<span class="n">model</span><span class="p">,</span> <span class="n">optimizers</span><span class="p">,</span> <span class="n">opt_level</span><span class="o">=</span><span class="n">amp_level</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="k">return</span> <span class="n">model</span><span class="p">,</span> <span class="n">optimizers</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="configure_ddp">configure_ddp</h4>
|
||||
<p>Overwrite to define your own DDP implementation init.
|
||||
The only requirement is that:
|
||||
1. On a validation batch the call goes to model.validation_step. <br />
|
||||
2. On a training batch the call goes to model.training_step. <br />
|
||||
3. On a testing batch, the call goes to model.test_step</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">configure_ddp</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">model</span><span class="p">,</span> <span class="n">device_ids</span><span class="p">):</span>
|
||||
<span class="sd">"""</span>
|
||||
<span class="sd"> Override to init DDP in a different way or use your own wrapper.</span>
|
||||
<span class="sd"> Must return model.</span>
|
||||
<span class="sd"> :param model:</span>
|
||||
<span class="sd"> :param device_ids:</span>
|
||||
<span class="sd"> :return: DDP wrapped model</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="c1"># Lightning DDP simply routes to test_step, val_step, etc...</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">LightningDistributedDataParallel</span><span class="p">(</span>
|
||||
<span class="n">model</span><span class="p">,</span>
|
||||
<span class="n">device_ids</span><span class="o">=</span><span class="n">device_ids</span><span class="p">,</span>
|
||||
<span class="n">find_unused_parameters</span><span class="o">=</span><span class="bp">True</span>
|
||||
<span class="p">)</span>
|
||||
<span class="k">return</span> <span class="n">model</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="init_ddp_connection">init_ddp_connection</h4>
|
||||
<p>Override to init DDP in your own way. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16
|
||||
17
|
||||
18
|
||||
19
|
||||
20
|
||||
21
|
||||
22
|
||||
23
|
||||
24
|
||||
25
|
||||
26
|
||||
27
|
||||
28
|
||||
29
|
||||
30
|
||||
31
|
||||
32
|
||||
33
|
||||
34</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">init_ddp_connection</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="sd">"""</span>
|
||||
<span class="sd"> Connect all procs in the world using the env:// init</span>
|
||||
<span class="sd"> Use the first node as the root address</span>
|
||||
<span class="sd"> """</span>
|
||||
|
||||
<span class="c1"># use slurm job id for the port number</span>
|
||||
<span class="c1"># guarantees unique ports across jobs from same grid search</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="c1"># use the last 4 numbers in the job id as the id</span>
|
||||
<span class="n">default_port</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'SLURM_JOB_ID'</span><span class="p">]</span>
|
||||
<span class="n">default_port</span> <span class="o">=</span> <span class="n">default_port</span><span class="p">[</span><span class="o">-</span><span class="mi">4</span><span class="p">:]</span>
|
||||
|
||||
<span class="c1"># all ports should be in the 10k+ range</span>
|
||||
<span class="n">default_port</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">default_port</span><span class="p">)</span> <span class="o">+</span> <span class="mi">15000</span>
|
||||
|
||||
<span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
|
||||
<span class="n">default_port</span> <span class="o">=</span> <span class="mi">12910</span>
|
||||
|
||||
<span class="c1"># if user gave a port number, use that one instead</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">default_port</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'MASTER_PORT'</span><span class="p">]</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'MASTER_PORT'</span><span class="p">]</span> <span class="o">=</span> <span class="nb">str</span><span class="p">(</span><span class="n">default_port</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># figure out the root node addr</span>
|
||||
<span class="k">try</span><span class="p">:</span>
|
||||
<span class="n">root_node</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'SLURM_NODELIST'</span><span class="p">]</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">' '</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
|
||||
<span class="n">root_node</span> <span class="o">=</span> <span class="s1">'127.0.0.2'</span>
|
||||
|
||||
<span class="n">root_node</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">resolve_root_node_address</span><span class="p">(</span><span class="n">root_node</span><span class="p">)</span>
|
||||
<span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s1">'MASTER_ADDR'</span><span class="p">]</span> <span class="o">=</span> <span class="n">root_node</span>
|
||||
<span class="n">dist</span><span class="o">.</span><span class="n">init_process_group</span><span class="p">(</span><span class="s1">'nccl'</span><span class="p">,</span> <span class="n">rank</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">proc_rank</span><span class="p">,</span> <span class="n">world_size</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">world_size</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
|
||||
|
||||
@@ -897,7 +1186,7 @@ This is the ideal place to inspect or log gradient information </p>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.ac79c3b0.js"></script>
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user