mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-18 12:50:58 +08:00
877 lines
23 KiB
HTML
877 lines
23 KiB
HTML
|
|
|
|
|
|
|
|
<!doctype html>
|
|
<html lang="en" class="no-js">
|
|
<head>
|
|
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
|
|
|
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
|
|
|
|
|
|
|
|
|
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
|
|
|
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
|
|
|
<meta name="lang:search.language" content="en">
|
|
|
|
<meta name="lang:search.pipeline.stopwords" content="True">
|
|
|
|
<meta name="lang:search.pipeline.trimmer" content="True">
|
|
|
|
<meta name="lang:search.result.none" content="No matching documents">
|
|
|
|
<meta name="lang:search.result.one" content="1 matching document">
|
|
|
|
<meta name="lang:search.result.other" content="# matching documents">
|
|
|
|
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
|
|
|
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
|
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
|
|
|
|
|
|
|
<title>Distributed training - PyTorch lightning Documentation</title>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
|
|
|
|
|
|
|
|
|
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
|
|
|
|
|
|
|
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
|
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</head>
|
|
|
|
<body dir="ltr">
|
|
|
|
<svg class="md-svg">
|
|
<defs>
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
|
|
|
</defs>
|
|
</svg>
|
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
|
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
|
|
|
<a href="#choosing-a-backend" tabindex="1" class="md-skip">
|
|
Skip to content
|
|
</a>
|
|
|
|
|
|
<header class="md-header" data-md-component="header">
|
|
<nav class="md-header-nav md-grid">
|
|
<div class="md-flex">
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
|
|
|
<i class="md-icon"></i>
|
|
|
|
</a>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--stretch">
|
|
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
|
|
|
<span class="md-header-nav__topic">
|
|
PyTorch lightning Documentation
|
|
</span>
|
|
<span class="md-header-nav__topic">
|
|
|
|
Distributed training
|
|
|
|
</span>
|
|
|
|
</div>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
|
|
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
|
|
|
<div class="md-search" data-md-component="search" role="dialog">
|
|
<label class="md-search__overlay" for="__search"></label>
|
|
<div class="md-search__inner" role="search">
|
|
<form class="md-search__form" name="search">
|
|
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
|
<label class="md-icon md-search__icon" for="__search"></label>
|
|
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
|

|
|
</button>
|
|
</form>
|
|
<div class="md-search__output">
|
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
|
<div class="md-search-result" data-md-component="result">
|
|
<div class="md-search-result__meta">
|
|
Type to start searching
|
|
</div>
|
|
<ol class="md-search-result__list"></ol>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<div class="md-header-nav__source">
|
|
|
|
|
|
|
|
|
|
|
|
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
|
|
|
<div class="md-source__icon">
|
|
<svg viewBox="0 0 24 24" width="24" height="24">
|
|
<use xlink:href="#__github" width="24" height="24"></use>
|
|
</svg>
|
|
</div>
|
|
|
|
<div class="md-source__repository">
|
|
GitHub
|
|
</div>
|
|
</a>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</nav>
|
|
</header>
|
|
|
|
<div class="md-container">
|
|
|
|
|
|
|
|
|
|
<main class="md-main">
|
|
<div class="md-main__inner md-grid" data-md-component="container">
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
<nav class="md-nav md-nav--primary" data-md-level="0">
|
|
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
|
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
|
|
|
<i class="md-icon"></i>
|
|
|
|
</a>
|
|
PyTorch lightning Documentation
|
|
</label>
|
|
|
|
<div class="md-nav__source">
|
|
|
|
|
|
|
|
|
|
|
|
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
|
|
|
<div class="md-source__icon">
|
|
<svg viewBox="0 0 24 24" width="24" height="24">
|
|
<use xlink:href="#__github" width="24" height="24"></use>
|
|
</svg>
|
|
</div>
|
|
|
|
<div class="md-source__repository">
|
|
GitHub
|
|
</div>
|
|
</a>
|
|
</div>
|
|
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../.." title="Home" class="md-nav__link">
|
|
Home
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
|
|
|
<label class="md-nav__link" for="nav-2">
|
|
LightningModule
|
|
</label>
|
|
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
|
<label class="md-nav__title" for="nav-2">
|
|
LightningModule
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
|
Lightning Module interface
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../LightningModule/methods/" title="Methods" class="md-nav__link">
|
|
Methods
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../LightningModule/properties/" title="Properties" class="md-nav__link">
|
|
Properties
|
|
</a>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
|
|
|
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
|
|
|
|
<label class="md-nav__link" for="nav-3">
|
|
Trainer
|
|
</label>
|
|
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
|
<label class="md-nav__title" for="nav-3">
|
|
Trainer
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../" title="Trainer" class="md-nav__link">
|
|
Trainer
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../Checkpointing/" title="Checkpointing" class="md-nav__link">
|
|
Checkpointing
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active">
|
|
|
|
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
|
|
|
|
|
<label class="md-nav__link md-nav__link--active" for="__toc">
|
|
Distributed training
|
|
</label>
|
|
|
|
<a href="./" title="Distributed training" class="md-nav__link md-nav__link--active">
|
|
Distributed training
|
|
</a>
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary">
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">Table of contents</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#choosing-a-backend" title="Choosing a backend" class="md-nav__link">
|
|
Choosing a backend
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#distributed-and-16-bit-precision" title="Distributed and 16-bit precision." class="md-nav__link">
|
|
Distributed and 16-bit precision.
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#cuda-flags" title="CUDA flags" class="md-nav__link">
|
|
CUDA flags
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#16-bit-mixed-precision" title="16-bit mixed precision" class="md-nav__link">
|
|
16-bit mixed precision
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#single-gpu" title="Single-gpu" class="md-nav__link">
|
|
Single-gpu
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#multi-gpu" title="multi-gpu" class="md-nav__link">
|
|
multi-gpu
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#multi-node" title="Multi-node" class="md-nav__link">
|
|
Multi-node
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#self-balancing-architecture" title="Self-balancing architecture" class="md-nav__link">
|
|
Self-balancing architecture
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../Logging/" title="Logging" class="md-nav__link">
|
|
Logging
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
|
SLURM Managed Cluster
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../Training Loop/" title="Training Loop" class="md-nav__link">
|
|
Training Loop
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../Validation loop/" title="Validation loop" class="md-nav__link">
|
|
Validation loop
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../debugging/" title="Debugging" class="md-nav__link">
|
|
Debugging
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../hooks/" title="Hooks" class="md-nav__link">
|
|
Hooks
|
|
</a>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
|
|
|
<label class="md-nav__link" for="nav-4">
|
|
Examples
|
|
</label>
|
|
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
|
<label class="md-nav__title" for="nav-4">
|
|
Examples
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
|
Examples
|
|
</a>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
<nav class="md-nav md-nav--secondary">
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">Table of contents</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#choosing-a-backend" title="Choosing a backend" class="md-nav__link">
|
|
Choosing a backend
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#distributed-and-16-bit-precision" title="Distributed and 16-bit precision." class="md-nav__link">
|
|
Distributed and 16-bit precision.
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#cuda-flags" title="CUDA flags" class="md-nav__link">
|
|
CUDA flags
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#16-bit-mixed-precision" title="16-bit mixed precision" class="md-nav__link">
|
|
16-bit mixed precision
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#single-gpu" title="Single-gpu" class="md-nav__link">
|
|
Single-gpu
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#multi-gpu" title="multi-gpu" class="md-nav__link">
|
|
multi-gpu
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#multi-node" title="Multi-node" class="md-nav__link">
|
|
Multi-node
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#self-balancing-architecture" title="Self-balancing architecture" class="md-nav__link">
|
|
Self-balancing architecture
|
|
</a>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="md-content">
|
|
<article class="md-content__inner md-typeset">
|
|
|
|
|
|
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Distributed training.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
|
|
|
|
|
<h1>Distributed training</h1>
|
|
|
|
<p>Lightning makes multi-gpu training and 16 bit training trivial.</p>
|
|
<p><em>Note:</em> <br />
|
|
None of the flags below require changing anything about your lightningModel definition. </p>
|
|
<hr />
|
|
<h4 id="choosing-a-backend">Choosing a backend</h4>
|
|
<p>Lightning supports two backends. DataParallel and DistributedDataParallel. Both can be used for single-node multi-GPU training.
|
|
For multi-node training you must use DistributedDataParallel. </p>
|
|
<p>You can toggle between each mode by setting this flag.</p>
|
|
<pre><code class="python"># DEFAULT uses DataParallel
|
|
trainer = Trainer(distributed_backend='dp')
|
|
|
|
# change to distributed data parallel
|
|
trainer = Trainer(distributed_backend='ddp')
|
|
</code></pre>
|
|
|
|
<p>If you request multiple nodes, the back-end will auto-switch to ddp.
|
|
We recommend you use DistributedDataparallel even for single-node multi-GPU training. It is MUCH faster than DP but <em>may</em>
|
|
have configuration issues depending on your cluster.</p>
|
|
<p>For a deeper understanding of what lightning is doing, feel free to read <a href="https://medium.com/@_willfalcon/9-tips-for-training-lightning-fast-neural-networks-in-pytorch-8e63a502f565">this guide</a>. </p>
|
|
<hr />
|
|
<h4 id="distributed-and-16-bit-precision">Distributed and 16-bit precision.</h4>
|
|
<p>Due to an issue with apex and DistributedDataParallel (PyTorch and NVIDIA issue), Lightning does
|
|
not allow 16-bit and DP training. We tried to get this to work, but it's an issue on their end. </p>
|
|
<p>Below are the possible configurations we support. </p>
|
|
<table>
|
|
<thead>
|
|
<tr>
|
|
<th>1 GPU</th>
|
|
<th>1+ GPUs</th>
|
|
<th>DP</th>
|
|
<th>DDP</th>
|
|
<th>16-bit</th>
|
|
<th>command</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<tr>
|
|
<td>Y</td>
|
|
<td></td>
|
|
<td></td>
|
|
<td></td>
|
|
<td></td>
|
|
<td><code>Trainer(gpus=[0])</code></td>
|
|
</tr>
|
|
<tr>
|
|
<td>Y</td>
|
|
<td></td>
|
|
<td></td>
|
|
<td></td>
|
|
<td>Y</td>
|
|
<td><code>Trainer(gpus=[0], use_amp=True)</code></td>
|
|
</tr>
|
|
<tr>
|
|
<td></td>
|
|
<td>Y</td>
|
|
<td>Y</td>
|
|
<td></td>
|
|
<td></td>
|
|
<td><code>Trainer(gpus=[0, ...])</code></td>
|
|
</tr>
|
|
<tr>
|
|
<td></td>
|
|
<td>Y</td>
|
|
<td></td>
|
|
<td>Y</td>
|
|
<td></td>
|
|
<td><code>Trainer(gpus=[0, ...], distributed_backend='ddp')</code></td>
|
|
</tr>
|
|
<tr>
|
|
<td></td>
|
|
<td>Y</td>
|
|
<td></td>
|
|
<td>Y</td>
|
|
<td>Y</td>
|
|
<td><code>Trainer(gpus=[0, ...], distributed_backend='ddp', use_amp=True)</code></td>
|
|
</tr>
|
|
</tbody>
|
|
</table>
|
|
<hr />
|
|
<h4 id="cuda-flags">CUDA flags</h4>
|
|
<p>CUDA flags make certain GPUs visible to your script.
|
|
Lightning sets these for you automatically, there's NO NEED to do this yourself.</p>
|
|
<pre><code class="python"># lightning will set according to what you give the trainer
|
|
# os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
|
|
# os.environ["CUDA_VISIBLE_DEVICES"] = "0"
|
|
</code></pre>
|
|
|
|
<hr />
|
|
<h4 id="16-bit-mixed-precision">16-bit mixed precision</h4>
|
|
<p>16 bit precision can cut your memory footprint by half. If using volta architecture GPUs it can give a dramatic training speed-up as well. <br />
|
|
First, install apex (if install fails, look <a href="https://github.com/NVIDIA/apex">here</a>):</p>
|
|
<pre><code class="bash">$ git clone https://github.com/NVIDIA/apex
|
|
$ cd apex
|
|
$ pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./
|
|
</code></pre>
|
|
|
|
<p>then set this use_amp to True.</p>
|
|
<pre><code class="python"># DEFAULT
|
|
trainer = Trainer(amp_level='O2', use_amp=False)
|
|
</code></pre>
|
|
|
|
<hr />
|
|
<h4 id="single-gpu">Single-gpu</h4>
|
|
<p>Make sure you're on a GPU machine. </p>
|
|
<pre><code class="python"># DEFAULT
|
|
trainer = Trainer(gpus=[0])
|
|
</code></pre>
|
|
|
|
<hr />
|
|
<h4 id="multi-gpu">multi-gpu</h4>
|
|
<p>Make sure you're on a GPU machine. You can set as many GPUs as you want.
|
|
In this setting, the model will run on all 8 GPUs at once using DataParallel under the hood.</p>
|
|
<pre><code class="python"># to use DataParallel (default)
|
|
trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='dp')
|
|
|
|
# RECOMMENDED use DistributedDataParallel
|
|
trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], distributed_backend='ddp')
|
|
</code></pre>
|
|
|
|
<hr />
|
|
<h4 id="multi-node">Multi-node</h4>
|
|
<p>Multi-node training is easily done by specifying these flags.</p>
|
|
<pre><code class="python"># train on 12*8 GPUs
|
|
trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], nb_gpu_nodes=12)
|
|
</code></pre>
|
|
|
|
<p>In addition, make sure to set up your SLURM job correctly via the <a href="https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/">SlurmClusterObject</a>. In particular, specify the number of tasks per node correctly.</p>
|
|
<pre><code class="python">cluster = SlurmCluster(
|
|
hyperparam_optimizer=test_tube.HyperOptArgumentParser(),
|
|
log_path='/some/path/to/save',
|
|
)
|
|
|
|
# OPTIONAL FLAGS WHICH MAY BE CLUSTER DEPENDENT
|
|
# which interface your nodes use for communication
|
|
cluster.add_command('export NCCL_SOCKET_IFNAME=^docker0,lo')
|
|
|
|
# see output of the NCCL connection process
|
|
# NCCL is how the nodes talk to each other
|
|
cluster.add_command('export NCCL_DEBUG=INFO')
|
|
|
|
# setting a master port here is a good idea.
|
|
cluster.add_command('export MASTER_PORT=%r' % PORT)
|
|
|
|
# good to load the latest NCCL version
|
|
cluster.load_modules(['NCCL/2.4.7-1-cuda.10.0'])
|
|
|
|
# configure cluster
|
|
cluster.per_experiment_nb_nodes = 12
|
|
cluster.per_experiment_nb_gpus = 8
|
|
|
|
cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu')
|
|
</code></pre>
|
|
|
|
<p>Finally, make sure to add a distributed sampler to your dataset. The distributed sampler copies a
|
|
portion of your dataset onto each GPU. (World_size = gpus_per_node * nb_nodes). </p>
|
|
<pre><code class="python"># ie: this:
|
|
dataset = myDataset()
|
|
dataloader = Dataloader(dataset)
|
|
|
|
# becomes:
|
|
dataset = myDataset()
|
|
dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
|
|
dataloader = Dataloader(dataset, sampler=dist_sampler)
|
|
</code></pre>
|
|
|
|
<hr />
|
|
<h4 id="self-balancing-architecture">Self-balancing architecture</h4>
|
|
<p>Here lightning distributes parts of your module across available GPUs to optimize for speed and memory. </p>
|
|
<p>COMING SOON.</p>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</article>
|
|
</div>
|
|
</div>
|
|
</main>
|
|
|
|
|
|
<footer class="md-footer">
|
|
|
|
<div class="md-footer-nav">
|
|
<nav class="md-footer-nav__inner md-grid">
|
|
|
|
<a href="../Checkpointing/" title="Checkpointing" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
|
<span class="md-flex__ellipsis">
|
|
<span class="md-footer-nav__direction">
|
|
Previous
|
|
</span>
|
|
Checkpointing
|
|
</span>
|
|
</div>
|
|
</a>
|
|
|
|
|
|
<a href="../Logging/" title="Logging" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
|
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
|
<span class="md-flex__ellipsis">
|
|
<span class="md-footer-nav__direction">
|
|
Next
|
|
</span>
|
|
Logging
|
|
</span>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
|
</div>
|
|
</a>
|
|
|
|
</nav>
|
|
</div>
|
|
|
|
<div class="md-footer-meta md-typeset">
|
|
<div class="md-footer-meta__inner md-grid">
|
|
<div class="md-footer-copyright">
|
|
|
|
powered by
|
|
<a href="https://www.mkdocs.org">MkDocs</a>
|
|
and
|
|
<a href="https://squidfunk.github.io/mkdocs-material/">
|
|
Material for MkDocs</a>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
|
|
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
|
|
|
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
|
|
|
|
|
</body>
|
|
</html> |