mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-12 12:40:20 +08:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 William Falcon
|
2cc732ca16 | ||
|
William Falcon
|
65095b6cac | ||
|
William Falcon
|
0675db8074 | ||
|
William Falcon
|
9b5ef41046 | ||
|
William Falcon
|
5f6a2c6d65 | ||
|
William Falcon
|
09017dfd67 | ||
|
William Falcon
|
c5d8042012 | ||
|
William Falcon
|
736c747c21 | ||
|
William Falcon
|
3c34e029ed | ||
|
William Falcon
|
e77709ff8b | ||
|
William Falcon
|
6d7eb70130 | ||
|
William Falcon
|
452c972d07 | ||
|
William Falcon
|
a5cc9ec2dc | ||
|
William Falcon
|
8f53e506cf | ||
|
William Falcon
|
5f229145c7 | ||
|
William Falcon
|
ab35762fa1 | ||
|
William Falcon
|
167e9f7aa0 | ||
|
William Falcon
|
06abe3f1b9 | ||
|
William Falcon
|
ba18f5f26a | ||
|
William Falcon
|
9696342652 | ||
|
William Falcon
|
2a8ef263a7 | ||
|
William Falcon
|
f70f01b8a8 | ||
|
William Falcon
|
6104ba74a9 | ||
|
William Falcon
|
65a4a54205 | ||
|
William Falcon
|
55692eb3e3 | ||
|
William Falcon
|
56cd3dd176 | ||
|
William Falcon
|
971d59efe8 | ||
|
William Falcon
|
3e511d129c | ||
|
William Falcon
|
61dd3eb069 | ||
|
William Falcon
|
a485420572 | ||
|
William Falcon
|
9eae41f903 | ||
|
William Falcon
|
8d6f8d8146 | ||
|
William Falcon
|
9b8a06bfdb | ||
|
William Falcon
|
d32c548cf8 | ||
|
William Falcon
|
bf44c4838d | ||
|
William Falcon
|
5856805d27 | ||
|
William Falcon
|
25bf79b9da | ||
|
William Falcon
|
6412446781 | ||
|
William Falcon
|
e4a503a016 | ||
|
William Falcon
|
5a6d112ed6 | ||
|
William Falcon
|
cc924cd8eb | ||
|
William Falcon
|
ef857d6f84 | ||
|
William Falcon
|
589e5381be | ||
|
William Falcon
|
2b89c3513b | ||
|
William Falcon
|
71d2fcb7cf | ||
|
William Falcon
|
c2ee2951c6 | ||
|
William Falcon
|
0bbd238d2d | ||
|
William Falcon
|
caeca5da25 | ||
|
William Falcon
|
d43f670f3c | ||
|
William Falcon
|
6ccd5b10ac | ||
|
William Falcon
|
bdda196694 | ||
|
William Falcon
|
9b7fab5324 | ||
|
William Falcon
|
3d1af47cb7 | ||
|
William Falcon
|
6f0d078ce3 | ||
|
William Falcon
|
4e9d3fc53b | ||
|
William Falcon
|
343e3df1c0 | ||
|
William Falcon
|
69a655280a | ||
|
William Falcon
|
e9e43b28f1 | ||
|
William Falcon
|
887a0cb091 | ||
|
William Falcon
|
6be87ec596 | ||
|
William Falcon
|
3f686c44b7 | ||
|
William Falcon
|
f6d5a2edcf | ||
|
William Falcon
|
74945262d0 | ||
|
William Falcon
|
76d0d72f1f | ||
|
William Falcon
|
c53607c954 | ||
|
William Falcon
|
bb6e103009 | ||
|
William Falcon
|
3e11a68de1 | ||
|
William Falcon
|
8822eb1e98 | ||
|
William Falcon
|
291e962a4d | ||
|
William Falcon
|
0dc5dda716 | ||
|
William Falcon
|
a1170c602c | ||
|
William Falcon
|
b06311544a | ||
|
William Falcon
|
2b304b30b2 | ||
|
William Falcon
|
359c5d369a | ||
|
William Falcon
|
3007636435 | ||
|
William Falcon
|
0985c7a1ca | ||
|
William Falcon
|
d5a3d380ff | ||
|
William Falcon
|
92b8d2056d | ||
|
William Falcon
|
a19dd86e4e | ||
|
William Falcon
|
4c60f2e8c3 | ||
|
William Falcon
|
f3a4149d50 | ||
|
William Falcon
|
c94858db8e | ||
|
William Falcon
|
f6f494585d | ||
|
William Falcon
|
88bae42294 | ||
|
William Falcon
|
3536c7c427 | ||
|
William Falcon
|
6eea1eac31 | ||
|
William Falcon
|
9c6730ea84 | ||
|
William Falcon
|
10846e93a6 | ||
|
William Falcon
|
91a717e3f4 | ||
|
William Falcon
|
e817783396 | ||
|
William Falcon
|
e5b6e5ad08 | ||
|
William Falcon
|
c042b56203 | ||
|
William Falcon
|
23edb866f3 | ||
|
William Falcon
|
2025d71c49 | ||
|
William Falcon
|
2df57315b2 | ||
|
William Falcon
|
4437d8387d | ||
|
William Falcon
|
2b3b9b948a | ||
|
William Falcon
|
3f13a6fde6 | ||
|
William Falcon
|
d68e76aec1 | ||
|
William Falcon
|
9c491427e2 | ||
|
William Falcon
|
168332a3ff | ||
|
William Falcon
|
0e3726f08e | ||
|
William Falcon
|
6b419bf27c | ||
|
William Falcon
|
f54e2b7823 | ||
|
William Falcon
|
418c0057f8 | ||
|
William Falcon
|
37e2372acd | ||
|
William Falcon
|
ee71cdbcfa | ||
|
William Falcon
|
f1d00ca90d | ||
|
William Falcon
|
c679e6fd24 | ||
|
William Falcon
|
74c3c9ed0a | ||
|
William Falcon
|
e117e9bad1 | ||
|
William Falcon
|
f324ebf0ad | ||
|
William Falcon
|
ccbbdb083a | ||
|
William Falcon
|
49a2a87c4d | ||
|
William Falcon
|
118d37068b | ||
|
William Falcon
|
3af10d404c | ||
|
William Falcon
|
b4af72625e |
-122
@@ -1,122 +0,0 @@
|
||||
# project
|
||||
.DS_Store
|
||||
.data/
|
||||
run_configs/
|
||||
test_tube_logs/
|
||||
test_tube_data/
|
||||
datasets/
|
||||
model_weights/
|
||||
app/models/
|
||||
pip-wheel-metadata/
|
||||
test_tube_exp/
|
||||
|
||||
# Byte-compiled / optimized / DLL files
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
example.py
|
||||
timit_data/
|
||||
LJSpeech-1.1/
|
||||
|
||||
# C extensions
|
||||
*.so
|
||||
|
||||
.idea/
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
env/
|
||||
ide_layouts/
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# PyInstaller
|
||||
# Usually these files are written by a python script from a template
|
||||
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
||||
*.manifest
|
||||
*.spec
|
||||
|
||||
# Installer logs
|
||||
pip-log.txt
|
||||
pip-delete-this-directory.txt
|
||||
|
||||
# Unit test / coverage reports
|
||||
htmlcov/
|
||||
.tox/
|
||||
.coverage
|
||||
.coverage.*
|
||||
.cache
|
||||
nosetests.xml
|
||||
coverage.xml
|
||||
*.cover
|
||||
.hypothesis/
|
||||
|
||||
# Translations
|
||||
*.mo
|
||||
*.pot
|
||||
|
||||
# Django stuff:
|
||||
*.log
|
||||
local_settings.py
|
||||
|
||||
# Flask stuff:
|
||||
instance/
|
||||
.webassets-cache
|
||||
|
||||
# Scrapy stuff:
|
||||
.scrapy
|
||||
|
||||
# Sphinx documentation
|
||||
docs/_build/
|
||||
|
||||
# PyBuilder
|
||||
target/
|
||||
|
||||
# Jupyter Notebook
|
||||
.ipynb_checkpoints
|
||||
|
||||
# pyenv
|
||||
.python-version
|
||||
|
||||
# celery beat schedule file
|
||||
celerybeat-schedule
|
||||
|
||||
# SageMath parsed files
|
||||
*.sage.py
|
||||
|
||||
# dotenv
|
||||
.env
|
||||
|
||||
# virtualenv
|
||||
.venv
|
||||
venv/
|
||||
ENV/
|
||||
|
||||
# Spyder project settings
|
||||
.spyderproject
|
||||
.spyproject
|
||||
|
||||
# Rope project settings
|
||||
.ropeproject
|
||||
|
||||
# mkdocs documentation
|
||||
/site
|
||||
|
||||
# mypy
|
||||
.mypy_cache/
|
||||
|
||||
# data
|
||||
mnist/
|
||||
@@ -0,0 +1,510 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="/assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="/assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="/assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="/assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="/." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="/." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3">
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/Trainer/hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="/examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
<h1>404 - Not found</h1>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="/assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"/"}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,21 +0,0 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2019 William Falcon
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,762 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>Methods - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#freeze" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Methods
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../.." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
Methods
|
||||
</label>
|
||||
|
||||
<a href="./" title="Methods" class="md-nav__link md-nav__link--active">
|
||||
Methods
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#freeze" title="freeze" class="md-nav__link">
|
||||
freeze
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#load_from_metrics" title="load_from_metrics" class="md-nav__link">
|
||||
load_from_metrics
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#load_from_metrics_1" title="load_from_metrics" class="md-nav__link">
|
||||
load_from_metrics
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#unfreeze" title="unfreeze" class="md-nav__link">
|
||||
unfreeze
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3">
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#freeze" title="freeze" class="md-nav__link">
|
||||
freeze
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#load_from_metrics" title="load_from_metrics" class="md-nav__link">
|
||||
load_from_metrics
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#load_from_metrics_1" title="load_from_metrics" class="md-nav__link">
|
||||
load_from_metrics
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#unfreeze" title="unfreeze" class="md-nav__link">
|
||||
unfreeze
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/LightningModule/methods.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>Methods</h1>
|
||||
|
||||
<p>Lightning modules are strict superclasses of torch.nn.Module. A LightningModule offers the following in addition to that API.</p>
|
||||
<hr />
|
||||
<h3 id="freeze">freeze</h3>
|
||||
<p>Freeze all params for inference</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">MyLightningModule</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
||||
<span class="n">model</span><span class="o">.</span><span class="n">freeze</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h3 id="load_from_metrics">load_from_metrics</h3>
|
||||
<p>This is the easiest/fastest way which loads hyperparameters and weights from a checkpoint,
|
||||
such as the one saved by the <code>ModelCheckpoint</code> callback</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">pretrained_model</span> <span class="o">=</span> <span class="n">MyLightningModule</span><span class="o">.</span><span class="n">load_from_checkpoint</span><span class="p">(</span>
|
||||
<span class="n">checkpoint_path</span><span class="o">=</span><span class="s1">'/path/to/pytorch_checkpoint.ckpt'</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># predict</span>
|
||||
<span class="n">pretrained_model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
|
||||
<span class="n">pretrained_model</span><span class="o">.</span><span class="n">freeze</span><span class="p">()</span>
|
||||
<span class="n">y_hat</span> <span class="o">=</span> <span class="n">pretrained_model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h3 id="load_from_metrics_1">load_from_metrics</h3>
|
||||
<p>If you're using test tube, there is an alternate method which uses the meta_tags.csv
|
||||
file from test-tube to rebuild the model. The meta_tags.csv file can be found in the
|
||||
test-tube experiment save_dir. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">pretrained_model</span> <span class="o">=</span> <span class="n">MyLightningModule</span><span class="o">.</span><span class="n">load_from_metrics</span><span class="p">(</span>
|
||||
<span class="n">weights_path</span><span class="o">=</span><span class="s1">'/path/to/pytorch_checkpoint.ckpt'</span><span class="p">,</span>
|
||||
<span class="n">tags_csv</span><span class="o">=</span><span class="s1">'/path/to/test_tube/experiment/version/meta_tags.csv'</span><span class="p">,</span>
|
||||
<span class="n">on_gpu</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
|
||||
<span class="n">map_location</span><span class="o">=</span><span class="bp">None</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># predict</span>
|
||||
<span class="n">pretrained_model</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
|
||||
<span class="n">pretrained_model</span><span class="o">.</span><span class="n">freeze</span><span class="p">()</span>
|
||||
<span class="n">y_hat</span> <span class="o">=</span> <span class="n">pretrained_model</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p><strong>Params</strong> </p>
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Param</th>
|
||||
<th>description</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>weights_path</td>
|
||||
<td>Path to a PyTorch checkpoint</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>tags_csv</td>
|
||||
<td>Path to meta_tags.csv file generated by the test-tube Experiment</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>on_gpu</td>
|
||||
<td>if True, puts model on GPU. Make sure to use transforms option if model devices have changed</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>map_location</td>
|
||||
<td>A dictionary mapping saved weight GPU devices to new GPU devices</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<p><strong>Returns</strong> </p>
|
||||
<p>LightningModule - The pretrained LightningModule</p>
|
||||
<hr />
|
||||
<h3 id="unfreeze">unfreeze</h3>
|
||||
<p>Unfreeze all params for inference</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">MyLightningModule</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
||||
<span class="n">model</span><span class="o">.</span><span class="n">unfreeze</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../RequiredTrainerInterface/" title="Lightning Module interface" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
Lightning Module interface
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../properties/" title="Properties" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Properties
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,807 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>Properties - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#current_epoch" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Properties
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../.." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
Properties
|
||||
</label>
|
||||
|
||||
<a href="./" title="Properties" class="md-nav__link md-nav__link--active">
|
||||
Properties
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#current_epoch" title="current_epoch" class="md-nav__link">
|
||||
current_epoch
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#dtype" title="dtype" class="md-nav__link">
|
||||
dtype
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#logger" title="logger" class="md-nav__link">
|
||||
logger
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#global_step" title="global_step" class="md-nav__link">
|
||||
global_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#gradient_clip_val" title="gradient_clip_val" class="md-nav__link">
|
||||
gradient_clip_val
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_gpu" title="on_gpu" class="md-nav__link">
|
||||
on_gpu
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#trainer" title="trainer" class="md-nav__link">
|
||||
trainer
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#debugging" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
|
||||
<nav class="md-nav">
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#example_input_array" title="example_input_array" class="md-nav__link">
|
||||
example_input_array
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3">
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#current_epoch" title="current_epoch" class="md-nav__link">
|
||||
current_epoch
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#dtype" title="dtype" class="md-nav__link">
|
||||
dtype
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#logger" title="logger" class="md-nav__link">
|
||||
logger
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#global_step" title="global_step" class="md-nav__link">
|
||||
global_step
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#gradient_clip_val" title="gradient_clip_val" class="md-nav__link">
|
||||
gradient_clip_val
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#on_gpu" title="on_gpu" class="md-nav__link">
|
||||
on_gpu
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#trainer" title="trainer" class="md-nav__link">
|
||||
trainer
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#debugging" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
|
||||
<nav class="md-nav">
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#example_input_array" title="example_input_array" class="md-nav__link">
|
||||
example_input_array
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/LightningModule/properties.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>Properties</h1>
|
||||
|
||||
<p>A LightningModule has the following properties which you can access at any time</p>
|
||||
<hr />
|
||||
<h4 id="current_epoch">current_epoch</h4>
|
||||
<p>The current epoch </p>
|
||||
<hr />
|
||||
<h4 id="dtype">dtype</h4>
|
||||
<p>Current dtype </p>
|
||||
<hr />
|
||||
<h4 id="logger">logger</h4>
|
||||
<p>A reference to the logger you passed into trainer.
|
||||
Passing a logger is optional. If you don't pass one in, Lightning will create one for you automatically.
|
||||
This logger saves logs to '''/os.getcwd()/lightning_logs'''</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">your_logger</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>Call it from anywhere in your LightningModule to add metrics, images, etc... whatever your logger supports. </p>
|
||||
<p>Here is an example using the TestTubeLogger (which is a wrapper on <a href="https://pytorch.org/docs/stable/tensorboard.html">PyTorch SummaryWriter</a> with versioned folder structure). </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># if logger is a tensorboard logger or TestTubeLogger</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_embedding</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">log</span><span class="p">({</span><span class="s1">'val_loss'</span><span class="p">:</span> <span class="mf">0.9</span><span class="p">})</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_scalars</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="global_step">global_step</h4>
|
||||
<p>Total training batches seen across all epochs </p>
|
||||
<hr />
|
||||
<h4 id="gradient_clip_val">gradient_clip_val</h4>
|
||||
<p>The current gradient clip value </p>
|
||||
<hr />
|
||||
<h4 id="on_gpu">on_gpu</h4>
|
||||
<p>True if your model is currently running on GPUs. Useful to set flags around the LightningModule for different CPU vs GPU behavior. </p>
|
||||
<hr />
|
||||
<h4 id="trainer">trainer</h4>
|
||||
<p>Last resort access to any state the trainer has. Changing certain properties here could affect your training run.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">optimizers</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">trainer</span><span class="o">.</span><span class="n">current_epoch</span>
|
||||
<span class="o">...</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<h2 id="debugging">Debugging</h2>
|
||||
<p>The LightningModule also offers these tricks to help debug. </p>
|
||||
<hr />
|
||||
<h4 id="example_input_array">example_input_array</h4>
|
||||
<p>In the LightningModule init, you can set a dummy tensor for this property
|
||||
to get a print out of sizes coming into and out of every layer. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="c1"># put the dimensions of the first input to your system</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">example_input_array</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">rand</span><span class="p">(</span><span class="mi">5</span><span class="p">,</span> <span class="mi">28</span> <span class="o">*</span> <span class="mi">28</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../methods/" title="Methods" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
Methods
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../../Trainer/" title="Trainer" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Trainer
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,9 +0,0 @@
|
||||
graft docs
|
||||
|
||||
include COPYING
|
||||
include AUTHORS
|
||||
|
||||
recursive-include src/einsteinpy/tests *.py *.html
|
||||
|
||||
prune docs/source/examples/.ipynb_checkpoints
|
||||
global-exclude *.py[cod] __pycache__ *.so *.dylib
|
||||
@@ -1,233 +0,0 @@
|
||||
<p align="center">
|
||||
<a href="https://williamfalcon.github.io/pytorch-lightning/">
|
||||
<img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/lightning_logo.png" width="50">
|
||||
</a>
|
||||
</p>
|
||||
<h3 align="center">
|
||||
Pytorch Lightning
|
||||
</h3>
|
||||
<p align="center">
|
||||
The Keras for ML researchers using PyTorch. More control. Less boilerplate.
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://badge.fury.io/py/pytorch-lightning"><img src="https://badge.fury.io/py/pytorch-lightning.svg" alt="PyPI version" height="18"></a>
|
||||
<!-- <a href="https://travis-ci.org/williamFalcon/test-tube"><img src="https://travis-ci.org/williamFalcon/pytorch-lightning.svg?branch=master"></a> -->
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/COPYING"><img src="https://img.shields.io/badge/License-MIT-yellow.svg"></a>
|
||||
</p>
|
||||
|
||||
```bash
|
||||
pip install pytorch-lightning
|
||||
```
|
||||
|
||||
## Docs
|
||||
**[View the docs here](https://williamfalcon.github.io/pytorch-lightning/)**
|
||||
|
||||
## What is it?
|
||||
Keras and fast.ai are too abstract for researchers. Lightning abstracts the full training loop but gives you control in the critical points.
|
||||
|
||||
|
||||
## Why do I want to use lightning?
|
||||
Because you don't want to define a training loop, validation loop, gradient clipping, checkpointing, loading,
|
||||
gpu training, etc... every time you start a project. Let lightning handle all of that for you! Just define your
|
||||
data and what happens in the training, testing and validation loop and lightning will do the rest.
|
||||
|
||||
To use lightning do 2 things:
|
||||
1. [Define a Trainer](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/trainer_cpu_template.py).
|
||||
2. [Define a LightningModel](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py).
|
||||
|
||||
## What does lightning control for me?
|
||||
Everything!
|
||||
Except for these 6 core functions which you define:
|
||||
|
||||
```{.python}
|
||||
# what to do in the training loop
|
||||
def training_step(self, data_batch, batch_nb):
|
||||
|
||||
# what to do in the validation loop
|
||||
def validation_step(self, data_batch, batch_nb):
|
||||
|
||||
# how to aggregate validation_step outputs
|
||||
def validation_end(self, outputs):
|
||||
|
||||
# and your dataloaders
|
||||
def tng_dataloader():
|
||||
def val_dataloader():
|
||||
def test_dataloader():
|
||||
```
|
||||
|
||||
**Could be as complex as seq-2-seq + attention**
|
||||
|
||||
```python
|
||||
# define what happens for training here
|
||||
def training_step(self, data_batch, batch_nb):
|
||||
x, y = data_batch
|
||||
|
||||
# define your own forward and loss calculation
|
||||
hidden_states = self.encoder(x)
|
||||
|
||||
# even as complex as a seq-2seq + attn model
|
||||
# (this is just a toy, non-working example to illustrate)
|
||||
start_token = '<SOS>'
|
||||
last_hidden = torch.zeros(...)
|
||||
loss = 0
|
||||
for step in range(max_seq_len):
|
||||
attn_context = self.attention_nn(hidden_states, start_token)
|
||||
pred = self.decoder(start_token, attn_context, last_hidden)
|
||||
last_hidden = pred
|
||||
pred = self.predict_nn(pred)
|
||||
loss += self.loss(last_hidden, y[step])
|
||||
|
||||
#toy example as well
|
||||
loss = loss / max_seq_len
|
||||
return {'loss': loss}
|
||||
```
|
||||
|
||||
**Or as basic as CNN image classification**
|
||||
|
||||
```python
|
||||
# define what happens for validation here
|
||||
def validation_step(self, data_batch, batch_nb):
|
||||
x, y = data_batch
|
||||
|
||||
# or as basic as a CNN classification
|
||||
out = self.forward(x)
|
||||
loss = my_loss(out, y)
|
||||
return {'loss': loss}
|
||||
```
|
||||
|
||||
**And you also decide how to collate the output of all validation steps**
|
||||
|
||||
```python
|
||||
def validation_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
val_loss_mean = 0
|
||||
val_acc_mean = 0
|
||||
for output in outputs:
|
||||
val_loss_mean += output['val_loss']
|
||||
val_acc_mean += output['val_acc']
|
||||
|
||||
val_loss_mean /= len(outputs)
|
||||
val_acc_mean /= len(outputs)
|
||||
tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
```
|
||||
|
||||
## Tensorboard
|
||||
Lightning is fully integrated with tensorboard.
|
||||
|
||||
<p align="center">
|
||||
<a href="https://williamfalcon.github.io/pytorch-lightning/">
|
||||
<img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/tf_loss.png" width="900px">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
Lightning also adds a text column with all the hyperparameters for this experiment.
|
||||
|
||||
<p align="center">
|
||||
<a href="https://williamfalcon.github.io/pytorch-lightning/">
|
||||
<img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/docs/source/_static/tf_tags.png" width="900px">
|
||||
</a>
|
||||
</p>
|
||||
|
||||
Simply note the path you set for the Experiment
|
||||
``` {.python}
|
||||
from test_tube import Experiment
|
||||
from pytorch-lightning import Trainer
|
||||
|
||||
exp = Experiment(save_dir='/some/path')
|
||||
trainer = Trainer(experiment=exp)
|
||||
...
|
||||
```
|
||||
|
||||
And run tensorboard from that dir
|
||||
```bash
|
||||
tensorboard --logdir /some/path
|
||||
```
|
||||
|
||||
## Lightning automatically automates all of the following ([each is also configurable](https://williamfalcon.github.io/pytorch-lightning/Trainer/)):
|
||||
|
||||
###### Checkpointing
|
||||
|
||||
- [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
|
||||
- [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics)
|
||||
|
||||
###### Computing cluster (SLURM)
|
||||
|
||||
- [Running grid search on a cluster](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#running-grid-search-on-a-cluster)
|
||||
- [Walltime auto-resubmit](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#walltime-auto-resubmit)
|
||||
|
||||
###### Debugging
|
||||
|
||||
- [Fast dev run](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#fast-dev-run)
|
||||
- [Inspect gradient norms](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#inspect-gradient-norms)
|
||||
- [Log GPU usage](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage)
|
||||
- [Make model overfit on subset of data](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data)
|
||||
- [Print the parameter count by layer](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer)
|
||||
- [Pring which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
|
||||
|
||||
|
||||
###### Distributed training
|
||||
|
||||
- [16-bit mixed precision](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision)
|
||||
- [Multi-GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU)
|
||||
- [Multi-node](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node)
|
||||
- [Single GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#single-gpu)
|
||||
- [Self-balancing architecture](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#self-balancing-architecture)
|
||||
|
||||
|
||||
###### Experiment Logging
|
||||
|
||||
- [Display metrics in progress bar](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#display-metrics-in-progress-bar)
|
||||
- Log arbitrary metrics
|
||||
- [Log metric row every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#log-metric-row-every-k-batches)
|
||||
- [Process position](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#process-position)
|
||||
- [Save a snapshot of all hyperparameters](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#save-a-snapshot-of-all-hyperparameters)
|
||||
- [Snapshot code for a training run](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#snapshot-code-for-a-training-run)
|
||||
- [Write logs file to csv every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#write-logs-file-to-csv-every-k-batches)
|
||||
|
||||
###### Training loop
|
||||
|
||||
- [Accumulate gradients](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients)
|
||||
- [Anneal Learning rate](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#anneal-learning-rate)
|
||||
- [Force training for min or max epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs)
|
||||
- [Force disable early stop](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop)
|
||||
- [Gradient Clipping](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping)
|
||||
- [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/Pytorch-Lightning/LightningModule/#configure_optimizers)
|
||||
- [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check)
|
||||
|
||||
###### Validation loop
|
||||
|
||||
- [Check validation every n epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#check-validation-every-n-epochs)
|
||||
- [Set how much of the validation set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-validation-set-to-check)
|
||||
- [Set how much of the test set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-test-set-to-check)
|
||||
- [Set validation check frequency within 1 training epoch](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-validation-check-frequency-within-1-training-epoch)
|
||||
- [Set the number of validation sanity steps](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-the-number-of-validation-sanity-steps)
|
||||
|
||||
|
||||
## Demo
|
||||
```bash
|
||||
# install lightning
|
||||
pip install pytorch-lightning
|
||||
|
||||
# clone lightning for the demo
|
||||
git clone https://github.com/williamFalcon/pytorch-lightning.git
|
||||
cd examples/new_project_templates/
|
||||
|
||||
# run demo (on cpu)
|
||||
python trainer_gpu_cluster_template.py
|
||||
```
|
||||
|
||||
Without changing the model AT ALL, you can run the model on a single gpu, over multiple gpus, or over multiple nodes.
|
||||
```bash
|
||||
# run a grid search on two gpus
|
||||
python fully_featured_trainer.py --gpus "0;1"
|
||||
|
||||
# run single model on multiple gpus
|
||||
python fully_featured_trainer.py --gpus "0;1" --interactive
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,756 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>Checkpointing - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#model-saving" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Checkpointing
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../.." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
Checkpointing
|
||||
</label>
|
||||
|
||||
<a href="./" title="Checkpointing" class="md-nav__link md-nav__link--active">
|
||||
Checkpointing
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#model-saving" title="Model saving" class="md-nav__link">
|
||||
Model saving
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#restoring-training-session" title="Restoring training session" class="md-nav__link">
|
||||
Restoring training session
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#model-saving" title="Model saving" class="md-nav__link">
|
||||
Model saving
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#restoring-training-session" title="Restoring training session" class="md-nav__link">
|
||||
Restoring training session
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Checkpointing.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>Checkpointing</h1>
|
||||
|
||||
<p>Lightning can automate saving and loading checkpoints.</p>
|
||||
<hr />
|
||||
<h3 id="model-saving">Model saving</h3>
|
||||
<p>Checkpointing is enabled by default to the current working directory.
|
||||
To change the checkpoint path pass in :</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">Trainer</span><span class="p">(</span><span class="n">default_save_path</span><span class="o">=</span><span class="s1">'/your/path/to/save/checkpoints'</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>To modify the behavior of checkpointing pass in your own callback.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.callbacks</span> <span class="kn">import</span> <span class="n">ModelCheckpoint</span>
|
||||
|
||||
<span class="c1"># DEFAULTS used by the Trainer</span>
|
||||
<span class="n">checkpoint_callback</span> <span class="o">=</span> <span class="n">ModelCheckpoint</span><span class="p">(</span>
|
||||
<span class="n">filepath</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">getcwd</span><span class="p">(),</span>
|
||||
<span class="n">save_best_only</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
|
||||
<span class="n">verbose</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
|
||||
<span class="n">monitor</span><span class="o">=</span><span class="s1">'val_loss'</span><span class="p">,</span>
|
||||
<span class="n">mode</span><span class="o">=</span><span class="s1">'min'</span><span class="p">,</span>
|
||||
<span class="n">prefix</span><span class="o">=</span><span class="s1">''</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">checkpoint_callback</span><span class="o">=</span><span class="n">checkpoint_callback</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h3 id="restoring-training-session">Restoring training session</h3>
|
||||
<p>You might want to not only load a model but also continue training it. Use this method to
|
||||
restore the trainer state as well. This will continue from the epoch and global step you last left off.<br />
|
||||
However, the dataloaders will start from the first batch again (if you shuffled it shouldn't matter). </p>
|
||||
<p>Lightning will restore the session if you pass a logger with the same version and there's a saved checkpoint. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning</span> <span class="kn">import</span> <span class="n">Trainer</span>
|
||||
<span class="kn">from</span> <span class="nn">pytorch_lightning.logging</span> <span class="kn">import</span> <span class="n">TestTubeLogger</span>
|
||||
|
||||
<span class="n">logger</span> <span class="o">=</span> <span class="n">TestTubeLogger</span><span class="p">(</span>
|
||||
<span class="n">save_dir</span><span class="o">=</span><span class="s1">'./savepath'</span><span class="p">,</span>
|
||||
<span class="n">version</span><span class="o">=</span><span class="mi">1</span> <span class="c1"># An existing version with a saved checkpoint</span>
|
||||
<span class="p">)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span>
|
||||
<span class="n">logger</span><span class="o">=</span><span class="n">logger</span><span class="p">,</span>
|
||||
<span class="n">default_save_path</span><span class="o">=</span><span class="s1">'./savepath'</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># this fit call loads model weights and trainer state</span>
|
||||
<span class="c1"># the trainer continues seamlessly from where you left off</span>
|
||||
<span class="c1"># without having to do anything else.</span>
|
||||
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>The trainer restores: </p>
|
||||
<ul>
|
||||
<li>global_step </li>
|
||||
<li>current_epoch </li>
|
||||
<li>All optimizers </li>
|
||||
<li>All lr_schedulers </li>
|
||||
<li>Model weights</li>
|
||||
</ul>
|
||||
<p>You can even change the logic of your model as long as the weights and "architecture" of
|
||||
the system isn't different. If you add a layer, for instance, it might not work. </p>
|
||||
<p>At a rough level, here's <a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/root_module/model_saving.py#L63">what happens inside Trainer</a>: </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="bp">self</span><span class="o">.</span><span class="n">global_step</span> <span class="o">=</span> <span class="n">checkpoint</span><span class="p">[</span><span class="s1">'global_step'</span><span class="p">]</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">current_epoch</span> <span class="o">=</span> <span class="n">checkpoint</span><span class="p">[</span><span class="s1">'epoch'</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># restore the optimizers</span>
|
||||
<span class="n">optimizer_states</span> <span class="o">=</span> <span class="n">checkpoint</span><span class="p">[</span><span class="s1">'optimizer_states'</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">opt_state</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">optimizers</span><span class="p">,</span> <span class="n">optimizer_states</span><span class="p">):</span>
|
||||
<span class="n">optimizer</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">opt_state</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># restore the lr schedulers</span>
|
||||
<span class="n">lr_schedulers</span> <span class="o">=</span> <span class="n">checkpoint</span><span class="p">[</span><span class="s1">'lr_schedulers'</span><span class="p">]</span>
|
||||
<span class="k">for</span> <span class="n">scheduler</span><span class="p">,</span> <span class="n">lrs_state</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_schedulers</span><span class="p">,</span> <span class="n">lr_schedulers</span><span class="p">):</span>
|
||||
<span class="n">scheduler</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">lrs_state</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># uses the model you passed into trainer </span>
|
||||
<span class="n">model</span><span class="o">.</span><span class="n">load_state_dict</span><span class="p">(</span><span class="n">checkpoint</span><span class="p">[</span><span class="s1">'state_dict'</span><span class="p">])</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../" title="Trainer" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
Trainer
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Distributed training/" title="Distributed training" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Distributed training
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,786 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>SLURM Managed Cluster - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#running-grid-search-on-a-cluster" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
SLURM Managed Cluster
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../.." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
SLURM Managed Cluster
|
||||
</label>
|
||||
|
||||
<a href="./" title="SLURM Managed Cluster" class="md-nav__link md-nav__link--active">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#running-grid-search-on-a-cluster" title="Running grid search on a cluster" class="md-nav__link">
|
||||
Running grid search on a cluster
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#walltime-auto-resubmit" title="Walltime auto-resubmit" class="md-nav__link">
|
||||
Walltime auto-resubmit
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#running-grid-search-on-a-cluster" title="Running grid search on a cluster" class="md-nav__link">
|
||||
Running grid search on a cluster
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#walltime-auto-resubmit" title="Walltime auto-resubmit" class="md-nav__link">
|
||||
Walltime auto-resubmit
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/SLURM Managed Cluster.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>SLURM Managed Cluster</h1>
|
||||
|
||||
<p>Lightning supports model training on a cluster managed by SLURM in the following cases: </p>
|
||||
<ol>
|
||||
<li>Training on a single cpu or single GPU.</li>
|
||||
<li>Train on multiple GPUs on the same node using DataParallel or DistributedDataParallel</li>
|
||||
<li>Training across multiple GPUs on multiple different nodes via DistributedDataParallel.</li>
|
||||
</ol>
|
||||
<p><strong>Note: A node means a machine with multiple GPUs</strong></p>
|
||||
<hr />
|
||||
<h4 id="running-grid-search-on-a-cluster">Running grid search on a cluster</h4>
|
||||
<p>To use lightning to run a hyperparameter search (grid-search or random-search) on a cluster do 4 things: </p>
|
||||
<p>(1). Define the parameters for the grid search </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">test_tube</span> <span class="kn">import</span> <span class="n">HyperOptArgumentParser</span>
|
||||
|
||||
<span class="c1"># subclass of argparse</span>
|
||||
<span class="n">parser</span> <span class="o">=</span> <span class="n">HyperOptArgumentParser</span><span class="p">(</span><span class="n">strategy</span><span class="o">=</span><span class="s1">'random_search'</span><span class="p">)</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">add_argument</span><span class="p">(</span><span class="s1">'--learning_rate'</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="mf">0.002</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">float</span><span class="p">,</span> <span class="n">help</span><span class="o">=</span><span class="s1">'the learning rate'</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># let's enable optimizing over the number of layers in the network</span>
|
||||
<span class="n">parser</span><span class="o">.</span><span class="n">opt_list</span><span class="p">(</span><span class="s1">'--nb_layers'</span><span class="p">,</span> <span class="n">default</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="nb">type</span><span class="o">=</span><span class="nb">int</span><span class="p">,</span> <span class="n">tunable</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">options</span><span class="o">=</span><span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">8</span><span class="p">])</span>
|
||||
|
||||
<span class="n">hparams</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_args</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p><strong>NOTE</strong> You must set <code>Tunable=True</code> for that argument to be considered in the permutation set. Otherwise
|
||||
test-tube will use the default value. This flag is useful when you don't want to search over an argument and
|
||||
want to use the default instead. </p>
|
||||
<p>(2). Define the cluster options in the <a href="https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/">SlurmCluster object</a> (over 5 nodes and 8 gpus) </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16
|
||||
17
|
||||
18
|
||||
19
|
||||
20
|
||||
21
|
||||
22
|
||||
23
|
||||
24
|
||||
25
|
||||
26</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">test_tube.hpc</span> <span class="kn">import</span> <span class="n">SlurmCluster</span>
|
||||
|
||||
<span class="c1"># hyperparameters is a test-tube hyper params object</span>
|
||||
<span class="c1"># see https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/</span>
|
||||
<span class="n">hyperparams</span> <span class="o">=</span> <span class="n">args</span><span class="o">.</span><span class="n">parse</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># init cluster</span>
|
||||
<span class="n">cluster</span> <span class="o">=</span> <span class="n">SlurmCluster</span><span class="p">(</span>
|
||||
<span class="n">hyperparam_optimizer</span><span class="o">=</span><span class="n">hyperparams</span><span class="p">,</span>
|
||||
<span class="n">log_path</span><span class="o">=</span><span class="s1">'/path/to/log/results/to'</span><span class="p">,</span>
|
||||
<span class="n">python_cmd</span><span class="o">=</span><span class="s1">'python3'</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># let the cluster know where to email for a change in job status (ie: complete, fail, etc...)</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">notify_job_status</span><span class="p">(</span><span class="n">email</span><span class="o">=</span><span class="s1">'some@email.com'</span><span class="p">,</span> <span class="n">on_done</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">on_fail</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># set the job options. In this instance, we'll run 20 different models</span>
|
||||
<span class="c1"># each with its own set of hyperparameters giving each one 1 GPU (ie: taking up 20 GPUs)</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">per_experiment_nb_gpus</span> <span class="o">=</span> <span class="mi">8</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">per_experiment_nb_nodes</span> <span class="o">=</span> <span class="mi">5</span>
|
||||
|
||||
<span class="c1"># we'll request 10GB of memory per node</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">memory_mb_per_node</span> <span class="o">=</span> <span class="mi">10000</span>
|
||||
|
||||
<span class="c1"># set a walltime of 10 minues</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">job_time</span> <span class="o">=</span> <span class="s1">'10:00'</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>(3). Make a main function with your model and trainer. Each job will call this function with a particular
|
||||
hparams configuration. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning</span> <span class="kn">import</span> <span class="n">Trainer</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">train_fx</span><span class="p">(</span><span class="n">trial_hparams</span><span class="p">,</span> <span class="n">cluster_manager</span><span class="p">,</span> <span class="n">_</span><span class="p">):</span>
|
||||
<span class="c1"># hparams has a specific set of hyperparams</span>
|
||||
|
||||
<span class="n">my_model</span> <span class="o">=</span> <span class="n">MyLightningModel</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># give the trainer the cluster object</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">()</span>
|
||||
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">my_model</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>(3). Start the grid/random search </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># run the models on the cluster</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">optimize_parallel_cluster_gpu</span><span class="p">(</span>
|
||||
<span class="n">train_fx</span><span class="p">,</span>
|
||||
<span class="n">nb_trials</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span>
|
||||
<span class="n">job_name</span><span class="o">=</span><span class="s1">'my_grid_search_exp_name'</span><span class="p">,</span>
|
||||
<span class="n">job_display_name</span><span class="o">=</span><span class="s1">'my_exp'</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p><strong>NOTE</strong> nb_trials specifies how many of the possible permutations to use. If using <code>grid_search</code> it will use
|
||||
the depth first ordering. If using <code>random_search</code> it will use the first k shuffled options. FYI, random search
|
||||
has been shown to be just as good as any Bayesian optimization method when using a reasonable number of samples (60),
|
||||
<a href="http://www.jmlr.org/papers/volume13/bergstra12a/bergstra12a.pdf">see this paper for more information</a>.</p>
|
||||
<hr />
|
||||
<h4 id="walltime-auto-resubmit">Walltime auto-resubmit</h4>
|
||||
<p>Lightning automatically resubmits jobs when they reach the walltime. Make sure to set the SIGUSR1 signal in
|
||||
your SLURM script. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># 90 seconds before training ends</span>
|
||||
<span class="c1">#SBATCH --signal=SIGUSR1@90</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>When lightning receives the SIGUSR1 signal it will:
|
||||
1. save a checkpoint with 'hpc_ckpt' in the name.
|
||||
2. resubmit the job using the SLURM_JOB_ID </p>
|
||||
<p>When the script starts again, Lightning will:
|
||||
1. search for a 'hpc_ckpt' checkpoint.
|
||||
2. restore the model, optimizers, schedulers, epoch, etc... </p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../Logging/" title="Logging" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
Logging
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Testing loop/" title="Testing loop" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Testing loop
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,667 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>Testing loop - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#test" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Testing loop
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../.." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
Testing loop
|
||||
</label>
|
||||
|
||||
<a href="./" title="Testing loop" class="md-nav__link md-nav__link--active">
|
||||
Testing loop
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#test" title="test" class="md-nav__link">
|
||||
test
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#test" title="test" class="md-nav__link">
|
||||
test
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Testing loop.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>Testing loop</h1>
|
||||
|
||||
<p>To ensure you don't accidentally use test data to guide training decisions Lightning makes running the test set deliberate. </p>
|
||||
<hr />
|
||||
<h4 id="test">test</h4>
|
||||
<p>You have two options to run the test set.
|
||||
First case is where you test right after a full training routine.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># run full training</span>
|
||||
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># run test set</span>
|
||||
<span class="n">trainer</span><span class="o">.</span><span class="n">test</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>Second case is where you load a model and run the test set </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">model</span> <span class="o">=</span> <span class="n">MyLightningModule</span><span class="o">.</span><span class="n">load_from_metrics</span><span class="p">(</span>
|
||||
<span class="n">weights_path</span><span class="o">=</span><span class="s1">'/path/to/pytorch_checkpoint.ckpt'</span><span class="p">,</span>
|
||||
<span class="n">tags_csv</span><span class="o">=</span><span class="s1">'/path/to/test_tube/experiment/version/meta_tags.csv'</span><span class="p">,</span>
|
||||
<span class="n">on_gpu</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span>
|
||||
<span class="n">map_location</span><span class="o">=</span><span class="bp">None</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># init trainer with whatever options</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># test (pass in the model)</span>
|
||||
<span class="n">trainer</span><span class="o">.</span><span class="n">test</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>In this second case, the options you pass to trainer will be used when running the test set (ie: 16-bit, dp, ddp, etc...) </p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
SLURM Managed Cluster
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Training Loop/" title="Training Loop" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Training Loop
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,896 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>Training Loop - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#accumulated-gradients" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Training Loop
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../.." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
Training Loop
|
||||
</label>
|
||||
|
||||
<a href="./" title="Training Loop" class="md-nav__link md-nav__link--active">
|
||||
Training Loop
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#accumulated-gradients" title="Accumulated gradients" class="md-nav__link">
|
||||
Accumulated gradients
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#force-training-for-min-or-max-epochs" title="Force training for min or max epochs" class="md-nav__link">
|
||||
Force training for min or max epochs
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#early-stopping" title="Early stopping" class="md-nav__link">
|
||||
Early stopping
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#force-disable-early-stop" title="Force disable early stop" class="md-nav__link">
|
||||
Force disable early stop
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#gradient-clipping" title="Gradient Clipping" class="md-nav__link">
|
||||
Gradient Clipping
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
|
||||
Inspect gradient norms
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-how-much-of-the-training-set-to-check" title="Set how much of the training set to check" class="md-nav__link">
|
||||
Set how much of the training set to check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#packed-sequences-as-inputs" title="Packed sequences as inputs" class="md-nav__link">
|
||||
Packed sequences as inputs
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#truncated-back-propagation-through-time" title="Truncated Back Propagation Through Time" class="md-nav__link">
|
||||
Truncated Back Propagation Through Time
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#accumulated-gradients" title="Accumulated gradients" class="md-nav__link">
|
||||
Accumulated gradients
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#force-training-for-min-or-max-epochs" title="Force training for min or max epochs" class="md-nav__link">
|
||||
Force training for min or max epochs
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#early-stopping" title="Early stopping" class="md-nav__link">
|
||||
Early stopping
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#force-disable-early-stop" title="Force disable early stop" class="md-nav__link">
|
||||
Force disable early stop
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#gradient-clipping" title="Gradient Clipping" class="md-nav__link">
|
||||
Gradient Clipping
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
|
||||
Inspect gradient norms
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-how-much-of-the-training-set-to-check" title="Set how much of the training set to check" class="md-nav__link">
|
||||
Set how much of the training set to check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#packed-sequences-as-inputs" title="Packed sequences as inputs" class="md-nav__link">
|
||||
Packed sequences as inputs
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#truncated-back-propagation-through-time" title="Truncated Back Propagation Through Time" class="md-nav__link">
|
||||
Truncated Back Propagation Through Time
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Training Loop.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>Training Loop</h1>
|
||||
|
||||
<p>The lightning training loop handles everything except the actual computations of your model. To decide what will happen in your training loop, define the <a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#training_step">training_step function</a>.</p>
|
||||
<p>Below are all the things lightning automates for you in the training loop.</p>
|
||||
<hr />
|
||||
<h4 id="accumulated-gradients">Accumulated gradients</h4>
|
||||
<p>Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (ie: no accumulated grads)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">accumulate_grad_batches</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="force-training-for-min-or-max-epochs">Force training for min or max epochs</h4>
|
||||
<p>It can be useful to force training for a minimum number of epochs or limit to a max number</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">min_nb_epochs</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">max_nb_epochs</span><span class="o">=</span><span class="mi">1000</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="early-stopping">Early stopping</h4>
|
||||
<p>The trainer already sets up default early stopping for you.
|
||||
To modify this behavior, pass in your own EarlyStopping callback.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16
|
||||
17
|
||||
18
|
||||
19</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.callbacks</span> <span class="kn">import</span> <span class="n">EarlyStopping</span>
|
||||
|
||||
<span class="c1"># DEFAULTS used by Trainer</span>
|
||||
<span class="n">early_stop_callback</span> <span class="o">=</span> <span class="n">EarlyStopping</span><span class="p">(</span>
|
||||
<span class="n">monitor</span><span class="o">=</span><span class="s1">'val_loss'</span><span class="p">,</span>
|
||||
<span class="n">min_delta</span><span class="o">=</span><span class="mf">0.00</span><span class="p">,</span>
|
||||
<span class="n">patience</span><span class="o">=</span><span class="mi">3</span><span class="p">,</span>
|
||||
<span class="n">verbose</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
|
||||
<span class="n">mode</span><span class="o">=</span><span class="s1">'min'</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># without passing anything in, uses the default callback above</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># pass in your own to override the default callback</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">early_stop_callback</span><span class="o">=</span><span class="n">early_stop_callback</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># pass in None to disable it</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">early_stop_callback</span><span class="o">=</span><span class="bp">None</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="force-disable-early-stop">Force disable early stop</h4>
|
||||
<p>To disable early stopping pass None to the early_stop_callback</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">early_stop_callback</span><span class="o">=</span><span class="bp">None</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="gradient-clipping">Gradient Clipping</h4>
|
||||
<p>Gradient clipping may be enabled to avoid exploding gradients.
|
||||
Specifically, this will <a href="https://pytorch.org/docs/stable/nn.html#torch.nn.utils.clip_grad_norm_">clip the gradient norm computed over all model parameters <em>together</em></a>.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (ie: don't clip)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">gradient_clip_val</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># clip gradients with norm above 0.5</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">gradient_clip_val</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="inspect-gradient-norms">Inspect gradient norms</h4>
|
||||
<p>Looking at grad norms can help you figure out where training might be going wrong.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (-1 doesn't track norms)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">track_grad_norm</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># track the LP norm (P=2 here)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">track_grad_norm</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="set-how-much-of-the-training-set-to-check">Set how much of the training set to check</h4>
|
||||
<p>If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag.</p>
|
||||
<p>train_percent_check will be overwritten by overfit_pct if <code>overfit_pct > 0</code></p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">train_percent_check</span><span class="o">=</span><span class="mf">1.0</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># check 10% only</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">train_percent_check</span><span class="o">=</span><span class="mf">0.1</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="packed-sequences-as-inputs">Packed sequences as inputs</h4>
|
||||
<p>When using PackedSequence, do 2 things:
|
||||
1. return either a padded tensor in dataset or a list of variable length tensors in the dataloader collate_fn (example above shows the list implementation). <br />
|
||||
2. Pack the sequence in forward or training and validation steps depending on use case.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># For use in dataloader</span>
|
||||
<span class="k">def</span> <span class="nf">collate_fn</span><span class="p">(</span><span class="n">batch</span><span class="p">):</span>
|
||||
<span class="n">x</span> <span class="o">=</span> <span class="p">[</span><span class="n">item</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">batch</span><span class="p">]</span>
|
||||
<span class="n">y</span> <span class="o">=</span> <span class="p">[</span><span class="n">item</span><span class="p">[</span><span class="mi">1</span><span class="p">]</span> <span class="k">for</span> <span class="n">item</span> <span class="ow">in</span> <span class="n">batch</span><span class="p">]</span>
|
||||
<span class="k">return</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span>
|
||||
|
||||
<span class="c1"># In module</span>
|
||||
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
|
||||
<span class="n">x</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">pack_sequence</span><span class="p">(</span><span class="n">batch</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">enforce_sorted</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
|
||||
<span class="n">y</span> <span class="o">=</span> <span class="n">rnn</span><span class="o">.</span><span class="n">pack_sequence</span><span class="p">(</span><span class="n">batch</span><span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="n">enforce_sorted</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="truncated-back-propagation-through-time">Truncated Back Propagation Through Time</h4>
|
||||
<p>There are times when multiple backwards passes are needed for each batch. For example, it may save memory to use Truncated Back Propagation Through Time when training RNNs.</p>
|
||||
<p>When this flag is enabled each batch is split into sequences of size truncated_bptt_steps and passed to training_step(...) separately. A default splitting function is provided, however, you can override it for more flexibility. See <a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks#tbptt_split_batch">tbptt_split_batch</a>.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (single backwards pass per batch)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">truncated_bptt_steps</span><span class="o">=</span><span class="bp">None</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># (split batch into sequences of size 2)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">truncated_bptt_steps</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../Testing loop/" title="Testing loop" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
Testing loop
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../Validation loop/" title="Validation loop" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Validation loop
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,757 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>Validation loop - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#check-validation-every-n-epochs" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Validation loop
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../.." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
Validation loop
|
||||
</label>
|
||||
|
||||
<a href="./" title="Validation loop" class="md-nav__link md-nav__link--active">
|
||||
Validation loop
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#check-validation-every-n-epochs" title="Check validation every n epochs" class="md-nav__link">
|
||||
Check validation every n epochs
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-how-much-of-the-validation-set-to-check" title="Set how much of the validation set to check" class="md-nav__link">
|
||||
Set how much of the validation set to check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-how-much-of-the-test-set-to-check" title="Set how much of the test set to check" class="md-nav__link">
|
||||
Set how much of the test set to check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-validation-check-frequency-within-1-training-epoch" title="Set validation check frequency within 1 training epoch" class="md-nav__link">
|
||||
Set validation check frequency within 1 training epoch
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-the-number-of-validation-sanity-steps" title="Set the number of validation sanity steps" class="md-nav__link">
|
||||
Set the number of validation sanity steps
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#check-validation-every-n-epochs" title="Check validation every n epochs" class="md-nav__link">
|
||||
Check validation every n epochs
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-how-much-of-the-validation-set-to-check" title="Set how much of the validation set to check" class="md-nav__link">
|
||||
Set how much of the validation set to check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-how-much-of-the-test-set-to-check" title="Set how much of the test set to check" class="md-nav__link">
|
||||
Set how much of the test set to check
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-validation-check-frequency-within-1-training-epoch" title="Set validation check frequency within 1 training epoch" class="md-nav__link">
|
||||
Set validation check frequency within 1 training epoch
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#set-the-number-of-validation-sanity-steps" title="Set the number of validation sanity steps" class="md-nav__link">
|
||||
Set the number of validation sanity steps
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Validation loop.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>Validation loop</h1>
|
||||
|
||||
<p>The lightning validation loop handles everything except the actual computations of your model. To decide what will happen in your validation loop, define the <a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#validation_step">validation_step function</a>.
|
||||
Below are all the things lightning automates for you in the validation loop.</p>
|
||||
<p><strong>Note</strong> <br />
|
||||
Lightning will run 5 steps of validation in the beginning of training as a sanity check so you don't have to wait until a full epoch to catch possible validation issues.</p>
|
||||
<hr />
|
||||
<h4 id="check-validation-every-n-epochs">Check validation every n epochs</h4>
|
||||
<p>If you have a small dataset you might want to check validation every n epochs</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">check_val_every_n_epoch</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="set-how-much-of-the-validation-set-to-check">Set how much of the validation set to check</h4>
|
||||
<p>If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag</p>
|
||||
<p>val_percent_check will be overwritten by overfit_pct if <code>overfit_pct > 0</code></p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">val_percent_check</span><span class="o">=</span><span class="mf">1.0</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># check 10% only</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">val_percent_check</span><span class="o">=</span><span class="mf">0.1</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="set-how-much-of-the-test-set-to-check">Set how much of the test set to check</h4>
|
||||
<p>If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag</p>
|
||||
<p>test_percent_check will be overwritten by overfit_pct if <code>overfit_pct > 0</code></p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">test_percent_check</span><span class="o">=</span><span class="mf">1.0</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># check 10% only</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">test_percent_check</span><span class="o">=</span><span class="mf">0.1</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="set-validation-check-frequency-within-1-training-epoch">Set validation check frequency within 1 training epoch</h4>
|
||||
<p>For large datasets it's often desirable to check validation multiple times within a training loop.
|
||||
Pass in a float to check that often within 1 training epoch.
|
||||
Pass in an int k to check every k training batches. Must use an int if using
|
||||
an IterableDataset.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">val_check_interval</span><span class="o">=</span><span class="mf">0.95</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># check every .25 of an epoch </span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">val_check_interval</span><span class="o">=</span><span class="mf">0.25</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># check every 100 train batches (ie: for IterableDatasets or fixed frequency)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">val_check_interval</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="set-the-number-of-validation-sanity-steps">Set the number of validation sanity steps</h4>
|
||||
<p>Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">nb_sanity_val_steps</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>You can use <code>Trainer(nb_sanity_val_steps=0)</code> to skip the sanity check.</p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../Training Loop/" title="Training Loop" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
Training Loop
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../debugging/" title="Debugging" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Debugging
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,761 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>Debugging - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#fast-dev-run" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Debugging
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../.." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
Debugging
|
||||
</label>
|
||||
|
||||
<a href="./" title="Debugging" class="md-nav__link md-nav__link--active">
|
||||
Debugging
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#fast-dev-run" title="Fast dev run" class="md-nav__link">
|
||||
Fast dev run
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
|
||||
Inspect gradient norms
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#make-model-overfit-on-subset-of-data" title="Make model overfit on subset of data" class="md-nav__link">
|
||||
Make model overfit on subset of data
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#print-the-parameter-count-by-layer" title="Print the parameter count by layer" class="md-nav__link">
|
||||
Print the parameter count by layer
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#print-which-gradients-are-nan" title="Print which gradients are nan" class="md-nav__link">
|
||||
Print which gradients are nan
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#log-gpu-usage" title="Log GPU usage" class="md-nav__link">
|
||||
Log GPU usage
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#fast-dev-run" title="Fast dev run" class="md-nav__link">
|
||||
Fast dev run
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#inspect-gradient-norms" title="Inspect gradient norms" class="md-nav__link">
|
||||
Inspect gradient norms
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#make-model-overfit-on-subset-of-data" title="Make model overfit on subset of data" class="md-nav__link">
|
||||
Make model overfit on subset of data
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#print-the-parameter-count-by-layer" title="Print the parameter count by layer" class="md-nav__link">
|
||||
Print the parameter count by layer
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#print-which-gradients-are-nan" title="Print which gradients are nan" class="md-nav__link">
|
||||
Print which gradients are nan
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#log-gpu-usage" title="Log GPU usage" class="md-nav__link">
|
||||
Log GPU usage
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/debugging.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>Debugging</h1>
|
||||
|
||||
<p>These flags are useful to help debug a model.</p>
|
||||
<hr />
|
||||
<h4 id="fast-dev-run">Fast dev run</h4>
|
||||
<p>This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch.
|
||||
Use this to debug a full run of your program quickly</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">fast_dev_run</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="inspect-gradient-norms">Inspect gradient norms</h4>
|
||||
<p>Looking at grad norms can help you figure out where training might be going wrong.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (-1 doesn't track norms)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">track_grad_norm</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># track the LP norm (P=2 here)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">track_grad_norm</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="make-model-overfit-on-subset-of-data">Make model overfit on subset of data</h4>
|
||||
<p>A useful debugging trick is to make your model overfit a tiny fraction of the data.</p>
|
||||
<p>setting <code>overfit_pct > 0</code> will overwrite train_percent_check, val_percent_check, test_percent_check</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT don't overfit (ie: normal training)</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">overfit_pct</span><span class="o">=</span><span class="mf">0.0</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># overfit on 1% of data </span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">overfit_pct</span><span class="o">=</span><span class="mf">0.01</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="print-the-parameter-count-by-layer">Print the parameter count by layer</h4>
|
||||
<p>By default lightning prints a list of parameters <em>and submodules</em> when it starts training.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT print a full list of all submodules and their parameters.</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">weights_summary</span><span class="o">=</span><span class="s1">'full'</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># only print the top-level modules (i.e. the children of LightningModule).</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">weights_summary</span><span class="o">=</span><span class="s1">'top'</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="print-which-gradients-are-nan">Print which gradients are nan</h4>
|
||||
<p>This option prints a list of tensors with nan gradients.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">print_nan_grads</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="log-gpu-usage">Log GPU usage</h4>
|
||||
<p>Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.</p>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../Validation loop/" title="Validation loop" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
Validation loop
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="../hooks/" title="Hooks" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Hooks
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,669 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>Trainer - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#trainer" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href=".." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Trainer
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href=".." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href=".." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="./" title="Trainer" class="md-nav__link md-nav__link--active">
|
||||
Trainer
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/index.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1 id="trainer">Trainer</h1>
|
||||
<p>[<a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/trainer/trainer.py">Github Code</a>]</p>
|
||||
<p>The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research.</p>
|
||||
<p>This is the basic use of the trainer:</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning</span> <span class="kn">import</span> <span class="n">Trainer</span>
|
||||
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">LightningTemplate</span><span class="p">()</span>
|
||||
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">()</span>
|
||||
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>But of course the fun is in all the advanced things it can do:</p>
|
||||
<p><strong>Checkpointing</strong> </p>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving">Checkpoint callback</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving">Model saving</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics">Model loading</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#restoring-training-session">Restoring training session</a></li>
|
||||
</ul>
|
||||
<p><strong>Computing cluster (SLURM)</strong> </p>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#running-grid-search-on-a-cluster">Running grid search on a cluster</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#walltime-auto-resubmit">Walltime auto-resubmit</a> </li>
|
||||
</ul>
|
||||
<p><strong>Debugging</strong> </p>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#fast-dev-run">Fast dev run</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#inspect-gradient-norms">Inspect gradient norms</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage">Log GPU usage</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data">Make model overfit on subset of data</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer">Print the parameter count by layer</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan">Print which gradients are nan</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/properties/#example_input_array">Print input and output size of every module in system</a></li>
|
||||
</ul>
|
||||
<p><strong>Distributed training</strong> </p>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#init_ddp_connection">Implement Your Own Distributed (DDP) training</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision">16-bit mixed precision</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU">Multi-GPU</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node">Multi-node</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#single-gpu">Single GPU</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#self-balancing-architecture">Self-balancing architecture</a></li>
|
||||
</ul>
|
||||
<p><strong>Experiment Logging</strong> </p>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#display-metrics-in-progress-bar">Display metrics in progress bar</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#log-metric-row-every-k-batches">Log metric row every k batches</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#process-position">Process position</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#tensorboard-support">Tensorboard support</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#save-a-snapshot-of-all-hyperparameters">Save a snapshot of all hyperparameters</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#snapshot-code-for-a-training-run">Snapshot code for a training run</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#write-logs-file-to-csv-every-k-batches">Write logs file to csv every k batches</a></li>
|
||||
</ul>
|
||||
<p><strong>Training loop</strong> </p>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients">Accumulate gradients</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs">Force training for min or max epochs</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#early-stopping">Early stopping callback</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop">Force disable early stop</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping">Gradient Clipping</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/">Hooks</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers">Learning rate scheduling</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers">Use multiple optimizers (like GANs)</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check">Set how much of the training set to check (1-100%)</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step">Step optimizers at arbitrary intervals</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#packed-sequences-as-inputs">Packed sequences</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning//Training%20Loop/#truncated-back-propation-through-time">Truncated Back Propagation Through Time</a></li>
|
||||
</ul>
|
||||
<p><strong>Validation loop</strong> </p>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#check-validation-every-n-epochs">Check validation every n epochs</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/">Hooks</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-validation-set-to-check">Set how much of the validation set to check</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-test-set-to-check">Set how much of the test set to check</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-validation-check-frequency-within-1-training-epoch">Set validation check frequency within 1 training epoch</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-the-number-of-validation-sanity-steps">Set the number of validation sanity steps</a></li>
|
||||
</ul>
|
||||
<p><strong>Testing loop</strong> </p>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Testing%20loop/">Run test set</a> </li>
|
||||
</ul>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../LightningModule/properties/" title="Properties" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
Properties
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
<a href="Checkpointing/" title="Checkpointing" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Checkpointing
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:".."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
Vendored
+4
File diff suppressed because one or more lines are too long
@@ -0,0 +1,13 @@
|
||||
/*!
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
|
||||
* use this file except in compliance with the License. You may obtain a copy
|
||||
* of the License at:
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING, SOFTWARE
|
||||
* DISTRIBUTED UNDER THE LICENSE IS DISTRIBUTED ON AN "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
|
||||
* SEE THE LICENSE FOR THE SPECIFIC LANGUAGE GOVERNING PERMISSIONS AND
|
||||
* LIMITATIONS UNDER THE LICENSE.
|
||||
*/@font-face{font-family:"Material Icons";font-style:normal;font-weight:400;src:local("Material Icons"),local("MaterialIcons-Regular"),url("specimen/MaterialIcons-Regular.woff2") format("woff2"),url("specimen/MaterialIcons-Regular.woff") format("woff"),url("specimen/MaterialIcons-Regular.ttf") format("truetype")}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 521 B |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="352" height="448" viewBox="0 0 352 448" id="__bitbucket"><path fill="currentColor" d="M203.75 214.75q2 15.75-12.625 25.25t-27.875 1.5q-9.75-4.25-13.375-14.5t-.125-20.5 13-14.5q9-4.5 18.125-3t16 8.875 6.875 16.875zm27.75-5.25q-3.5-26.75-28.25-41T154 165.25q-15.75 7-25.125 22.125t-8.625 32.375q1 22.75 19.375 38.75t41.375 14q22.75-2 38-21t12.5-42zM291.25 74q-5-6.75-14-11.125t-14.5-5.5T245 54.25q-72.75-11.75-141.5.5-10.75 1.75-16.5 3t-13.75 5.5T60.75 74q7.5 7 19 11.375t18.375 5.5T120 93.75Q177 101 232 94q15.75-2 22.375-3t18.125-5.375T291.25 74zm14.25 258.75q-2 6.5-3.875 19.125t-3.5 21-7.125 17.5-14.5 14.125q-21.5 12-47.375 17.875t-50.5 5.5-50.375-4.625q-11.5-2-20.375-4.5T88.75 412 70.5 401.125t-13-15.375q-6.25-24-14.25-73l1.5-4 4.5-2.25q55.75 37 126.625 37t126.875-37q5.25 1.5 6 5.75t-1.25 11.25-2 9.25zM350.75 92.5q-6.5 41.75-27.75 163.75-1.25 7.5-6.75 14t-10.875 10T291.75 288q-63 31.5-152.5 22-62-6.75-98.5-34.75-3.75-3-6.375-6.625t-4.25-8.75-2.25-8.5-1.5-9.875T25 232.75q-2.25-12.5-6.625-37.5t-7-40.375T5.5 118 0 78.5Q.75 72 4.375 66.375T12.25 57t11.25-7.5T35 43.875t12-4.625q31.25-11.5 78.25-16 94.75-9.25 169 12.5Q333 47.25 348 66.25q4 5 4.125 12.75t-1.375 13.5z"/></svg>
|
||||
|
After Width: | Height: | Size: 1.2 KiB |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
After Width: | Height: | Size: 993 B |
@@ -0,0 +1 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="500" height="500" viewBox="0 0 500 500" id="__gitlab"><path fill="currentColor" d="M93.667 473.347l90.684-279.097H2.983l90.684 279.097z" transform="translate(156.198 1.16)"/><path fill="currentColor" d="M221.333 473.345L130.649 194.25H3.557l217.776 279.095z" transform="translate(28.531 1.16)" opacity=".7"/><path fill="currentColor" d="M32 195.155L4.441 279.97a18.773 18.773 0 0 0 6.821 20.99l238.514 173.29L32 195.155z" transform="translate(.089 .256)" opacity=".5"/><path fill="currentColor" d="M2.667-84.844h127.092L75.14-252.942c-2.811-8.649-15.047-8.649-17.856 0L2.667-84.844z" transform="translate(29.422 280.256)"/><path fill="currentColor" d="M2.667 473.345L93.351 194.25h127.092L2.667 473.345z" transform="translate(247.198 1.16)" opacity=".7"/><path fill="currentColor" d="M221.334 195.155l27.559 84.815a18.772 18.772 0 0 1-6.821 20.99L3.557 474.25l217.777-279.095z" transform="translate(246.307 .256)" opacity=".5"/><path fill="currentColor" d="M130.667-84.844H3.575l54.618-168.098c2.811-8.649 15.047-8.649 17.856 0l54.618 168.098z" transform="translate(336.974 280.256)"/></svg>
|
||||
|
After Width: | Height: | Size: 1.1 KiB |
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r,m,i;e.da=function(){this.pipeline.reset(),this.pipeline.add(e.da.trimmer,e.da.stopWordFilter,e.da.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.da.stemmer))},e.da.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stＡ-Ｚａ-ｚ",e.da.trimmer=e.trimmerSupport.generateTrimmer(e.da.wordCharacters),e.Pipeline.registerFunction(e.da.trimmer,"trimmer-da"),e.da.stemmer=(r=e.stemmerSupport.Among,m=e.stemmerSupport.SnowballProgram,i=new function(){var i,t,n,s=[new r("hed",-1,1),new r("ethed",0,1),new r("ered",-1,1),new r("e",-1,1),new r("erede",3,1),new r("ende",3,1),new r("erende",5,1),new r("ene",3,1),new r("erne",3,1),new r("ere",3,1),new r("en",-1,1),new r("heden",10,1),new r("eren",10,1),new r("er",-1,1),new r("heder",13,1),new r("erer",13,1),new r("s",-1,2),new r("heds",16,1),new r("es",16,1),new r("endes",18,1),new r("erendes",19,1),new r("enes",18,1),new r("ernes",18,1),new r("eres",18,1),new r("ens",16,1),new r("hedens",24,1),new r("erens",24,1),new r("ers",16,1),new r("ets",16,1),new r("erets",28,1),new r("et",-1,1),new r("eret",30,1)],o=[new r("gd",-1,-1),new r("dt",-1,-1),new r("gt",-1,-1),new r("kt",-1,-1)],a=[new r("ig",-1,1),new r("lig",0,1),new r("elig",1,1),new r("els",-1,1),new r("løst",-1,2)],d=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,48,0,128],u=[239,254,42,3,0,0,0,0,0,0,0,0,0,0,0,0,16],c=new m;function l(){var e,r=c.limit-c.cursor;c.cursor>=t&&(e=c.limit_backward,c.limit_backward=t,c.ket=c.cursor,c.find_among_b(o,4)?(c.bra=c.cursor,c.limit_backward=e,c.cursor=c.limit-r,c.cursor>c.limit_backward&&(c.cursor--,c.bra=c.cursor,c.slice_del())):c.limit_backward=e)}this.setCurrent=function(e){c.setCurrent(e)},this.getCurrent=function(){return c.getCurrent()},this.stem=function(){var e,r=c.cursor;return function(){var e,r=c.cursor+3;if(t=c.limit,0<=r&&r<=c.limit){for(i=r;;){if(e=c.cursor,c.in_grouping(d,97,248)){c.cursor=e;break}if((c.cursor=e)>=c.limit)return;c.cursor++}for(;!c.out_grouping(d,97,248);){if(c.cursor>=c.limit)return;c.cursor++}(t=c.cursor)<i&&(t=i)}}(),c.limit_backward=r,c.cursor=c.limit,function(){var e,r;if(c.cursor>=t&&(r=c.limit_backward,c.limit_backward=t,c.ket=c.cursor,e=c.find_among_b(s,32),c.limit_backward=r,e))switch(c.bra=c.cursor,e){case 1:c.slice_del();break;case 2:c.in_grouping_b(u,97,229)&&c.slice_del()}}(),c.cursor=c.limit,l(),c.cursor=c.limit,function(){var e,r,i,n=c.limit-c.cursor;if(c.ket=c.cursor,c.eq_s_b(2,"st")&&(c.bra=c.cursor,c.eq_s_b(2,"ig")&&c.slice_del()),c.cursor=c.limit-n,c.cursor>=t&&(r=c.limit_backward,c.limit_backward=t,c.ket=c.cursor,e=c.find_among_b(a,5),c.limit_backward=r,e))switch(c.bra=c.cursor,e){case 1:c.slice_del(),i=c.limit-c.cursor,l(),c.cursor=c.limit-i;break;case 2:c.slice_from("løs")}}(),c.cursor=c.limit,c.cursor>=t&&(e=c.limit_backward,c.limit_backward=t,c.ket=c.cursor,c.out_grouping_b(d,97,248)?(c.bra=c.cursor,n=c.slice_to(n),c.limit_backward=e,c.eq_v_b(n)&&c.slice_del()):c.limit_backward=e),!0}},function(e){return"function"==typeof e.update?e.update(function(e){return i.setCurrent(e),i.stem(),i.getCurrent()}):(i.setCurrent(e),i.stem(),i.getCurrent())}),e.Pipeline.registerFunction(e.da.stemmer,"stemmer-da"),e.da.stopWordFilter=e.generateStopWordFilter("ad af alle alt anden at blev blive bliver da de dem den denne der deres det dette dig din disse dog du efter eller en end er et for fra ham han hans har havde have hende hendes her hos hun hvad hvis hvor i ikke ind jeg jer jo kunne man mange med meget men mig min mine mit mod ned noget nogle nu når og også om op os over på selv sig sin sine sit skal skulle som sådan thi til ud under var vi vil ville vor være været".split(" ")),e.Pipeline.registerFunction(e.da.stopWordFilter,"stopWordFilter-da")}});
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(m){if(void 0===m)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===m.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var l="2"==m.version[0];m.ja=function(){this.pipeline.reset(),this.pipeline.add(m.ja.trimmer,m.ja.stopWordFilter,m.ja.stemmer),l?this.tokenizer=m.ja.tokenizer:(m.tokenizer&&(m.tokenizer=m.ja.tokenizer),this.tokenizerFn&&(this.tokenizerFn=m.ja.tokenizer))};var j=new m.TinySegmenter;m.ja.tokenizer=function(e){var r,t,i,n,o,s,p,a,u;if(!arguments.length||null==e||null==e)return[];if(Array.isArray(e))return e.map(function(e){return l?new m.Token(e.toLowerCase()):e.toLowerCase()});for(r=(t=e.toString().toLowerCase().replace(/^\s+/,"")).length-1;0<=r;r--)if(/\S/.test(t.charAt(r))){t=t.substring(0,r+1);break}for(o=[],i=t.length,p=a=0;a<=i;a++)if(s=a-p,t.charAt(a).match(/\s/)||a==i){if(0<s)for(n=j.segment(t.slice(p,a)).filter(function(e){return!!e}),u=p,r=0;r<n.length;r++)l?o.push(new m.Token(n[r],{position:[u,n[r].length],index:o.length})):o.push(n[r]),u+=n[r].length;p=a+1}return o},m.ja.stemmer=function(e){return e},m.Pipeline.registerFunction(m.ja.stemmer,"stemmer-ja"),m.ja.wordCharacters="一二三四五六七八九十百千万億兆一-龠々〆ヵヶぁ-んァ-ヴーｱ-ﾝﾞa-zA-Zａ-ｚＡ-Ｚ0-9０-９",m.ja.trimmer=m.trimmerSupport.generateTrimmer(m.ja.wordCharacters),m.Pipeline.registerFunction(m.ja.trimmer,"trimmer-ja"),m.ja.stopWordFilter=m.generateStopWordFilter("これ それ あれ この その あの ここ そこ あそこ こちら どこ だれ なに なん 何 私 貴方 貴方方 我々 私達 あの人 あのかた 彼女 彼 です あります おります います は が の に を で え から まで より も どの と し それで しかし".split(" ")),m.Pipeline.registerFunction(m.ja.stopWordFilter,"stopWordFilter-ja"),m.jp=m.ja,m.Pipeline.registerFunction(m.jp.stemmer,"stemmer-jp"),m.Pipeline.registerFunction(m.jp.trimmer,"trimmer-jp"),m.Pipeline.registerFunction(m.jp.stopWordFilter,"stopWordFilter-jp")}});
|
||||
@@ -0,0 +1 @@
|
||||
module.exports=require("./lunr.ja");
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,i){"function"==typeof define&&define.amd?define(i):"object"==typeof exports?module.exports=i():i()(e.lunr)}(this,function(){return function(o){o.multiLanguage=function(){for(var e=Array.prototype.slice.call(arguments),i=e.join("-"),t="",r=[],n=[],s=0;s<e.length;++s)"en"==e[s]?(t+="\\w",r.unshift(o.stopWordFilter),r.push(o.stemmer),n.push(o.stemmer)):(t+=o[e[s]].wordCharacters,r.unshift(o[e[s]].stopWordFilter),r.push(o[e[s]].stemmer),n.push(o[e[s]].stemmer));var p=o.trimmerSupport.generateTrimmer(t);return o.Pipeline.registerFunction(p,"lunr-multi-trimmer-"+i),r.unshift(p),function(){this.pipeline.reset(),this.pipeline.add.apply(this.pipeline,r),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add.apply(this.searchPipeline,n))}}}});
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r,n,i;e.no=function(){this.pipeline.reset(),this.pipeline.add(e.no.trimmer,e.no.stopWordFilter,e.no.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.no.stemmer))},e.no.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stＡ-Ｚａ-ｚ",e.no.trimmer=e.trimmerSupport.generateTrimmer(e.no.wordCharacters),e.Pipeline.registerFunction(e.no.trimmer,"trimmer-no"),e.no.stemmer=(r=e.stemmerSupport.Among,n=e.stemmerSupport.SnowballProgram,i=new function(){var o,s,a=[new r("a",-1,1),new r("e",-1,1),new r("ede",1,1),new r("ande",1,1),new r("ende",1,1),new r("ane",1,1),new r("ene",1,1),new r("hetene",6,1),new r("erte",1,3),new r("en",-1,1),new r("heten",9,1),new r("ar",-1,1),new r("er",-1,1),new r("heter",12,1),new r("s",-1,2),new r("as",14,1),new r("es",14,1),new r("edes",16,1),new r("endes",16,1),new r("enes",16,1),new r("hetenes",19,1),new r("ens",14,1),new r("hetens",21,1),new r("ers",14,1),new r("ets",14,1),new r("et",-1,1),new r("het",25,1),new r("ert",-1,3),new r("ast",-1,1)],m=[new r("dt",-1,-1),new r("vt",-1,-1)],l=[new r("leg",-1,1),new r("eleg",0,1),new r("ig",-1,1),new r("eig",2,1),new r("lig",2,1),new r("elig",4,1),new r("els",-1,1),new r("lov",-1,1),new r("elov",7,1),new r("slov",7,1),new r("hetslov",9,1)],u=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,48,0,128],d=[119,125,149,1],c=new n;this.setCurrent=function(e){c.setCurrent(e)},this.getCurrent=function(){return c.getCurrent()},this.stem=function(){var e,r,n,i,t=c.cursor;return function(){var e,r=c.cursor+3;if(s=c.limit,0<=r||r<=c.limit){for(o=r;;){if(e=c.cursor,c.in_grouping(u,97,248)){c.cursor=e;break}if(e>=c.limit)return;c.cursor=e+1}for(;!c.out_grouping(u,97,248);){if(c.cursor>=c.limit)return;c.cursor++}(s=c.cursor)<o&&(s=o)}}(),c.limit_backward=t,c.cursor=c.limit,function(){var e,r,n;if(c.cursor>=s&&(r=c.limit_backward,c.limit_backward=s,c.ket=c.cursor,e=c.find_among_b(a,29),c.limit_backward=r,e))switch(c.bra=c.cursor,e){case 1:c.slice_del();break;case 2:n=c.limit-c.cursor,c.in_grouping_b(d,98,122)?c.slice_del():(c.cursor=c.limit-n,c.eq_s_b(1,"k")&&c.out_grouping_b(u,97,248)&&c.slice_del());break;case 3:c.slice_from("er")}}(),c.cursor=c.limit,r=c.limit-c.cursor,c.cursor>=s&&(e=c.limit_backward,c.limit_backward=s,c.ket=c.cursor,c.find_among_b(m,2)?(c.bra=c.cursor,c.limit_backward=e,c.cursor=c.limit-r,c.cursor>c.limit_backward&&(c.cursor--,c.bra=c.cursor,c.slice_del())):c.limit_backward=e),c.cursor=c.limit,c.cursor>=s&&(i=c.limit_backward,c.limit_backward=s,c.ket=c.cursor,(n=c.find_among_b(l,11))?(c.bra=c.cursor,c.limit_backward=i,1==n&&c.slice_del()):c.limit_backward=i),!0}},function(e){return"function"==typeof e.update?e.update(function(e){return i.setCurrent(e),i.stem(),i.getCurrent()}):(i.setCurrent(e),i.stem(),i.getCurrent())}),e.Pipeline.registerFunction(e.no.stemmer,"stemmer-no"),e.no.stopWordFilter=e.generateStopWordFilter("alle at av bare begge ble blei bli blir blitt både båe da de deg dei deim deira deires dem den denne der dere deres det dette di din disse ditt du dykk dykkar då eg ein eit eitt eller elles en enn er et ett etter for fordi fra før ha hadde han hans har hennar henne hennes her hjå ho hoe honom hoss hossen hun hva hvem hver hvilke hvilken hvis hvor hvordan hvorfor i ikke ikkje ikkje ingen ingi inkje inn inni ja jeg kan kom korleis korso kun kunne kva kvar kvarhelst kven kvi kvifor man mange me med medan meg meget mellom men mi min mine mitt mot mykje ned no noe noen noka noko nokon nokor nokre nå når og også om opp oss over på samme seg selv si si sia sidan siden sin sine sitt sjøl skal skulle slik so som som somme somt så sånn til um upp ut uten var vart varte ved vere verte vi vil ville vore vors vort vår være være vært å".split(" ")),e.Pipeline.registerFunction(e.no.stopWordFilter,"stopWordFilter-no")}});
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1 @@
|
||||
!function(r,t){"function"==typeof define&&define.amd?define(t):"object"==typeof exports?module.exports=t():t()(r.lunr)}(this,function(){return function(r){r.stemmerSupport={Among:function(r,t,i,s){if(this.toCharArray=function(r){for(var t=r.length,i=new Array(t),s=0;s<t;s++)i[s]=r.charCodeAt(s);return i},!r&&""!=r||!t&&0!=t||!i)throw"Bad Among initialisation: s:"+r+", substring_i: "+t+", result: "+i;this.s_size=r.length,this.s=this.toCharArray(r),this.substring_i=t,this.result=i,this.method=s},SnowballProgram:function(){var b;return{bra:0,ket:0,limit:0,cursor:0,limit_backward:0,setCurrent:function(r){b=r,this.cursor=0,this.limit=r.length,this.limit_backward=0,this.bra=this.cursor,this.ket=this.limit},getCurrent:function(){var r=b;return b=null,r},in_grouping:function(r,t,i){if(this.cursor<this.limit){var s=b.charCodeAt(this.cursor);if(s<=i&&t<=s&&r[(s-=t)>>3]&1<<(7&s))return this.cursor++,!0}return!1},in_grouping_b:function(r,t,i){if(this.cursor>this.limit_backward){var s=b.charCodeAt(this.cursor-1);if(s<=i&&t<=s&&r[(s-=t)>>3]&1<<(7&s))return this.cursor--,!0}return!1},out_grouping:function(r,t,i){if(this.cursor<this.limit){var s=b.charCodeAt(this.cursor);if(i<s||s<t)return this.cursor++,!0;if(!(r[(s-=t)>>3]&1<<(7&s)))return this.cursor++,!0}return!1},out_grouping_b:function(r,t,i){if(this.cursor>this.limit_backward){var s=b.charCodeAt(this.cursor-1);if(i<s||s<t)return this.cursor--,!0;if(!(r[(s-=t)>>3]&1<<(7&s)))return this.cursor--,!0}return!1},eq_s:function(r,t){if(this.limit-this.cursor<r)return!1;for(var i=0;i<r;i++)if(b.charCodeAt(this.cursor+i)!=t.charCodeAt(i))return!1;return this.cursor+=r,!0},eq_s_b:function(r,t){if(this.cursor-this.limit_backward<r)return!1;for(var i=0;i<r;i++)if(b.charCodeAt(this.cursor-r+i)!=t.charCodeAt(i))return!1;return this.cursor-=r,!0},find_among:function(r,t){for(var i=0,s=t,e=this.cursor,n=this.limit,u=0,o=0,h=!1;;){for(var c=i+(s-i>>1),a=0,f=u<o?u:o,l=r[c],_=f;_<l.s_size;_++){if(e+f==n){a=-1;break}if(a=b.charCodeAt(e+f)-l.s[_])break;f++}if(a<0?(s=c,o=f):(i=c,u=f),s-i<=1){if(0<i||s==i||h)break;h=!0}}for(;;){if(u>=(l=r[i]).s_size){if(this.cursor=e+l.s_size,!l.method)return l.result;var m=l.method();if(this.cursor=e+l.s_size,m)return l.result}if((i=l.substring_i)<0)return 0}},find_among_b:function(r,t){for(var i=0,s=t,e=this.cursor,n=this.limit_backward,u=0,o=0,h=!1;;){for(var c=i+(s-i>>1),a=0,f=u<o?u:o,l=(_=r[c]).s_size-1-f;0<=l;l--){if(e-f==n){a=-1;break}if(a=b.charCodeAt(e-1-f)-_.s[l])break;f++}if(a<0?(s=c,o=f):(i=c,u=f),s-i<=1){if(0<i||s==i||h)break;h=!0}}for(;;){var _;if(u>=(_=r[i]).s_size){if(this.cursor=e-_.s_size,!_.method)return _.result;var m=_.method();if(this.cursor=e-_.s_size,m)return _.result}if((i=_.substring_i)<0)return 0}},replace_s:function(r,t,i){var s=i.length-(t-r);return b=b.substring(0,r)+i+b.substring(t),this.limit+=s,this.cursor>=t?this.cursor+=s:this.cursor>r&&(this.cursor=r),s},slice_check:function(){if(this.bra<0||this.bra>this.ket||this.ket>this.limit||this.limit>b.length)throw"faulty slice operation"},slice_from:function(r){this.slice_check(),this.replace_s(this.bra,this.ket,r)},slice_del:function(){this.slice_from("")},insert:function(r,t,i){var s=this.replace_s(r,t,i);r<=this.bra&&(this.bra+=s),r<=this.ket&&(this.ket+=s)},slice_to:function(){return this.slice_check(),b.substring(this.bra,this.ket)},eq_v_b:function(r){return this.eq_s_b(r.length,r)}}}},r.trimmerSupport={generateTrimmer:function(r){var t=new RegExp("^[^"+r+"]+"),i=new RegExp("[^"+r+"]+$");return function(r){return"function"==typeof r.update?r.update(function(r){return r.replace(t,"").replace(i,"")}):r.replace(t,"").replace(i,"")}}}}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(e){if(void 0===e)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===e.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var r,l,n;e.sv=function(){this.pipeline.reset(),this.pipeline.add(e.sv.trimmer,e.sv.stopWordFilter,e.sv.stemmer),this.searchPipeline&&(this.searchPipeline.reset(),this.searchPipeline.add(e.sv.stemmer))},e.sv.wordCharacters="A-Za-zªºÀ-ÖØ-öø-ʸˠ-ˤᴀ-ᴥᴬ-ᵜᵢ-ᵥᵫ-ᵷᵹ-ᶾḀ-ỿⁱⁿₐ-ₜKÅℲⅎⅠ-ↈⱠ-ⱿꜢ-ꞇꞋ-ꞭꞰ-ꞷꟷ-ꟿꬰ-ꭚꭜ-ꭤff-stＡ-Ｚａ-ｚ",e.sv.trimmer=e.trimmerSupport.generateTrimmer(e.sv.wordCharacters),e.Pipeline.registerFunction(e.sv.trimmer,"trimmer-sv"),e.sv.stemmer=(r=e.stemmerSupport.Among,l=e.stemmerSupport.SnowballProgram,n=new function(){var n,t,i=[new r("a",-1,1),new r("arna",0,1),new r("erna",0,1),new r("heterna",2,1),new r("orna",0,1),new r("ad",-1,1),new r("e",-1,1),new r("ade",6,1),new r("ande",6,1),new r("arne",6,1),new r("are",6,1),new r("aste",6,1),new r("en",-1,1),new r("anden",12,1),new r("aren",12,1),new r("heten",12,1),new r("ern",-1,1),new r("ar",-1,1),new r("er",-1,1),new r("heter",18,1),new r("or",-1,1),new r("s",-1,2),new r("as",21,1),new r("arnas",22,1),new r("ernas",22,1),new r("ornas",22,1),new r("es",21,1),new r("ades",26,1),new r("andes",26,1),new r("ens",21,1),new r("arens",29,1),new r("hetens",29,1),new r("erns",21,1),new r("at",-1,1),new r("andet",-1,1),new r("het",-1,1),new r("ast",-1,1)],s=[new r("dd",-1,-1),new r("gd",-1,-1),new r("nn",-1,-1),new r("dt",-1,-1),new r("gt",-1,-1),new r("kt",-1,-1),new r("tt",-1,-1)],a=[new r("ig",-1,1),new r("lig",0,1),new r("els",-1,1),new r("fullt",-1,3),new r("löst",-1,2)],o=[17,65,16,1,0,0,0,0,0,0,0,0,0,0,0,0,24,0,32],u=[119,127,149],m=new l;this.setCurrent=function(e){m.setCurrent(e)},this.getCurrent=function(){return m.getCurrent()},this.stem=function(){var e,r=m.cursor;return function(){var e,r=m.cursor+3;if(t=m.limit,0<=r||r<=m.limit){for(n=r;;){if(e=m.cursor,m.in_grouping(o,97,246)){m.cursor=e;break}if(m.cursor=e,m.cursor>=m.limit)return;m.cursor++}for(;!m.out_grouping(o,97,246);){if(m.cursor>=m.limit)return;m.cursor++}(t=m.cursor)<n&&(t=n)}}(),m.limit_backward=r,m.cursor=m.limit,function(){var e,r=m.limit_backward;if(m.cursor>=t&&(m.limit_backward=t,m.cursor=m.limit,m.ket=m.cursor,e=m.find_among_b(i,37),m.limit_backward=r,e))switch(m.bra=m.cursor,e){case 1:m.slice_del();break;case 2:m.in_grouping_b(u,98,121)&&m.slice_del()}}(),m.cursor=m.limit,e=m.limit_backward,m.cursor>=t&&(m.limit_backward=t,m.cursor=m.limit,m.find_among_b(s,7)&&(m.cursor=m.limit,m.ket=m.cursor,m.cursor>m.limit_backward&&(m.bra=--m.cursor,m.slice_del())),m.limit_backward=e),m.cursor=m.limit,function(){var e,r;if(m.cursor>=t){if(r=m.limit_backward,m.limit_backward=t,m.cursor=m.limit,m.ket=m.cursor,e=m.find_among_b(a,5))switch(m.bra=m.cursor,e){case 1:m.slice_del();break;case 2:m.slice_from("lös");break;case 3:m.slice_from("full")}m.limit_backward=r}}(),!0}},function(e){return"function"==typeof e.update?e.update(function(e){return n.setCurrent(e),n.stem(),n.getCurrent()}):(n.setCurrent(e),n.stem(),n.getCurrent())}),e.Pipeline.registerFunction(e.sv.stemmer,"stemmer-sv"),e.sv.stopWordFilter=e.generateStopWordFilter("alla allt att av blev bli blir blivit de dem den denna deras dess dessa det detta dig din dina ditt du där då efter ej eller en er era ert ett från för ha hade han hans har henne hennes hon honom hur här i icke ingen inom inte jag ju kan kunde man med mellan men mig min mina mitt mot mycket ni nu när någon något några och om oss på samma sedan sig sin sina sitta själv skulle som så sådan sådana sådant till under upp ut utan vad var vara varför varit varje vars vart vem vi vid vilka vilkas vilken vilket vår våra vårt än är åt över".split(" ")),e.Pipeline.registerFunction(e.sv.stopWordFilter,"stopWordFilter-sv")}});
|
||||
@@ -0,0 +1 @@
|
||||
!function(e,r){"function"==typeof define&&define.amd?define(r):"object"==typeof exports?module.exports=r():r()(e.lunr)}(this,function(){return function(t){if(void 0===t)throw new Error("Lunr is not present. Please include / require Lunr before this script.");if(void 0===t.stemmerSupport)throw new Error("Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.");var i="2"==t.version[0];t.th=function(){this.pipeline.reset(),this.pipeline.add(t.th.trimmer),i?this.tokenizer=t.th.tokenizer:(t.tokenizer&&(t.tokenizer=t.th.tokenizer),this.tokenizerFn&&(this.tokenizerFn=t.th.tokenizer))},t.th.wordCharacters="[-]",t.th.trimmer=t.trimmerSupport.generateTrimmer(t.th.wordCharacters),t.Pipeline.registerFunction(t.th.trimmer,"trimmer-th");var n=t.wordcut;n.init(),t.th.tokenizer=function(e){if(!arguments.length||null==e||null==e)return[];if(Array.isArray(e))return e.map(function(e){return i?new t.Token(e):e});var r=e.toString().replace(/^\s+/,"");return n.cut(r).split("|")}}});
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -1,394 +0,0 @@
|
||||
# Lightning Module interface
|
||||
[[Github Code](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/root_module/root_module.py)]
|
||||
|
||||
A lightning module is a strict superclass of nn.Module, it provides a standard interface for the trainer to interact with the model.
|
||||
|
||||
The easiest thing to do is copy [this template](../../examples/new_project_templates/lightning_module_template.py) and modify accordingly.
|
||||
|
||||
Otherwise, to Define a Lightning Module, implement the following methods:
|
||||
|
||||
**Required**:
|
||||
|
||||
- [training_step](RequiredTrainerInterface.md#training_step)
|
||||
- [validation_step](RequiredTrainerInterface.md#validation_step)
|
||||
- [validation_end](RequiredTrainerInterface.md#validation_end)
|
||||
|
||||
- [configure_optimizers](RequiredTrainerInterface.md#configure_optimizers)
|
||||
- [get_save_dict](RequiredTrainerInterface.md#get_save_dict)
|
||||
- [load_model_specific](RequiredTrainerInterface.md#load_model_specific)
|
||||
|
||||
- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)
|
||||
- [tng_dataloader](RequiredTrainerInterface.md#tng_dataloader)
|
||||
- [test_dataloader](RequiredTrainerInterface.md#test_dataloader)
|
||||
|
||||
**Optional**:
|
||||
|
||||
- [update_tng_log_metrics](RequiredTrainerInterface.md#update_tng_log_metrics)
|
||||
- [add_model_specific_args](RequiredTrainerInterface.md#add_model_specific_args)
|
||||
|
||||
---
|
||||
|
||||
### training_step
|
||||
|
||||
``` {.python}
|
||||
def training_step(self, data_batch, batch_nb)
|
||||
```
|
||||
|
||||
In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model.
|
||||
|
||||
**Params**
|
||||
|
||||
| Param | description |
|
||||
|---|---|
|
||||
| data_batch | The output of your dataloader. A tensor, tuple or list |
|
||||
| batch_nb | Integer displaying which batch this is |
|
||||
|
||||
**Return**
|
||||
|
||||
Dictionary or OrderedDict
|
||||
|
||||
| key | value | is required |
|
||||
|---|---|---|
|
||||
| loss | tensor scalar | Y |
|
||||
| prog | Dict for progress bar display. Must have only tensors | N |
|
||||
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
def training_step(self, data_batch, batch_nb):
|
||||
x, y, z = data_batch
|
||||
|
||||
# implement your own
|
||||
out = self.forward(x)
|
||||
loss = self.loss(out, x)
|
||||
|
||||
output = {
|
||||
'loss': loss, # required
|
||||
'prog': {'tng_loss': loss, 'batch_nb': batch_nb} # optional
|
||||
}
|
||||
|
||||
# return a dict
|
||||
return output
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### validation_step
|
||||
|
||||
``` {.python}
|
||||
def validation_step(self, data_batch, batch_nb)
|
||||
```
|
||||
|
||||
In this step you'd normally do the forward pass and calculate the loss for a batch. You can also do fancier things like multiple forward passes or something specific to your model.
|
||||
This is most likely the same as your training_step. But unlike training step, the outputs from here will go to validation_end for collation.
|
||||
|
||||
**Params**
|
||||
|
||||
| Param | description |
|
||||
|---|---|
|
||||
| data_batch | The output of your dataloader. A tensor, tuple or list |
|
||||
| batch_nb | Integer displaying which batch this is |
|
||||
|
||||
**Return**
|
||||
|
||||
| Return | description | optional |
|
||||
|---|---|---|
|
||||
| dict | Dict of OrderedDict with metrics to display in progress bar. All keys must be tensors. | Y |
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
def validation_step(self, data_batch, batch_nb):
|
||||
x, y, z = data_batch
|
||||
|
||||
# implement your own
|
||||
out = self.forward(x)
|
||||
loss = self.loss(out, x)
|
||||
|
||||
# calculate acc
|
||||
labels_hat = torch.argmax(out, dim=1)
|
||||
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
|
||||
# all optional...
|
||||
# return whatever you need for the collation function validation_end
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': torch.tensor(val_acc), # everything must be a tensor
|
||||
})
|
||||
|
||||
# return an optional dict
|
||||
return output
|
||||
```
|
||||
|
||||
---
|
||||
### validation_end
|
||||
|
||||
``` {.python}
|
||||
def validation_end(self, outputs)
|
||||
```
|
||||
|
||||
Called at the end of the validation loop with the output of each validation_step.
|
||||
|
||||
**Params**
|
||||
|
||||
| Param | description |
|
||||
|---|---|
|
||||
| outputs | List of outputs you defined in validation_step |
|
||||
|
||||
**Return**
|
||||
|
||||
| Return | description | optional |
|
||||
|---|---|---|
|
||||
| dict | Dict of OrderedDict with metrics to display in progress bar | Y |
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
def validation_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
val_loss_mean = 0
|
||||
val_acc_mean = 0
|
||||
for output in outputs:
|
||||
val_loss_mean += output['val_loss']
|
||||
val_acc_mean += output['val_acc']
|
||||
|
||||
val_loss_mean /= len(outputs)
|
||||
val_acc_mean /= len(outputs)
|
||||
tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
```
|
||||
|
||||
---
|
||||
### configure_optimizers
|
||||
|
||||
``` {.python}
|
||||
def configure_optimizers(self)
|
||||
```
|
||||
|
||||
Set up as many optimizers as you need. Normally you'd need one. But in the case of GANs or something more esoteric you might have multiple.
|
||||
Lightning will call .backward() and .step() on each one. If you use 16 bit precision it will also handle that.
|
||||
|
||||
|
||||
##### Return
|
||||
List - List of optimizers
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
# most cases
|
||||
def configure_optimizers(self):
|
||||
opt = Adam(lr=0.01)
|
||||
return [opt]
|
||||
|
||||
# gan example
|
||||
def configure_optimizers(self):
|
||||
generator_opt = Adam(lr=0.01)
|
||||
disriminator_opt = Adam(lr=0.02)
|
||||
return [generator_opt, disriminator_opt]
|
||||
```
|
||||
|
||||
---
|
||||
### get_save_dict
|
||||
|
||||
``` {.python}
|
||||
def get_save_dict(self)
|
||||
```
|
||||
Called by lightning to checkpoint your model. Lightning saves current epoch, current batch nb, etc...
|
||||
All you have to return is what specifically about your lightning model you want to checkpoint.
|
||||
|
||||
##### Return
|
||||
Dictionary - No required keys. Most of the time as described in this example.
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
def get_save_dict(self):
|
||||
# 99% of use cases this is all you need to return
|
||||
checkpoint = {'state_dict': self.state_dict()}
|
||||
return checkpoint
|
||||
```
|
||||
|
||||
---
|
||||
### load_model_specific
|
||||
|
||||
``` {.python}
|
||||
def load_model_specific(self, checkpoint)
|
||||
```
|
||||
Called by lightning to restore your model. This is your chance to restore your model using the keys you added in get_save_dict.
|
||||
Lightning will automatically restore current epoch, batch nb, etc.
|
||||
|
||||
##### Return
|
||||
Nothing
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
def load_model_specific(self, checkpoint):
|
||||
# you defined 'state_dict' in get_save_dict()
|
||||
self.load_state_dict(checkpoint['state_dict'])
|
||||
```
|
||||
|
||||
---
|
||||
### tng_dataloader
|
||||
|
||||
``` {.python}
|
||||
@property
|
||||
def tng_dataloader(self)
|
||||
```
|
||||
Called by lightning during training loop. Define it as a property.
|
||||
|
||||
##### Return
|
||||
Pytorch DataLoader
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
@property
|
||||
def tng_dataloader(self):
|
||||
if self._tng_dataloader is None:
|
||||
try:
|
||||
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
|
||||
dataset = MNIST(root='/path/to/mnist/', train=True, transform=transform, download=True)
|
||||
loader = torch.utils.data.DataLoader(
|
||||
dataset=dataset,
|
||||
batch_size=self.hparams.batch_size,
|
||||
shuffle=True
|
||||
)
|
||||
self._tng_dataloader = loader
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
return self._tng_dataloader
|
||||
```
|
||||
|
||||
---
|
||||
### val_dataloader
|
||||
|
||||
``` {.python}
|
||||
@property
|
||||
def tng_dataloader(self)
|
||||
```
|
||||
Called by lightning during validation loop. Define it as a property.
|
||||
|
||||
##### Return
|
||||
Pytorch DataLoader
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
@property
|
||||
def val_dataloader(self):
|
||||
if self._val_dataloader is None:
|
||||
try:
|
||||
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
|
||||
dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True)
|
||||
loader = torch.utils.data.DataLoader(
|
||||
dataset=dataset,
|
||||
batch_size=self.hparams.batch_size,
|
||||
shuffle=True
|
||||
)
|
||||
self._val_dataloader = loader
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
return self._val_dataloader
|
||||
```
|
||||
|
||||
---
|
||||
### test_dataloader
|
||||
|
||||
``` {.python}
|
||||
@property
|
||||
def test_dataloader(self)
|
||||
```
|
||||
Called by lightning during test loop. Define it as a property.
|
||||
|
||||
##### Return
|
||||
Pytorch DataLoader
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
@property
|
||||
def test_dataloader(self):
|
||||
if self._test_dataloader is None:
|
||||
try:
|
||||
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
|
||||
dataset = MNIST(root='/path/to/mnist/', train=False, transform=transform, download=True)
|
||||
loader = torch.utils.data.DataLoader(
|
||||
dataset=dataset,
|
||||
batch_size=self.hparams.batch_size,
|
||||
shuffle=True
|
||||
)
|
||||
self._test_dataloader = loader
|
||||
except Exception as e:
|
||||
raise e
|
||||
|
||||
return self._test_dataloader
|
||||
```
|
||||
|
||||
---
|
||||
### update_tng_log_metrics
|
||||
|
||||
``` {.python}
|
||||
def update_tng_log_metrics(self, logs)
|
||||
```
|
||||
Called by lightning right before it logs metrics for this batch.
|
||||
This is a chance to ammend or add to the metrics about to be logged.
|
||||
|
||||
##### Return
|
||||
Dict
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
def update_tng_log_metrics(self, logs):
|
||||
# modify or add to logs
|
||||
return logs
|
||||
```
|
||||
|
||||
---
|
||||
### add_model_specific_args
|
||||
|
||||
``` {.python}
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser, root_dir)
|
||||
```
|
||||
Lightning has a list of default argparse commands.
|
||||
This method is your chance to add or modify commands specific to your model.
|
||||
The [hyperparameter argument parser](https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/) is available anywhere in your model by calling self.hparams.
|
||||
|
||||
##### Return
|
||||
An argument parser
|
||||
|
||||
**Example**
|
||||
|
||||
``` {.python}
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser, root_dir):
|
||||
parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
|
||||
|
||||
# param overwrites
|
||||
# parser.set_defaults(gradient_clip=5.0)
|
||||
|
||||
# network params
|
||||
parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
|
||||
parser.add_argument('--in_features', default=28*28)
|
||||
parser.add_argument('--out_features', default=10)
|
||||
parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference
|
||||
|
||||
# data
|
||||
parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
|
||||
|
||||
# training params (opt)
|
||||
parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
|
||||
tunable=False)
|
||||
parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False)
|
||||
parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
|
||||
return parser
|
||||
```
|
||||
@@ -1,49 +0,0 @@
|
||||
Lightning modules are strict superclasses of torch.nn.Module. A LightningModule offers the following in addition to that API.
|
||||
|
||||
---
|
||||
### freeze
|
||||
Freeze all params for inference
|
||||
```{.python}
|
||||
model = MyLightningModule(...)
|
||||
model.freeze()
|
||||
```
|
||||
|
||||
---
|
||||
### load_from_metrics
|
||||
This is the easiest/fastest way which uses the meta_tags.csv file from test-tube to rebuild the model.
|
||||
The meta_tags.csv file can be found in the test-tube experiment save_dir.
|
||||
|
||||
```{.python}
|
||||
pretrained_model = MyLightningModule.load_from_metrics(
|
||||
weights_path='/path/to/pytorch_checkpoint.ckpt',
|
||||
tags_csv='/path/to/test_tube/experiment/version/meta_tags.csv',
|
||||
on_gpu=True,
|
||||
map_location=None
|
||||
)
|
||||
|
||||
# predict
|
||||
pretrained_model.freeze()
|
||||
y_hat = pretrained_model(x)
|
||||
```
|
||||
|
||||
**Params**
|
||||
|
||||
| Param | description |
|
||||
|---|---|
|
||||
| weights_path | Path to a pytorch checkpoint |
|
||||
| tags_csv | Path to meta_tags.csv file generated by the test-tube Experiment |
|
||||
| on_gpu | if True, puts model on GPU. Make sure to use transforms option if model devices have changed |
|
||||
| map_location | A dictionary mapping saved weight GPU devices to new GPU devices |
|
||||
|
||||
**Returns**
|
||||
|
||||
LightningModule - The pretrained LightningModule
|
||||
|
||||
---
|
||||
### unfreeze
|
||||
Unfreeze all params for inference
|
||||
```{.python}
|
||||
model = MyLightningModule(...)
|
||||
model.unfreeze()
|
||||
```
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
A LightningModule has the following properties which you can access at any time
|
||||
|
||||
---
|
||||
#### current_epoch
|
||||
The current epoch
|
||||
|
||||
---
|
||||
#### dtype
|
||||
Current dtype
|
||||
|
||||
---
|
||||
#### experiment
|
||||
An instance of test-tube Experiment which you can use to log anything for tensorboarX.
|
||||
```{.python}
|
||||
self.experiment.add_embedding(...)
|
||||
self.experiment.log({'val_loss': 0.9})
|
||||
self.experiment.add_scalars(...)
|
||||
```
|
||||
|
||||
---
|
||||
#### global_step
|
||||
Total training batches seen across all epochs
|
||||
|
||||
---
|
||||
#### gradient_clip
|
||||
The current gradient clip value
|
||||
|
||||
---
|
||||
#### on_gpu
|
||||
True if your model is currently running on GPUs. Useful to set flags around the LightningModule for different CPU vs GPU behavior.
|
||||
|
||||
---
|
||||
#### trainer
|
||||
Last resort access to any state the trainer has. Changing certain properties here could affect your training run.
|
||||
```{.python}
|
||||
self.trainer.optimizers
|
||||
self.trainer.current_epoch
|
||||
...
|
||||
```
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
Lightning can automate saving and loading checkpoints.
|
||||
|
||||
---
|
||||
### Model saving
|
||||
To enable checkpointing, define the checkpoint callback and give it to the trainer.
|
||||
|
||||
``` {.python}
|
||||
from pytorch_lightning.utils.pt_callbacks import ModelCheckpoint
|
||||
|
||||
checkpoint_callback = ModelCheckpoint(
|
||||
filepath='/path/to/store/weights.ckpt',
|
||||
save_best_only=True,
|
||||
verbose=True,
|
||||
monitor='val_loss',
|
||||
mode='min'
|
||||
)
|
||||
|
||||
trainer = Trainer(checkpoint_callback=checkpoint_callback)
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
Lightning makes multi-gpu training and 16 bit training trivial.
|
||||
|
||||
*Note:*
|
||||
None of the flags below require changing anything about your lightningModel definition.
|
||||
|
||||
---
|
||||
#### 16-bit mixed precision
|
||||
16 bit precision can cut your memory footprint by half. If using volta architecture GPUs it can give a dramatic training speed-up as well.
|
||||
First, install apex (if install fails, look [here](https://github.com/NVIDIA/apex)):
|
||||
```bash
|
||||
$ git clone https://github.com/NVIDIA/apex
|
||||
$ cd apex
|
||||
$ pip install -v --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./
|
||||
```
|
||||
|
||||
then set this use_amp to True.
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(amp_level='O2', use_amp=False)
|
||||
```
|
||||
|
||||
---
|
||||
#### Single-gpu
|
||||
Make sure you're on a GPU machine.
|
||||
```python
|
||||
# set these flags
|
||||
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
|
||||
|
||||
# DEFAULT
|
||||
trainer = Trainer(gpus=[0])
|
||||
```
|
||||
|
||||
---
|
||||
#### multi-gpu
|
||||
Make sure you're on a GPU machine. You can set as many GPUs as you want.
|
||||
In this setting, the model will run on all 8 GPUs at once using DataParallel under the hood.
|
||||
```python
|
||||
# set these flags
|
||||
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1,2,3,4,5,6,7"
|
||||
|
||||
|
||||
trainer = Trainer(gpus=[0,1,2,3,4,5,6,7])
|
||||
```
|
||||
|
||||
---
|
||||
#### Multi-node
|
||||
Multi-node training is easily done by specifying these flags.
|
||||
```python
|
||||
# train on 12*8 GPUs
|
||||
trainer = Trainer(gpus=[0,1,2,3,4,5,6,7], nb_gpu_nodes=12)
|
||||
```
|
||||
|
||||
In addition, make sure to set up your SLURM job correctly via the [SlurmClusterObject](https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/). In particular, specify the number of tasks per node correctly.
|
||||
|
||||
```python
|
||||
cluster = SlurmCluster(
|
||||
hyperparam_optimizer=test_tube.HyperOptArgumentParser(),
|
||||
log_path='/some/path/to/save',
|
||||
)
|
||||
|
||||
# configure cluster
|
||||
cluster.per_experiment_nb_nodes = 12
|
||||
cluster.per_experiment_nb_gpus = 8
|
||||
|
||||
cluster.add_slurm_cmd(cmd='ntasks-per-node', value=8, comment='1 task per gpu')
|
||||
```
|
||||
|
||||
---
|
||||
#### Self-balancing architecture
|
||||
Here lightning distributes parts of your module across available GPUs to optimize for speed and memory.
|
||||
|
||||
COMING SOON.
|
||||
@@ -1,61 +0,0 @@
|
||||
Lighting offers a few options for logging information about model, gpu usage, etc (via test-tube). It also offers printing options for training monitoring.
|
||||
|
||||
|
||||
---
|
||||
#### Display metrics in progress bar
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(progress_bar=True)
|
||||
```
|
||||
|
||||
---
|
||||
#### Log metric row every k batches
|
||||
Every k batches lightning will make an entry in the metrics log
|
||||
``` {.python}
|
||||
# DEFAULT (ie: save a .csv log file every 10 batches)
|
||||
trainer = Trainer(add_log_row_interval=10)
|
||||
```
|
||||
|
||||
---
|
||||
#### Process position
|
||||
When running multiple models on the same machine we want to decide which progress bar to use.
|
||||
Lightning will stack progress bars according to this value.
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(process_position=0)
|
||||
|
||||
# if this is the second model on the node, show the second progress bar below
|
||||
trainer = Trainer(process_position=1)
|
||||
```
|
||||
|
||||
---
|
||||
#### Save a snapshot of all hyperparameters
|
||||
Whenever you call .save() on the test-tube experiment it logs all the hyperparameters in current use.
|
||||
Give lightning a test-tube Experiment object to automate this for you.
|
||||
``` {.python}
|
||||
from test-tube import Experiment
|
||||
|
||||
exp = Experiment(...)
|
||||
Trainer(experiment=exp)
|
||||
```
|
||||
|
||||
---
|
||||
#### Snapshot code for a training run
|
||||
Whenever you call .save() on the test-tube experiment it snapshows all code and pushes to a git tag.
|
||||
Give lightning a test-tube Experiment object to automate this for you.
|
||||
``` {.python}
|
||||
from test-tube import Experiment
|
||||
|
||||
exp = Experiment(create_git_tag=True)
|
||||
Trainer(experiment=exp)
|
||||
```
|
||||
|
||||
|
||||
---
|
||||
#### Write logs file to csv every k batches
|
||||
Every k batches, lightning will write the new logs to disk
|
||||
``` {.python}
|
||||
# DEFAULT (ie: save a .csv log file every 100 batches)
|
||||
trainer = Trainer(log_save_interval=100)
|
||||
```
|
||||
|
||||
@@ -1,104 +0,0 @@
|
||||
Lightning supports model training on a cluster managed by SLURM in the following cases:
|
||||
|
||||
1. Training on single or multi-cpus only.
|
||||
2. Training on single or multi-gpus on the same node.
|
||||
3. Coming SOON: Training across multiple nodes.
|
||||
|
||||
---
|
||||
#### Running grid search on a cluster
|
||||
To use lightning to run a hyperparameter search (grid-search or random-search) on a cluster do 4 things:
|
||||
|
||||
(1). Define the parameters for the grid search
|
||||
|
||||
```{.python}
|
||||
from test_tube import HyperOptArgumentParser
|
||||
|
||||
# subclass of argparse
|
||||
parser = HyperOptArgumentParser(strategy='random_search')
|
||||
parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate')
|
||||
|
||||
# let's enable optimizing over the number of layers in the network
|
||||
parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8])
|
||||
|
||||
hparams = parser.parse_args()
|
||||
```
|
||||
|
||||
|
||||
(2). Define the cluster options in the [SlurmCluster object](https://williamfalcon.github.io/test-tube/hpc/SlurmCluster/) (over 5 nodes and 8 gpus)
|
||||
|
||||
```{.python}
|
||||
from test_tube.hpc import SlurmCluster
|
||||
|
||||
# hyperparameters is a test-tube hyper params object
|
||||
# see https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/
|
||||
hyperparams = args.parse()
|
||||
|
||||
# init cluster
|
||||
cluster = SlurmCluster(
|
||||
hyperparam_optimizer=hyperparams,
|
||||
log_path='/path/to/log/results/to',
|
||||
python_cmd='python3'
|
||||
)
|
||||
|
||||
# let the cluster know where to email for a change in job status (ie: complete, fail, etc...)
|
||||
cluster.notify_job_status(email='some@email.com', on_done=True, on_fail=True)
|
||||
|
||||
# set the job options. In this instance, we'll run 20 different models
|
||||
# each with its own set of hyperparameters giving each one 1 GPU (ie: taking up 20 GPUs)
|
||||
cluster.per_experiment_nb_gpus = 8
|
||||
cluster.per_experiment_nb_nodes = 5
|
||||
|
||||
# we'll request 10GB of memory per node
|
||||
cluster.memory_mb_per_node = 10000
|
||||
|
||||
# set a walltime of 10 minues
|
||||
cluster.job_time = '10:00'
|
||||
```
|
||||
|
||||
(3). Give trainer the cluster_manager in your main function:
|
||||
|
||||
```{.python}
|
||||
from pytorch_lightning import Trainer
|
||||
|
||||
def train_fx(trial_hparams, cluster_manager, _):
|
||||
# hparams has a specific set of hyperparams
|
||||
|
||||
my_model = MyLightningModel()
|
||||
|
||||
# give the trainer the cluster object
|
||||
trainer = Trainer(cluster=cluster_manager)
|
||||
trainer.fit(my_model)
|
||||
|
||||
```
|
||||
|
||||
(4). Start the grid search
|
||||
```{.python}
|
||||
# run the models on the cluster
|
||||
cluster.optimize_parallel_cluster_gpu(
|
||||
train_fx,
|
||||
nb_trials=20,
|
||||
job_name='my_grid_search_exp_name',
|
||||
job_display_name='my_exp')
|
||||
```
|
||||
|
||||
That's it! The SlurmCluster object will automatically checkpoint the lightning model and resubmit if it runs into the walltime!
|
||||
|
||||
|
||||
---
|
||||
#### Walltime auto-resubmit
|
||||
Lightning automatically resubmits jobs when they reach the walltime. You get this behavior for free if you give lightning
|
||||
a slurm cluster object.
|
||||
|
||||
```{.python}
|
||||
def my_main_fx(hparams, slurm_manager, _):
|
||||
trainer = Trainer(cluster=slurm_manager)
|
||||
```
|
||||
|
||||
(See the grid search example above for cluster configuration).
|
||||
With this feature lightning will:
|
||||
|
||||
1. automatically checkpoint the model
|
||||
2. checkpoint the trainer session
|
||||
3. resubmit a continuation job.
|
||||
4. load the checkpoint and trainer session in the new model
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
The lightning training loop handles everything except the actual computations of your model. To decide what will happen in your training loop, define the [training_step function](../../Pytorch-lightning/LightningModule/#training_step).
|
||||
|
||||
Below are all the things lightning automates for you in the training loop.
|
||||
|
||||
---
|
||||
#### Accumulated gradients
|
||||
Accumulated gradients runs K small batches of size N before doing a backwards pass. The effect is a large effective batch size of size KxN.
|
||||
|
||||
``` {.python}
|
||||
# DEFAULT (ie: no accumulated grads)
|
||||
trainer = Trainer(accumulate_grad_batches=1)
|
||||
```
|
||||
|
||||
---
|
||||
#### Anneal Learning rate
|
||||
Cut the learning rate by 10 at every epoch listed in this list.
|
||||
``` {.python}
|
||||
# DEFAULT (don't anneal)
|
||||
trainer = Trainer(lr_scheduler_milestones=None)
|
||||
|
||||
# cut LR by 10 at 100, 200, and 300 epochs
|
||||
trainer = Trainer(lr_scheduler_milestones='100, 200, 300')
|
||||
```
|
||||
|
||||
---
|
||||
#### Force training for min or max epochs
|
||||
It can be useful to force training for a minimum number of epochs or limit to a max number
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)
|
||||
```
|
||||
|
||||
---
|
||||
#### Force disable early stop
|
||||
Use this to turn off early stopping and run training to the [max_epoch](#force-training-for-min-or-max-epochs)
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(enable_early_stop=True)
|
||||
```
|
||||
|
||||
---
|
||||
#### Gradient Clipping
|
||||
Use this to turn off early stopping and run training to the [max_epoch](#force-training-for-min-or-max-epochs)
|
||||
``` {.python}
|
||||
# DEFAULT (ie: don't clip)
|
||||
trainer = Trainer(gradient_clip=0)
|
||||
```
|
||||
|
||||
|
||||
|
||||
---
|
||||
#### Inspect gradient norms
|
||||
Looking at grad norms can help you figure out where training might be going wrong.
|
||||
``` {.python}
|
||||
# DEFAULT (-1 doesn't track norms)
|
||||
trainer = Trainer(track_grad_norm=-1)
|
||||
|
||||
# track the LP norm (P=2 here)
|
||||
trainer = Trainer(track_grad_norm=2)
|
||||
```
|
||||
|
||||
|
||||
---
|
||||
#### Set how much of the training set to check
|
||||
If you don't want to check 100% of the training set (for debugging or if it's huge), set this flag
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(train_percent_check=1.0)
|
||||
|
||||
# check 10% only
|
||||
trainer = Trainer(train_percent_check=0.1)
|
||||
```
|
||||
@@ -1,57 +0,0 @@
|
||||
The lightning validation loop handles everything except the actual computations of your model. To decide what will happen in your validation loop, define the [validation_step function](../../Pytorch-lightning/LightningModule/#validation_step).
|
||||
Below are all the things lightning automates for you in the validation loop.
|
||||
|
||||
**Note**
|
||||
Lightning will run 5 steps of validation in the beginning of training as a sanity check so you don't have to wait until a full epoch to catch possible validation issues.
|
||||
|
||||
|
||||
|
||||
|
||||
---
|
||||
#### Check validation every n epochs
|
||||
If you have a small dataset you might want to check validation every n epochs
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(check_val_every_n_epoch=1)
|
||||
```
|
||||
|
||||
---
|
||||
#### Set how much of the validation set to check
|
||||
If you don't want to check 100% of the validation set (for debugging or if it's huge), set this flag
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(val_percent_check=1.0)
|
||||
|
||||
# check 10% only
|
||||
trainer = Trainer(val_percent_check=0.1)
|
||||
```
|
||||
|
||||
---
|
||||
#### Set how much of the test set to check
|
||||
If you don't want to check 100% of the test set (for debugging or if it's huge), set this flag
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(test_percent_check=1.0)
|
||||
|
||||
# check 10% only
|
||||
trainer = Trainer(test_percent_check=0.1)
|
||||
```
|
||||
|
||||
---
|
||||
#### Set validation check frequency within 1 training epoch
|
||||
For large datasets it's often desirable to check validation multiple times within a training loop
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(val_check_interval=0.95)
|
||||
|
||||
# check every .25 of an epoch
|
||||
trainer = Trainer(val_check_interval=0.25)
|
||||
```
|
||||
|
||||
---
|
||||
#### Set the number of validation sanity steps
|
||||
Lightning runs a few steps of validation in the beginning of training. This avoids crashing in the validation loop sometime deep into a lengthy training loop.
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(nb_sanity_val_steps=5)
|
||||
```
|
||||
@@ -1,48 +0,0 @@
|
||||
These flags are useful to help debug a model.
|
||||
|
||||
---
|
||||
#### Fast dev run
|
||||
This flag is meant for debugging a full train/val/test loop. It'll activate callbacks, everything but only with 1 training and 1 validation batch.
|
||||
Use this to debug a full run of your program quickly
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(fast_dev_run=False)
|
||||
```
|
||||
|
||||
---
|
||||
#### Inspect gradient norms
|
||||
Looking at grad norms can help you figure out where training might be going wrong.
|
||||
``` {.python}
|
||||
# DEFAULT (-1 doesn't track norms)
|
||||
trainer = Trainer(track_grad_norm=-1)
|
||||
|
||||
# track the LP norm (P=2 here)
|
||||
trainer = Trainer(track_grad_norm=2)
|
||||
```
|
||||
|
||||
---
|
||||
#### Make model overfit on subset of data
|
||||
A useful debugging trick is to make your model overfit a tiny fraction of the data.
|
||||
``` {.python}
|
||||
# DEFAULT don't overfit (ie: normal training)
|
||||
trainer = Trainer(overfit_pct=0.0)
|
||||
|
||||
# overfit on 1% of data
|
||||
trainer = Trainer(overfit_pct=0.01)
|
||||
```
|
||||
|
||||
---
|
||||
#### Print the parameter count by layer
|
||||
By default lightning prints a list of parameters *and submodules* when it starts training.
|
||||
|
||||
---
|
||||
#### Print which gradients are nan
|
||||
This option prints a list of tensors with nan gradients.
|
||||
``` {.python}
|
||||
# DEFAULT
|
||||
trainer = Trainer(print_nan_grads=False)
|
||||
```
|
||||
|
||||
---
|
||||
#### Log GPU usage
|
||||
Lightning automatically logs gpu usage to the test tube logs. It'll only do it at the metric logging interval, so it doesn't slow down training.
|
||||
@@ -1,74 +0,0 @@
|
||||
# Trainer
|
||||
[[Github Code](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/models/trainer.py)]
|
||||
|
||||
The lightning trainer abstracts best practices for running a training, val, test routine. It calls parts of your model when it wants to hand over full control and otherwise makes training assumptions which are now standard practice in AI research.
|
||||
|
||||
This is the basic use of the trainer:
|
||||
|
||||
``` {.python}
|
||||
from pytorch_lightning import Trainer
|
||||
|
||||
model = LightningTemplate()
|
||||
|
||||
trainer = Trainer()
|
||||
trainer.fit(model)
|
||||
```
|
||||
|
||||
But of course the fun is in all the advanced things it can do:
|
||||
|
||||
|
||||
**Checkpointing**
|
||||
|
||||
- Model saving
|
||||
- Model loading
|
||||
|
||||
**Computing cluster (SLURM)**
|
||||
|
||||
- [Running grid search on a cluster](SLURM%20Managed%20Cluster/#running-grid-search-on-a-cluster)
|
||||
- [Walltime auto-resubmit](SLURM%20Managed%20Cluster/#walltime-auto-resubmit)
|
||||
|
||||
**Debugging**
|
||||
|
||||
- [Fast dev run](Debugging/#fast-dev-run)
|
||||
- [Inspect gradient norms](Debugging/#inspect-gradient-norms)
|
||||
- [Log GPU usage](Debugging/#Log-gpu-usage)
|
||||
- [Make model overfit on subset of data](Debugging/#make-model-overfit-on-subset-of-data)
|
||||
- [Print the parameter count by layer](Debugging/#print-the-parameter-count-by-layer)
|
||||
- [Pring which gradients are nan](Debugging/#print-which-gradients-are-nan)
|
||||
|
||||
|
||||
**Distributed training**
|
||||
|
||||
- [16-bit mixed precision](Distributed%20training/#16-bit-mixed-precision)
|
||||
- [Multi-GPU](Distributed%20training/#Multi-GPU)
|
||||
- [Multi-node](Distributed%20training/#Multi-node)
|
||||
- [Single GPU](Distributed%20training/#single-gpu)
|
||||
- [Self-balancing architecture](Distributed%20training/#self-balancing-architecture)
|
||||
|
||||
|
||||
**Experiment Logging**
|
||||
|
||||
- [Display metrics in progress bar](Logging/#display-metrics-in-progress-bar)
|
||||
- Log arbitrary metrics
|
||||
- [Log metric row every k batches](Logging/#log-metric-row-every-k-batches)
|
||||
- [Process position](Logging/#process-position)
|
||||
- [Save a snapshot of all hyperparameters](Logging/#save-a-snapshot-of-all-hyperparameters)
|
||||
- [Snapshot code for a training run](Logging/#snapshot-code-for-a-training-run)
|
||||
- [Write logs file to csv every k batches](Logging/#write-logs-file-to-csv-every-k-batches)
|
||||
|
||||
**Training loop**
|
||||
|
||||
- [Accumulate gradients](Training%20Loop/#accumulated-gradients)
|
||||
- [Anneal Learning rate](Training%20Loop/#anneal-learning-rate)
|
||||
- [Force training for min or max epochs](Training%20Loop/#force-training-for-min-or-max-epochs)
|
||||
- [Force disable early stop](Training%20Loop/#force-disable-early-stop)
|
||||
- [Use multiple optimizers (like GANs)](../Pytorch-lightning/LightningModule/#configure_optimizers)
|
||||
- [Set how much of the training set to check (1-100%)](Training%20Loop/#set-how-much-of-the-training-set-to-check)
|
||||
|
||||
**Validation loop**
|
||||
|
||||
- [Check validation every n epochs](Validation%20Loop/#check-validation-every-n-epochs)
|
||||
- [Set how much of the validation set to check](Validation%20Loop/#set-how-much-of-the-validation-set-to-check)
|
||||
- [Set how much of the test set to check](Validation%20Loop/#set-how-much-of-the-test-set-to-check)
|
||||
- [Set validation check frequency within 1 training epoch](Validation%20Loop/#set-validation-check-frequency-within-1-training-epoch)
|
||||
- [Set the number of validation sanity steps](Validation%20Loop/#set-the-number-of-validation-sanity-steps)
|
||||
@@ -1,171 +0,0 @@
|
||||
### Template model definition
|
||||
In 99% of cases you want to just copy [this template](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py) to start a new lightningModule and change the core of what your model is actually trying to do.
|
||||
|
||||
```bash
|
||||
# get a copy of the module template
|
||||
wget https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py
|
||||
```
|
||||
|
||||
---
|
||||
### Trainer Example
|
||||
|
||||
** \_\_main__ function**
|
||||
|
||||
Normally, we want to let the \_\_main__ function start the training.
|
||||
Inside the main we parse training arguments with whatever hyperparameters we want. Your LightningModule will have a
|
||||
chance to add hyperparameters.
|
||||
|
||||
```{.python}
|
||||
from test_tube import HyperOptArgumentParser
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# use default args given by lightning
|
||||
root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
|
||||
parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False)
|
||||
add_default_args(parent_parser, root_dir)
|
||||
|
||||
# allow model to overwrite or extend args
|
||||
parser = ExampleModel.add_model_specific_args(parent_parser)
|
||||
hyperparams = parser.parse_args()
|
||||
|
||||
# train model
|
||||
main(hyperparams)
|
||||
```
|
||||
**Main Function**
|
||||
|
||||
The main function is your entry into the program. This is where you init your model, checkpoint directory, and launch the training.
|
||||
The main function should have 3 arguments:
|
||||
- hparams: a configuration of hyperparameters.
|
||||
- slurm_manager: Slurm cluster manager object (can be None)
|
||||
- dict: for you to return any values you want (useful in meta-learning, otherwise set to _)
|
||||
|
||||
```{}
|
||||
def main(hparams, cluster, results_dict):
|
||||
"""
|
||||
Main training routine specific for this project
|
||||
:param hparams:
|
||||
:return:
|
||||
"""
|
||||
# init experiment
|
||||
log_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
exp = Experiment(
|
||||
name='test_tube_exp',
|
||||
debug=True,
|
||||
save_dir=log_dir,
|
||||
version=0,
|
||||
autosave=False,
|
||||
description='test demo'
|
||||
)
|
||||
|
||||
# set the hparams for the experiment
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
|
||||
# build model
|
||||
model = MyLightningModule(hparams)
|
||||
|
||||
# callbacks
|
||||
early_stop = EarlyStopping(
|
||||
monitor=hparams.early_stop_metric,
|
||||
patience=hparams.early_stop_patience,
|
||||
verbose=True,
|
||||
mode=hparams.early_stop_mode
|
||||
)
|
||||
|
||||
model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
|
||||
checkpoint = ModelCheckpoint(
|
||||
filepath=model_save_path,
|
||||
save_function=None,
|
||||
save_best_only=True,
|
||||
verbose=True,
|
||||
monitor=hparams.model_save_monitor_value,
|
||||
mode=hparams.model_save_monitor_mode
|
||||
)
|
||||
|
||||
# configure trainer
|
||||
trainer = Trainer(
|
||||
experiment=exp,
|
||||
cluster=cluster,
|
||||
checkpoint_callback=checkpoint,
|
||||
early_stop_callback=early_stop,
|
||||
)
|
||||
|
||||
# train model
|
||||
trainer.fit(model)
|
||||
```
|
||||
|
||||
|
||||
|
||||
|
||||
The __main__ function will start training on your **main** function. If you use the HyperParameterOptimizer
|
||||
in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple
|
||||
argument parser you get the default arguments in the argument parser.
|
||||
|
||||
So, calling main(hyperparams) runs the model with the default argparse arguments.
|
||||
```{.python}
|
||||
main(hyperparams)
|
||||
```
|
||||
|
||||
---
|
||||
#### CPU hyperparameter search
|
||||
|
||||
```{.python}
|
||||
# run a grid search over 20 hyperparameter combinations.
|
||||
hyperparams.optimize_parallel_cpu(
|
||||
main_local,
|
||||
nb_trials=20,
|
||||
nb_workers=1
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
#### Hyperparameter search on a single or multiple GPUs
|
||||
```{.python}
|
||||
# run a grid search over 20 hyperparameter combinations.
|
||||
hyperparams.optimize_parallel_gpu(
|
||||
main_local,
|
||||
nb_trials=20,
|
||||
nb_workers=1,
|
||||
gpus=[0,1,2,3]
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
#### Hyperparameter search on a SLURM HPC cluster
|
||||
```{.python}
|
||||
def optimize_on_cluster(hyperparams):
|
||||
# enable cluster training
|
||||
cluster = SlurmCluster(
|
||||
hyperparam_optimizer=hyperparams,
|
||||
log_path=hyperparams.tt_save_path,
|
||||
test_tube_exp_name=hyperparams.tt_name
|
||||
)
|
||||
|
||||
# email for cluster coms
|
||||
cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True)
|
||||
|
||||
# configure cluster
|
||||
cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus
|
||||
cluster.job_time = '48:00:00'
|
||||
cluster.gpu_type = '1080ti'
|
||||
cluster.memory_mb_per_node = 48000
|
||||
|
||||
# any modules for code to run in env
|
||||
cluster.add_command('source activate pytorch_lightning')
|
||||
|
||||
# name of exp
|
||||
job_display_name = hyperparams.tt_name.split('_')[0]
|
||||
job_display_name = job_display_name[0:3]
|
||||
|
||||
# run hopt
|
||||
print('submitting jobs...')
|
||||
cluster.optimize_parallel_cluster_gpu(
|
||||
main,
|
||||
nb_trials=hyperparams.nb_hopt_trials,
|
||||
job_name=job_display_name
|
||||
)
|
||||
|
||||
# run cluster hyperparameter search
|
||||
optimize_on_cluster(hyperparams)
|
||||
```
|
||||
@@ -1,76 +0,0 @@
|
||||
###### New project Quick Start
|
||||
To start a new project define these two files.
|
||||
|
||||
1. [Define a LightningModule](/LightningModule/RequiredTrainerInterface/#template-model-definition)
|
||||
2. Pick a trainer
|
||||
- [Basic CPU Trainer](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/trainer_cpu_template.py)
|
||||
- [GPU cluster Trainer](https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/trainer_gpu_cluster_template.py)
|
||||
|
||||
###### Docs shortcuts
|
||||
- [LightningModule](LightningModule/RequiredTrainerInterface/)
|
||||
- [Trainer](Trainer/)
|
||||
|
||||
###### Quick start examples
|
||||
- [CPU example](examples/Examples/#cpu-hyperparameter-search)
|
||||
- [Hyperparameter search on single GPU](examples/Examples/#hyperparameter-search-on-a-single-or-multiple-gpus)
|
||||
- [Hyperparameter search on multiple GPUs on same node](examples/Examples/#hyperparameter-search-on-a-single-or-multiple-gpus)
|
||||
- [Hyperparameter search on a SLURM HPC cluster](examples/Examples/#Hyperparameter search on a SLURM HPC cluster)
|
||||
|
||||
|
||||
###### Checkpointing
|
||||
|
||||
- [Model saving](https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving)
|
||||
- [Model loading](https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics)
|
||||
|
||||
###### Computing cluster (SLURM)
|
||||
|
||||
- [Running grid search on a cluster](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#running-grid-search-on-a-cluster)
|
||||
- [Walltime auto-resubmit](https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#walltime-auto-resubmit)
|
||||
|
||||
###### Debugging
|
||||
|
||||
- [Fast dev run](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#fast-dev-run)
|
||||
- [Inspect gradient norms](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#inspect-gradient-norms)
|
||||
- [Log GPU usage](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage)
|
||||
- [Make model overfit on subset of data](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data)
|
||||
- [Print the parameter count by layer](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer)
|
||||
- [Pring which gradients are nan](https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan)
|
||||
|
||||
|
||||
###### Distributed training
|
||||
|
||||
- [16-bit mixed precision](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision)
|
||||
- [Multi-GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU)
|
||||
- [Multi-node](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node)
|
||||
- [Single GPU](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#single-gpu)
|
||||
- [Self-balancing architecture](https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#self-balancing-architecture)
|
||||
|
||||
|
||||
###### Experiment Logging
|
||||
|
||||
- [Display metrics in progress bar](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#display-metrics-in-progress-bar)
|
||||
- Log arbitrary metrics
|
||||
- [Log metric row every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#log-metric-row-every-k-batches)
|
||||
- [Process position](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#process-position)
|
||||
- [Save a snapshot of all hyperparameters](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#save-a-snapshot-of-all-hyperparameters)
|
||||
- [Snapshot code for a training run](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#snapshot-code-for-a-training-run)
|
||||
- [Write logs file to csv every k batches](https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#write-logs-file-to-csv-every-k-batches)
|
||||
|
||||
###### Training loop
|
||||
|
||||
- [Accumulate gradients](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients)
|
||||
- [Anneal Learning rate](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#anneal-learning-rate)
|
||||
- [Force training for min or max epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs)
|
||||
- [Force disable early stop](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop)
|
||||
- [Gradient Clipping](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping)
|
||||
- [Use multiple optimizers (like GANs)](https://williamfalcon.github.io/pytorch-lightning/Pytorch-Lightning/LightningModule/#configure_optimizers)
|
||||
- [Set how much of the training set to check (1-100%)](https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check)
|
||||
|
||||
######Validation loop
|
||||
|
||||
- [Check validation every n epochs](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#check-validation-every-n-epochs)
|
||||
- [Set how much of the validation set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-validation-set-to-check)
|
||||
- [Set how much of the test set to check](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-test-set-to-check)
|
||||
- [Set validation check frequency within 1 training epoch](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-validation-check-frequency-within-1-training-epoch)
|
||||
- [Set the number of validation sanity steps](https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-the-number-of-validation-sanity-steps)
|
||||
|
||||
@@ -0,0 +1,871 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>Examples - PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#template-model-definition" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Examples
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../.." title="Home" class="md-nav__link">
|
||||
Home
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3">
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="../../Trainer/hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4" checked>
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
Examples
|
||||
</label>
|
||||
|
||||
<a href="./" title="Examples" class="md-nav__link md-nav__link--active">
|
||||
Examples
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#template-model-definition" title="Template model definition" class="md-nav__link">
|
||||
Template model definition
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#trainer-example" title="Trainer Example" class="md-nav__link">
|
||||
Trainer Example
|
||||
</a>
|
||||
|
||||
<nav class="md-nav">
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#cpu-hyperparameter-search" title="CPU hyperparameter search" class="md-nav__link">
|
||||
CPU hyperparameter search
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" title="Hyperparameter search on a single or multiple GPUs" class="md-nav__link">
|
||||
Hyperparameter search on a single or multiple GPUs
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" title="Hyperparameter search on a SLURM HPC cluster" class="md-nav__link">
|
||||
Hyperparameter search on a SLURM HPC cluster
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#template-model-definition" title="Template model definition" class="md-nav__link">
|
||||
Template model definition
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#trainer-example" title="Trainer Example" class="md-nav__link">
|
||||
Trainer Example
|
||||
</a>
|
||||
|
||||
<nav class="md-nav">
|
||||
<ul class="md-nav__list">
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#cpu-hyperparameter-search" title="CPU hyperparameter search" class="md-nav__link">
|
||||
CPU hyperparameter search
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#hyperparameter-search-on-a-single-or-multiple-gpus" title="Hyperparameter search on a single or multiple GPUs" class="md-nav__link">
|
||||
Hyperparameter search on a single or multiple GPUs
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#hyperparameter-search-on-a-slurm-hpc-cluster" title="Hyperparameter search on a SLURM HPC cluster" class="md-nav__link">
|
||||
Hyperparameter search on a SLURM HPC cluster
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/examples/Examples.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>Examples</h1>
|
||||
|
||||
<h3 id="template-model-definition">Template model definition</h3>
|
||||
<p>In 99% of cases you want to just copy <a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples">one of the examples</a> to start a new lightningModule and change the core of what your model is actually trying to do.</p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># get a copy of the module template</span>
|
||||
wget https://raw.githubusercontent.com/williamFalcon/pytorch-lightning/master/pl_examples/new_project_templates/lightning_module_template.py
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h3 id="trainer-example">Trainer Example</h3>
|
||||
<p><strong> __main__ function</strong> </p>
|
||||
<p>Normally, we want to let the __main__ function start the training.
|
||||
Inside the main we parse training arguments with whatever hyperparameters we want. Your LightningModule will have a
|
||||
chance to add hyperparameters. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">test_tube</span> <span class="kn">import</span> <span class="n">HyperOptArgumentParser</span>
|
||||
|
||||
<span class="k">if</span> <span class="vm">__name__</span> <span class="o">==</span> <span class="s1">'__main__'</span><span class="p">:</span>
|
||||
|
||||
<span class="c1"># use default args given by lightning</span>
|
||||
<span class="n">root_dir</span> <span class="o">=</span> <span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">dirname</span><span class="p">(</span><span class="n">sys</span><span class="o">.</span><span class="n">modules</span><span class="p">[</span><span class="s1">'__main__'</span><span class="p">]</span><span class="o">.</span><span class="vm">__file__</span><span class="p">))[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">parent_parser</span> <span class="o">=</span> <span class="n">HyperOptArgumentParser</span><span class="p">(</span><span class="n">strategy</span><span class="o">=</span><span class="s1">'random_search'</span><span class="p">,</span> <span class="n">add_help</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
|
||||
<span class="n">add_default_args</span><span class="p">(</span><span class="n">parent_parser</span><span class="p">,</span> <span class="n">root_dir</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># allow model to overwrite or extend args</span>
|
||||
<span class="n">parser</span> <span class="o">=</span> <span class="n">ExampleModel</span><span class="o">.</span><span class="n">add_model_specific_args</span><span class="p">(</span><span class="n">parent_parser</span><span class="p">)</span>
|
||||
<span class="n">hyperparams</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_args</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># train model</span>
|
||||
<span class="n">main</span><span class="p">(</span><span class="n">hyperparams</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p><strong>Main Function</strong> </p>
|
||||
<p>The main function is your entry into the program. This is where you init your model, checkpoint directory, and launch the training.
|
||||
The main function should have 3 arguments: <br />
|
||||
- hparams: a configuration of hyperparameters. <br />
|
||||
- slurm_manager: Slurm cluster manager object (can be None)
|
||||
- dict: for you to return any values you want (useful in meta-learning, otherwise set to _) </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">main</span><span class="p">(</span><span class="n">hparams</span><span class="p">,</span> <span class="n">cluster</span><span class="p">,</span> <span class="n">results_dict</span><span class="p">):</span>
|
||||
<span class="sd">"""</span>
|
||||
<span class="sd"> Main training routine specific for this project</span>
|
||||
<span class="sd"> :param hparams:</span>
|
||||
<span class="sd"> :return:</span>
|
||||
<span class="sd"> """</span>
|
||||
<span class="c1"># build model</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">MyLightningModule</span><span class="p">(</span><span class="n">hparams</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># configure trainer</span>
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">()</span>
|
||||
|
||||
<span class="c1"># train model</span>
|
||||
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>The <strong>main</strong> function will start training on your <strong>main</strong> function. If you use the HyperParameterOptimizer
|
||||
in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple
|
||||
argument parser you get the default arguments in the argument parser.</p>
|
||||
<p>So, calling main(hyperparams) runs the model with the default argparse arguments. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">main</span><span class="p">(</span><span class="n">hyperparams</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="cpu-hyperparameter-search">CPU hyperparameter search</h4>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># run a grid search over 20 hyperparameter combinations.</span>
|
||||
<span class="n">hyperparams</span><span class="o">.</span><span class="n">optimize_parallel_cpu</span><span class="p">(</span>
|
||||
<span class="n">main_local</span><span class="p">,</span>
|
||||
<span class="n">nb_trials</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span>
|
||||
<span class="n">nb_workers</span><span class="o">=</span><span class="mi">1</span>
|
||||
<span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="hyperparameter-search-on-a-single-or-multiple-gpus">Hyperparameter search on a single or multiple GPUs</h4>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># run a grid search over 20 hyperparameter combinations.</span>
|
||||
<span class="n">hyperparams</span><span class="o">.</span><span class="n">optimize_parallel_gpu</span><span class="p">(</span>
|
||||
<span class="n">main_local</span><span class="p">,</span>
|
||||
<span class="n">nb_trials</span><span class="o">=</span><span class="mi">20</span><span class="p">,</span>
|
||||
<span class="n">nb_workers</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
|
||||
<span class="n">gpus</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">2</span><span class="p">,</span><span class="mi">3</span><span class="p">]</span>
|
||||
<span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<hr />
|
||||
<h4 id="hyperparameter-search-on-a-slurm-hpc-cluster">Hyperparameter search on a SLURM HPC cluster</h4>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16
|
||||
17
|
||||
18
|
||||
19
|
||||
20
|
||||
21
|
||||
22
|
||||
23
|
||||
24
|
||||
25
|
||||
26
|
||||
27
|
||||
28
|
||||
29
|
||||
30
|
||||
31
|
||||
32
|
||||
33
|
||||
34</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">optimize_on_cluster</span><span class="p">(</span><span class="n">hyperparams</span><span class="p">):</span>
|
||||
<span class="c1"># enable cluster training</span>
|
||||
<span class="n">cluster</span> <span class="o">=</span> <span class="n">SlurmCluster</span><span class="p">(</span>
|
||||
<span class="n">hyperparam_optimizer</span><span class="o">=</span><span class="n">hyperparams</span><span class="p">,</span>
|
||||
<span class="n">log_path</span><span class="o">=</span><span class="n">hyperparams</span><span class="o">.</span><span class="n">tt_save_path</span><span class="p">,</span>
|
||||
<span class="n">test_tube_exp_name</span><span class="o">=</span><span class="n">hyperparams</span><span class="o">.</span><span class="n">tt_name</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># email for cluster coms</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">notify_job_status</span><span class="p">(</span><span class="n">email</span><span class="o">=</span><span class="s1">'add_email_here'</span><span class="p">,</span> <span class="n">on_done</span><span class="o">=</span><span class="bp">True</span><span class="p">,</span> <span class="n">on_fail</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># configure cluster</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">per_experiment_nb_gpus</span> <span class="o">=</span> <span class="n">hyperparams</span><span class="o">.</span><span class="n">per_experiment_nb_gpus</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">job_time</span> <span class="o">=</span> <span class="s1">'48:00:00'</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">gpu_type</span> <span class="o">=</span> <span class="s1">'1080ti'</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">memory_mb_per_node</span> <span class="o">=</span> <span class="mi">48000</span>
|
||||
|
||||
<span class="c1"># any modules for code to run in env</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">add_command</span><span class="p">(</span><span class="s1">'source activate pytorch_lightning'</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># name of exp</span>
|
||||
<span class="n">job_display_name</span> <span class="o">=</span> <span class="n">hyperparams</span><span class="o">.</span><span class="n">tt_name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="s1">'_'</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>
|
||||
<span class="n">job_display_name</span> <span class="o">=</span> <span class="n">job_display_name</span><span class="p">[</span><span class="mi">0</span><span class="p">:</span><span class="mi">3</span><span class="p">]</span>
|
||||
|
||||
<span class="c1"># run hopt</span>
|
||||
<span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">'submitting jobs...'</span><span class="p">)</span>
|
||||
<span class="n">cluster</span><span class="o">.</span><span class="n">optimize_parallel_cluster_gpu</span><span class="p">(</span>
|
||||
<span class="n">main</span><span class="p">,</span>
|
||||
<span class="n">nb_trials</span><span class="o">=</span><span class="n">hyperparams</span><span class="o">.</span><span class="n">nb_hopt_trials</span><span class="p">,</span>
|
||||
<span class="n">job_name</span><span class="o">=</span><span class="n">job_display_name</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
<span class="c1"># run cluster hyperparameter search </span>
|
||||
<span class="n">optimize_on_cluster</span><span class="p">(</span><span class="n">hyperparams</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
<a href="../../Trainer/hooks/" title="Hooks" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Previous
|
||||
</span>
|
||||
Hooks
|
||||
</span>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
@@ -1 +0,0 @@
|
||||
from .lightning_module_template import LightningTemplateModel
|
||||
@@ -1,249 +0,0 @@
|
||||
import os
|
||||
from collections import OrderedDict
|
||||
import torch.nn as nn
|
||||
from torchvision.datasets import MNIST
|
||||
import torchvision.transforms as transforms
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from test_tube import HyperOptArgumentParser
|
||||
from torch import optim
|
||||
from torch.utils.data import DataLoader
|
||||
from torch.utils.data.distributed import DistributedSampler
|
||||
|
||||
from pytorch_lightning.root_module.root_module import LightningModule
|
||||
|
||||
|
||||
class LightningTemplateModel(LightningModule):
|
||||
"""
|
||||
Sample model to show how to define a template
|
||||
"""
|
||||
|
||||
def __init__(self, hparams):
|
||||
"""
|
||||
Pass in parsed HyperOptArgumentParser to the model
|
||||
:param hparams:
|
||||
"""
|
||||
# init superclass
|
||||
super(LightningTemplateModel, self).__init__(hparams)
|
||||
|
||||
self.batch_size = hparams.batch_size
|
||||
|
||||
# build model
|
||||
self.__build_model()
|
||||
|
||||
# ---------------------
|
||||
# MODEL SETUP
|
||||
# ---------------------
|
||||
def __build_model(self):
|
||||
"""
|
||||
Layout model
|
||||
:return:
|
||||
"""
|
||||
self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
|
||||
self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
|
||||
self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
|
||||
|
||||
self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)
|
||||
|
||||
# ---------------------
|
||||
# TRAINING
|
||||
# ---------------------
|
||||
def forward(self, x):
|
||||
"""
|
||||
No special modification required for lightning, define as you normally would
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
|
||||
x = self.c_d1(x)
|
||||
x = torch.tanh(x)
|
||||
x = self.c_d1_bn(x)
|
||||
x = self.c_d1_drop(x)
|
||||
|
||||
x = self.c_d2(x)
|
||||
logits = F.log_softmax(x, dim=1)
|
||||
|
||||
return logits
|
||||
|
||||
def loss(self, labels, logits):
|
||||
nll = F.nll_loss(logits, labels)
|
||||
return nll
|
||||
|
||||
def training_step(self, data_batch, batch_i):
|
||||
"""
|
||||
Lightning calls this inside the training loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
# forward pass
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
# calculate loss
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
output = OrderedDict({
|
||||
'loss': loss_val
|
||||
})
|
||||
|
||||
# can also return just a scalar instead of a dict (return loss_val)
|
||||
return output
|
||||
|
||||
def validation_step(self, data_batch, batch_i):
|
||||
"""
|
||||
Lightning calls this inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
|
||||
output = OrderedDict({
|
||||
'val_loss': loss_val,
|
||||
'val_acc': torch.tensor(val_acc).cuda(loss_val.device.index),
|
||||
})
|
||||
|
||||
# can also return just a scalar instead of a dict (return loss_val)
|
||||
return output
|
||||
|
||||
def validation_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
# if returned a scalar from validation_step, outputs is a list of tensor scalars
|
||||
# we return just the average in this case (if we want)
|
||||
# return torch.stack(outputs).mean()
|
||||
|
||||
val_loss_mean = 0
|
||||
val_acc_mean = 0
|
||||
for output in outputs:
|
||||
val_loss_mean += output['val_loss']
|
||||
val_acc_mean += output['val_acc']
|
||||
|
||||
val_loss_mean /= len(outputs)
|
||||
val_acc_mean /= len(outputs)
|
||||
tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
|
||||
return tqdm_dic
|
||||
|
||||
def update_tng_log_metrics(self, logs):
|
||||
return logs
|
||||
|
||||
# ---------------------
|
||||
# MODEL SAVING
|
||||
# ---------------------
|
||||
def get_save_dict(self):
|
||||
checkpoint = {'state_dict': self.state_dict()}
|
||||
return checkpoint
|
||||
|
||||
def load_model_specific(self, checkpoint):
|
||||
self.load_state_dict(checkpoint['state_dict'])
|
||||
pass
|
||||
|
||||
# ---------------------
|
||||
# TRAINING SETUP
|
||||
# ---------------------
|
||||
def configure_optimizers(self):
|
||||
"""
|
||||
return whatever optimizers we want here
|
||||
:return: list of optimizers
|
||||
"""
|
||||
optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
|
||||
return [optimizer]
|
||||
|
||||
def __dataloader(self, train):
|
||||
# init data generators
|
||||
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
|
||||
dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)
|
||||
|
||||
# when using multi-node we need to add the datasampler
|
||||
train_sampler = None
|
||||
batch_size = self.hparams.batch_size
|
||||
|
||||
try:
|
||||
if self.on_gpu:
|
||||
train_sampler = DistributedSampler(dataset, rank=self.trainer.proc_rank)
|
||||
batch_size = batch_size // self.trainer.world_size # scale batch size
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
should_shuffle = train_sampler is None
|
||||
loader = DataLoader(
|
||||
dataset=dataset,
|
||||
batch_size=batch_size,
|
||||
shuffle=should_shuffle,
|
||||
sampler=train_sampler
|
||||
)
|
||||
|
||||
return loader
|
||||
|
||||
@property
|
||||
def tng_dataloader(self):
|
||||
if self._tng_dataloader is None:
|
||||
try:
|
||||
self._tng_dataloader = self.__dataloader(train=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise e
|
||||
return self._tng_dataloader
|
||||
|
||||
@property
|
||||
def val_dataloader(self):
|
||||
if self._val_dataloader is None:
|
||||
try:
|
||||
self._val_dataloader = self.__dataloader(train=False)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise e
|
||||
return self._val_dataloader
|
||||
|
||||
@property
|
||||
def test_dataloader(self):
|
||||
if self._test_dataloader is None:
|
||||
try:
|
||||
self._test_dataloader = self.__dataloader(train=False)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise e
|
||||
return self._test_dataloader
|
||||
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser, root_dir):
|
||||
"""
|
||||
Parameters you define here will be available to your model through self.hparams
|
||||
:param parent_parser:
|
||||
:param root_dir:
|
||||
:return:
|
||||
"""
|
||||
parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
|
||||
|
||||
# param overwrites
|
||||
# parser.set_defaults(gradient_clip=5.0)
|
||||
|
||||
# network params
|
||||
parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
|
||||
parser.add_argument('--in_features', default=28*28, type=int)
|
||||
parser.add_argument('--out_features', default=10, type=int)
|
||||
parser.add_argument('--hidden_dim', default=50000, type=int) # use 500 for CPU, 50000 for GPU to see speed difference
|
||||
|
||||
# data
|
||||
parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
|
||||
|
||||
# training params (opt)
|
||||
parser.opt_list('--learning_rate', default=0.001*8, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
|
||||
tunable=False)
|
||||
parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
|
||||
|
||||
# if using 2 nodes with 4 gpus each the batch size here (256) will be 256 / (2*8) = 16 per gpu
|
||||
parser.opt_list('--batch_size', default=256*8, type=int, options=[32, 64, 128, 256], tunable=False,
|
||||
help='batch size will be divided over all the gpus being used across all nodes')
|
||||
return parser
|
||||
@@ -1,172 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
import numpy as np
|
||||
from time import sleep
|
||||
import torch
|
||||
|
||||
from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
|
||||
from pytorch_lightning.models.trainer import Trainer
|
||||
from pytorch_lightning.utils.arg_parse import add_default_args
|
||||
|
||||
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
|
||||
|
||||
SEED = 2334
|
||||
torch.manual_seed(SEED)
|
||||
np.random.seed(SEED)
|
||||
|
||||
# ---------------------
|
||||
# DEFINE MODEL HERE
|
||||
# ---------------------
|
||||
from lightning_module_template import LightningTemplateModel
|
||||
# ---------------------
|
||||
|
||||
"""
|
||||
Allows training by using command line arguments
|
||||
Run by:
|
||||
# TYPE YOUR RUN COMMAND HERE
|
||||
"""
|
||||
|
||||
|
||||
def main_local(hparams):
|
||||
main(hparams, None, None)
|
||||
|
||||
|
||||
def main(hparams, cluster, results_dict):
|
||||
"""
|
||||
Main training routine specific for this project
|
||||
:param hparams:
|
||||
:return:
|
||||
"""
|
||||
# ------------------------
|
||||
# 1 INIT LIGHTNING MODEL
|
||||
# ------------------------
|
||||
print('loading model...')
|
||||
model = LightningTemplateModel(hparams)
|
||||
print('model built')
|
||||
|
||||
# ------------------------
|
||||
# 2 INIT TEST TUBE EXP
|
||||
# ------------------------
|
||||
# when using grid search, it's possible for all models to start at once
|
||||
# and use the same test tube experiment version
|
||||
relative_node_id = int(os.environ['SLURM_NODEID'])
|
||||
sleep(relative_node_id + 1)
|
||||
|
||||
# init experiment
|
||||
exp = Experiment(
|
||||
name=hyperparams.experiment_name,
|
||||
save_dir=hyperparams.test_tube_save_path,
|
||||
autosave=False,
|
||||
description='test demo'
|
||||
)
|
||||
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
|
||||
# ------------------------
|
||||
# 3 DEFINE CALLBACKS
|
||||
# ------------------------
|
||||
model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
|
||||
early_stop = EarlyStopping(
|
||||
monitor='val_acc',
|
||||
patience=3,
|
||||
verbose=True,
|
||||
mode='max'
|
||||
)
|
||||
|
||||
checkpoint = ModelCheckpoint(
|
||||
filepath=model_save_path,
|
||||
save_best_only=True,
|
||||
verbose=True,
|
||||
monitor='val_loss',
|
||||
mode='min'
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 4 INIT TRAINER
|
||||
# ------------------------
|
||||
trainer = Trainer(
|
||||
experiment=exp,
|
||||
cluster=cluster,
|
||||
checkpoint_callback=checkpoint,
|
||||
early_stop_callback=early_stop,
|
||||
gpus=hparams.gpus,
|
||||
nb_gpu_nodes=hyperparams.nb_gpu_nodes
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 5 START TRAINING
|
||||
# ------------------------
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
def optimize_on_cluster(hyperparams):
|
||||
# enable cluster training
|
||||
# log all scripts to the test tube folder
|
||||
cluster = SlurmCluster(
|
||||
hyperparam_optimizer=hyperparams,
|
||||
log_path=hyperparams.slurm_log_path,
|
||||
)
|
||||
|
||||
# email for cluster coms
|
||||
cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True)
|
||||
|
||||
# configure cluster
|
||||
cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus
|
||||
cluster.per_experiment_nb_nodes = hyperparams.nb_gpu_nodes
|
||||
cluster.job_time = '2:00:00'
|
||||
cluster.gpu_type = 'volta'
|
||||
cluster.memory_mb_per_node = 0
|
||||
|
||||
# any modules for code to run in env
|
||||
cluster.add_command('source activate lightning')
|
||||
|
||||
# run only on 32GB voltas
|
||||
cluster.add_slurm_cmd(cmd='constraint', value='volta32gb', comment='use 32gb gpus')
|
||||
cluster.add_slurm_cmd(cmd='partition', value=hyperparams.gpu_partition, comment='use 32gb gpus')
|
||||
|
||||
# run hopt
|
||||
# creates and submits jobs to slurm
|
||||
cluster.optimize_parallel_cluster_gpu(
|
||||
main,
|
||||
nb_trials=hyperparams.nb_hopt_trials,
|
||||
job_name=hyperparams.experiment_name
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# use default args
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
|
||||
|
||||
checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
|
||||
test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
|
||||
slurm_out_dir = os.path.join(demo_log_dir, 'slurm_scripts')
|
||||
|
||||
parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
|
||||
|
||||
# cluster args not defined inside the model
|
||||
parent_parser.add_argument('--gpu_partition', type=str, help='consult your cluster manual')
|
||||
|
||||
# TODO: make 1 param
|
||||
parent_parser.add_argument('--per_experiment_nb_gpus', type=int, help='how many gpus to use in a node')
|
||||
parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node')
|
||||
|
||||
parent_parser.add_argument('--nb_gpu_nodes', type=int, default=1, help='how many nodes to use in a cluster')
|
||||
parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
|
||||
parent_parser.add_argument('--slurm_log_path', type=str, default=slurm_out_dir, help='where to save slurm meta')
|
||||
parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
|
||||
parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
|
||||
parent_parser.add_argument('--nb_hopt_trials', type=int, default=1, help='how many grid search trials to run')
|
||||
|
||||
# allow model to overwrite or extend args
|
||||
parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
|
||||
hyperparams = parser.parse_args()
|
||||
|
||||
# ---------------------
|
||||
# RUN TRAINING
|
||||
# ---------------------
|
||||
# run on HPC cluster
|
||||
print('RUNNING ON SLURM CLUSTER')
|
||||
optimize_on_cluster(hyperparams)
|
||||
@@ -1,110 +0,0 @@
|
||||
"""
|
||||
Runs a model on a single node across N-gpus.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import numpy as np
|
||||
from time import sleep
|
||||
import torch
|
||||
|
||||
from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
|
||||
from pytorch_lightning.models.trainer import Trainer
|
||||
from pytorch_lightning.utils.arg_parse import add_default_args
|
||||
|
||||
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
|
||||
|
||||
SEED = 2334
|
||||
torch.manual_seed(SEED)
|
||||
np.random.seed(SEED)
|
||||
|
||||
from lightning_module_template import LightningTemplateModel
|
||||
|
||||
|
||||
def main(hparams):
|
||||
"""
|
||||
Main training routine specific for this project
|
||||
:param hparams:
|
||||
:return:
|
||||
"""
|
||||
# ------------------------
|
||||
# 1 INIT LIGHTNING MODEL
|
||||
# ------------------------
|
||||
print('loading model...')
|
||||
model = LightningTemplateModel(hparams)
|
||||
print('model built')
|
||||
|
||||
# ------------------------
|
||||
# 2 INIT TEST TUBE EXP
|
||||
# ------------------------
|
||||
|
||||
# init experiment
|
||||
exp = Experiment(
|
||||
name=hyperparams.experiment_name,
|
||||
save_dir=hyperparams.test_tube_save_path,
|
||||
autosave=False,
|
||||
description='test demo'
|
||||
)
|
||||
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
|
||||
# ------------------------
|
||||
# 3 DEFINE CALLBACKS
|
||||
# ------------------------
|
||||
model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
|
||||
early_stop = EarlyStopping(
|
||||
monitor='val_acc',
|
||||
patience=3,
|
||||
verbose=True,
|
||||
mode='max'
|
||||
)
|
||||
|
||||
checkpoint = ModelCheckpoint(
|
||||
filepath=model_save_path,
|
||||
save_best_only=True,
|
||||
verbose=True,
|
||||
monitor='val_loss',
|
||||
mode='min'
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 4 INIT TRAINER
|
||||
# ------------------------
|
||||
trainer = Trainer(
|
||||
experiment=exp,
|
||||
checkpoint_callback=checkpoint,
|
||||
early_stop_callback=early_stop,
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 5 START TRAINING
|
||||
# ------------------------
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# dirs
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
|
||||
checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
|
||||
test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
|
||||
|
||||
# although we user hyperOptParser, we are using it only as argparse right now
|
||||
parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
|
||||
|
||||
# gpu args
|
||||
parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
|
||||
parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
|
||||
parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
|
||||
|
||||
# allow model to overwrite or extend args
|
||||
parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
|
||||
hyperparams = parser.parse_args()
|
||||
|
||||
# ---------------------
|
||||
# RUN TRAINING
|
||||
# ---------------------
|
||||
# run on HPC cluster
|
||||
print(f'RUNNING ON CPU')
|
||||
main(hyperparams)
|
||||
@@ -1,113 +0,0 @@
|
||||
"""
|
||||
Runs a model on a single node across N-gpus.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import numpy as np
|
||||
from time import sleep
|
||||
import torch
|
||||
|
||||
from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
|
||||
from pytorch_lightning.models.trainer import Trainer
|
||||
from pytorch_lightning.utils.arg_parse import add_default_args
|
||||
|
||||
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
|
||||
|
||||
SEED = 2334
|
||||
torch.manual_seed(SEED)
|
||||
np.random.seed(SEED)
|
||||
|
||||
from lightning_module_template import LightningTemplateModel
|
||||
|
||||
|
||||
def main(hparams):
|
||||
"""
|
||||
Main training routine specific for this project
|
||||
:param hparams:
|
||||
:return:
|
||||
"""
|
||||
# ------------------------
|
||||
# 1 INIT LIGHTNING MODEL
|
||||
# ------------------------
|
||||
print('loading model...')
|
||||
model = LightningTemplateModel(hparams)
|
||||
print('model built')
|
||||
|
||||
# ------------------------
|
||||
# 2 INIT TEST TUBE EXP
|
||||
# ------------------------
|
||||
|
||||
# init experiment
|
||||
exp = Experiment(
|
||||
name=hyperparams.experiment_name,
|
||||
save_dir=hyperparams.test_tube_save_path,
|
||||
autosave=False,
|
||||
description='test demo'
|
||||
)
|
||||
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
|
||||
# ------------------------
|
||||
# 3 DEFINE CALLBACKS
|
||||
# ------------------------
|
||||
model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
|
||||
early_stop = EarlyStopping(
|
||||
monitor='val_acc',
|
||||
patience=3,
|
||||
verbose=True,
|
||||
mode='max'
|
||||
)
|
||||
|
||||
checkpoint = ModelCheckpoint(
|
||||
filepath=model_save_path,
|
||||
save_best_only=True,
|
||||
verbose=True,
|
||||
monitor='val_loss',
|
||||
mode='min'
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 4 INIT TRAINER
|
||||
# ------------------------
|
||||
trainer = Trainer(
|
||||
experiment=exp,
|
||||
checkpoint_callback=checkpoint,
|
||||
early_stop_callback=early_stop,
|
||||
gpus=hparams.gpus,
|
||||
use_amp=True
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 5 START TRAINING
|
||||
# ------------------------
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# dirs
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
|
||||
checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
|
||||
test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
|
||||
|
||||
# although we user hyperOptParser, we are using it only as argparse right now
|
||||
parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
|
||||
|
||||
# gpu args
|
||||
parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node. -1 uses all the gpus on the node')
|
||||
parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
|
||||
parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
|
||||
parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
|
||||
|
||||
# allow model to overwrite or extend args
|
||||
parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
|
||||
hyperparams = parser.parse_args()
|
||||
|
||||
# ---------------------
|
||||
# RUN TRAINING
|
||||
# ---------------------
|
||||
# run on HPC cluster
|
||||
print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
|
||||
main(hyperparams)
|
||||
@@ -1,112 +0,0 @@
|
||||
"""
|
||||
Runs a model on a single node across N-gpus.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import numpy as np
|
||||
from time import sleep
|
||||
import torch
|
||||
|
||||
from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
|
||||
from pytorch_lightning.models.trainer import Trainer
|
||||
from pytorch_lightning.utils.arg_parse import add_default_args
|
||||
|
||||
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
|
||||
|
||||
SEED = 2334
|
||||
torch.manual_seed(SEED)
|
||||
np.random.seed(SEED)
|
||||
|
||||
from lightning_module_template import LightningTemplateModel
|
||||
|
||||
|
||||
def main(hparams):
|
||||
"""
|
||||
Main training routine specific for this project
|
||||
:param hparams:
|
||||
:return:
|
||||
"""
|
||||
# ------------------------
|
||||
# 1 INIT LIGHTNING MODEL
|
||||
# ------------------------
|
||||
print('loading model...')
|
||||
model = LightningTemplateModel(hparams)
|
||||
print('model built')
|
||||
|
||||
# ------------------------
|
||||
# 2 INIT TEST TUBE EXP
|
||||
# ------------------------
|
||||
|
||||
# init experiment
|
||||
exp = Experiment(
|
||||
name=hyperparams.experiment_name,
|
||||
save_dir=hyperparams.test_tube_save_path,
|
||||
autosave=False,
|
||||
description='test demo'
|
||||
)
|
||||
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
|
||||
# ------------------------
|
||||
# 3 DEFINE CALLBACKS
|
||||
# ------------------------
|
||||
model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
|
||||
early_stop = EarlyStopping(
|
||||
monitor='val_acc',
|
||||
patience=3,
|
||||
verbose=True,
|
||||
mode='max'
|
||||
)
|
||||
|
||||
checkpoint = ModelCheckpoint(
|
||||
filepath=model_save_path,
|
||||
save_best_only=True,
|
||||
verbose=True,
|
||||
monitor='val_loss',
|
||||
mode='min'
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 4 INIT TRAINER
|
||||
# ------------------------
|
||||
trainer = Trainer(
|
||||
experiment=exp,
|
||||
checkpoint_callback=checkpoint,
|
||||
early_stop_callback=early_stop,
|
||||
gpus=hparams.gpus,
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 5 START TRAINING
|
||||
# ------------------------
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# dirs
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
|
||||
checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
|
||||
test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
|
||||
|
||||
# although we user hyperOptParser, we are using it only as argparse right now
|
||||
parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
|
||||
|
||||
# gpu args
|
||||
parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node. -1 uses all the gpus on the node')
|
||||
parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
|
||||
parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
|
||||
parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
|
||||
|
||||
# allow model to overwrite or extend args
|
||||
parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
|
||||
hyperparams = parser.parse_args()
|
||||
|
||||
# ---------------------
|
||||
# RUN TRAINING
|
||||
# ---------------------
|
||||
# run on HPC cluster
|
||||
print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
|
||||
main(hyperparams)
|
||||
@@ -1,112 +0,0 @@
|
||||
"""
|
||||
Runs a model on a single node across N-gpus.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import numpy as np
|
||||
from time import sleep
|
||||
import torch
|
||||
|
||||
from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
|
||||
from pytorch_lightning.models.trainer import Trainer
|
||||
from pytorch_lightning.utils.arg_parse import add_default_args
|
||||
|
||||
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
|
||||
|
||||
SEED = 2334
|
||||
torch.manual_seed(SEED)
|
||||
np.random.seed(SEED)
|
||||
|
||||
from lightning_module_template import LightningTemplateModel
|
||||
|
||||
|
||||
def main(hparams):
|
||||
"""
|
||||
Main training routine specific for this project
|
||||
:param hparams:
|
||||
:return:
|
||||
"""
|
||||
# ------------------------
|
||||
# 1 INIT LIGHTNING MODEL
|
||||
# ------------------------
|
||||
print('loading model...')
|
||||
model = LightningTemplateModel(hparams)
|
||||
print('model built')
|
||||
|
||||
# ------------------------
|
||||
# 2 INIT TEST TUBE EXP
|
||||
# ------------------------
|
||||
|
||||
# init experiment
|
||||
exp = Experiment(
|
||||
name=hyperparams.experiment_name,
|
||||
save_dir=hyperparams.test_tube_save_path,
|
||||
autosave=False,
|
||||
description='test demo'
|
||||
)
|
||||
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
|
||||
# ------------------------
|
||||
# 3 DEFINE CALLBACKS
|
||||
# ------------------------
|
||||
model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
|
||||
early_stop = EarlyStopping(
|
||||
monitor='val_acc',
|
||||
patience=3,
|
||||
verbose=True,
|
||||
mode='max'
|
||||
)
|
||||
|
||||
checkpoint = ModelCheckpoint(
|
||||
filepath=model_save_path,
|
||||
save_best_only=True,
|
||||
verbose=True,
|
||||
monitor='val_loss',
|
||||
mode='min'
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 4 INIT TRAINER
|
||||
# ------------------------
|
||||
trainer = Trainer(
|
||||
experiment=exp,
|
||||
checkpoint_callback=checkpoint,
|
||||
early_stop_callback=early_stop,
|
||||
gpus=hparams.gpus,
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 5 START TRAINING
|
||||
# ------------------------
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# dirs
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
|
||||
checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
|
||||
test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
|
||||
|
||||
# although we user hyperOptParser, we are using it only as argparse right now
|
||||
parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
|
||||
|
||||
# gpu args
|
||||
parent_parser.add_argument('--gpus', type=str, default='-1', help='how many gpus to use in the node. -1 uses all the gpus on the node')
|
||||
parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
|
||||
parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
|
||||
parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
|
||||
|
||||
# allow model to overwrite or extend args
|
||||
parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
|
||||
hyperparams = parser.parse_args()
|
||||
|
||||
# ---------------------
|
||||
# RUN TRAINING
|
||||
# ---------------------
|
||||
# run on HPC cluster
|
||||
print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
|
||||
main(hyperparams)
|
||||
@@ -1,112 +0,0 @@
|
||||
"""
|
||||
Runs a model on a single node across N-gpus.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
import numpy as np
|
||||
from time import sleep
|
||||
import torch
|
||||
|
||||
from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
|
||||
from pytorch_lightning.models.trainer import Trainer
|
||||
from pytorch_lightning.utils.arg_parse import add_default_args
|
||||
|
||||
from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
|
||||
|
||||
SEED = 2334
|
||||
torch.manual_seed(SEED)
|
||||
np.random.seed(SEED)
|
||||
|
||||
from lightning_module_template import LightningTemplateModel
|
||||
|
||||
|
||||
def main(hparams):
|
||||
"""
|
||||
Main training routine specific for this project
|
||||
:param hparams:
|
||||
:return:
|
||||
"""
|
||||
# ------------------------
|
||||
# 1 INIT LIGHTNING MODEL
|
||||
# ------------------------
|
||||
print('loading model...')
|
||||
model = LightningTemplateModel(hparams)
|
||||
print('model built')
|
||||
|
||||
# ------------------------
|
||||
# 2 INIT TEST TUBE EXP
|
||||
# ------------------------
|
||||
|
||||
# init experiment
|
||||
exp = Experiment(
|
||||
name=hyperparams.experiment_name,
|
||||
save_dir=hyperparams.test_tube_save_path,
|
||||
autosave=False,
|
||||
description='test demo'
|
||||
)
|
||||
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
|
||||
# ------------------------
|
||||
# 3 DEFINE CALLBACKS
|
||||
# ------------------------
|
||||
model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
|
||||
early_stop = EarlyStopping(
|
||||
monitor='val_acc',
|
||||
patience=3,
|
||||
verbose=True,
|
||||
mode='max'
|
||||
)
|
||||
|
||||
checkpoint = ModelCheckpoint(
|
||||
filepath=model_save_path,
|
||||
save_best_only=True,
|
||||
verbose=True,
|
||||
monitor='val_loss',
|
||||
mode='min'
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 4 INIT TRAINER
|
||||
# ------------------------
|
||||
trainer = Trainer(
|
||||
experiment=exp,
|
||||
checkpoint_callback=checkpoint,
|
||||
early_stop_callback=early_stop,
|
||||
gpus=hparams.gpus,
|
||||
)
|
||||
|
||||
# ------------------------
|
||||
# 5 START TRAINING
|
||||
# ------------------------
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# dirs
|
||||
root_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
demo_log_dir = os.path.join(root_dir, 'pt_lightning_demo_logs')
|
||||
checkpoint_dir = os.path.join(demo_log_dir, 'model_weights')
|
||||
test_tube_dir = os.path.join(demo_log_dir, 'test_tube_data')
|
||||
|
||||
# although we user hyperOptParser, we are using it only as argparse right now
|
||||
parent_parser = HyperOptArgumentParser(strategy='grid_search', add_help=False)
|
||||
|
||||
# gpu args
|
||||
parent_parser.add_argument('--gpus', type=str, default='0', help='how many gpus to use in the node. -1 uses all the gpus on the node')
|
||||
parent_parser.add_argument('--test_tube_save_path', type=str, default=test_tube_dir, help='where to save logs')
|
||||
parent_parser.add_argument('--model_save_path', type=str, default=checkpoint_dir, help='where to save model')
|
||||
parent_parser.add_argument('--experiment_name', type=str, default='pt_lightning_exp_a', help='test tube exp name')
|
||||
|
||||
# allow model to overwrite or extend args
|
||||
parser = LightningTemplateModel.add_model_specific_args(parent_parser, root_dir)
|
||||
hyperparams = parser.parse_args()
|
||||
|
||||
# ---------------------
|
||||
# RUN TRAINING
|
||||
# ---------------------
|
||||
# run on HPC cluster
|
||||
print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {hyperparams.gpus}')
|
||||
main(hyperparams)
|
||||
@@ -1,73 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
from test_tube import HyperOptArgumentParser, Experiment
|
||||
from pytorch_lightning.models.trainer import Trainer
|
||||
from pytorch_lightning.utils.arg_parse import add_default_args
|
||||
from pytorch_lightning.callbacks.pt_callbacks import EarlyStopping, ModelCheckpoint
|
||||
from docs.source.examples.example_model import ExampleModel
|
||||
|
||||
|
||||
def main(hparams):
|
||||
"""
|
||||
Main training routine specific for this project
|
||||
:param hparams:
|
||||
:return:
|
||||
"""
|
||||
# init experiment
|
||||
exp = Experiment(
|
||||
name=hparams.tt_name,
|
||||
debug=hparams.debug,
|
||||
save_dir=hparams.tt_save_path,
|
||||
version=hparams.hpc_exp_number,
|
||||
autosave=False,
|
||||
description=hparams.tt_description
|
||||
)
|
||||
|
||||
exp.argparse(hparams)
|
||||
exp.save()
|
||||
|
||||
# build model
|
||||
model = ExampleModel(hparams)
|
||||
|
||||
# callbacks
|
||||
early_stop = EarlyStopping(
|
||||
monitor='val_acc',
|
||||
patience=3,
|
||||
mode='min',
|
||||
verbose=True,
|
||||
)
|
||||
|
||||
model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
|
||||
checkpoint = ModelCheckpoint(
|
||||
filepath=model_save_path,
|
||||
save_best_only=True,
|
||||
verbose=True,
|
||||
monitor='val_acc',
|
||||
mode='min'
|
||||
)
|
||||
|
||||
# configure trainer
|
||||
trainer = Trainer(
|
||||
experiment=exp,
|
||||
checkpoint_callback=checkpoint,
|
||||
early_stop_callback=early_stop,
|
||||
)
|
||||
|
||||
# train model
|
||||
trainer.fit(model)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
# use default args given by lightning
|
||||
root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
|
||||
parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False)
|
||||
add_default_args(parent_parser, root_dir)
|
||||
|
||||
# allow model to overwrite or extend args
|
||||
parser = ExampleModel.add_model_specific_args(parent_parser)
|
||||
hyperparams = parser.parse_args()
|
||||
|
||||
# train model
|
||||
main(hyperparams)
|
||||
+973
@@ -0,0 +1,973 @@
|
||||
|
||||
|
||||
|
||||
|
||||
<!doctype html>
|
||||
<html lang="en" class="no-js">
|
||||
<head>
|
||||
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
||||
|
||||
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
||||
|
||||
|
||||
|
||||
|
||||
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
||||
|
||||
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
||||
|
||||
<meta name="lang:search.language" content="en">
|
||||
|
||||
<meta name="lang:search.pipeline.stopwords" content="True">
|
||||
|
||||
<meta name="lang:search.pipeline.trimmer" content="True">
|
||||
|
||||
<meta name="lang:search.result.none" content="No matching documents">
|
||||
|
||||
<meta name="lang:search.result.one" content="1 matching document">
|
||||
|
||||
<meta name="lang:search.result.other" content="# matching documents">
|
||||
|
||||
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
||||
|
||||
<link rel="shortcut icon" href="assets/images/favicon.png">
|
||||
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
||||
|
||||
|
||||
|
||||
<title>PyTorch lightning Documentation</title>
|
||||
|
||||
|
||||
|
||||
<link rel="stylesheet" href="assets/stylesheets/application.0284f74d.css">
|
||||
|
||||
|
||||
|
||||
|
||||
<script src="assets/javascripts/modernizr.74668098.js"></script>
|
||||
|
||||
|
||||
|
||||
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
||||
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
||||
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
||||
|
||||
|
||||
<link rel="stylesheet" href="assets/fonts/material-icons.css">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</head>
|
||||
|
||||
<body dir="ltr">
|
||||
|
||||
<svg class="md-svg">
|
||||
<defs>
|
||||
|
||||
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
||||
|
||||
</defs>
|
||||
</svg>
|
||||
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
||||
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
||||
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
||||
|
||||
<a href="#new-project-quick-start" tabindex="1" class="md-skip">
|
||||
Skip to content
|
||||
</a>
|
||||
|
||||
|
||||
<header class="md-header" data-md-component="header">
|
||||
<nav class="md-header-nav md-grid">
|
||||
<div class="md-flex">
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<a href="." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--stretch">
|
||||
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
||||
|
||||
<span class="md-header-nav__topic">
|
||||
PyTorch lightning Documentation
|
||||
</span>
|
||||
<span class="md-header-nav__topic">
|
||||
|
||||
Home
|
||||
|
||||
</span>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
|
||||
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
||||
|
||||
<div class="md-search" data-md-component="search" role="dialog">
|
||||
<label class="md-search__overlay" for="__search"></label>
|
||||
<div class="md-search__inner" role="search">
|
||||
<form class="md-search__form" name="search">
|
||||
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
||||
<label class="md-icon md-search__icon" for="__search"></label>
|
||||
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
||||

|
||||
</button>
|
||||
</form>
|
||||
<div class="md-search__output">
|
||||
<div class="md-search__scrollwrap" data-md-scrollfix>
|
||||
<div class="md-search-result" data-md-component="result">
|
||||
<div class="md-search-result__meta">
|
||||
Type to start searching
|
||||
</div>
|
||||
<ol class="md-search-result__list"></ol>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<div class="md-header-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</nav>
|
||||
</header>
|
||||
|
||||
<div class="md-container">
|
||||
|
||||
|
||||
|
||||
|
||||
<main class="md-main">
|
||||
<div class="md-main__inner md-grid" data-md-component="container">
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
<nav class="md-nav md-nav--primary" data-md-level="0">
|
||||
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
||||
<a href="." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
||||
|
||||
<i class="md-icon"></i>
|
||||
|
||||
</a>
|
||||
PyTorch lightning Documentation
|
||||
</label>
|
||||
|
||||
<div class="md-nav__source">
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
||||
|
||||
<div class="md-source__icon">
|
||||
<svg viewBox="0 0 24 24" width="24" height="24">
|
||||
<use xlink:href="#__github" width="24" height="24"></use>
|
||||
</svg>
|
||||
</div>
|
||||
|
||||
<div class="md-source__repository">
|
||||
williamFalcon/pytorch-lightning
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--active">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
||||
|
||||
|
||||
<label class="md-nav__link md-nav__link--active" for="__toc">
|
||||
Home
|
||||
</label>
|
||||
|
||||
<a href="." title="Home" class="md-nav__link md-nav__link--active">
|
||||
Home
|
||||
</a>
|
||||
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#new-project-quick-start" title="New project Quick Start" class="md-nav__link">
|
||||
New project Quick Start
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#case-1-bert" title="Case 1: BERT" class="md-nav__link">
|
||||
Case 1: BERT
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#case-2-cooler-not-bert" title="Case 2: COOLER NOT BERT" class="md-nav__link">
|
||||
Case 2: COOLER NOT BERT
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#rapid-research-flow" title="Rapid research flow" class="md-nav__link">
|
||||
Rapid research flow
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#templates" title="Templates" class="md-nav__link">
|
||||
Templates
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#docs-shortcuts" title="Docs shortcuts" class="md-nav__link">
|
||||
Docs shortcuts
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#quick-start-examples" title="Quick start examples" class="md-nav__link">
|
||||
Quick start examples
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#checkpointing" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#computing-cluster-slurm" title="Computing cluster (SLURM)" class="md-nav__link">
|
||||
Computing cluster (SLURM)
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#debugging" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#distributed-training" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#experiment-logging" title="Experiment Logging" class="md-nav__link">
|
||||
Experiment Logging
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#training-loop" title="Training loop" class="md-nav__link">
|
||||
Training loop
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#validation-loop" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#testing-loop" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
||||
|
||||
<label class="md-nav__link" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-2">
|
||||
LightningModule
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
||||
Lightning Module interface
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="LightningModule/methods/" title="Methods" class="md-nav__link">
|
||||
Methods
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="LightningModule/properties/" title="Properties" class="md-nav__link">
|
||||
Properties
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3">
|
||||
|
||||
<label class="md-nav__link" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-3">
|
||||
Trainer
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/" title="Trainer" class="md-nav__link">
|
||||
Trainer
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/Checkpointing/" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/Distributed training/" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/Logging/" title="Logging" class="md-nav__link">
|
||||
Logging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
||||
SLURM Managed Cluster
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/Testing loop/" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/Training Loop/" title="Training Loop" class="md-nav__link">
|
||||
Training Loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/Validation loop/" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/debugging/" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="Trainer/hooks/" title="Hooks" class="md-nav__link">
|
||||
Hooks
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item md-nav__item--nested">
|
||||
|
||||
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
||||
|
||||
<label class="md-nav__link" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
||||
<label class="md-nav__title" for="nav-4">
|
||||
Examples
|
||||
</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="examples/Examples/" title="Examples" class="md-nav__link">
|
||||
Examples
|
||||
</a>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</li>
|
||||
|
||||
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
||||
<div class="md-sidebar__scrollwrap">
|
||||
<div class="md-sidebar__inner">
|
||||
|
||||
<nav class="md-nav md-nav--secondary">
|
||||
|
||||
|
||||
|
||||
<label class="md-nav__title" for="__toc">Table of contents</label>
|
||||
<ul class="md-nav__list" data-md-scrollfix>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#new-project-quick-start" title="New project Quick Start" class="md-nav__link">
|
||||
New project Quick Start
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#case-1-bert" title="Case 1: BERT" class="md-nav__link">
|
||||
Case 1: BERT
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#case-2-cooler-not-bert" title="Case 2: COOLER NOT BERT" class="md-nav__link">
|
||||
Case 2: COOLER NOT BERT
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#rapid-research-flow" title="Rapid research flow" class="md-nav__link">
|
||||
Rapid research flow
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#templates" title="Templates" class="md-nav__link">
|
||||
Templates
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#docs-shortcuts" title="Docs shortcuts" class="md-nav__link">
|
||||
Docs shortcuts
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#quick-start-examples" title="Quick start examples" class="md-nav__link">
|
||||
Quick start examples
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#checkpointing" title="Checkpointing" class="md-nav__link">
|
||||
Checkpointing
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#computing-cluster-slurm" title="Computing cluster (SLURM)" class="md-nav__link">
|
||||
Computing cluster (SLURM)
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#debugging" title="Debugging" class="md-nav__link">
|
||||
Debugging
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#distributed-training" title="Distributed training" class="md-nav__link">
|
||||
Distributed training
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#experiment-logging" title="Experiment Logging" class="md-nav__link">
|
||||
Experiment Logging
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#training-loop" title="Training loop" class="md-nav__link">
|
||||
Training loop
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#validation-loop" title="Validation loop" class="md-nav__link">
|
||||
Validation loop
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
<li class="md-nav__item">
|
||||
<a href="#testing-loop" title="Testing loop" class="md-nav__link">
|
||||
Testing loop
|
||||
</a>
|
||||
|
||||
</li>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</ul>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="md-content">
|
||||
<article class="md-content__inner md-typeset">
|
||||
|
||||
|
||||
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/index.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
||||
|
||||
|
||||
<h1>Home</h1>
|
||||
|
||||
<h6 id="new-project-quick-start">New project Quick Start</h6>
|
||||
<p>To start a new project define two files, a LightningModule and a Trainer file. <br />
|
||||
To illustrate Lightning power and simplicity, here's an example of a typical research flow. </p>
|
||||
<h6 id="case-1-bert">Case 1: BERT</h6>
|
||||
<p>Let's say you're working on something like BERT but want to try different ways of training or even different networks.<br />
|
||||
You would define a single LightningModule and use flags to switch between your different ideas. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7
|
||||
8
|
||||
9
|
||||
10
|
||||
11
|
||||
12
|
||||
13
|
||||
14
|
||||
15
|
||||
16
|
||||
17</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">class</span> <span class="nc">BERT</span><span class="p">(</span><span class="n">pl</span><span class="o">.</span><span class="n">LightningModule</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">model_name</span><span class="p">,</span> <span class="n">task</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">=</span> <span class="n">task</span>
|
||||
|
||||
<span class="k">if</span> <span class="n">model_name</span> <span class="o">==</span> <span class="s1">'transformer'</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">net</span> <span class="o">=</span> <span class="n">Transformer</span><span class="p">()</span>
|
||||
<span class="k">elif</span> <span class="n">model_name</span> <span class="o">==</span> <span class="s1">'my_cool_version'</span><span class="p">:</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">net</span> <span class="o">=</span> <span class="n">MyCoolVersion</span><span class="p">()</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">==</span> <span class="s1">'standard_bert'</span><span class="p">:</span>
|
||||
<span class="c1"># do standard bert training with self.net...</span>
|
||||
<span class="c1"># return loss</span>
|
||||
|
||||
<span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">task</span> <span class="o">==</span> <span class="s1">'my_cool_task'</span><span class="p">:</span>
|
||||
<span class="c1"># do my own version with self.net</span>
|
||||
<span class="c1"># return loss</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<h6 id="case-2-cooler-not-bert">Case 2: COOLER NOT BERT</h6>
|
||||
<p>But if you wanted to try something <strong>completely</strong> different, you'd define a new module for that. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">class</span> <span class="nc">CoolerNotBERT</span><span class="p">(</span><span class="n">pl</span><span class="o">.</span><span class="n">LightningModule</span><span class="p">):</span>
|
||||
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
||||
<span class="bp">self</span><span class="o">.</span><span class="n">net</span> <span class="o">=</span> <span class="o">...</span>
|
||||
|
||||
<span class="k">def</span> <span class="nf">training_step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">batch</span><span class="p">,</span> <span class="n">batch_nb</span><span class="p">):</span>
|
||||
<span class="c1"># do some other cool task</span>
|
||||
<span class="c1"># return loss </span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<h6 id="rapid-research-flow">Rapid research flow</h6>
|
||||
<p>Then you could do rapid research by switching between these two and using the same trainer. </p>
|
||||
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
||||
2
|
||||
3
|
||||
4
|
||||
5
|
||||
6
|
||||
7</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">if</span> <span class="n">use_bert</span><span class="p">:</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">BERT</span><span class="p">()</span>
|
||||
<span class="k">else</span><span class="p">:</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">CoolerNotBERT</span><span class="p">()</span>
|
||||
|
||||
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">gpus</span><span class="o">=</span><span class="mi">4</span><span class="p">,</span> <span class="n">use_amp</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
|
||||
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
|
||||
</pre></div>
|
||||
</td></tr></table>
|
||||
|
||||
<p>Notice a few things about this flow: <br />
|
||||
1. You're writing pure PyTorch... no unnecessary abstractions or new libraries to learn. <br />
|
||||
2. You get free GPU and 16-bit support without writing any of that code in your model. <br />
|
||||
3. You also get all of the capabilities below (without coding or testing yourself). </p>
|
||||
<hr />
|
||||
<h6 id="templates">Templates</h6>
|
||||
<ol>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#minimal-example">MNIST LightningModule</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/">Trainer</a><ul>
|
||||
<li><a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/basic_examples">Basic CPU, GPU Trainer Template</a></li>
|
||||
<li><a href="https://github.com/williamFalcon/pytorch-lightning/tree/master/pl_examples/multi_node_examples">GPU cluster Trainer Template</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
</ol>
|
||||
<h6 id="docs-shortcuts">Docs shortcuts</h6>
|
||||
<ul>
|
||||
<li><a href="LightningModule/RequiredTrainerInterface/">LightningModule</a> </li>
|
||||
<li><a href="Trainer/">Trainer</a> </li>
|
||||
</ul>
|
||||
<h6 id="quick-start-examples">Quick start examples</h6>
|
||||
<ul>
|
||||
<li><a href="examples/Examples/#cpu-hyperparameter-search">CPU example</a> </li>
|
||||
<li><a href="examples/Examples/#hyperparameter-search-on-a-single-or-multiple-gpus">Hyperparameter search on single GPU</a> </li>
|
||||
<li><a href="examples/Examples/#hyperparameter-search-on-a-single-or-multiple-gpus">Hyperparameter search on multiple GPUs on same node</a> </li>
|
||||
<li><a href="examples/Examples/#Hyperparameter search on a SLURM HPC cluster">Hyperparameter search on a SLURM HPC cluster</a> </li>
|
||||
</ul>
|
||||
<h6 id="checkpointing">Checkpointing</h6>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving">Checkpoint callback</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#model-saving">Model saving</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/methods/#load-from-metrics">Model loading</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Checkpointing/#restoring-training-session">Restoring training session</a></li>
|
||||
</ul>
|
||||
<h6 id="computing-cluster-slurm">Computing cluster (SLURM)</h6>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#running-grid-search-on-a-cluster">Running grid search on a cluster</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#walltime-auto-resubmit">Walltime auto-resubmit</a> </li>
|
||||
</ul>
|
||||
<h6 id="debugging">Debugging</h6>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#fast-dev-run">Fast dev run</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#inspect-gradient-norms">Inspect gradient norms</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#Log-gpu-usage">Log GPU usage</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#make-model-overfit-on-subset-of-data">Make model overfit on subset of data</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-the-parameter-count-by-layer">Print the parameter count by layer</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/debugging/#print-which-gradients-are-nan">Pring which gradients are nan</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/properties/#example_input_array">Print input and output size of every module in system</a></li>
|
||||
</ul>
|
||||
<h6 id="distributed-training">Distributed training</h6>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#init_ddp_connection">Implement Your Own Distributed (DDP) training</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#16-bit-mixed-precision">16-bit mixed precision</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-GPU">Multi-GPU</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#Multi-node">Multi-node</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#single-gpu">Single GPU</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Distributed%20training/#self-balancing-architecture">Self-balancing architecture</a></li>
|
||||
</ul>
|
||||
<h6 id="experiment-logging">Experiment Logging</h6>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#display-metrics-in-progress-bar">Display metrics in progress bar</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#log-metric-row-every-k-batches">Log metric row every k batches</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#process-position">Process position</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#tensorboard-support">Tensorboard support</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#save-a-snapshot-of-all-hyperparameters">Save a snapshot of all hyperparameters</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#snapshot-code-for-a-training-run">Snapshot code for a training run</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Logging/#write-logs-file-to-csv-every-k-batches">Write logs file to csv every k batches</a></li>
|
||||
</ul>
|
||||
<h6 id="training-loop">Training loop</h6>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#accumulated-gradients">Accumulate gradients</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs">Force training for min or max epochs</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#early-stopping">Early stopping callback</a> </li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop">Force disable early stop</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping">Gradient Clipping</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/">Hooks</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers">Learning rate scheduling</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/LightningModule/RequiredTrainerInterface/#configure_optimizers">Use multiple optimizers (like GANs)</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check">Set how much of the training set to check (1-100%)</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/#optimizer_step">Step optimizers at arbitrary intervals</a></li>
|
||||
</ul>
|
||||
<h6 id="validation-loop">Validation loop</h6>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#check-validation-every-n-epochs">Check validation every n epochs</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/hooks/">Hooks</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-validation-set-to-check">Set how much of the validation set to check</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-how-much-of-the-test-set-to-check">Set how much of the test set to check</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-validation-check-frequency-within-1-training-epoch">Set validation check frequency within 1 training epoch</a></li>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Validation%20loop/#set-the-number-of-validation-sanity-steps">Set the number of validation sanity steps</a></li>
|
||||
</ul>
|
||||
<h6 id="testing-loop">Testing loop</h6>
|
||||
<ul>
|
||||
<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Testing%20loop/">Run test set</a> </li>
|
||||
</ul>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</article>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
||||
<footer class="md-footer">
|
||||
|
||||
<div class="md-footer-nav">
|
||||
<nav class="md-footer-nav__inner md-grid">
|
||||
|
||||
|
||||
<a href="LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
||||
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
||||
<span class="md-flex__ellipsis">
|
||||
<span class="md-footer-nav__direction">
|
||||
Next
|
||||
</span>
|
||||
Lightning Module interface
|
||||
</span>
|
||||
</div>
|
||||
<div class="md-flex__cell md-flex__cell--shrink">
|
||||
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
||||
</div>
|
||||
</a>
|
||||
|
||||
</nav>
|
||||
</div>
|
||||
|
||||
<div class="md-footer-meta md-typeset">
|
||||
<div class="md-footer-meta__inner md-grid">
|
||||
<div class="md-footer-copyright">
|
||||
|
||||
powered by
|
||||
<a href="https://www.mkdocs.org">MkDocs</a>
|
||||
and
|
||||
<a href="https://squidfunk.github.io/mkdocs-material/">
|
||||
Material for MkDocs</a>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</footer>
|
||||
|
||||
</div>
|
||||
|
||||
<script src="assets/javascripts/application.245445c6.js"></script>
|
||||
|
||||
<script>app.initialize({version:"1.0.4",url:{base:"."}})</script>
|
||||
|
||||
|
||||
</body>
|
||||
</html>
|
||||
-10
@@ -1,10 +0,0 @@
|
||||
site_name: Pytorch lightning Documentation
|
||||
theme:
|
||||
name: 'material'
|
||||
docs_dir: docs
|
||||
repo_url: https://github.com/williamFalcon/pytorch-lightning
|
||||
site_dir: 'site'
|
||||
site_description: 'Documentation for Pytorch LightningModule, the researcher version of keras.'
|
||||
|
||||
dev_addr: '0.0.0.0:8000'
|
||||
#google_analytics: ['UA-aasd', 'sitename']
|
||||
@@ -1,5 +0,0 @@
|
||||
[build-system]
|
||||
requires = [
|
||||
"setuptools",
|
||||
"wheel",
|
||||
]
|
||||
@@ -1,2 +0,0 @@
|
||||
from .models import Trainer
|
||||
from .root_module.root_module import LightningModule
|
||||
@@ -1 +0,0 @@
|
||||
from .pt_callbacks import EarlyStopping, ModelCheckpoint
|
||||
@@ -1,263 +0,0 @@
|
||||
import numpy as np
|
||||
import os, shutil
|
||||
from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel
|
||||
|
||||
|
||||
class Callback(object):
|
||||
"""Abstract base class used to build new callbacks.
|
||||
# Properties
|
||||
params: dict. Training parameters
|
||||
(eg. verbosity, batch size, number of epochs...).
|
||||
model: instance of `keras.models.Model`.
|
||||
Reference of the model being trained.
|
||||
The `logs` dictionary that callback methods
|
||||
take as argument will contain keys for quantities relevant to
|
||||
the current batch or epoch.
|
||||
Currently, the `.fit()` method of the `Sequential` model class
|
||||
will include the following quantities in the `logs` that
|
||||
it passes to its callbacks:
|
||||
on_epoch_end: logs include `acc` and `loss`, and
|
||||
optionally include `val_loss`
|
||||
(if validation is enabled in `fit`), and `val_acc`
|
||||
(if validation and accuracy monitoring are enabled).
|
||||
on_batch_begin: logs include `size`,
|
||||
the number of samples in the current batch.
|
||||
on_batch_end: logs include `loss`, and optionally `acc`
|
||||
(if accuracy monitoring is enabled).
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.validation_data = None
|
||||
self.model = None
|
||||
|
||||
def set_params(self, params):
|
||||
self.params = params
|
||||
|
||||
def set_model(self, model):
|
||||
if type(model) is LightningDistributedDataParallel:
|
||||
model = model.module
|
||||
self.model = model
|
||||
|
||||
def on_epoch_begin(self, epoch, logs=None):
|
||||
pass
|
||||
|
||||
def on_epoch_end(self, epoch, logs=None):
|
||||
pass
|
||||
|
||||
def on_batch_begin(self, batch, logs=None):
|
||||
pass
|
||||
|
||||
def on_batch_end(self, batch, logs=None):
|
||||
pass
|
||||
|
||||
def on_train_begin(self, logs=None):
|
||||
pass
|
||||
|
||||
def on_train_end(self, logs=None):
|
||||
pass
|
||||
|
||||
|
||||
class EarlyStopping(Callback):
|
||||
"""Stop training when a monitored quantity has stopped improving.
|
||||
# Arguments
|
||||
monitor: quantity to be monitored.
|
||||
min_delta: minimum change in the monitored quantity
|
||||
to qualify as an improvement, i.e. an absolute
|
||||
change of less than min_delta, will count as no
|
||||
improvement.
|
||||
patience: number of epochs with no improvement
|
||||
after which training will be stopped.
|
||||
verbose: verbosity mode.
|
||||
mode: one of {auto, min, max}. In `min` mode,
|
||||
training will stop when the quantity
|
||||
monitored has stopped decreasing; in `max`
|
||||
mode it will stop when the quantity
|
||||
monitored has stopped increasing; in `auto`
|
||||
mode, the direction is automatically inferred
|
||||
from the name of the monitored quantity.
|
||||
"""
|
||||
|
||||
def __init__(self, monitor='val_loss',
|
||||
min_delta=0.0, patience=0, verbose=0, mode='auto'):
|
||||
super(EarlyStopping, self).__init__()
|
||||
|
||||
self.monitor = monitor
|
||||
self.patience = patience
|
||||
self.verbose = verbose
|
||||
self.min_delta = min_delta
|
||||
self.wait = 0
|
||||
self.stopped_epoch = 0
|
||||
|
||||
if mode not in ['auto', 'min', 'max']:
|
||||
print('EarlyStopping mode %s is unknown, fallback to auto mode.' % mode)
|
||||
mode = 'auto'
|
||||
|
||||
if mode == 'min':
|
||||
self.monitor_op = np.less
|
||||
elif mode == 'max':
|
||||
self.monitor_op = np.greater
|
||||
else:
|
||||
if 'acc' in self.monitor:
|
||||
self.monitor_op = np.greater
|
||||
else:
|
||||
self.monitor_op = np.less
|
||||
|
||||
if self.monitor_op == np.greater:
|
||||
self.min_delta *= 1
|
||||
else:
|
||||
self.min_delta *= -1
|
||||
|
||||
self.on_train_begin()
|
||||
|
||||
def on_train_begin(self, logs=None):
|
||||
# Allow instances to be re-used
|
||||
self.wait = 0
|
||||
self.stopped_epoch = 0
|
||||
self.best = np.Inf if self.monitor_op == np.less else -np.Inf
|
||||
|
||||
def on_epoch_end(self, epoch, logs=None):
|
||||
current = logs.get(self.monitor)
|
||||
stop_training = False
|
||||
if current is None:
|
||||
print('Early stopping conditioned on metric `%s` ''which is not available. Available metrics are: %s' %
|
||||
(self.monitor, ','.join(list(logs.keys()))), RuntimeWarning
|
||||
)
|
||||
exit(-1)
|
||||
|
||||
if self.monitor_op(current - self.min_delta, self.best):
|
||||
self.best = current
|
||||
self.wait = 0
|
||||
else:
|
||||
self.wait += 1
|
||||
if self.wait >= self.patience:
|
||||
self.stopped_epoch = epoch
|
||||
stop_training = True
|
||||
self.on_train_end()
|
||||
|
||||
return stop_training
|
||||
|
||||
def on_train_end(self, logs=None):
|
||||
if self.stopped_epoch > 0 and self.verbose > 0:
|
||||
print('Epoch %05d: early stopping' % (self.stopped_epoch + 1))
|
||||
|
||||
|
||||
class ModelCheckpoint(Callback):
|
||||
"""Save the model after every epoch.
|
||||
`filepath` can contain named formatting options,
|
||||
which will be filled the value of `epoch` and
|
||||
keys in `logs` (passed in `on_epoch_end`).
|
||||
For example: if `filepath` is `weights.{epoch:02d}-{val_loss:.2f}.hdf5`,
|
||||
then the model checkpoints will be saved with the epoch number and
|
||||
the validation loss in the filename.
|
||||
# Arguments
|
||||
filepath: string, path to save the model file.
|
||||
monitor: quantity to monitor.
|
||||
verbose: verbosity mode, 0 or 1.
|
||||
save_best_only: if `save_best_only=True`,
|
||||
the latest best model according to
|
||||
the quantity monitored will not be overwritten.
|
||||
mode: one of {auto, min, max}.
|
||||
If `save_best_only=True`, the decision
|
||||
to overwrite the current save file is made
|
||||
based on either the maximization or the
|
||||
minimization of the monitored quantity. For `val_acc`,
|
||||
this should be `max`, for `val_loss` this should
|
||||
be `min`, etc. In `auto` mode, the direction is
|
||||
automatically inferred from the name of the monitored quantity.
|
||||
save_weights_only: if True, then only the model's weights will be
|
||||
saved (`model.save_weights(filepath)`), else the full model
|
||||
is saved (`model.save(filepath)`).
|
||||
period: Interval (number of epochs) between checkpoints.
|
||||
"""
|
||||
|
||||
def __init__(self, filepath, monitor='val_loss', verbose=0,
|
||||
save_best_only=False, save_weights_only=False,
|
||||
mode='auto', period=1, prefix=''):
|
||||
super(ModelCheckpoint, self).__init__()
|
||||
self.monitor = monitor
|
||||
self.verbose = verbose
|
||||
self.filepath = filepath
|
||||
self.save_best_only = save_best_only
|
||||
self.save_weights_only = save_weights_only
|
||||
self.period = period
|
||||
self.epochs_since_last_save = 0
|
||||
self.prefix = prefix
|
||||
|
||||
if mode not in ['auto', 'min', 'max']:
|
||||
print('ModelCheckpoint mode %s is unknown, '
|
||||
'fallback to auto mode.' % (mode),
|
||||
RuntimeWarning)
|
||||
mode = 'auto'
|
||||
|
||||
if mode == 'min':
|
||||
self.monitor_op = np.less
|
||||
self.best = np.Inf
|
||||
elif mode == 'max':
|
||||
self.monitor_op = np.greater
|
||||
self.best = -np.Inf
|
||||
else:
|
||||
if 'acc' in self.monitor or self.monitor.startswith('fmeasure'):
|
||||
self.monitor_op = np.greater
|
||||
self.best = -np.Inf
|
||||
else:
|
||||
self.monitor_op = np.less
|
||||
self.best = np.Inf
|
||||
|
||||
def save_model(self, filepath, overwrite):
|
||||
dirpath = '/'.join(filepath.split('/')[:-1])
|
||||
|
||||
# make paths
|
||||
os.makedirs(os.path.dirname(filepath), exist_ok=True)
|
||||
|
||||
if overwrite:
|
||||
for filename in os.listdir(dirpath):
|
||||
if self.prefix in filename:
|
||||
path_to_delete = os.path.join(dirpath, filename)
|
||||
try:
|
||||
shutil.rmtree(path_to_delete)
|
||||
except OSError:
|
||||
os.remove(path_to_delete)
|
||||
|
||||
# delegate the saving to the model
|
||||
self.save_function(filepath)
|
||||
|
||||
def on_epoch_end(self, epoch, logs=None):
|
||||
logs = logs or {}
|
||||
self.epochs_since_last_save += 1
|
||||
if self.epochs_since_last_save >= self.period:
|
||||
self.epochs_since_last_save = 0
|
||||
filepath = '{}/{}_ckpt_epoch_{}.ckpt'.format(self.filepath, self.prefix, epoch + 1)
|
||||
if self.save_best_only:
|
||||
current = logs.get(self.monitor)
|
||||
if current is None:
|
||||
print('Can save best model only with %s available, '
|
||||
'skipping.' % (self.monitor), RuntimeWarning)
|
||||
else:
|
||||
if self.monitor_op(current, self.best):
|
||||
if self.verbose > 0:
|
||||
print('\nEpoch %05d: %s improved from %0.5f to %0.5f,'
|
||||
' saving model to %s'
|
||||
% (epoch + 1, self.monitor, self.best,
|
||||
current, filepath))
|
||||
self.best = current
|
||||
self.save_model(filepath, overwrite=True)
|
||||
|
||||
else:
|
||||
if self.verbose > 0:
|
||||
print('\nEpoch %05d: %s did not improve' %
|
||||
(epoch + 1, self.monitor))
|
||||
else:
|
||||
if self.verbose > 0:
|
||||
print('\nEpoch %05d: saving model to %s' % (epoch + 1, filepath))
|
||||
self.save_model(filepath, overwrite=False)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
c = EarlyStopping(min_delta=0.9, patience=2, verbose=True)
|
||||
losses = [10, 9, 8, 8, 6, 4.3, 5, 4.4, 2.8, 2.5]
|
||||
for i, loss in enumerate(losses):
|
||||
should_stop = c.on_epoch_end(i, logs={'val_loss': loss})
|
||||
print(loss)
|
||||
if should_stop:
|
||||
break
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
from .trainer import Trainer
|
||||
@@ -1,203 +0,0 @@
|
||||
import torch.nn as nn
|
||||
import numpy as np
|
||||
from pytorch_lightning.root_module.root_module import LightningModule
|
||||
from test_tube import HyperOptArgumentParser
|
||||
from torchvision.datasets import MNIST
|
||||
import torchvision.transforms as transforms
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class ExampleModel1(LightningModule):
|
||||
"""
|
||||
Sample model to show how to define a template
|
||||
"""
|
||||
|
||||
def __init__(self, hparams):
|
||||
# init superclass
|
||||
super(ExampleModel1, self).__init__(hparams)
|
||||
|
||||
self.batch_size = hparams.batch_size
|
||||
|
||||
# build model
|
||||
self.__build_model()
|
||||
|
||||
# ---------------------
|
||||
# MODEL SETUP
|
||||
# ---------------------
|
||||
def __build_model(self):
|
||||
"""
|
||||
Layout model
|
||||
:return:
|
||||
"""
|
||||
self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
|
||||
self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
|
||||
self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
|
||||
|
||||
self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)
|
||||
|
||||
# ---------------------
|
||||
# TRAINING
|
||||
# ---------------------
|
||||
def forward(self, x):
|
||||
x = self.c_d1(x)
|
||||
x = F.tanh(x)
|
||||
x = self.c_d1_bn(x)
|
||||
x = self.c_d1_drop(x)
|
||||
|
||||
x = self.c_d2(x)
|
||||
logits = F.log_softmax(x, dim=1)
|
||||
|
||||
return logits
|
||||
|
||||
def loss(self, labels, logits):
|
||||
nll = F.nll_loss(logits, labels)
|
||||
return nll
|
||||
|
||||
def training_step(self, data_batch):
|
||||
"""
|
||||
Called inside the training loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
# forward pass
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
# calculate loss
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
tqdm_dic = {'jefe': 1}
|
||||
return loss_val, tqdm_dic
|
||||
|
||||
def validation_step(self, data_batch):
|
||||
"""
|
||||
Called inside the validation loop
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
x, y = data_batch
|
||||
x = x.view(x.size(0), -1)
|
||||
y_hat = self.forward(x)
|
||||
|
||||
loss_val = self.loss(y, y_hat)
|
||||
|
||||
# acc
|
||||
labels_hat = torch.argmax(y_hat, dim=1)
|
||||
val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
|
||||
|
||||
output = {'y_hat': y_hat, 'val_loss': loss_val.item(), 'val_acc': val_acc}
|
||||
return output
|
||||
|
||||
def validation_end(self, outputs):
|
||||
"""
|
||||
Called at the end of validation to aggregate outputs
|
||||
:param outputs: list of individual outputs of each validation step
|
||||
:return:
|
||||
"""
|
||||
val_loss_mean = 0
|
||||
accs = []
|
||||
for output in outputs:
|
||||
val_loss_mean += output['val_loss']
|
||||
accs.append(output['val_acc'])
|
||||
|
||||
val_loss_mean /= len(outputs)
|
||||
tqdm_dic = {'val_loss': val_loss_mean, 'val_acc': np.mean(accs)}
|
||||
return tqdm_dic
|
||||
|
||||
def update_tng_log_metrics(self, logs):
|
||||
return logs
|
||||
|
||||
# ---------------------
|
||||
# MODEL SAVING
|
||||
# ---------------------
|
||||
def get_save_dict(self):
|
||||
checkpoint = {
|
||||
'state_dict': self.state_dict(),
|
||||
}
|
||||
|
||||
return checkpoint
|
||||
|
||||
def load_model_specific(self, checkpoint):
|
||||
self.load_state_dict(checkpoint['state_dict'])
|
||||
pass
|
||||
|
||||
# ---------------------
|
||||
# TRAINING SETUP
|
||||
# ---------------------
|
||||
def configure_optimizers(self):
|
||||
"""
|
||||
return whatever optimizers we want here
|
||||
:return: list of optimizers
|
||||
"""
|
||||
optimizer = self.choose_optimizer(self.hparams.optimizer_name, self.parameters(), {'lr': self.hparams.learning_rate}, 'optimizer')
|
||||
self.optimizers = [optimizer]
|
||||
return self.optimizers
|
||||
|
||||
def __dataloader(self, train):
|
||||
# init data generators
|
||||
transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
|
||||
|
||||
dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)
|
||||
|
||||
loader = torch.utils.data.DataLoader(
|
||||
dataset=dataset,
|
||||
batch_size=self.hparams.batch_size,
|
||||
shuffle=True
|
||||
)
|
||||
|
||||
return loader
|
||||
|
||||
@property
|
||||
def tng_dataloader(self):
|
||||
if self._tng_dataloader is None:
|
||||
try:
|
||||
self._tng_dataloader = self.__dataloader(train=True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise e
|
||||
return self._tng_dataloader
|
||||
|
||||
@property
|
||||
def val_dataloader(self):
|
||||
if self._val_dataloader is None:
|
||||
try:
|
||||
self._val_dataloader = self.__dataloader(train=False)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise e
|
||||
return self._val_dataloader
|
||||
|
||||
@property
|
||||
def test_dataloader(self):
|
||||
if self._test_dataloader is None:
|
||||
try:
|
||||
self._test_dataloader = self.__dataloader(train=False)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise e
|
||||
return self._test_dataloader
|
||||
|
||||
@staticmethod
|
||||
def add_model_specific_args(parent_parser):
|
||||
parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
|
||||
|
||||
# param overwrites
|
||||
# parser.set_defaults(gradient_clip=5.0)
|
||||
|
||||
# network params
|
||||
parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
|
||||
parser.add_argument('--in_features', default=28*28)
|
||||
parser.add_argument('--hidden_dim', default=500)
|
||||
parser.add_argument('--out_features', default=10)
|
||||
|
||||
# data
|
||||
parser.add_argument('--data_root', default='/Users/williamfalcon/Developer/personal/research_lib/research_proj/datasets/mnist', type=str)
|
||||
|
||||
# training params (opt)
|
||||
parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
|
||||
tunable=False)
|
||||
parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False)
|
||||
parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
|
||||
return parser
|
||||
@@ -1,862 +0,0 @@
|
||||
"""
|
||||
The trainer handles all the logic for running a val loop, training loop, distributing, etc...
|
||||
"""
|
||||
import subprocess
|
||||
import traceback
|
||||
import warnings
|
||||
import os
|
||||
import pdb
|
||||
import re
|
||||
|
||||
import torch
|
||||
from torch.utils.data.distributed import DistributedSampler
|
||||
from torch.optim.lr_scheduler import MultiStepLR
|
||||
import torch.multiprocessing as mp
|
||||
import torch.distributed as dist
|
||||
import numpy as np
|
||||
import tqdm
|
||||
|
||||
from pytorch_lightning.root_module.memory import get_gpu_memory_map
|
||||
from pytorch_lightning.root_module.model_saving import TrainerIO
|
||||
from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel
|
||||
|
||||
|
||||
try:
|
||||
from apex import amp
|
||||
APEX_AVAILABLE = True
|
||||
except ModuleNotFoundError:
|
||||
APEX_AVAILABLE = False
|
||||
|
||||
|
||||
def reduce_distributed_output(output, nb_gpus):
|
||||
if nb_gpus <= 1:
|
||||
return output
|
||||
|
||||
# when using DP, we get one output per gpu
|
||||
# average outputs and return
|
||||
if type(output) is torch.Tensor:
|
||||
return output.mean()
|
||||
|
||||
for k, v in output.items():
|
||||
# recurse on nested dics
|
||||
if isinstance(output[k], dict):
|
||||
output[k] = reduce_distributed_output(output[k], nb_gpus)
|
||||
|
||||
# reduce only metrics that have the same nb of gpus
|
||||
elif output[k].size(0) == nb_gpus:
|
||||
reduced = torch.mean(output[k])
|
||||
output[k] = reduced
|
||||
return output
|
||||
|
||||
|
||||
class Trainer(TrainerIO):
|
||||
|
||||
def __init__(self,
|
||||
experiment,
|
||||
early_stop_callback=None,
|
||||
checkpoint_callback=None,
|
||||
gradient_clip=0,
|
||||
cluster=None,
|
||||
process_position=0,
|
||||
current_gpu_name=0,
|
||||
nb_gpu_nodes=1,
|
||||
gpus=None,
|
||||
progress_bar=True,
|
||||
overfit_pct=0.0,
|
||||
track_grad_norm=-1,
|
||||
check_val_every_n_epoch=1,
|
||||
fast_dev_run=False,
|
||||
accumulate_grad_batches=1,
|
||||
max_nb_epochs=1000, min_nb_epochs=1,
|
||||
train_percent_check=1.0, val_percent_check=1.0, test_percent_check=1.0,
|
||||
val_check_interval=0.95,
|
||||
log_save_interval=100, add_log_row_interval=10,
|
||||
lr_scheduler_milestones=None,
|
||||
distributed_backend='dp',
|
||||
use_amp=False,
|
||||
print_nan_grads=False,
|
||||
print_weights_summary=True,
|
||||
amp_level='O2',
|
||||
nb_sanity_val_steps=5):
|
||||
|
||||
"""
|
||||
|
||||
:param experiment: Test-tube experiment
|
||||
:param early_stop_callback: from pytorch_lightning import EarlyStopping
|
||||
:param checkpoint_callback: from pytorch_lightning import Checkpoint
|
||||
:param gradient_clip:
|
||||
:param cluster:
|
||||
:param process_position:
|
||||
:param current_gpu_name:
|
||||
:param nb_gpu_nodes:
|
||||
:param gpus:
|
||||
:param progress_bar:
|
||||
:param overfit_pct:
|
||||
:param track_grad_norm:
|
||||
:param check_val_every_n_epoch:
|
||||
:param fast_dev_run:
|
||||
:param accumulate_grad_batches:
|
||||
:param max_nb_epochs:
|
||||
:param min_nb_epochs:
|
||||
:param train_percent_check:
|
||||
:param val_percent_check:
|
||||
:param test_percent_check:
|
||||
:param val_check_interval:
|
||||
:param log_save_interval:
|
||||
:param add_log_row_interval:
|
||||
:param lr_scheduler_milestones:
|
||||
:param distributed_backend: 'np' to use DistributedParallel, 'ddp' to use DistributedDataParallel
|
||||
:param use_amp:
|
||||
:param print_nan_grads:
|
||||
:param print_weights_summary:
|
||||
:param amp_level:
|
||||
:param nb_sanity_val_steps:
|
||||
"""
|
||||
|
||||
# Transfer params
|
||||
self.nb_gpu_nodes = nb_gpu_nodes
|
||||
self.gradient_clip = gradient_clip
|
||||
self.check_val_every_n_epoch = check_val_every_n_epoch
|
||||
self.enable_early_stop = early_stop_callback is not None
|
||||
self.track_grad_norm = track_grad_norm
|
||||
self.fast_dev_run = fast_dev_run
|
||||
self.on_gpu = gpus is not None and torch.cuda.is_available()
|
||||
self.progress_bar = progress_bar
|
||||
self.experiment = experiment
|
||||
self.exp_save_path = experiment.get_data_path(experiment.name, experiment.version)
|
||||
self.cluster = cluster
|
||||
self.process_position = process_position
|
||||
self.current_gpu_name = current_gpu_name
|
||||
self.print_weights_summary = print_weights_summary
|
||||
self.checkpoint_callback = checkpoint_callback
|
||||
|
||||
if self.checkpoint_callback is not None:
|
||||
self.checkpoint_callback.save_function = self.save_checkpoint
|
||||
|
||||
self.early_stop = early_stop_callback
|
||||
self.model = None
|
||||
self.max_nb_epochs = max_nb_epochs
|
||||
self.accumulate_grad_batches = accumulate_grad_batches
|
||||
self.early_stop_callback = early_stop_callback
|
||||
self.min_nb_epochs = min_nb_epochs
|
||||
self.nb_sanity_val_steps = nb_sanity_val_steps
|
||||
self.lr_scheduler_milestones = [] if lr_scheduler_milestones is None else [int(x.strip()) for x in lr_scheduler_milestones.split(',')]
|
||||
self.lr_schedulers = []
|
||||
self.amp_level = amp_level
|
||||
self.print_nan_grads = print_nan_grads
|
||||
self.data_parallel_device_ids = None
|
||||
self.world_size = 1
|
||||
self.node_rank = 0
|
||||
self.use_ddp = False
|
||||
self.use_dp = False
|
||||
|
||||
|
||||
# gpus come in as a string.
|
||||
# if gpus = -1 then use all available devices
|
||||
# otherwise, split the string using commas
|
||||
if gpus is not None:
|
||||
if type(gpus) is list:
|
||||
self.data_parallel_device_ids = gpus
|
||||
elif type(gpus) is str:
|
||||
if gpus == '-1':
|
||||
self.data_parallel_device_ids = list(range(0, torch.cuda.device_count()))
|
||||
else:
|
||||
self.data_parallel_device_ids = [int(x.strip()) for x in gpus.split(',')]
|
||||
else:
|
||||
raise Exception('gpus has to be a string or list of ids')
|
||||
|
||||
# set the correct cuda visible devices (using pci order)
|
||||
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = ','.join([str(x) for x in self.data_parallel_device_ids])
|
||||
print(f'VISIBLE GPUS: {os.environ["CUDA_VISIBLE_DEVICES"]}')
|
||||
|
||||
# make DP and DDP mutually exclusive
|
||||
# single GPU will also use DP with devices=[0]
|
||||
have_gpus = self.data_parallel_device_ids is not None and len(self.data_parallel_device_ids) > 0
|
||||
if have_gpus:
|
||||
self.use_dp = distributed_backend == 'dp'
|
||||
self.use_ddp = distributed_backend == 'ddp'
|
||||
|
||||
# use ddp automatically if nb_gpu_nodes > 1
|
||||
if nb_gpu_nodes > 1:
|
||||
self.use_ddp = True
|
||||
self.use_ddp = False
|
||||
w = 'DataParallel does not support nb_gpu_nodes > 1. ' \
|
||||
'Switching to DistributedDataParallel for you. ' \
|
||||
'To silence this warning set distributed_backend=ddp'
|
||||
warnings.warn(w)
|
||||
|
||||
# process info
|
||||
self.proc_rank = 0
|
||||
|
||||
# training state
|
||||
self.optimizers = None
|
||||
self.prog_bar = None
|
||||
self.global_step = 0
|
||||
self.current_epoch = 0
|
||||
self.total_batches = 0
|
||||
|
||||
# logging
|
||||
self.log_save_interval = log_save_interval
|
||||
self.val_check_interval = val_check_interval
|
||||
self.add_log_row_interval = add_log_row_interval
|
||||
|
||||
# dataloaders
|
||||
self.tng_dataloader = None
|
||||
self.test_dataloader = None
|
||||
self.val_dataloader = None
|
||||
|
||||
# how much of the data to use
|
||||
self.__determine_data_use_amount(train_percent_check, val_percent_check, test_percent_check, overfit_pct)
|
||||
print('gpu available: {}, used: {}'.format(torch.cuda.is_available(), self.on_gpu))
|
||||
|
||||
# 16 bit mixed precision training using apex
|
||||
self.use_amp = use_amp and APEX_AVAILABLE
|
||||
if self.use_amp:
|
||||
print('using 16bit precision')
|
||||
|
||||
if use_amp and not APEX_AVAILABLE:
|
||||
msg = '''
|
||||
You set use_amp=True but do not have apex installed.
|
||||
Install apex first using this guide and rerun with use_amp=True:
|
||||
https://github.com/NVIDIA/apex#linux
|
||||
|
||||
this run will NOT use 16 bit precision
|
||||
'''
|
||||
warnings.warn(msg)
|
||||
|
||||
@property
|
||||
def data_parallel(self):
|
||||
return self.use_dp or self.use_ddp
|
||||
|
||||
def __determine_data_use_amount(self, train_percent_check, val_percent_check, test_percent_check, overfit_pct):
|
||||
"""
|
||||
Use less data for debugging purposes
|
||||
"""
|
||||
self.train_percent_check = train_percent_check
|
||||
self.val_percent_check = val_percent_check
|
||||
self.test_percent_check = test_percent_check
|
||||
if overfit_pct > 0:
|
||||
self.train_percent_check = overfit_pct
|
||||
self.val_percent_check = overfit_pct
|
||||
self.test_percent_check = overfit_pct
|
||||
|
||||
def __get_model(self):
|
||||
return self.model.module if self.data_parallel else self.model
|
||||
|
||||
def __is_function_implemented(self, f_name):
|
||||
model = self.__get_model()
|
||||
f_op = getattr(model, f_name, None)
|
||||
return callable(f_op)
|
||||
|
||||
@property
|
||||
def __tng_tqdm_dic(self):
|
||||
tqdm_dic = {
|
||||
'tng_loss': '{0:.3f}'.format(self.avg_loss),
|
||||
'v_nb': '{}'.format(self.experiment.version),
|
||||
'epoch': '{}'.format(self.current_epoch),
|
||||
'batch_nb':'{}'.format(self.batch_nb),
|
||||
}
|
||||
tqdm_dic.update(self.tqdm_metrics)
|
||||
|
||||
if self.on_gpu:
|
||||
tqdm_dic['gpu'] = '{}'.format(self.current_gpu_name)
|
||||
|
||||
return tqdm_dic
|
||||
|
||||
def __layout_bookeeping(self):
|
||||
# training bookeeping
|
||||
self.total_batch_nb = 0
|
||||
self.running_loss = []
|
||||
self.avg_loss = 0
|
||||
self.batch_nb = 0
|
||||
self.tqdm_metrics = {}
|
||||
|
||||
# determine number of training batches
|
||||
self.nb_tng_batches = len(self.tng_dataloader)
|
||||
self.nb_tng_batches = int(self.nb_tng_batches * self.train_percent_check)
|
||||
|
||||
# determine number of validation batches
|
||||
self.nb_val_batches = len(self.val_dataloader)
|
||||
self.nb_val_batches = int(self.nb_val_batches * self.val_percent_check)
|
||||
self.nb_val_batches = max(1, self.nb_val_batches)
|
||||
self.nb_val_batches = self.nb_val_batches
|
||||
|
||||
# determine number of test batches
|
||||
self.nb_test_batches = len(self.test_dataloader)
|
||||
self.nb_test_batches = int(self.nb_test_batches * self.test_percent_check)
|
||||
|
||||
# determine when to check validation
|
||||
self.val_check_batch = int(self.nb_tng_batches * self.val_check_interval)
|
||||
|
||||
def __add_tqdm_metrics(self, metrics):
|
||||
for k, v in metrics.items():
|
||||
if type(v) is torch.Tensor:
|
||||
v = v.item()
|
||||
|
||||
self.tqdm_metrics[k] = v
|
||||
|
||||
def validate(self, model, dataloader, max_batches):
|
||||
"""
|
||||
Run validation code
|
||||
:param model: PT model
|
||||
:param dataloader: PT dataloader
|
||||
:param max_batches: Scalar
|
||||
:return:
|
||||
"""
|
||||
# enable eval mode
|
||||
model.zero_grad()
|
||||
model.eval()
|
||||
|
||||
# disable gradients to save memory
|
||||
torch.set_grad_enabled(False)
|
||||
|
||||
# bookkeeping
|
||||
outputs = []
|
||||
|
||||
# run training
|
||||
for batch_i, data_batch in enumerate(dataloader):
|
||||
|
||||
if data_batch is None:
|
||||
continue
|
||||
|
||||
# stop short when on fast dev run
|
||||
if max_batches is not None and batch_i >= max_batches:
|
||||
break
|
||||
|
||||
# -----------------
|
||||
# RUN VALIDATION STEP
|
||||
# -----------------
|
||||
if self.use_ddp:
|
||||
output = model(data_batch, batch_i)
|
||||
elif self.use_dp:
|
||||
output = model(data_batch, batch_i)
|
||||
output = reduce_distributed_output(output, len(self.data_parallel_device_ids))
|
||||
|
||||
else:
|
||||
output = model.validation_step(data_batch, batch_i)
|
||||
|
||||
outputs.append(output)
|
||||
|
||||
# batch done
|
||||
if self.progress_bar and self.prog_bar is not None:
|
||||
self.prog_bar.update(1)
|
||||
|
||||
# give model a chance to do something with the outputs
|
||||
if self.data_parallel:
|
||||
val_results = model.module.validation_end(outputs)
|
||||
else:
|
||||
val_results = model.validation_end(outputs)
|
||||
|
||||
# enable train mode again
|
||||
model.train()
|
||||
|
||||
# enable gradients to save memory
|
||||
torch.set_grad_enabled(True)
|
||||
|
||||
return val_results
|
||||
|
||||
def __get_dataloaders(self, model):
|
||||
"""
|
||||
Dataloaders are provided by the model
|
||||
:param model:
|
||||
:return:
|
||||
"""
|
||||
self.tng_dataloader = model.tng_dataloader
|
||||
self.test_dataloader = model.test_dataloader
|
||||
self.val_dataloader = model.val_dataloader
|
||||
|
||||
if self.use_ddp and not isinstance(self.tng_dataloader.sampler, DistributedSampler):
|
||||
msg = '''
|
||||
when using multiple gpus and multiple nodes you must pass a DistributedSampler to DataLoader(sampler).
|
||||
|
||||
ie: this:
|
||||
dataset = myDataset()
|
||||
dataloader = Dataloader(dataset)
|
||||
|
||||
becomes:
|
||||
dataset = myDataset()
|
||||
dist_sampler = torch.utils.data.distributed.DistributedSampler(dataset)
|
||||
dataloader = Dataloader(dataset, sampler=dist_sampler)
|
||||
'''
|
||||
raise Exception(msg)
|
||||
|
||||
# -----------------------------
|
||||
# MODEL TRAINING
|
||||
# -----------------------------
|
||||
def fit(self, model):
|
||||
|
||||
# when using multi-node or DDP within a node start each module in a separate process
|
||||
if self.use_ddp:
|
||||
# must copy only the meta of the exp so it survives pickle/unpickle when going to new process
|
||||
self.experiment = self.experiment.get_meta_copy()
|
||||
|
||||
# whenever we have the correct number of tasks, we let slurm manage processes
|
||||
# otherwise we launch the required number of processes
|
||||
try:
|
||||
nb_slurm_tasks = int(os.environ['SLURM_NTASKS'])
|
||||
nb_requested_gpus = len(self.data_parallel_device_ids) * self.nb_gpu_nodes
|
||||
is_slurm_managing_tasks = nb_slurm_tasks == nb_requested_gpus
|
||||
except Exception as e:
|
||||
# likely not on slurm, so set the slurm managed flag to false
|
||||
is_slurm_managing_tasks = False
|
||||
|
||||
if is_slurm_managing_tasks:
|
||||
task = int(os.environ['SLURM_LOCALID'])
|
||||
self.ddp_train(task, model)
|
||||
else:
|
||||
msg = f"""
|
||||
You requested {nb_requested_gpus} GPUs but launched {nb_slurm_tasks} slurm tasks.
|
||||
We will launch {nb_requested_gpus} processes for you.
|
||||
We recommend you let slurm manage the processes by setting: --ntasks-per-node={nb_requested_gpus}
|
||||
If you're not using SLURM, ignore this message!
|
||||
"""
|
||||
warnings.warn(msg)
|
||||
mp.spawn(self.ddp_train, nprocs=len(self.data_parallel_device_ids), args=(model, ))
|
||||
|
||||
# 1 gpu or dp option triggers training using DP module
|
||||
# easier to avoid NCCL issues
|
||||
elif self.use_dp:
|
||||
self.dp_train(model)
|
||||
|
||||
# ON CPU
|
||||
else:
|
||||
# CHOOSE OPTIMIZER
|
||||
# filter out the weights that were done on gpu so we can load on good old cpus
|
||||
self.optimizers = model.configure_optimizers()
|
||||
|
||||
# run through amp wrapper
|
||||
if self.use_amp:
|
||||
# An example
|
||||
model, optimizers = amp.initialize(
|
||||
model, self.optimizers, opt_level=self.amp_level,
|
||||
)
|
||||
self.optimizers = optimizers
|
||||
|
||||
self.__run_pretrain_routine(model)
|
||||
|
||||
def dp_train(self, model):
|
||||
|
||||
# CHOOSE OPTIMIZER
|
||||
# filter out the weights that were done on gpu so we can load on good old cpus
|
||||
self.optimizers = model.configure_optimizers()
|
||||
|
||||
model.cuda(self.data_parallel_device_ids[0])
|
||||
model = LightningDataParallel(model, device_ids=self.data_parallel_device_ids)
|
||||
|
||||
# run through amp wrapper
|
||||
if self.use_amp:
|
||||
# An example
|
||||
model, optimizers = amp.initialize(
|
||||
model, self.optimizers, opt_level=self.amp_level,
|
||||
)
|
||||
self.optimizers = optimizers
|
||||
|
||||
self.__run_pretrain_routine(model)
|
||||
|
||||
def ddp_train(self, gpu_nb, model):
|
||||
"""
|
||||
Entry point into a DP thread
|
||||
:param gpu_nb:
|
||||
:param model:
|
||||
:param cluster_obj:
|
||||
:return:
|
||||
"""
|
||||
# node rank using relative slurm id
|
||||
# otherwise default to node rank 0
|
||||
try:
|
||||
node_id = os.environ['SLURM_NODEID']
|
||||
self.node_rank = int(node_id)
|
||||
except Exception as e:
|
||||
self.node_rank = 0
|
||||
|
||||
# recover original exp before went into process
|
||||
# init in write mode only on proc 0
|
||||
self.experiment.debug = self.proc_rank > 0
|
||||
self.experiment = self.experiment.get_non_ddp_exp()
|
||||
|
||||
# show progbar only on prog_rank 0
|
||||
self.prog_bar = self.prog_bar and self.node_rank == 0 and gpu_nb == 0
|
||||
|
||||
# determine which process we are and world size
|
||||
self.proc_rank = self.node_rank * len(self.data_parallel_device_ids) + gpu_nb
|
||||
self.world_size = self.nb_gpu_nodes * len(self.data_parallel_device_ids)
|
||||
|
||||
# set up server using proc 0's ip address
|
||||
# try to init for 20 times at max in case ports are taken
|
||||
# where to store ip_table
|
||||
self.__init_tcp_connection()
|
||||
|
||||
# CHOOSE OPTIMIZER
|
||||
# filter out the weights that were done on gpu so we can load on good old cpus
|
||||
self.optimizers = model.configure_optimizers()
|
||||
|
||||
# MODEL
|
||||
# copy model to each gpu
|
||||
torch.cuda.set_device(gpu_nb)
|
||||
model.cuda(gpu_nb)
|
||||
|
||||
# AMP
|
||||
# run through amp wrapper before going to distributed DP
|
||||
if self.use_amp:
|
||||
# An example
|
||||
model, optimizers = amp.initialize(
|
||||
model, self.optimizers, opt_level=self.amp_level,
|
||||
)
|
||||
self.optimizers = optimizers
|
||||
|
||||
model = LightningDistributedDataParallel(model, device_ids=[gpu_nb])
|
||||
|
||||
# continue training routine
|
||||
self.__run_pretrain_routine(model)
|
||||
|
||||
def __init_tcp_connection(self):
|
||||
"""
|
||||
Connect all procs in the world using the env:// init
|
||||
Use the first node as the root address
|
||||
:param port:
|
||||
:param tries:
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
port = os.environ['MASTER_PORT']
|
||||
except Exception as e:
|
||||
port = 12910
|
||||
os.environ['MASTER_PORT'] = f'{port}'
|
||||
|
||||
root_node = self.__resolve_root_node_address()
|
||||
os.environ['MASTER_ADDR'] = root_node
|
||||
|
||||
dist.init_process_group("nccl", rank=self.proc_rank, world_size=self.world_size)
|
||||
|
||||
|
||||
def __resolve_root_node_address(self):
|
||||
try:
|
||||
root_node = os.environ['SLURM_NODELIST'].split(' ')[0]
|
||||
|
||||
if '[' in root_node:
|
||||
name = root_node.split('[')[0]
|
||||
number = root_node.split(',')[0]
|
||||
if '-' in number:
|
||||
number = number.split('-')[0]
|
||||
|
||||
number = re.sub('[^0-9]', '', number)
|
||||
root_node = name + number
|
||||
|
||||
except Exception as e:
|
||||
root_node = '127.0.0.2'
|
||||
|
||||
return root_node
|
||||
|
||||
def __run_pretrain_routine(self, model):
|
||||
"""
|
||||
Sanity check a few things before starting actual training
|
||||
:param model:
|
||||
:return:
|
||||
"""
|
||||
ref_model = model
|
||||
if self.data_parallel:
|
||||
ref_model = model.module
|
||||
|
||||
ref_model.trainer = self
|
||||
|
||||
# set local properties on the model
|
||||
ref_model.on_gpu = self.on_gpu
|
||||
|
||||
# transfer data loaders from model
|
||||
self.__get_dataloaders(ref_model)
|
||||
|
||||
# init training constants
|
||||
self.__layout_bookeeping()
|
||||
|
||||
# add lr schedulers
|
||||
if self.lr_scheduler_milestones is not None:
|
||||
for optimizer in self.optimizers:
|
||||
scheduler = MultiStepLR(optimizer, self.lr_scheduler_milestones)
|
||||
self.lr_schedulers.append(scheduler)
|
||||
|
||||
# print model summary
|
||||
if self.proc_rank == 0 and self.print_weights_summary:
|
||||
ref_model.summarize()
|
||||
|
||||
# give model convenience properties
|
||||
ref_model.trainer = self
|
||||
ref_model.experiment = self.experiment
|
||||
|
||||
# run tiny validation to make sure program won't crash during val
|
||||
_ = self.validate(model, self.val_dataloader, max_batches=self.nb_sanity_val_steps)
|
||||
|
||||
# save exp to get started
|
||||
if self.proc_rank == 0:
|
||||
self.experiment.save()
|
||||
|
||||
# enable cluster checkpointing
|
||||
if self.cluster is not None:
|
||||
self.enable_auto_hpc_walltime_manager()
|
||||
|
||||
# ---------------------------
|
||||
# CORE TRAINING LOOP
|
||||
# ---------------------------
|
||||
self.model = model
|
||||
self.__train()
|
||||
|
||||
def __train(self):
|
||||
# run all epochs
|
||||
for epoch_nb in range(self.current_epoch, self.max_nb_epochs):
|
||||
# update the lr scheduler
|
||||
for lr_scheduler in self.lr_schedulers:
|
||||
lr_scheduler.step()
|
||||
|
||||
model = self.__get_model()
|
||||
model.current_epoch = epoch_nb
|
||||
|
||||
# hook
|
||||
if self.__is_function_implemented('on_epoch_start'):
|
||||
model = self.__get_model()
|
||||
model.on_epoch_start()
|
||||
|
||||
self.current_epoch = epoch_nb
|
||||
self.total_batches = self.nb_tng_batches + self.nb_val_batches
|
||||
self.batch_loss_value = 0 # accumulated grads
|
||||
|
||||
# init progbar when requested
|
||||
if self.progress_bar:
|
||||
self.prog_bar = tqdm.tqdm(range(self.total_batches), position=self.process_position)
|
||||
|
||||
for batch_nb, data_batch in enumerate(self.tng_dataloader):
|
||||
self.batch_nb = batch_nb
|
||||
self.global_step += 1
|
||||
|
||||
model = self.__get_model()
|
||||
model.global_step = self.global_step
|
||||
|
||||
# stop when the flag is changed or we've gone past the amount requested in the batches
|
||||
self.total_batch_nb += 1
|
||||
met_batch_limit = batch_nb > self.nb_tng_batches
|
||||
if met_batch_limit:
|
||||
break
|
||||
|
||||
# ---------------
|
||||
# RUN TRAIN STEP
|
||||
# ---------------
|
||||
batch_result = self.__run_tng_batch(data_batch, batch_nb)
|
||||
early_stop_epoch = batch_result == -1
|
||||
|
||||
# ---------------
|
||||
# RUN VAL STEP
|
||||
# ---------------
|
||||
is_val_check_batch = (batch_nb + 1) % self.val_check_batch == 0
|
||||
if self.fast_dev_run or is_val_check_batch or early_stop_epoch:
|
||||
self.__run_validation()
|
||||
|
||||
# when batch should be saved
|
||||
if (batch_nb + 1) % self.log_save_interval == 0 or early_stop_epoch:
|
||||
if self.proc_rank == 0:
|
||||
self.experiment.save()
|
||||
|
||||
# when metrics should be logged
|
||||
if batch_nb % self.add_log_row_interval == 0 or early_stop_epoch:
|
||||
# count items in memory
|
||||
# nb_params, nb_tensors = count_mem_items()
|
||||
|
||||
model = self.__get_model()
|
||||
metrics = model.update_tng_log_metrics(self.__tng_tqdm_dic)
|
||||
|
||||
# add gpu memory
|
||||
if self.on_gpu:
|
||||
mem_map = get_gpu_memory_map()
|
||||
metrics.update(mem_map)
|
||||
|
||||
# add norms
|
||||
if self.track_grad_norm > 0:
|
||||
model = self.__get_model()
|
||||
grad_norm_dic = model.grad_norm(self.track_grad_norm)
|
||||
metrics.update(grad_norm_dic)
|
||||
|
||||
if self.__is_function_implemented('on_tng_metrics'):
|
||||
model.on_tng_metrics(metrics)
|
||||
|
||||
# log metrics
|
||||
scalar_metrics = self.__metrics_to_scalars(metrics, blacklist=self.__log_vals_blacklist())
|
||||
if self.proc_rank == 0:
|
||||
self.experiment.log(scalar_metrics, global_step=self.global_step)
|
||||
self.experiment.save()
|
||||
|
||||
# hook
|
||||
if self.__is_function_implemented('on_batch_end'):
|
||||
model = self.__get_model()
|
||||
model.on_batch_end()
|
||||
|
||||
# end epoch early
|
||||
if early_stop_epoch:
|
||||
break
|
||||
|
||||
# hook
|
||||
if self.__is_function_implemented('on_epoch_end'):
|
||||
model = self.__get_model()
|
||||
model.on_epoch_end()
|
||||
|
||||
# early stopping
|
||||
met_min_epochs = epoch_nb > self.min_nb_epochs
|
||||
if self.enable_early_stop and met_min_epochs:
|
||||
should_stop = self.early_stop_callback.on_epoch_end(epoch=epoch_nb, logs=self.__tng_tqdm_dic)
|
||||
|
||||
# stop training
|
||||
stop = should_stop and met_min_epochs
|
||||
if stop:
|
||||
return
|
||||
|
||||
def __metrics_to_scalars(self, metrics, blacklist=[]):
|
||||
new_metrics = {}
|
||||
for k, v in metrics.items():
|
||||
if type(v) is torch.Tensor:
|
||||
v = v.item()
|
||||
|
||||
if type(v) is dict:
|
||||
v = self.__metrics_to_scalars(v)
|
||||
|
||||
if k not in blacklist:
|
||||
new_metrics[k] = float(v)
|
||||
|
||||
return new_metrics
|
||||
|
||||
def __log_vals_blacklist(self):
|
||||
"""avoid logging some vals lightning uses to maintain state"""
|
||||
blacklist = {'batch_nb', 'v_nb', 'gpu'}
|
||||
return blacklist
|
||||
|
||||
def __run_tng_batch(self, data_batch, batch_nb):
|
||||
if data_batch is None:
|
||||
return 0
|
||||
|
||||
# hook
|
||||
if self.__is_function_implemented('on_batch_start'):
|
||||
model_ref = self.__get_model()
|
||||
response = model_ref.on_batch_start(data_batch)
|
||||
|
||||
if response == -1:
|
||||
return -1
|
||||
|
||||
if self.progress_bar:
|
||||
self.prog_bar.update(1)
|
||||
|
||||
# forward pass
|
||||
# return a scalar value and a dic with tqdm metrics
|
||||
if self.use_ddp:
|
||||
output = self.model(data_batch, batch_nb)
|
||||
elif self.use_dp:
|
||||
output = self.model(data_batch, batch_nb)
|
||||
output = reduce_distributed_output(output, len(self.data_parallel_device_ids))
|
||||
else:
|
||||
output = self.model.training_step(data_batch, batch_nb)
|
||||
|
||||
try:
|
||||
model_specific_tqdm_metrics_dic = output['tqdm_metrics']
|
||||
except Exception as e:
|
||||
model_specific_tqdm_metrics_dic = {}
|
||||
|
||||
# if output dict doesn't have the keyword loss
|
||||
# then assume the output=loss if scalar
|
||||
try:
|
||||
loss = output['loss']
|
||||
except Exception as e:
|
||||
if type(output) is torch.Tensor:
|
||||
loss = output
|
||||
|
||||
self.__add_tqdm_metrics(model_specific_tqdm_metrics_dic)
|
||||
|
||||
# backward pass
|
||||
if self.use_amp:
|
||||
# scale loss when using amp
|
||||
for optimizer in self.optimizers:
|
||||
with amp.scale_loss(loss, optimizer) as scaled_loss:
|
||||
scaled_loss.backward()
|
||||
else:
|
||||
loss.backward()
|
||||
|
||||
if self.print_nan_grads:
|
||||
model = self.__get_model()
|
||||
for param in model.parameters():
|
||||
print(param.grad.float().sum())
|
||||
|
||||
# avoid memory leaks
|
||||
self.batch_loss_value += loss.item()
|
||||
|
||||
# gradient update with accumulated gradients
|
||||
if (self.batch_nb + 1) % self.accumulate_grad_batches == 0:
|
||||
|
||||
# clip gradients
|
||||
if self.gradient_clip > 0:
|
||||
model = self.__get_model()
|
||||
torch.nn.utils.clip_grad_norm(model.parameters(), self.gradient_clip)
|
||||
|
||||
# update gradients across all optimizers
|
||||
for optimizer in self.optimizers:
|
||||
optimizer.step()
|
||||
|
||||
# clear gradients
|
||||
optimizer.zero_grad()
|
||||
|
||||
# queuing loss across batches blows it up proportionally... divide out the number accumulated
|
||||
self.batch_loss_value = self.batch_loss_value / self.accumulate_grad_batches
|
||||
|
||||
# track loss
|
||||
self.running_loss.append(self.batch_loss_value)
|
||||
self.batch_loss_value = 0
|
||||
self.avg_loss = np.mean(self.running_loss[-100:])
|
||||
|
||||
# update progbar
|
||||
if self.progress_bar:
|
||||
# add model specific metrics
|
||||
tqdm_metrics = self.__tng_tqdm_dic
|
||||
self.prog_bar.set_postfix(**tqdm_metrics)
|
||||
|
||||
# activate batch end hook
|
||||
if self.__is_function_implemented('on_batch_end'):
|
||||
model = self.__get_model()
|
||||
model.on_batch_end()
|
||||
|
||||
return 0
|
||||
|
||||
def __run_validation(self):
|
||||
# decide if can check epochs
|
||||
can_check_epoch = (self.current_epoch + 1) % self.check_val_every_n_epoch == 0
|
||||
if self.fast_dev_run:
|
||||
print('skipping to check performance bc of --fast_dev_run')
|
||||
elif not can_check_epoch:
|
||||
return
|
||||
|
||||
try:
|
||||
# hook
|
||||
if self.__is_function_implemented('on_pre_performance_check'):
|
||||
model = self.__get_model()
|
||||
model.on_pre_performance_check()
|
||||
|
||||
# use full val set on end of epoch
|
||||
# use a small portion otherwise
|
||||
max_batches = None if not self.fast_dev_run else 1
|
||||
model_specific_tqdm_metrics_dic = self.validate(
|
||||
self.model,
|
||||
self.val_dataloader,
|
||||
max_batches
|
||||
)
|
||||
self.__add_tqdm_metrics(model_specific_tqdm_metrics_dic)
|
||||
|
||||
# hook
|
||||
if self.__is_function_implemented('on_post_performance_check'):
|
||||
model = self.__get_model()
|
||||
model.on_post_performance_check()
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(traceback.print_exc())
|
||||
|
||||
if self.progress_bar:
|
||||
# add model specific metrics
|
||||
tqdm_metrics = self.__tng_tqdm_dic
|
||||
self.prog_bar.set_postfix(**tqdm_metrics)
|
||||
|
||||
# model checkpointing
|
||||
if self.proc_rank == 0 and self.checkpoint_callback:
|
||||
print('save callback...')
|
||||
self.checkpoint_callback.on_epoch_end(epoch=self.current_epoch, logs=self.__tng_tqdm_dic)
|
||||
@@ -1,188 +0,0 @@
|
||||
from torch.nn import DataParallel
|
||||
from torch.nn.parallel import DistributedDataParallel
|
||||
import itertools
|
||||
from itertools import chain
|
||||
|
||||
import threading
|
||||
import torch
|
||||
from torch.cuda._utils import _get_device_index
|
||||
import pdb
|
||||
|
||||
|
||||
def _find_tensors(obj):
|
||||
r"""
|
||||
Recursively find all tensors contained in the specified object.
|
||||
"""
|
||||
if isinstance(obj, torch.Tensor):
|
||||
return [obj]
|
||||
if isinstance(obj, (list, tuple)):
|
||||
return itertools.chain(*map(_find_tensors, obj))
|
||||
if isinstance(obj, dict):
|
||||
return itertools.chain(*map(_find_tensors, obj.values()))
|
||||
return []
|
||||
|
||||
|
||||
|
||||
def get_a_var(obj):
|
||||
if isinstance(obj, torch.Tensor):
|
||||
return obj
|
||||
|
||||
if isinstance(obj, list) or isinstance(obj, tuple):
|
||||
for result in map(get_a_var, obj):
|
||||
if isinstance(result, torch.Tensor):
|
||||
return result
|
||||
if isinstance(obj, dict):
|
||||
for result in map(get_a_var, obj.items()):
|
||||
if isinstance(result, torch.Tensor):
|
||||
return result
|
||||
return None
|
||||
|
||||
|
||||
class LightningDataParallel(DataParallel):
|
||||
"""
|
||||
Override the forward call in lightning so it goes to training and validation step respectively
|
||||
"""
|
||||
|
||||
def forward(self, *inputs, **kwargs):
|
||||
if not self.device_ids:
|
||||
return self.module(*inputs, **kwargs)
|
||||
|
||||
for t in chain(self.module.parameters(), self.module.buffers()):
|
||||
if t.device != self.src_device_obj:
|
||||
raise RuntimeError("module must have its parameters and buffers "
|
||||
"on device {} (device_ids[0]) but found one of "
|
||||
"them on device: {}".format(self.src_device_obj, t.device))
|
||||
|
||||
inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
|
||||
if len(self.device_ids) == 1:
|
||||
# lightning
|
||||
if self.module.training:
|
||||
return self.module.training_step(*inputs[0], **kwargs[0])
|
||||
else:
|
||||
return self.module.validation_step(*inputs[0], **kwargs[0])
|
||||
|
||||
replicas = self.replicate(self.module, self.device_ids[:len(inputs)])
|
||||
outputs = self.parallel_apply(replicas, inputs, kwargs)
|
||||
return self.gather(outputs, self.output_device)
|
||||
|
||||
|
||||
def parallel_apply(self, replicas, inputs, kwargs):
|
||||
return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
|
||||
|
||||
|
||||
class LightningDistributedDataParallel(DistributedDataParallel):
|
||||
"""
|
||||
Override the forward call in lightning so it goes to training and validation step respectively
|
||||
"""
|
||||
|
||||
def parallel_apply(self, replicas, inputs, kwargs):
|
||||
return parallel_apply(replicas, inputs, kwargs, self.device_ids[:len(replicas)])
|
||||
|
||||
def forward(self, *inputs, **kwargs):
|
||||
self._sync_params()
|
||||
if self.device_ids:
|
||||
inputs, kwargs = self.scatter(inputs, kwargs, self.device_ids)
|
||||
if len(self.device_ids) == 1:
|
||||
# --------------
|
||||
# LIGHTNING MOD
|
||||
# --------------
|
||||
# normal
|
||||
# output = self.module(*inputs[0], **kwargs[0])
|
||||
|
||||
# lightning
|
||||
if self.module.training:
|
||||
output = self.module.training_step(*inputs[0], **kwargs[0])
|
||||
else:
|
||||
output = self.module.validation_step(*inputs[0], **kwargs[0])
|
||||
else:
|
||||
outputs = self.parallel_apply(self._module_copies[:len(inputs)], inputs, kwargs)
|
||||
output = self.gather(outputs, self.output_device)
|
||||
else:
|
||||
output = self.module(*inputs, **kwargs)
|
||||
|
||||
if torch.is_grad_enabled():
|
||||
# We'll return the output object verbatim since it is a freeform
|
||||
# object. We need to find any tensors in this object, though,
|
||||
# because we need to figure out which parameters were used during
|
||||
# this forward pass, to ensure we short circuit reduction for any
|
||||
# unused parameters. Only if `find_unused_parameters` is set.
|
||||
if self.find_unused_parameters:
|
||||
self.reducer.prepare_for_backward(list(_find_tensors(output)))
|
||||
else:
|
||||
self.reducer.prepare_for_backward([])
|
||||
return output
|
||||
|
||||
|
||||
def parallel_apply(modules, inputs, kwargs_tup=None, devices=None):
|
||||
r"""Applies each `module` in :attr:`modules` in parallel on arguments
|
||||
contained in :attr:`inputs` (positional) and :attr:`kwargs_tup` (keyword)
|
||||
on each of :attr:`devices`.
|
||||
|
||||
Args:
|
||||
modules (Module): modules to be parallelized
|
||||
inputs (tensor): inputs to the modules
|
||||
devices (list of int or torch.device): CUDA devices
|
||||
|
||||
:attr:`modules`, :attr:`inputs`, :attr:`kwargs_tup` (if given), and
|
||||
:attr:`devices` (if given) should all have same length. Moreover, each
|
||||
element of :attr:`inputs` can either be a single object as the only argument
|
||||
to a module, or a collection of positional arguments.
|
||||
"""
|
||||
assert len(modules) == len(inputs)
|
||||
if kwargs_tup is not None:
|
||||
assert len(modules) == len(kwargs_tup)
|
||||
else:
|
||||
kwargs_tup = ({},) * len(modules)
|
||||
if devices is not None:
|
||||
assert len(modules) == len(devices)
|
||||
else:
|
||||
devices = [None] * len(modules)
|
||||
devices = list(map(lambda x: _get_device_index(x, True), devices))
|
||||
lock = threading.Lock()
|
||||
results = {}
|
||||
grad_enabled = torch.is_grad_enabled()
|
||||
|
||||
def _worker(i, module, input, kwargs, device=None):
|
||||
torch.set_grad_enabled(grad_enabled)
|
||||
if device is None:
|
||||
device = get_a_var(input).get_device()
|
||||
try:
|
||||
with torch.cuda.device(device):
|
||||
# this also avoids accidental slicing of `input` if it is a Tensor
|
||||
if not isinstance(input, (list, tuple)):
|
||||
input = (input,)
|
||||
|
||||
# ---------------
|
||||
# CHANGE
|
||||
if module.training:
|
||||
output = module.training_step(*input, **kwargs)
|
||||
else:
|
||||
output = module.validation_step(*input, **kwargs)
|
||||
# ---------------
|
||||
|
||||
with lock:
|
||||
results[i] = output
|
||||
except Exception as e:
|
||||
with lock:
|
||||
results[i] = e
|
||||
|
||||
if len(modules) > 1:
|
||||
threads = [threading.Thread(target=_worker,
|
||||
args=(i, module, input, kwargs, device))
|
||||
for i, (module, input, kwargs, device) in
|
||||
enumerate(zip(modules, inputs, kwargs_tup, devices))]
|
||||
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
else:
|
||||
_worker(0, modules[0], inputs[0], kwargs_tup[0], devices[0])
|
||||
|
||||
outputs = []
|
||||
for i in range(len(inputs)):
|
||||
output = results[i]
|
||||
if isinstance(output, Exception):
|
||||
raise output
|
||||
outputs.append(output)
|
||||
return outputs
|
||||
@@ -1,40 +0,0 @@
|
||||
import numpy as np
|
||||
from torch import nn
|
||||
|
||||
"""
|
||||
Module to describe gradients
|
||||
"""
|
||||
|
||||
|
||||
class GradInformation(nn.Module):
|
||||
|
||||
def grad_norm(self, norm_type):
|
||||
results = {}
|
||||
total_norm = 0
|
||||
for i, p in enumerate(self.parameters()):
|
||||
if p.requires_grad:
|
||||
try:
|
||||
param_norm = p.grad.data.norm(norm_type)
|
||||
total_norm += param_norm ** norm_type
|
||||
norm = param_norm ** (1 / norm_type)
|
||||
|
||||
results['grad_{}_norm_{}'.format(norm_type, i)] = round(norm.data.cpu().numpy().flatten()[0], 3)
|
||||
except Exception as e:
|
||||
# this param had no grad
|
||||
pass
|
||||
|
||||
total_norm = total_norm ** (1. / norm_type)
|
||||
results['grad_{}_norm_total'.format(norm_type)] = round(total_norm.data.cpu().numpy().flatten()[0], 3)
|
||||
return results
|
||||
|
||||
|
||||
def describe_grads(self):
|
||||
for p in self.parameters():
|
||||
g = p.grad.data.numpy().flatten()
|
||||
print(np.max(g), np.min(g), np.mean(g))
|
||||
|
||||
|
||||
def describe_params(self):
|
||||
for p in self.parameters():
|
||||
g = p.data.numpy().flatten()
|
||||
print(np.max(g), np.min(g), np.mean(g))
|
||||
@@ -1,24 +0,0 @@
|
||||
import torch
|
||||
|
||||
class ModelHooks(torch.nn.Module):
|
||||
def on_batch_start(self, data_batch):
|
||||
pass
|
||||
|
||||
def on_batch_end(self):
|
||||
pass
|
||||
|
||||
def on_epoch_start(self):
|
||||
pass
|
||||
|
||||
def on_epoch_end(self):
|
||||
pass
|
||||
|
||||
def on_pre_performance_check(self):
|
||||
pass
|
||||
|
||||
def on_post_performance_check(self):
|
||||
pass
|
||||
|
||||
def on_tng_metrics(self, metrics):
|
||||
pass
|
||||
|
||||
@@ -1,180 +0,0 @@
|
||||
import torch
|
||||
import gc
|
||||
import subprocess
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
|
||||
'''
|
||||
Generates a summary of a model's layers and dimensionality
|
||||
'''
|
||||
|
||||
|
||||
class ModelSummary(object):
|
||||
|
||||
def __init__(self, model):
|
||||
'''
|
||||
Generates summaries of model layers and dimensions.
|
||||
'''
|
||||
self.model = model
|
||||
self.in_sizes = []
|
||||
self.out_sizes = []
|
||||
|
||||
self.summarize()
|
||||
|
||||
def __str__(self):
|
||||
return self.summary.__str__()
|
||||
|
||||
def __repr__(self):
|
||||
return self.summary.__str__()
|
||||
|
||||
def get_variable_sizes(self):
|
||||
'''Run sample input through each layer to get output sizes'''
|
||||
mods = list(self.model.modules())
|
||||
in_sizes = []
|
||||
out_sizes = []
|
||||
input_ = self.example_input_array
|
||||
for i in range(1, len(mods)):
|
||||
m = mods[i]
|
||||
if type(input_) is list or type(input_) is tuple:
|
||||
out = m(*input_)
|
||||
else:
|
||||
out = m(input_)
|
||||
|
||||
if type(input_) is tuple or type(input_) is list:
|
||||
in_size = []
|
||||
for x in input_:
|
||||
if type(x) is list:
|
||||
in_size.append(len(x))
|
||||
else:
|
||||
in_size.append(x.size())
|
||||
else:
|
||||
in_size = np.array(input_.size())
|
||||
|
||||
in_sizes.append(in_size)
|
||||
|
||||
if type(out) is tuple or type(out) is list:
|
||||
out_size = np.asarray([x.size() for x in out])
|
||||
else:
|
||||
out_size = np.array(out.size())
|
||||
|
||||
out_sizes.append(out_size)
|
||||
input_ = out
|
||||
|
||||
self.in_sizes = in_sizes
|
||||
self.out_sizes = out_sizes
|
||||
return
|
||||
|
||||
def get_layer_names(self):
|
||||
'''Collect Layer Names'''
|
||||
mods = list(self.model.named_modules())
|
||||
names = []
|
||||
layers = []
|
||||
for m in mods[1:]:
|
||||
names += [m[0]]
|
||||
layers += [str(m[1].__class__)]
|
||||
|
||||
layer_types = [x.split('.')[-1][:-2] for x in layers]
|
||||
|
||||
self.layer_names = names
|
||||
self.layer_types = layer_types
|
||||
return
|
||||
|
||||
def get_parameter_sizes(self):
|
||||
'''Get sizes of all parameters in `model`'''
|
||||
mods = list(self.model.modules())
|
||||
sizes = []
|
||||
|
||||
for i in range(1,len(mods)):
|
||||
m = mods[i]
|
||||
p = list(m.parameters())
|
||||
modsz = []
|
||||
for j in range(len(p)):
|
||||
modsz.append(np.array(p[j].size()))
|
||||
sizes.append(modsz)
|
||||
|
||||
self.param_sizes = sizes
|
||||
return
|
||||
|
||||
def get_parameter_nums(self):
|
||||
'''Get number of parameters in each layer'''
|
||||
param_nums = []
|
||||
for mod in self.param_sizes:
|
||||
all_params = 0
|
||||
for p in mod:
|
||||
all_params += np.prod(p)
|
||||
param_nums.append(all_params)
|
||||
self.param_nums = param_nums
|
||||
return
|
||||
|
||||
def make_summary(self):
|
||||
'''
|
||||
Makes a summary listing with:
|
||||
|
||||
Layer Name, Layer Type, Input Size, Output Size, Number of Parameters
|
||||
'''
|
||||
|
||||
df = pd.DataFrame( np.zeros( (len(self.layer_names), 3) ) )
|
||||
df.columns = ['Name', 'Type', 'Params']
|
||||
|
||||
df['Name'] = self.layer_names
|
||||
df['Type'] = self.layer_types
|
||||
df['Params'] = self.param_nums
|
||||
|
||||
self.summary = df
|
||||
return
|
||||
|
||||
def summarize(self):
|
||||
self.get_layer_names()
|
||||
self.get_parameter_sizes()
|
||||
self.get_parameter_nums()
|
||||
self.make_summary()
|
||||
|
||||
|
||||
def print_mem_stack():
|
||||
for obj in gc.get_objects():
|
||||
try:
|
||||
if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
|
||||
print(type(obj), obj.size())
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
|
||||
def count_mem_items():
|
||||
nb_params = 0
|
||||
nb_tensors = 0
|
||||
for obj in gc.get_objects():
|
||||
try:
|
||||
if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
|
||||
obj_type = str(type(obj))
|
||||
if 'parameter' in obj_type:
|
||||
nb_params += 1
|
||||
else:
|
||||
nb_tensors += 1
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return nb_params, nb_tensors
|
||||
|
||||
|
||||
def get_gpu_memory_map():
|
||||
"""Get the current gpu usage.
|
||||
|
||||
Returns
|
||||
-------
|
||||
usage: dict
|
||||
Keys are device ids as integers.
|
||||
Values are memory usage as integers in MB.
|
||||
"""
|
||||
result = subprocess.check_output(
|
||||
[
|
||||
'nvidia-smi', '--query-gpu=memory.used',
|
||||
'--format=csv,nounits,noheader'
|
||||
], encoding='utf-8')
|
||||
# Convert lines into a dictionary
|
||||
gpu_memory = [int(x) for x in result.strip().split('\n')]
|
||||
gpu_memory_map = {}
|
||||
for k, v in zip(range(len(gpu_memory)), gpu_memory):
|
||||
k = f'gpu_{k}'
|
||||
gpu_memory_map[k] = v
|
||||
return gpu_memory_map
|
||||
@@ -1,206 +0,0 @@
|
||||
import torch
|
||||
import os
|
||||
import re
|
||||
import pdb
|
||||
from pytorch_lightning.pt_overrides.override_data_parallel import LightningDistributedDataParallel, LightningDataParallel
|
||||
|
||||
class ModelIO(object):
|
||||
|
||||
def load_model_specific(self, checkpoint):
|
||||
"""
|
||||
Do something with the checkpoint
|
||||
:param checkpoint:
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_save_dict(self):
|
||||
"""
|
||||
Return specific things for the model
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# -------------------------
|
||||
# OPTIONAL HOOKS
|
||||
# -------------------------
|
||||
def on_hpc_save(self):
|
||||
"""
|
||||
Hook to do whatever you need right before Slurm manager saves the model
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
def on_hpc_load(self):
|
||||
"""
|
||||
Hook to do whatever you need right before Slurm manager loads the model
|
||||
:return:
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
class TrainerIO(object):
|
||||
|
||||
# --------------------
|
||||
# MODEL SAVE CHECKPOINT
|
||||
# --------------------
|
||||
def save_checkpoint(self, filepath):
|
||||
checkpoint = self.dump_checkpoint()
|
||||
|
||||
# do the actual save
|
||||
torch.save(checkpoint, filepath)
|
||||
|
||||
def dump_checkpoint(self):
|
||||
checkpoint = {
|
||||
'epoch': self.current_epoch,
|
||||
'checkpoint_callback_best': self.checkpoint_callback.best,
|
||||
'early_stop_callback_wait': self.early_stop_callback.wait,
|
||||
'early_stop_callback_patience': self.early_stop_callback.patience,
|
||||
'global_step': self.global_step
|
||||
}
|
||||
|
||||
optimizer_states = []
|
||||
for i, optimizer in enumerate(self.optimizers):
|
||||
optimizer_states.append(optimizer.state_dict())
|
||||
|
||||
checkpoint['optimizer_states'] = optimizer_states
|
||||
|
||||
# request what to save from the model
|
||||
is_dp_module = type(self.model) is LightningDistributedDataParallel or type(self.model) is LightningDataParallel
|
||||
model = self.model.module if is_dp_module else self.model
|
||||
checkpoint_dict = model.get_save_dict()
|
||||
|
||||
# merge trainer and model saving items
|
||||
checkpoint.update(checkpoint_dict)
|
||||
return checkpoint
|
||||
|
||||
# --------------------
|
||||
# HPC IO
|
||||
# --------------------
|
||||
def enable_auto_hpc_walltime_manager(self):
|
||||
if self.cluster is None:
|
||||
return
|
||||
|
||||
# allow test tube to handle model check pointing automatically
|
||||
self.cluster.set_checkpoint_save_function(
|
||||
self.hpc_save,
|
||||
kwargs={
|
||||
'folderpath': self.checkpoint_callback.filepath,
|
||||
'experiment': self.experiment
|
||||
}
|
||||
)
|
||||
self.cluster.set_checkpoint_load_function(
|
||||
self.hpc_load,
|
||||
kwargs={
|
||||
'folderpath': self.checkpoint_callback.filepath,
|
||||
'on_gpu': self.on_gpu
|
||||
}
|
||||
)
|
||||
|
||||
def restore_training_state(self, checkpoint):
|
||||
"""
|
||||
Restore trainer state.
|
||||
Model will get its change to update
|
||||
:param checkpoint:
|
||||
:return:
|
||||
"""
|
||||
self.checkpoint_callback.best = checkpoint['checkpoint_callback_best']
|
||||
self.early_stop_callback.wait = checkpoint['early_stop_callback_wait']
|
||||
self.early_stop_callback.patience = checkpoint['early_stop_callback_patience']
|
||||
self.global_step = checkpoint['global_step']
|
||||
self.current_epoch = checkpoint['epoch']
|
||||
|
||||
# restore the optimizers
|
||||
optimizer_states = checkpoint['optimizer_states']
|
||||
for optimizer, opt_state in zip(self.optimizers, optimizer_states):
|
||||
optimizer.load_state_dict(opt_state)
|
||||
|
||||
# ----------------------------------
|
||||
# PRIVATE OPS
|
||||
# ----------------------------------
|
||||
def hpc_save(self, folderpath, experiment):
|
||||
# make sure the checkpoint folder exists
|
||||
os.makedirs(folderpath, exist_ok=True)
|
||||
|
||||
# save exp to make sure we get all the metrics
|
||||
experiment.save()
|
||||
|
||||
# close experiment to avoid issues
|
||||
experiment.close()
|
||||
|
||||
ckpt_number = self.max_ckpt_in_folder(folderpath) + 1
|
||||
|
||||
if not os.path.exists(folderpath):
|
||||
os.makedirs(folderpath, exist_ok=True)
|
||||
filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, ckpt_number)
|
||||
|
||||
# give model a chance to do something on hpc_save
|
||||
self.on_hpc_save()
|
||||
|
||||
# request what to save from the model
|
||||
checkpoint_dict = self.dump_checkpoint()
|
||||
|
||||
# do the actual save
|
||||
torch.save(checkpoint_dict, filepath)
|
||||
|
||||
def hpc_load(self, folderpath, on_gpu):
|
||||
filepath = '{}/hpc_ckpt_{}.ckpt'.format(folderpath, self.max_ckpt_in_folder(folderpath))
|
||||
|
||||
if on_gpu:
|
||||
checkpoint = torch.load(filepath)
|
||||
else:
|
||||
checkpoint = torch.load(filepath, map_location=lambda storage, loc: storage)
|
||||
|
||||
# load training state
|
||||
self.restore_training_state(checkpoint)
|
||||
|
||||
# load model state
|
||||
model = self.model.module if type(self.model) is LightningDataParallel else self.model
|
||||
model.load_model_specific(checkpoint)
|
||||
|
||||
# call model hook
|
||||
self.on_hpc_load()
|
||||
|
||||
def max_ckpt_in_folder(self, path):
|
||||
files = os.listdir(path)
|
||||
files = [x for x in files if 'ckpt_' in x]
|
||||
if len(files) == 0:
|
||||
return 0
|
||||
|
||||
ckpt_vs = []
|
||||
for name in files:
|
||||
name = name.split('ckpt_')[-1]
|
||||
name = re.sub('[^0-9]', '', name)
|
||||
ckpt_vs.append(int(name))
|
||||
|
||||
return max(ckpt_vs)
|
||||
|
||||
|
||||
def load_hparams_from_tags_csv(tags_csv):
|
||||
from argparse import Namespace
|
||||
import pandas as pd
|
||||
|
||||
tags_df = pd.read_csv(tags_csv)
|
||||
dic = tags_df.to_dict(orient='records')
|
||||
|
||||
ns_dict = {row['key']: convert(row['value']) for row in dic}
|
||||
|
||||
ns = Namespace(**ns_dict)
|
||||
return ns
|
||||
|
||||
|
||||
def convert(val):
|
||||
constructors = [int, float, str]
|
||||
|
||||
if type(val) is str:
|
||||
if val.lower() == 'true':
|
||||
return True
|
||||
if val.lower() == 'false':
|
||||
return False
|
||||
|
||||
for c in constructors:
|
||||
try:
|
||||
return c(val)
|
||||
except ValueError:
|
||||
pass
|
||||
return val
|
||||
@@ -1,22 +0,0 @@
|
||||
from torch import nn
|
||||
from torch import optim
|
||||
|
||||
|
||||
class OptimizerConfig(nn.Module):
|
||||
|
||||
def choose_optimizer(self, optimizer, params, optimizer_params, opt_name_key):
|
||||
if optimizer == 'adam':
|
||||
optimizer = optim.Adam(params, **optimizer_params)
|
||||
if optimizer == 'sparse_adam':
|
||||
optimizer = optim.SparseAdam(params, **optimizer_params)
|
||||
if optimizer == 'sgd':
|
||||
optimizer = optim.SGD(params, **optimizer_params)
|
||||
if optimizer == 'adadelta':
|
||||
optimizer = optim.Adadelta(params, **optimizer_params)
|
||||
|
||||
# transfer opt state if loaded
|
||||
if opt_name_key in self.loaded_optimizer_states_dict:
|
||||
state = self.loaded_optimizer_states_dict[opt_name_key]
|
||||
optimizer.load_state_dict(state)
|
||||
|
||||
return optimizer
|
||||
@@ -1,164 +0,0 @@
|
||||
import os
|
||||
import torch
|
||||
import math
|
||||
|
||||
from pytorch_lightning.root_module.memory import ModelSummary
|
||||
from pytorch_lightning.root_module.grads import GradInformation
|
||||
from pytorch_lightning.root_module.model_saving import ModelIO, load_hparams_from_tags_csv
|
||||
from pytorch_lightning.root_module.optimization import OptimizerConfig
|
||||
from pytorch_lightning.root_module.hooks import ModelHooks
|
||||
|
||||
|
||||
class LightningModule(GradInformation, ModelIO, OptimizerConfig, ModelHooks):
|
||||
|
||||
def __init__(self, hparams):
|
||||
super(LightningModule, self).__init__()
|
||||
self.hparams = hparams
|
||||
|
||||
self.dtype = torch.FloatTensor
|
||||
self.exp_save_path = None
|
||||
self.current_epoch = 0
|
||||
self.global_step = 0
|
||||
self.loaded_optimizer_states_dict = {}
|
||||
self.trainer = None
|
||||
self.experiment = None
|
||||
|
||||
# track if gpu was requested for checkpointing
|
||||
self.on_gpu = False
|
||||
|
||||
# computed vars for the dataloaders
|
||||
self._tng_dataloader = None
|
||||
self._val_dataloader = None
|
||||
self._test_dataloader = None
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
"""
|
||||
Expand model in into whatever you need.
|
||||
Also need to return the target
|
||||
:param x:
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def validation_step(self, data_batch, batch_nb):
|
||||
"""
|
||||
return whatever outputs will need to be aggregated in validation_end
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def validation_end(self, outputs):
|
||||
"""
|
||||
Outputs has the appended output after each validation step
|
||||
:param outputs:
|
||||
:return: dic_with_metrics for tqdm
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def training_step(self, data_batch, batch_nb):
|
||||
"""
|
||||
return loss, dict with metrics for tqdm
|
||||
:param data_batch:
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def configure_optimizers(self):
|
||||
"""
|
||||
Return array of optimizers
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def update_tng_log_metrics(self, logs):
|
||||
"""
|
||||
Chance to update metrics to be logged for training step.
|
||||
For example, add music, images, etc... to log
|
||||
:param logs:
|
||||
:return:
|
||||
"""
|
||||
return logs
|
||||
|
||||
def loss(self, *args, **kwargs):
|
||||
"""
|
||||
Expand model_out into your components
|
||||
:param model_out:
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def summarize(self):
|
||||
model_summary = ModelSummary(self)
|
||||
print(model_summary)
|
||||
|
||||
|
||||
def freeze(self):
|
||||
for param in self.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
def unfreeze(self):
|
||||
for param in self.parameters():
|
||||
param.requires_grad = True
|
||||
|
||||
@property
|
||||
def tng_dataloader(self):
|
||||
"""
|
||||
Implement a function to load an h5py of this data
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def test_dataloader(self):
|
||||
"""
|
||||
Implement a function to load an h5py of this data
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def val_dataloader(self):
|
||||
"""
|
||||
Implement a function to load an h5py of this data
|
||||
:return:
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def get_process_position(gpus):
|
||||
try:
|
||||
current_gpu = os.environ["CUDA_VISIBLE_DEVICES"]
|
||||
gpu_ids = gpus.split(',')
|
||||
process_position = gpu_ids.index(current_gpu)
|
||||
return process_position, current_gpu
|
||||
except Exception as e:
|
||||
return 0, 0
|
||||
|
||||
@classmethod
|
||||
def load_from_metrics(cls, weights_path, tags_csv, on_gpu, map_location=None):
|
||||
"""
|
||||
Primary way of loading model from csv weights path
|
||||
:param weights_path:
|
||||
:param tags_csv:
|
||||
:param on_gpu:
|
||||
:param map_location: dic for mapping storage {'cuda:1':'cuda:0'}
|
||||
:return:
|
||||
"""
|
||||
hparams = load_hparams_from_tags_csv(tags_csv)
|
||||
hparams.__setattr__('on_gpu', on_gpu)
|
||||
|
||||
if on_gpu:
|
||||
if map_location is not None:
|
||||
checkpoint = torch.load(weights_path, map_location=map_location)
|
||||
else:
|
||||
checkpoint = torch.load(weights_path)
|
||||
else:
|
||||
checkpoint = torch.load(weights_path, map_location=lambda storage, loc: storage)
|
||||
|
||||
model = cls(hparams)
|
||||
|
||||
# allow model to load
|
||||
model.load_model_specific(checkpoint)
|
||||
model.load_state_dict(checkpoint['state_dict'], strict=False)
|
||||
return model
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user