mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-10 12:21:57 +08:00
1096 lines
39 KiB
HTML
1096 lines
39 KiB
HTML
|
|
|
|
|
|
|
|
<!doctype html>
|
|
<html lang="en" class="no-js">
|
|
<head>
|
|
|
|
<meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<meta http-equiv="x-ua-compatible" content="ie=edge">
|
|
|
|
<meta name="description" content="Documentation for PyTorch LightningModule, the researcher version of keras.">
|
|
|
|
|
|
|
|
|
|
<meta name="lang:clipboard.copy" content="Copy to clipboard">
|
|
|
|
<meta name="lang:clipboard.copied" content="Copied to clipboard">
|
|
|
|
<meta name="lang:search.language" content="en">
|
|
|
|
<meta name="lang:search.pipeline.stopwords" content="True">
|
|
|
|
<meta name="lang:search.pipeline.trimmer" content="True">
|
|
|
|
<meta name="lang:search.result.none" content="No matching documents">
|
|
|
|
<meta name="lang:search.result.one" content="1 matching document">
|
|
|
|
<meta name="lang:search.result.other" content="# matching documents">
|
|
|
|
<meta name="lang:search.tokenizer" content="[\s\-]+">
|
|
|
|
<link rel="shortcut icon" href="../../assets/images/favicon.png">
|
|
<meta name="generator" content="mkdocs-1.0.4, mkdocs-material-4.4.0">
|
|
|
|
|
|
|
|
<title>Logging - PyTorch lightning Documentation</title>
|
|
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/stylesheets/application.0284f74d.css">
|
|
|
|
|
|
|
|
|
|
<script src="../../assets/javascripts/modernizr.74668098.js"></script>
|
|
|
|
|
|
|
|
<link href="https://fonts.gstatic.com" rel="preconnect" crossorigin>
|
|
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,400i,700|Roboto+Mono&display=fallback">
|
|
<style>body,input{font-family:"Roboto","Helvetica Neue",Helvetica,Arial,sans-serif}code,kbd,pre{font-family:"Roboto Mono","Courier New",Courier,monospace}</style>
|
|
|
|
|
|
<link rel="stylesheet" href="../../assets/fonts/material-icons.css">
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</head>
|
|
|
|
<body dir="ltr">
|
|
|
|
<svg class="md-svg">
|
|
<defs>
|
|
|
|
|
|
<svg xmlns="http://www.w3.org/2000/svg" width="416" height="448" viewBox="0 0 416 448" id="__github"><path fill="currentColor" d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z"/></svg>
|
|
|
|
</defs>
|
|
</svg>
|
|
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
|
|
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
|
|
<label class="md-overlay" data-md-component="overlay" for="__drawer"></label>
|
|
|
|
<a href="#default_save_path" tabindex="1" class="md-skip">
|
|
Skip to content
|
|
</a>
|
|
|
|
|
|
<header class="md-header" data-md-component="header">
|
|
<nav class="md-header-nav md-grid">
|
|
<div class="md-flex">
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<a href="../.." title="PyTorch lightning Documentation" class="md-header-nav__button md-logo">
|
|
|
|
<i class="md-icon"></i>
|
|
|
|
</a>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<label class="md-icon md-icon--menu md-header-nav__button" for="__drawer"></label>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--stretch">
|
|
<div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
|
|
|
|
<span class="md-header-nav__topic">
|
|
PyTorch lightning Documentation
|
|
</span>
|
|
<span class="md-header-nav__topic">
|
|
|
|
Logging
|
|
|
|
</span>
|
|
|
|
</div>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
|
|
<label class="md-icon md-icon--search md-header-nav__button" for="__search"></label>
|
|
|
|
<div class="md-search" data-md-component="search" role="dialog">
|
|
<label class="md-search__overlay" for="__search"></label>
|
|
<div class="md-search__inner" role="search">
|
|
<form class="md-search__form" name="search">
|
|
<input type="text" class="md-search__input" name="query" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="query" data-md-state="active">
|
|
<label class="md-icon md-search__icon" for="__search"></label>
|
|
<button type="reset" class="md-icon md-search__icon" data-md-component="reset" tabindex="-1">
|
|

|
|
</button>
|
|
</form>
|
|
<div class="md-search__output">
|
|
<div class="md-search__scrollwrap" data-md-scrollfix>
|
|
<div class="md-search-result" data-md-component="result">
|
|
<div class="md-search-result__meta">
|
|
Type to start searching
|
|
</div>
|
|
<ol class="md-search-result__list"></ol>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<div class="md-header-nav__source">
|
|
|
|
|
|
|
|
|
|
|
|
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
|
|
|
<div class="md-source__icon">
|
|
<svg viewBox="0 0 24 24" width="24" height="24">
|
|
<use xlink:href="#__github" width="24" height="24"></use>
|
|
</svg>
|
|
</div>
|
|
|
|
<div class="md-source__repository">
|
|
williamFalcon/pytorch-lightning
|
|
</div>
|
|
</a>
|
|
</div>
|
|
</div>
|
|
|
|
</div>
|
|
</nav>
|
|
</header>
|
|
|
|
<div class="md-container">
|
|
|
|
|
|
|
|
|
|
<main class="md-main">
|
|
<div class="md-main__inner md-grid" data-md-component="container">
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
<nav class="md-nav md-nav--primary" data-md-level="0">
|
|
<label class="md-nav__title md-nav__title--site" for="__drawer">
|
|
<a href="../.." title="PyTorch lightning Documentation" class="md-nav__button md-logo">
|
|
|
|
<i class="md-icon"></i>
|
|
|
|
</a>
|
|
PyTorch lightning Documentation
|
|
</label>
|
|
|
|
<div class="md-nav__source">
|
|
|
|
|
|
|
|
|
|
|
|
<a href="https://github.com/williamFalcon/pytorch-lightning/" title="Go to repository" class="md-source" data-md-source="github">
|
|
|
|
<div class="md-source__icon">
|
|
<svg viewBox="0 0 24 24" width="24" height="24">
|
|
<use xlink:href="#__github" width="24" height="24"></use>
|
|
</svg>
|
|
</div>
|
|
|
|
<div class="md-source__repository">
|
|
williamFalcon/pytorch-lightning
|
|
</div>
|
|
</a>
|
|
</div>
|
|
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../.." title="Home" class="md-nav__link">
|
|
Home
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-2" type="checkbox" id="nav-2">
|
|
|
|
<label class="md-nav__link" for="nav-2">
|
|
LightningModule
|
|
</label>
|
|
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
|
<label class="md-nav__title" for="nav-2">
|
|
LightningModule
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../LightningModule/RequiredTrainerInterface/" title="Lightning Module interface" class="md-nav__link">
|
|
Lightning Module interface
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../LightningModule/methods/" title="Methods" class="md-nav__link">
|
|
Methods
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../LightningModule/properties/" title="Properties" class="md-nav__link">
|
|
Properties
|
|
</a>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
|
|
|
|
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-3" type="checkbox" id="nav-3" checked>
|
|
|
|
<label class="md-nav__link" for="nav-3">
|
|
Trainer
|
|
</label>
|
|
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
|
<label class="md-nav__title" for="nav-3">
|
|
Trainer
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../" title="Trainer" class="md-nav__link">
|
|
Trainer
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../Checkpointing/" title="Checkpointing" class="md-nav__link">
|
|
Checkpointing
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../Distributed training/" title="Distributed training" class="md-nav__link">
|
|
Distributed training
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--active">
|
|
|
|
<input class="md-toggle md-nav__toggle" data-md-toggle="toc" type="checkbox" id="__toc">
|
|
|
|
|
|
<label class="md-nav__link md-nav__link--active" for="__toc">
|
|
Logging
|
|
</label>
|
|
|
|
<a href="./" title="Logging" class="md-nav__link md-nav__link--active">
|
|
Logging
|
|
</a>
|
|
|
|
|
|
<nav class="md-nav md-nav--secondary">
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">Table of contents</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#default_save_path" title="default_save_path" class="md-nav__link">
|
|
default_save_path
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#setting-up-logging" title="Setting up logging" class="md-nav__link">
|
|
Setting up logging
|
|
</a>
|
|
|
|
<nav class="md-nav">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#test-tube" title="Test tube" class="md-nav__link">
|
|
Test tube
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#mlflow" title="MLFlow" class="md-nav__link">
|
|
MLFlow
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#cometml" title="Comet.ml" class="md-nav__link">
|
|
Comet.ml
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#custom-logger" title="Custom logger" class="md-nav__link">
|
|
Custom logger
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#using-loggers" title="Using loggers" class="md-nav__link">
|
|
Using loggers
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#display-metrics-in-progress-bar" title="Display metrics in progress bar" class="md-nav__link">
|
|
Display metrics in progress bar
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#log-metric-row-every-k-batches" title="Log metric row every k batches" class="md-nav__link">
|
|
Log metric row every k batches
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#log-gpu-memory" title="Log GPU memory" class="md-nav__link">
|
|
Log GPU memory
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#process-position" title="Process position" class="md-nav__link">
|
|
Process position
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#save-a-snapshot-of-all-hyperparameters" title="Save a snapshot of all hyperparameters" class="md-nav__link">
|
|
Save a snapshot of all hyperparameters
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#write-logs-file-to-csv-every-k-batches" title="Write logs file to csv every k batches" class="md-nav__link">
|
|
Write logs file to csv every k batches
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-nav__link">
|
|
SLURM Managed Cluster
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../Testing loop/" title="Testing loop" class="md-nav__link">
|
|
Testing loop
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../Training Loop/" title="Training Loop" class="md-nav__link">
|
|
Training Loop
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../Validation loop/" title="Validation loop" class="md-nav__link">
|
|
Validation loop
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../debugging/" title="Debugging" class="md-nav__link">
|
|
Debugging
|
|
</a>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../hooks/" title="Hooks" class="md-nav__link">
|
|
Hooks
|
|
</a>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item md-nav__item--nested">
|
|
|
|
<input class="md-toggle md-nav__toggle" data-md-toggle="nav-4" type="checkbox" id="nav-4">
|
|
|
|
<label class="md-nav__link" for="nav-4">
|
|
Examples
|
|
</label>
|
|
<nav class="md-nav" data-md-component="collapsible" data-md-level="1">
|
|
<label class="md-nav__title" for="nav-4">
|
|
Examples
|
|
</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
<li class="md-nav__item">
|
|
<a href="../../examples/Examples/" title="Examples" class="md-nav__link">
|
|
Examples
|
|
</a>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</li>
|
|
|
|
|
|
</ul>
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
|
|
<div class="md-sidebar__scrollwrap">
|
|
<div class="md-sidebar__inner">
|
|
|
|
<nav class="md-nav md-nav--secondary">
|
|
|
|
|
|
|
|
<label class="md-nav__title" for="__toc">Table of contents</label>
|
|
<ul class="md-nav__list" data-md-scrollfix>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#default_save_path" title="default_save_path" class="md-nav__link">
|
|
default_save_path
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#setting-up-logging" title="Setting up logging" class="md-nav__link">
|
|
Setting up logging
|
|
</a>
|
|
|
|
<nav class="md-nav">
|
|
<ul class="md-nav__list">
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#test-tube" title="Test tube" class="md-nav__link">
|
|
Test tube
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#mlflow" title="MLFlow" class="md-nav__link">
|
|
MLFlow
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#cometml" title="Comet.ml" class="md-nav__link">
|
|
Comet.ml
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#custom-logger" title="Custom logger" class="md-nav__link">
|
|
Custom logger
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#using-loggers" title="Using loggers" class="md-nav__link">
|
|
Using loggers
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#display-metrics-in-progress-bar" title="Display metrics in progress bar" class="md-nav__link">
|
|
Display metrics in progress bar
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#log-metric-row-every-k-batches" title="Log metric row every k batches" class="md-nav__link">
|
|
Log metric row every k batches
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#log-gpu-memory" title="Log GPU memory" class="md-nav__link">
|
|
Log GPU memory
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#process-position" title="Process position" class="md-nav__link">
|
|
Process position
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#save-a-snapshot-of-all-hyperparameters" title="Save a snapshot of all hyperparameters" class="md-nav__link">
|
|
Save a snapshot of all hyperparameters
|
|
</a>
|
|
|
|
</li>
|
|
|
|
<li class="md-nav__item">
|
|
<a href="#write-logs-file-to-csv-every-k-batches" title="Write logs file to csv every k batches" class="md-nav__link">
|
|
Write logs file to csv every k batches
|
|
</a>
|
|
|
|
</li>
|
|
|
|
</ul>
|
|
</nav>
|
|
|
|
</li>
|
|
|
|
|
|
|
|
|
|
|
|
</ul>
|
|
|
|
</nav>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<div class="md-content">
|
|
<article class="md-content__inner md-typeset">
|
|
|
|
|
|
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Trainer/Logging.md" title="Edit this page" class="md-icon md-content__icon"></a>
|
|
|
|
|
|
<h1>Logging</h1>
|
|
|
|
<p>Lighting offers options for logging information about model, gpu usage, etc, via several different logging frameworks. It also offers printing options for training monitoring.</p>
|
|
<hr />
|
|
<h3 id="default_save_path">default_save_path</h3>
|
|
<p>Lightning sets a default TestTubeLogger and CheckpointCallback for you which log to
|
|
<code>os.getcwd()</code> by default. To modify the logging path you can set:</p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">Trainer</span><span class="p">(</span><span class="n">default_save_path</span><span class="o">=</span><span class="s1">'/your/path/to/save/checkpoints'</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<p>If you need more custom behavior (different paths for both, different metrics, etc...)
|
|
from the logger and the checkpointCallback, pass in your own instances as explained below.</p>
|
|
<hr />
|
|
<h3 id="setting-up-logging">Setting up logging</h3>
|
|
<p>The trainer inits a default logger for you (TestTubeLogger). All logs will
|
|
go to the current working directory under a folder named <code>`os.getcwd()/lightning_logs</code>. </p>
|
|
<p>If you want to modify the default logging behavior even more, pass in a logger
|
|
(which should inherit from <code>LightningBaseLogger</code>). </p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="n">my_logger</span> <span class="o">=</span> <span class="n">MyLightningLogger</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">my_logger</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<p>The path in this logger will overwrite default_save_path.</p>
|
|
<p>Lightning supports several common experiment tracking frameworks out of the box</p>
|
|
<hr />
|
|
<h4 id="test-tube">Test tube</h4>
|
|
<p>Log using <a href="https://williamfalcon.github.io/test-tube/">test tube</a>. Test tube logger is
|
|
a strict subclass of <a href="https://pytorch.org/docs/stable/tensorboard.html">PyTorch SummaryWriter</a>, refer to their
|
|
documentation for all supported operations. The TestTubeLogger adds a nicer folder structure
|
|
to manage experiments and snapshots all hyperparameters you pass to a LightningModule.</p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6
|
|
7
|
|
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.logging</span> <span class="kn">import</span> <span class="n">TestTubeLogger</span>
|
|
<span class="n">tt_logger</span> <span class="o">=</span> <span class="n">TestTubeLogger</span><span class="p">(</span>
|
|
<span class="n">save_dir</span><span class="o">=</span><span class="s2">"."</span><span class="p">,</span>
|
|
<span class="n">name</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span>
|
|
<span class="n">debug</span><span class="o">=</span><span class="bp">False</span><span class="p">,</span>
|
|
<span class="n">create_git_tag</span><span class="o">=</span><span class="bp">False</span>
|
|
<span class="p">)</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">tt_logger</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<p>Use the logger anywhere in you LightningModule as follows:</p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">train_step</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
|
|
<span class="c1"># example</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">whatever_method_summary_writer_supports</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">any_lightning_module_function_or_hook</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_histogram</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<hr />
|
|
<h4 id="mlflow">MLFlow</h4>
|
|
<p>Log using <a href="https://mlflow.org">mlflow</a></p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.logging</span> <span class="kn">import</span> <span class="n">MLFlowLogger</span>
|
|
<span class="n">mlf_logger</span> <span class="o">=</span> <span class="n">MLFlowLogger</span><span class="p">(</span>
|
|
<span class="n">experiment_name</span><span class="o">=</span><span class="s2">"default"</span><span class="p">,</span>
|
|
<span class="n">tracking_uri</span><span class="o">=</span><span class="s2">"file:/."</span>
|
|
<span class="p">)</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">mlf_logger</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<p>Use the logger anywhere in you LightningModule as follows:</p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">train_step</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
|
|
<span class="c1"># example</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">whatever_ml_flow_supports</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">any_lightning_module_function_or_hook</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">whatever_ml_flow_supports</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<hr />
|
|
<h4 id="cometml">Comet.ml</h4>
|
|
<p>Log using <a href="https://www.comet.ml">comet</a></p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6
|
|
7</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.logging</span> <span class="kn">import</span> <span class="n">CometLogger</span>
|
|
<span class="c1"># arguments made to CometLogger are passed on to the comet_ml.Experiment class</span>
|
|
<span class="n">comet_logger</span> <span class="o">=</span> <span class="n">CometLogger</span><span class="p">(</span>
|
|
<span class="n">api_key</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"COMET_KEY"</span><span class="p">],</span>
|
|
<span class="n">workspace</span><span class="o">=</span><span class="n">os</span><span class="o">.</span><span class="n">environ</span><span class="p">[</span><span class="s2">"COMET_KEY"</span><span class="p">],</span>
|
|
<span class="p">)</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">comet_logger</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<p>Use the logger anywhere in you LightningModule as follows:</p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">train_step</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
|
|
<span class="c1"># example</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">whatever_comet_ml_supports</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">any_lightning_module_function_or_hook</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">whatever_comet_ml_supports</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<hr />
|
|
<h4 id="custom-logger">Custom logger</h4>
|
|
<p>You can implement your own logger by writing a class that inherits from
|
|
<code>LightningLoggerBase</code>. Use the <code>rank_zero_only</code> decorator to make sure that
|
|
only the first process in DDP training logs data.</p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6
|
|
7
|
|
8
|
|
9
|
|
10
|
|
11
|
|
12
|
|
13
|
|
14
|
|
15
|
|
16
|
|
17
|
|
18
|
|
19
|
|
20
|
|
21
|
|
22
|
|
23
|
|
24</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="kn">from</span> <span class="nn">pytorch_lightning.logging</span> <span class="kn">import</span> <span class="n">LightningLoggerBase</span><span class="p">,</span> <span class="n">rank_zero_only</span>
|
|
|
|
<span class="k">class</span> <span class="nc">MyLogger</span><span class="p">(</span><span class="n">LightningLoggerBase</span><span class="p">):</span>
|
|
|
|
<span class="nd">@rank_zero_only</span>
|
|
<span class="k">def</span> <span class="nf">log_hyperparams</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">params</span><span class="p">):</span>
|
|
<span class="c1"># params is an argparse.Namespace</span>
|
|
<span class="c1"># your code to record hyperparameters goes here</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="nd">@rank_zero_only</span>
|
|
<span class="k">def</span> <span class="nf">log_metrics</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">metrics</span><span class="p">,</span> <span class="n">step_num</span><span class="p">):</span>
|
|
<span class="c1"># metrics is a dictionary of metric names and values</span>
|
|
<span class="c1"># your code to record metrics goes here</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="k">def</span> <span class="nf">save</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
|
|
<span class="c1"># Optional. Any code necessary to save logger data goes here</span>
|
|
<span class="k">pass</span>
|
|
|
|
<span class="nd">@rank_zero_only</span>
|
|
<span class="k">def</span> <span class="nf">finalize</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">status</span><span class="p">):</span>
|
|
<span class="c1"># Optional. Any code that needs to be run after training</span>
|
|
<span class="c1"># finishes goes here</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<p>If you write a logger than may be useful to others, please send
|
|
a pull request to add it to Lighting!</p>
|
|
<hr />
|
|
<h4 id="using-loggers">Using loggers</h4>
|
|
<p>You can call the logger anywhere from your LightningModule by doing:</p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">def</span> <span class="nf">train_step</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
|
|
<span class="c1"># example</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">whatever_method_summary_writer_supports</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
|
|
<span class="k">def</span> <span class="nf">any_lightning_module_function_or_hook</span><span class="p">(</span><span class="o">...</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">logger</span><span class="o">.</span><span class="n">experiment</span><span class="o">.</span><span class="n">add_histogram</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<h4 id="display-metrics-in-progress-bar">Display metrics in progress bar</h4>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">show_progress_bar</span><span class="o">=</span><span class="bp">True</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<hr />
|
|
<h4 id="log-metric-row-every-k-batches">Log metric row every k batches</h4>
|
|
<p>Every k batches lightning will make an entry in the metrics log</p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (ie: save a .csv log file every 10 batches)</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">row_log_interval</span><span class="o">=</span><span class="mi">10</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<hr />
|
|
<h4 id="log-gpu-memory">Log GPU memory</h4>
|
|
<p>Logs GPU memory when metrics are logged. </p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6
|
|
7
|
|
8</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">log_gpu_memory</span><span class="o">=</span><span class="bp">None</span><span class="p">)</span>
|
|
|
|
<span class="c1"># log only the min/max utilization</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">log_gpu_memory</span><span class="o">=</span><span class="s1">'min_max'</span><span class="p">)</span>
|
|
|
|
<span class="c1"># log all the GPU memory (if on DDP, logs only that node)</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">log_gpu_memory</span><span class="o">=</span><span class="s1">'all'</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<hr />
|
|
<h4 id="process-position">Process position</h4>
|
|
<p>When running multiple models on the same machine we want to decide which progress bar to use.
|
|
Lightning will stack progress bars according to this value. </p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2
|
|
3
|
|
4
|
|
5</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">process_position</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
|
|
|
|
<span class="c1"># if this is the second model on the node, show the second progress bar below</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">process_position</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<hr />
|
|
<h4 id="save-a-snapshot-of-all-hyperparameters">Save a snapshot of all hyperparameters</h4>
|
|
<p>Automatically log hyperparameters stored in the <code>hparams</code> attribute as an <code>argparse.Namespace</code> </p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre> 1
|
|
2
|
|
3
|
|
4
|
|
5
|
|
6
|
|
7
|
|
8
|
|
9
|
|
10
|
|
11
|
|
12</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="k">class</span> <span class="nc">MyModel</span><span class="p">(</span><span class="n">pl</span><span class="o">.</span><span class="n">Lightning</span><span class="p">):</span>
|
|
<span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hparams</span><span class="p">):</span>
|
|
<span class="bp">self</span><span class="o">.</span><span class="n">hparams</span> <span class="o">=</span> <span class="n">hparams</span>
|
|
|
|
<span class="o">...</span>
|
|
|
|
<span class="n">args</span> <span class="o">=</span> <span class="n">parser</span><span class="o">.</span><span class="n">parse_args</span><span class="p">()</span>
|
|
<span class="n">model</span> <span class="o">=</span> <span class="n">MyModel</span><span class="p">(</span><span class="n">args</span><span class="p">)</span>
|
|
|
|
<span class="n">logger</span> <span class="o">=</span> <span class="n">TestTubeLogger</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
|
|
<span class="n">t</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">logger</span><span class="o">=</span><span class="n">logger</span><span class="p">)</span>
|
|
<span class="n">trainer</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
<hr />
|
|
<h4 id="write-logs-file-to-csv-every-k-batches">Write logs file to csv every k batches</h4>
|
|
<p>Every k batches, lightning will write the new logs to disk</p>
|
|
<table class="codehilitetable"><tr><td class="linenos"><div class="linenodiv"><pre>1
|
|
2</pre></div></td><td class="code"><div class="codehilite"><pre><span></span><span class="c1"># DEFAULT (ie: save a .csv log file every 100 batches)</span>
|
|
<span class="n">trainer</span> <span class="o">=</span> <span class="n">Trainer</span><span class="p">(</span><span class="n">log_save_interval</span><span class="o">=</span><span class="mi">100</span><span class="p">)</span>
|
|
</pre></div>
|
|
</td></tr></table>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
</article>
|
|
</div>
|
|
</div>
|
|
</main>
|
|
|
|
|
|
<footer class="md-footer">
|
|
|
|
<div class="md-footer-nav">
|
|
<nav class="md-footer-nav__inner md-grid">
|
|
|
|
<a href="../Distributed training/" title="Distributed training" class="md-flex md-footer-nav__link md-footer-nav__link--prev" rel="prev">
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<i class="md-icon md-icon--arrow-back md-footer-nav__button"></i>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
|
<span class="md-flex__ellipsis">
|
|
<span class="md-footer-nav__direction">
|
|
Previous
|
|
</span>
|
|
Distributed training
|
|
</span>
|
|
</div>
|
|
</a>
|
|
|
|
|
|
<a href="../SLURM Managed Cluster/" title="SLURM Managed Cluster" class="md-flex md-footer-nav__link md-footer-nav__link--next" rel="next">
|
|
<div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
|
|
<span class="md-flex__ellipsis">
|
|
<span class="md-footer-nav__direction">
|
|
Next
|
|
</span>
|
|
SLURM Managed Cluster
|
|
</span>
|
|
</div>
|
|
<div class="md-flex__cell md-flex__cell--shrink">
|
|
<i class="md-icon md-icon--arrow-forward md-footer-nav__button"></i>
|
|
</div>
|
|
</a>
|
|
|
|
</nav>
|
|
</div>
|
|
|
|
<div class="md-footer-meta md-typeset">
|
|
<div class="md-footer-meta__inner md-grid">
|
|
<div class="md-footer-copyright">
|
|
|
|
powered by
|
|
<a href="https://www.mkdocs.org">MkDocs</a>
|
|
and
|
|
<a href="https://squidfunk.github.io/mkdocs-material/">
|
|
Material for MkDocs</a>
|
|
</div>
|
|
|
|
</div>
|
|
</div>
|
|
</footer>
|
|
|
|
</div>
|
|
|
|
<script src="../../assets/javascripts/application.245445c6.js"></script>
|
|
|
|
<script>app.initialize({version:"1.0.4",url:{base:"../.."}})</script>
|
|
|
|
|
|
</body>
|
|
</html> |