mirror of
https://github.com/wassname/pytorch-lightning.git
synced 2026-09-18 12:50:58 +08:00
382 lines
13 KiB
HTML
382 lines
13 KiB
HTML
<!DOCTYPE html>
|
|
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
|
|
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<meta http-equiv="X-UA-Compatible" content="IE=edge">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
|
|
|
|
<link rel="shortcut icon" href="../img/favicon.ico">
|
|
<title>Examples - Pytorch lightning Documentation</title>
|
|
<link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
|
|
|
|
<link rel="stylesheet" href="../css/theme.css" type="text/css" />
|
|
<link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
|
|
<link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
|
|
|
|
<script>
|
|
// Current page data
|
|
var mkdocs_page_name = "Examples";
|
|
var mkdocs_page_input_path = "Examples.md";
|
|
var mkdocs_page_url = null;
|
|
</script>
|
|
|
|
<script src="../js/jquery-2.1.1.min.js" defer></script>
|
|
<script src="../js/modernizr-2.8.3.min.js" defer></script>
|
|
<script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
|
|
<script>hljs.initHighlightingOnLoad();</script>
|
|
|
|
</head>
|
|
|
|
<body class="wy-body-for-nav" role="document">
|
|
|
|
<div class="wy-grid-for-nav">
|
|
|
|
|
|
<nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
|
|
<div class="wy-side-nav-search">
|
|
<a href=".." class="icon icon-home"> Pytorch lightning Documentation</a>
|
|
<div role="search">
|
|
<form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
|
|
<input type="text" name="q" placeholder="Search docs" title="Type search term here" />
|
|
</form>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
|
|
<ul class="current">
|
|
|
|
|
|
<li class="toctree-l1">
|
|
|
|
<a class="" href="..">PYTORCH-LIGHTNING DOCUMENTATION</a>
|
|
</li>
|
|
|
|
<li class="toctree-l1 current">
|
|
|
|
<a class="current" href="./">Examples</a>
|
|
<ul class="subnav">
|
|
|
|
<li class="toctree-l2"><a href="#template-model-definition">Template model definition</a></li>
|
|
|
|
|
|
<li class="toctree-l2"><a href="#trainer-example">Trainer Example</a></li>
|
|
|
|
<ul>
|
|
|
|
<li><a class="toctree-l3" href="#cpu-hyperparameter-search">CPU hyperparameter search</a></li>
|
|
|
|
<li><a class="toctree-l3" href="#hyperparameter-search-on-a-single-or-multiple-gpus">Hyperparameter search on a single or multiple GPUs</a></li>
|
|
|
|
<li><a class="toctree-l3" href="#hyperparameter-search-on-a-slurm-hpc-cluster">Hyperparameter search on a SLURM HPC cluster</a></li>
|
|
|
|
</ul>
|
|
|
|
|
|
</ul>
|
|
</li>
|
|
|
|
<li class="toctree-l1">
|
|
|
|
<span class="caption-text">LightningModule</span>
|
|
<ul class="subnav">
|
|
<li class="">
|
|
|
|
<a class="" href="../LightningModule/RequiredTrainerInterface/">Lightning Module interface</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../LightningModule/methods/">Methods</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../LightningModule/properties/">Properties</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
|
|
<li class="toctree-l1">
|
|
|
|
<span class="caption-text">Trainer</span>
|
|
<ul class="subnav">
|
|
<li class="">
|
|
|
|
<a class="" href="../Trainer/">Trainer</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../Trainer/Checkpointing/">Checkpointing</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../Trainer/Distributed training/">Distributed training</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../Trainer/Logging/">Logging</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../Trainer/SLURM Managed Cluster/">SLURM Managed Cluster</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../Trainer/Training Loop/">Training Loop</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../Trainer/Validation loop/">Validation loop</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../Trainer/debugging/">Debugging</a>
|
|
</li>
|
|
<li class="">
|
|
|
|
<a class="" href="../Trainer/hooks/">Hooks</a>
|
|
</li>
|
|
</ul>
|
|
</li>
|
|
|
|
</ul>
|
|
</div>
|
|
|
|
</nav>
|
|
|
|
<section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
|
|
|
|
|
|
<nav class="wy-nav-top" role="navigation" aria-label="top navigation">
|
|
<i data-toggle="wy-nav-top" class="fa fa-bars"></i>
|
|
<a href="..">Pytorch lightning Documentation</a>
|
|
</nav>
|
|
|
|
|
|
<div class="wy-nav-content">
|
|
<div class="rst-content">
|
|
<div role="navigation" aria-label="breadcrumbs navigation">
|
|
<ul class="wy-breadcrumbs">
|
|
<li><a href="..">Docs</a> »</li>
|
|
|
|
|
|
|
|
<li>Examples</li>
|
|
<li class="wy-breadcrumbs-aside">
|
|
|
|
<a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Examples.md"
|
|
class="icon icon-github"> Edit on GitHub</a>
|
|
|
|
</li>
|
|
</ul>
|
|
<hr/>
|
|
</div>
|
|
<div role="main">
|
|
<div class="section">
|
|
|
|
<h3 id="template-model-definition">Template model definition</h3>
|
|
<p>In 99% of cases you want to just copy <a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py">this template</a> to start a new lightningModule and change the core of what your model is actually trying to do.</p>
|
|
<pre><code class="bash"># get a copy of the module template
|
|
wget https://github.com/williamFalcon/pytorch-lightning/blob/master/examples/new_project_templates/lightning_module_template.py
|
|
</code></pre>
|
|
|
|
<hr />
|
|
<h3 id="trainer-example">Trainer Example</h3>
|
|
<p><strong> __main__ function</strong> </p>
|
|
<p>Normally, we want to let the __main__ function start the training.
|
|
Inside the main we parse training arguments with whatever hyperparameters we want. Your LightningModule will have a
|
|
chance to add hyperparameters. </p>
|
|
<pre><code class="python">from test_tube import HyperOptArgumentParser
|
|
|
|
if __name__ == '__main__':
|
|
|
|
# use default args given by lightning
|
|
root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
|
|
parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False)
|
|
add_default_args(parent_parser, root_dir)
|
|
|
|
# allow model to overwrite or extend args
|
|
parser = ExampleModel.add_model_specific_args(parent_parser)
|
|
hyperparams = parser.parse_args()
|
|
|
|
# train model
|
|
main(hyperparams)
|
|
</code></pre>
|
|
|
|
<p><strong>Main Function</strong> </p>
|
|
<p>The main function is your entry into the program. This is where you init your model, checkpoint directory, and launch the training.
|
|
The main function should have 3 arguments: <br />
|
|
- hparams: a configuration of hyperparameters. <br />
|
|
- slurm_manager: Slurm cluster manager object (can be None)
|
|
- dict: for you to return any values you want (useful in meta-learning, otherwise set to _) </p>
|
|
<pre><code>def main(hparams, cluster, results_dict):
|
|
"""
|
|
Main training routine specific for this project
|
|
:param hparams:
|
|
:return:
|
|
"""
|
|
# init experiment
|
|
log_dir = os.path.dirname(os.path.realpath(__file__))
|
|
exp = Experiment(
|
|
name='test_tube_exp',
|
|
debug=True,
|
|
save_dir=log_dir,
|
|
version=0,
|
|
autosave=False,
|
|
description='test demo'
|
|
)
|
|
|
|
# set the hparams for the experiment
|
|
exp.argparse(hparams)
|
|
exp.save()
|
|
|
|
# build model
|
|
model = MyLightningModule(hparams)
|
|
|
|
# callbacks
|
|
early_stop = EarlyStopping(
|
|
monitor=hparams.early_stop_metric,
|
|
patience=hparams.early_stop_patience,
|
|
verbose=True,
|
|
mode=hparams.early_stop_mode
|
|
)
|
|
|
|
model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
|
|
checkpoint = ModelCheckpoint(
|
|
filepath=model_save_path,
|
|
save_function=None,
|
|
save_best_only=True,
|
|
verbose=True,
|
|
monitor=hparams.model_save_monitor_value,
|
|
mode=hparams.model_save_monitor_mode
|
|
)
|
|
|
|
# configure trainer
|
|
trainer = Trainer(
|
|
experiment=exp,
|
|
cluster=cluster,
|
|
checkpoint_callback=checkpoint,
|
|
early_stop_callback=early_stop,
|
|
)
|
|
|
|
# train model
|
|
trainer.fit(model)
|
|
</code></pre>
|
|
|
|
<p>The <strong>main</strong> function will start training on your <strong>main</strong> function. If you use the HyperParameterOptimizer
|
|
in hyper parameter optimization mode, this main function will get one set of hyperparameters. If you use it as a simple
|
|
argument parser you get the default arguments in the argument parser.</p>
|
|
<p>So, calling main(hyperparams) runs the model with the default argparse arguments. </p>
|
|
<pre><code class="python">main(hyperparams)
|
|
</code></pre>
|
|
|
|
<hr />
|
|
<h4 id="cpu-hyperparameter-search">CPU hyperparameter search</h4>
|
|
<pre><code class="python"># run a grid search over 20 hyperparameter combinations.
|
|
hyperparams.optimize_parallel_cpu(
|
|
main_local,
|
|
nb_trials=20,
|
|
nb_workers=1
|
|
)
|
|
</code></pre>
|
|
|
|
<hr />
|
|
<h4 id="hyperparameter-search-on-a-single-or-multiple-gpus">Hyperparameter search on a single or multiple GPUs</h4>
|
|
<pre><code class="python"># run a grid search over 20 hyperparameter combinations.
|
|
hyperparams.optimize_parallel_gpu(
|
|
main_local,
|
|
nb_trials=20,
|
|
nb_workers=1,
|
|
gpus=[0,1,2,3]
|
|
)
|
|
</code></pre>
|
|
|
|
<hr />
|
|
<h4 id="hyperparameter-search-on-a-slurm-hpc-cluster">Hyperparameter search on a SLURM HPC cluster</h4>
|
|
<pre><code class="python">def optimize_on_cluster(hyperparams):
|
|
# enable cluster training
|
|
cluster = SlurmCluster(
|
|
hyperparam_optimizer=hyperparams,
|
|
log_path=hyperparams.tt_save_path,
|
|
test_tube_exp_name=hyperparams.tt_name
|
|
)
|
|
|
|
# email for cluster coms
|
|
cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True)
|
|
|
|
# configure cluster
|
|
cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus
|
|
cluster.job_time = '48:00:00'
|
|
cluster.gpu_type = '1080ti'
|
|
cluster.memory_mb_per_node = 48000
|
|
|
|
# any modules for code to run in env
|
|
cluster.add_command('source activate pytorch_lightning')
|
|
|
|
# name of exp
|
|
job_display_name = hyperparams.tt_name.split('_')[0]
|
|
job_display_name = job_display_name[0:3]
|
|
|
|
# run hopt
|
|
print('submitting jobs...')
|
|
cluster.optimize_parallel_cluster_gpu(
|
|
main,
|
|
nb_trials=hyperparams.nb_hopt_trials,
|
|
job_name=job_display_name
|
|
)
|
|
|
|
# run cluster hyperparameter search
|
|
optimize_on_cluster(hyperparams)
|
|
</code></pre>
|
|
|
|
</div>
|
|
</div>
|
|
<footer>
|
|
|
|
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
|
|
|
|
<a href="../LightningModule/RequiredTrainerInterface/" class="btn btn-neutral float-right" title="Lightning Module interface">Next <span class="icon icon-circle-arrow-right"></span></a>
|
|
|
|
|
|
<a href=".." class="btn btn-neutral" title="PYTORCH-LIGHTNING DOCUMENTATION"><span class="icon icon-circle-arrow-left"></span> Previous</a>
|
|
|
|
</div>
|
|
|
|
|
|
<hr/>
|
|
|
|
<div role="contentinfo">
|
|
<!-- Copyright etc -->
|
|
|
|
</div>
|
|
|
|
Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
|
|
</footer>
|
|
|
|
</div>
|
|
</div>
|
|
|
|
</section>
|
|
|
|
</div>
|
|
|
|
<div class="rst-versions" role="note" style="cursor: pointer">
|
|
<span class="rst-current-version" data-toggle="rst-current-version">
|
|
|
|
<a href="https://github.com/williamFalcon/pytorch-lightning/" class="fa fa-github" style="float: left; color: #fcfcfc"> GitHub</a>
|
|
|
|
|
|
<span><a href=".." style="color: #fcfcfc;">« Previous</a></span>
|
|
|
|
|
|
<span style="margin-left: 15px"><a href="../LightningModule/RequiredTrainerInterface/" style="color: #fcfcfc">Next »</a></span>
|
|
|
|
</span>
|
|
</div>
|
|
<script>var base_url = '..';</script>
|
|
<script src="../js/theme.js" defer></script>
|
|
<script src="../search/main.js" defer></script>
|
|
|
|
</body>
|
|
</html>
|