Deployed a86ce39 with MkDocs version: 1.0.4

2026-09-09 11:32:07 +08:00 · 2019-06-28 17:01:13 -05:00
parent ee71cdbcfa
commit 37e2372acd
28 changed files with 533 additions and 506 deletions
@@ -48,6 +48,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="/Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -0,0 +1,439 @@
+<!DOCTYPE html>
+<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
+<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
+<head>
+  <meta charset="utf-8">
+  <meta http-equiv="X-UA-Compatible" content="IE=edge">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  
+  
+  <link rel="shortcut icon" href="../img/favicon.ico">
+  <title>Examples - Pytorch lightning Documentation</title>
+  <link href='https://fonts.googleapis.com/css?family=Lato:400,700|Roboto+Slab:400,700|Inconsolata:400,700' rel='stylesheet' type='text/css'>
+
+  <link rel="stylesheet" href="../css/theme.css" type="text/css" />
+  <link rel="stylesheet" href="../css/theme_extra.css" type="text/css" />
+  <link rel="stylesheet" href="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/styles/github.min.css">
+  
+  <script>
+    // Current page data
+    var mkdocs_page_name = "Examples";
+    var mkdocs_page_input_path = "Examples.md";
+    var mkdocs_page_url = null;
+  </script>
+  
+  <script src="../js/jquery-2.1.1.min.js" defer></script>
+  <script src="../js/modernizr-2.8.3.min.js" defer></script>
+  <script src="//cdnjs.cloudflare.com/ajax/libs/highlight.js/9.12.0/highlight.min.js"></script>
+  <script>hljs.initHighlightingOnLoad();</script> 
+  
+</head>
+
+<body class="wy-body-for-nav" role="document">
+
+  <div class="wy-grid-for-nav">
+
+    
+    <nav data-toggle="wy-nav-shift" class="wy-nav-side stickynav">
+      <div class="wy-side-nav-search">
+        <a href=".." class="icon icon-home"> Pytorch lightning Documentation</a>
+        <div role="search">
+  <form id ="rtd-search-form" class="wy-form" action="../search.html" method="get">
+    <input type="text" name="q" placeholder="Search docs" title="Type search term here" />
+  </form>
+</div>
+      </div>
+
+      <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
+	<ul class="current">
+	  
+          
+            <li class="toctree-l1">
+		
+    <a class="" href="..">PYTORCH-LIGHTNING DOCUMENTATION</a>
+	    </li>
+          
+            <li class="toctree-l1 current">
+		
+    <a class="current" href="./">Examples</a>
+    <ul class="subnav">
+            
+    <li class="toctree-l2"><a href="#template-model-definition">Template model definition</a></li>
+    
+
+    </ul>
+	    </li>
+          
+            <li class="toctree-l1">
+		
+    <span class="caption-text">LightningModule</span>
+    <ul class="subnav">
+                <li class="">
+                    
+    <a class="" href="../LightningModule/RequiredTrainerInterface/">Lightning Module interface</a>
+                </li>
+                <li class="">
+                    
+    <a class="" href="../LightningModule/methods/">Methods</a>
+                </li>
+    </ul>
+	    </li>
+          
+            <li class="toctree-l1">
+		
+    <span class="caption-text">Trainer</span>
+    <ul class="subnav">
+                <li class="">
+                    
+    <a class="" href="../Trainer/">Trainer</a>
+                </li>
+                <li class="">
+                    
+    <a class="" href="../Trainer/Checkpointing/">Checkpointing</a>
+                </li>
+                <li class="">
+                    
+    <a class="" href="../Trainer/Distributed training/">Distributed training</a>
+                </li>
+                <li class="">
+                    
+    <a class="" href="../Trainer/Logging/">Logging</a>
+                </li>
+                <li class="">
+                    
+    <a class="" href="../Trainer/SLURM Managed Cluster/">SLURM Managed Cluster</a>
+                </li>
+                <li class="">
+                    
+    <a class="" href="../Trainer/Training Loop/">Training Loop</a>
+                </li>
+                <li class="">
+                    
+    <a class="" href="../Trainer/Validation loop/">Validation loop</a>
+                </li>
+                <li class="">
+                    
+    <a class="" href="../Trainer/debugging/">Debugging</a>
+                </li>
+                <li class="">
+                    
+    <a class="" href="../Trainer/hooks/">Hooks</a>
+                </li>
+    </ul>
+	    </li>
+          
+        </ul>
+      </div>
+      &nbsp;
+    </nav>
+
+    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
+
+      
+      <nav class="wy-nav-top" role="navigation" aria-label="top navigation">
+        <i data-toggle="wy-nav-top" class="fa fa-bars"></i>
+        <a href="..">Pytorch lightning Documentation</a>
+      </nav>
+
+      
+      <div class="wy-nav-content">
+        <div class="rst-content">
+          <div role="navigation" aria-label="breadcrumbs navigation">
+  <ul class="wy-breadcrumbs">
+    <li><a href="..">Docs</a> &raquo;</li>
+    
+      
+    
+    <li>Examples</li>
+    <li class="wy-breadcrumbs-aside">
+      
+        <a href="https://github.com/williamFalcon/pytorch-lightning/edit/master/docs/Examples.md"
+          class="icon icon-github"> Edit on GitHub</a>
+      
+    </li>
+  </ul>
+  <hr/>
+</div>
+          <div role="main">
+            <div class="section">
+              
+                <h4 id="template-model-definition">Template model definition</h4>
+<p>In 99% of cases you want to just copy this template to start a new lightningModule and change the core of what your model is actually trying to do.</p>
+<pre><code class="python">import os
+from collections import OrderedDict
+import torch.nn as nn
+from torchvision.datasets import MNIST
+import torchvision.transforms as transforms
+import torch
+import torch.nn.functional as F
+from test_tube import HyperOptArgumentParser
+from torch import optim
+
+from pytorch_lightning.root_module.root_module import LightningModule
+
+
+class LightningTemplateModel(LightningModule):
+    &quot;&quot;&quot;
+    Sample model to show how to define a template
+    &quot;&quot;&quot;
+
+    def __init__(self, hparams):
+        &quot;&quot;&quot;
+        Pass in parsed HyperOptArgumentParser to the model
+        :param hparams:
+        &quot;&quot;&quot;
+        # init superclass
+        super(LightningTemplateModel, self).__init__(hparams)
+
+        self.batch_size = hparams.batch_size
+
+        # build model
+        self.__build_model()
+
+    # ---------------------
+    # MODEL SETUP
+    # ---------------------
+    def __build_model(self):
+        &quot;&quot;&quot;
+        Layout model
+        :return:
+        &quot;&quot;&quot;
+        self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
+        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
+        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
+
+        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)
+
+    # ---------------------
+    # TRAINING
+    # ---------------------
+    def forward(self, x):
+        &quot;&quot;&quot;
+        No special modification required for lightning, define as you normally would
+        :param x:
+        :return:
+        &quot;&quot;&quot;
+
+        x = self.c_d1(x)
+        x = torch.tanh(x)
+        x = self.c_d1_bn(x)
+        x = self.c_d1_drop(x)
+
+        x = self.c_d2(x)
+        logits = F.log_softmax(x, dim=1)
+
+        return logits
+
+    def loss(self, labels, logits):
+        nll = F.nll_loss(logits, labels)
+        return nll
+
+    def training_step(self, data_batch, batch_i):
+        &quot;&quot;&quot;
+        Lightning calls this inside the training loop
+        :param data_batch:
+        :return:
+        &quot;&quot;&quot;
+        # forward pass
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+        y_hat = self.forward(x)
+
+        # calculate loss
+        loss_val = self.loss(y, y_hat)
+
+        output = OrderedDict({
+            'loss': loss_val,
+            'tqdm_metrics': {}
+        })
+        return output
+
+    def validation_step(self, data_batch, batch_i):
+        &quot;&quot;&quot;
+        Lightning calls this inside the validation loop
+        :param data_batch:
+        :return:
+        &quot;&quot;&quot;
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+        y_hat = self.forward(x)
+
+        loss_val = self.loss(y, y_hat)
+
+        # acc
+        labels_hat = torch.argmax(y_hat, dim=1)
+        val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
+
+        output = OrderedDict({
+            'val_loss': loss_val,
+            'val_acc': torch.tensor(val_acc),
+        })
+        return output
+
+    def validation_end(self, outputs):
+        &quot;&quot;&quot;
+        Called at the end of validation to aggregate outputs
+        :param outputs: list of individual outputs of each validation step
+        :return:
+        &quot;&quot;&quot;
+        val_loss_mean = 0
+        val_acc_mean = 0
+        for output in outputs:
+            val_loss_mean += output['val_loss']
+            val_acc_mean += output['val_acc']
+
+        val_loss_mean /= len(outputs)
+        val_acc_mean /= len(outputs)
+        tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
+        return tqdm_dic
+
+    def update_tng_log_metrics(self, logs):
+        return logs
+
+    # ---------------------
+    # MODEL SAVING
+    # ---------------------
+    def get_save_dict(self):
+        checkpoint = {'state_dict': self.state_dict()}
+        return checkpoint
+
+    def load_model_specific(self, checkpoint):
+        self.load_state_dict(checkpoint['state_dict'])
+        pass
+
+    # ---------------------
+    # TRAINING SETUP
+    # ---------------------
+    def configure_optimizers(self):
+        &quot;&quot;&quot;
+        return whatever optimizers we want here
+        :return: list of optimizers
+        &quot;&quot;&quot;
+        optimizer = optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
+        return [optimizer]
+
+    def __dataloader(self, train):
+        # init data generators
+        transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+
+        dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)
+
+        loader = torch.utils.data.DataLoader(
+            dataset=dataset,
+            batch_size=self.hparams.batch_size,
+            shuffle=True
+        )
+
+        return loader
+
+    @property
+    def tng_dataloader(self):
+        if self._tng_dataloader is None:
+            try:
+                self._tng_dataloader = self.__dataloader(train=True)
+            except Exception as e:
+                print(e)
+                raise e
+        return self._tng_dataloader
+
+    @property
+    def val_dataloader(self):
+        if self._val_dataloader is None:
+            try:
+                self._val_dataloader = self.__dataloader(train=False)
+            except Exception as e:
+                print(e)
+                raise e
+        return self._val_dataloader
+
+    @property
+    def test_dataloader(self):
+        if self._test_dataloader is None:
+            try:
+                self._test_dataloader = self.__dataloader(train=False)
+            except Exception as e:
+                print(e)
+                raise e
+        return self._test_dataloader
+
+    @staticmethod
+    def add_model_specific_args(parent_parser, root_dir):
+        &quot;&quot;&quot;
+        Parameters you define here will be available to your model through self.hparams
+        :param parent_parser:
+        :param root_dir:
+        :return:
+        &quot;&quot;&quot;
+        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+
+        # param overwrites
+        # parser.set_defaults(gradient_clip=5.0)
+
+        # network params
+        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
+        parser.add_argument('--in_features', default=28*28)
+        parser.add_argument('--out_features', default=10)
+        parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference
+
+        # data
+        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
+
+        # training params (opt)
+        parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
+                        tunable=False)
+        parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False)
+        parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
+        return parser
+
+</code></pre>
+              
+            </div>
+          </div>
+          <footer>
+  
+    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
+      
+        <a href="../LightningModule/RequiredTrainerInterface/" class="btn btn-neutral float-right" title="Lightning Module interface">Next <span class="icon icon-circle-arrow-right"></span></a>
+      
+      
+        <a href=".." class="btn btn-neutral" title="PYTORCH-LIGHTNING DOCUMENTATION"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+      
+    </div>
+  
+
+  <hr/>
+
+  <div role="contentinfo">
+    <!-- Copyright etc -->
+    
+  </div>
+
+  Built with <a href="http://www.mkdocs.org">MkDocs</a> using a <a href="https://github.com/snide/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
+</footer>
+      
+        </div>
+      </div>
+
+    </section>
+
+  </div>
+
+  <div class="rst-versions" role="note" style="cursor: pointer">
+    <span class="rst-current-version" data-toggle="rst-current-version">
+      
+          <a href="https://github.com/williamFalcon/pytorch-lightning/" class="fa fa-github" style="float: left; color: #fcfcfc"> GitHub</a>
+      
+      
+        <span><a href=".." style="color: #fcfcfc;">&laquo; Previous</a></span>
+      
+      
+        <span style="margin-left: 15px"><a href="../LightningModule/RequiredTrainerInterface/" style="color: #fcfcfc">Next &raquo;</a></span>
+      
+    </span>
+</div>
+    <script>var base_url = '..';</script>
+    <script src="../js/theme.js" defer></script>
+      <script src="../search/main.js" defer></script>
+
+</body>
+</html>
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class=" current">
@@ -597,7 +602,7 @@ def add_model_specific_args(parent_parser, root_dir):
        <a href="../methods/" class="btn btn-neutral float-right" title="Methods">Next <span class="icon icon-circle-arrow-right"></span></a>
      
      
-        <a href="../.." class="btn btn-neutral" title="PYTORCH-LIGHTNING DOCUMENTATION"><span class="icon icon-circle-arrow-left"></span> Previous</a>
+        <a href="../../Examples/" class="btn btn-neutral" title="Examples"><span class="icon icon-circle-arrow-left"></span> Previous</a>
      
    </div>
  
@@ -625,7 +630,7 @@ def add_model_specific_args(parent_parser, root_dir):
          <a href="https://github.com/williamFalcon/pytorch-lightning/" class="fa fa-github" style="float: left; color: #fcfcfc"> GitHub</a>
      
      
-        <span><a href="../.." style="color: #fcfcfc;">&laquo; Previous</a></span>
+        <span><a href="../../Examples/" style="color: #fcfcfc;">&laquo; Previous</a></span>
      
      
        <span style="margin-left: 15px"><a href="../methods/" style="color: #fcfcfc">Next &raquo;</a></span>
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -109,6 +114,9 @@
    <li class="toctree-l3"><a href="#force-disable-early-stop">Force disable early stop</a></li>
    

+    <li class="toctree-l3"><a href="#gradient-clipping">Gradient Clipping</a></li>
+    
+
    <li class="toctree-l3"><a href="#inspect-gradient-norms">Inspect gradient norms</a></li>
    

@@ -204,6 +212,13 @@ trainer = Trainer(min_nb_epochs=1, max_nb_epochs=1000)
 trainer = Trainer(enable_early_stop=True)
 </code></pre>

+<hr />
+<h4 id="gradient-clipping">Gradient Clipping</h4>
+<p>Use this to turn off early stopping and run training to the <a href="#force-training-for-min-or-max-epochs">max_epoch</a></p>
+<pre><code class="python"># DEFAULT (ie: don't clip)
+trainer = Trainer(gradient_clip=0)
+</code></pre>
+
 <hr />
 <h4 id="inspect-gradient-norms">Inspect gradient norms</h4>
 <p>Looking at grad norms can help you figure out where training might be going wrong.</p>
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -55,6 +55,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="../Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -85,6 +85,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -205,10 +210,8 @@
 </ul>
 <h6 id="computing-cluster-slurm">Computing cluster (SLURM)</h6>
 <ul>
-<li>Automatic checkpointing   </li>
-<li>Automatic saving, loading  </li>
-<li>Running grid search on a cluster </li>
-<li>Walltime auto-resubmit   </li>
+<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#running-grid-search-on-a-cluster">Running grid search on a cluster</a> </li>
+<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/SLURM%20Managed%20Cluster#walltime-auto-resubmit">Walltime auto-resubmit</a>   </li>
 </ul>
 <h6 id="debugging">Debugging</h6>
 <ul>
@@ -243,6 +246,7 @@
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#anneal-learning-rate">Anneal Learning rate</a></li>
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-training-for-min-or-max-epochs">Force training for min or max epochs</a></li>
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#force-disable-early-stop">Force disable early stop</a></li>
+<li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#gradient-clipping">Gradient Clipping: DOC TODO</a></li>
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/Pytorch-Lightning/LightningModule/#configure_optimizers">Use multiple optimizers (like GANs)</a></li>
 <li><a href="https://williamfalcon.github.io/pytorch-lightning/Trainer/Training%20Loop/#set-how-much-of-the-training-set-to-check">Set how much of the training set to check (1-100%)</a></li>
 </ul>
@@ -261,7 +265,7 @@
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
-        <a href="LightningModule/RequiredTrainerInterface/" class="btn btn-neutral float-right" title="Lightning Module interface">Next <span class="icon icon-circle-arrow-right"></span></a>
+        <a href="Examples/" class="btn btn-neutral float-right" title="Examples">Next <span class="icon icon-circle-arrow-right"></span></a>
      
      
    </div>
@@ -291,7 +295,7 @@
      
      
      
-        <span style="margin-left: 15px"><a href="LightningModule/RequiredTrainerInterface/" style="color: #fcfcfc">Next &raquo;</a></span>
+        <span style="margin-left: 15px"><a href="Examples/" style="color: #fcfcfc">Next &raquo;</a></span>
      
    </span>
 </div>
@@ -304,5 +308,5 @@

 <!--
 MkDocs version : 1.0.4
-Build Date UTC : 2019-06-28 21:46:09
+Build Date UTC : 2019-06-28 22:01:13
 -->
@@ -48,6 +48,11 @@
          
            <li class="toctree-l1">
 		
+    <a class="" href="./Examples/">Examples</a>
+	    </li>
+          
+            <li class="toctree-l1">
+		
    <span class="caption-text">LightningModule</span>
    <ul class="subnav">
                <li class="">
@@ -60,4 +60,9 @@
     <lastmod>2019-06-28</lastmod>
     <changefreq>daily</changefreq>
    </url>
+    <url>
+     <loc>None</loc>
+     <lastmod>2019-06-28</lastmod>
+     <changefreq>daily</changefreq>
+    </url>
 </urlset>
@@ -1 +0,0 @@
-from .example_model import ExampleModel
@@ -1,74 +0,0 @@
-import os
-import sys
-
-from test_tube import HyperOptArgumentParser, Experiment
-from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-from pytorch_lightning.callbacks.pt_callbacks import EarlyStopping, ModelCheckpoint
-from docs.source.examples.example_model import ExampleModel
-
-
-def main(hparams):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    # init experiment
-    exp = Experiment(
-        name=hparams.tt_name,
-        debug=hparams.debug,
-        save_dir=hparams.tt_save_path,
-        version=hparams.hpc_exp_number,
-        autosave=False,
-        description=hparams.tt_description
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # build model
-    model = ExampleModel(hparams)
-
-    # callbacks
-    early_stop = EarlyStopping(
-        monitor='val_acc',
-        patience=3,
-        mode='min',
-        verbose=True,
-    )
-
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_function=None,
-        save_best_only=True,
-        verbose=True,
-        monitor='val_acc',
-        mode='min'
-    )
-
-    # configure trainer
-    trainer = Trainer(
-        experiment=exp,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-    )
-
-    # train model
-    trainer.fit(model)
-
-
-if __name__ == '__main__':
-
-    # use default args given by lightning
-    root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
-    parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False)
-    add_default_args(parent_parser, root_dir)
-
-    # allow model to overwrite or extend args
-    parser = ExampleModel.add_model_specific_args(parent_parser)
-    hyperparams = parser.parse_args()
-
-    # train model
-    main(hyperparams)
@@ -1,211 +0,0 @@
-import torch.nn as nn
-import numpy as np
-from pytorch_lightning.root_module.root_module import LightningModule
-from test_tube import HyperOptArgumentParser
-from torchvision.datasets import MNIST
-import torchvision.transforms as transforms
-import torch
-import torch.nn.functional as F
-import os, pdb
-from collections import OrderedDict
-
-
-class ExampleModel(LightningModule):
-    """
-    Sample model to show how to define a template
-    """
-
-    def __init__(self, hparams):
-        # init superclass
-        super(ExampleModel, self).__init__(hparams)
-
-        self.batch_size = hparams.batch_size
-
-        # build model
-        self.__build_model()
-
-    # ---------------------
-    # MODEL SETUP
-    # ---------------------
-    def __build_model(self):
-        """
-        Layout model
-        :return:
-        """
-        self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
-        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
-        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
-
-        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)
-
-    # ---------------------
-    # TRAINING
-    # ---------------------
-    def forward(self, x):
-
-        x = self.c_d1(x)
-        x = torch.tanh(x)
-        x = self.c_d1_bn(x)
-        x = self.c_d1_drop(x)
-
-        x = self.c_d2(x)
-        logits = F.log_softmax(x, dim=1)
-
-        return logits
-
-    def loss(self, labels, logits):
-        nll = F.nll_loss(logits, labels)
-        return nll
-
-    def training_step(self, data_batch, batch_i):
-        """
-        Called inside the training loop
-        :param data_batch:
-        :return:
-        """
-        # forward pass
-        x, y = data_batch
-        x = x.view(x.size(0), -1)
-        y_hat = self.forward(x)
-
-        # calculate loss
-        loss_val = self.loss(y, y_hat)
-
-        output = OrderedDict({
-            'loss': loss_val,
-            'tqdm_metrics': {}
-        })
-        return output
-
-    def validation_step(self, data_batch, batch_i):
-        """
-        Called inside the validation loop
-        :param data_batch:
-        :return:
-        """
-        x, y = data_batch
-        x = x.view(x.size(0), -1)
-        y_hat = self.forward(x)
-
-        loss_val = self.loss(y, y_hat)
-
-        # acc
-        labels_hat = torch.argmax(y_hat, dim=1)
-        val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
-
-        output = OrderedDict({
-            'val_loss': loss_val,
-            'val_acc': torch.tensor(val_acc),
-        })
-        return output
-
-
-    def validation_end(self, outputs):
-        """
-        Called at the end of validation to aggregate outputs
-        :param outputs: list of individual outputs of each validation step
-        :return:
-        """
-        val_loss_mean = 0
-        val_acc_mean = 0
-        for output in outputs:
-            val_loss_mean += output['val_loss']
-            val_acc_mean += output['val_acc']
-
-        val_loss_mean /= len(outputs)
-        val_acc_mean /= len(outputs)
-        tqdm_dic = {'val_loss': val_loss_mean.item(), 'val_acc': val_acc_mean.item()}
-        return tqdm_dic
-
-    def update_tng_log_metrics(self, logs):
-        return logs
-
-    # ---------------------
-    # MODEL SAVING
-    # ---------------------
-    def get_save_dict(self):
-        checkpoint = {'state_dict': self.state_dict()}
-        return checkpoint
-
-    def load_model_specific(self, checkpoint):
-        self.load_state_dict(checkpoint['state_dict'])
-        pass
-
-    # ---------------------
-    # TRAINING SETUP
-    # ---------------------
-    def configure_optimizers(self):
-        """
-        return whatever optimizers we want here
-        :return: list of optimizers
-        """
-        optimizer = self.choose_optimizer(self.hparams.optimizer_name, self.parameters(), {'lr': self.hparams.learning_rate}, 'optimizer')
-        self.optimizers = [optimizer]
-        return self.optimizers
-
-    def __dataloader(self, train):
-        # init data generators
-        transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
-
-        dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)
-
-        loader = torch.utils.data.DataLoader(
-            dataset=dataset,
-            batch_size=self.hparams.batch_size,
-            shuffle=True
-        )
-
-        return loader
-
-    @property
-    def tng_dataloader(self):
-        if self._tng_dataloader is None:
-            try:
-                self._tng_dataloader = self.__dataloader(train=True)
-            except Exception as e:
-                print(e)
-                raise e
-        return self._tng_dataloader
-
-    @property
-    def val_dataloader(self):
-        if self._val_dataloader is None:
-            try:
-                self._val_dataloader = self.__dataloader(train=False)
-            except Exception as e:
-                print(e)
-                raise e
-        return self._val_dataloader
-
-    @property
-    def test_dataloader(self):
-        if self._test_dataloader is None:
-            try:
-                self._test_dataloader = self.__dataloader(train=False)
-            except Exception as e:
-                print(e)
-                raise e
-        return self._test_dataloader
-
-    @staticmethod
-    def add_model_specific_args(parent_parser, root_dir):
-        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
-
-        # param overwrites
-        # parser.set_defaults(gradient_clip=5.0)
-
-        # network params
-        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
-        parser.add_argument('--in_features', default=28*28)
-        parser.add_argument('--out_features', default=10)
-        parser.add_argument('--hidden_dim', default=50000) # use 500 for CPU, 50000 for GPU to see speed difference
-
-        # data
-        parser.add_argument('--data_root', default=os.path.join(root_dir, 'mnist'), type=str)
-
-        # training params (opt)
-        parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
-                        tunable=False)
-        parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False)
-        parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
-        return parser
@@ -1,210 +0,0 @@
-import os
-import sys
-import numpy as np
-from time import sleep
-import torch
-
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
-from pytorch_lightning.models.trainer import Trainer
-from pytorch_lightning.utils.arg_parse import add_default_args
-
-from pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint
-
-SEED = 2334
-torch.manual_seed(SEED)
-np.random.seed(SEED)
-
-# ---------------------
-# DEFINE MODEL HERE
-# ---------------------
-from docs.source.examples.example_model import ExampleModel
-# ---------------------
-
-AVAILABLE_MODELS = {
-    'model_template': ExampleModel
-}
-
-
-"""
-Allows training by using command line arguments
-Run by:
-# TYPE YOUR RUN COMMAND HERE
-"""
-
-
-def main_local(hparams):
-    main(hparams, None, None)
-
-
-def main(hparams, cluster, results_dict):
-    """
-    Main training routine specific for this project
-    :param hparams:
-    :return:
-    """
-    on_gpu = hparams.gpus is not None and torch.cuda.is_available()
-
-    device = 'cuda' if on_gpu else 'cpu'
-    hparams.__setattr__('device', device)
-    hparams.__setattr__('on_gpu', on_gpu)
-    hparams.__setattr__('nb_gpus', torch.cuda.device_count())
-    hparams.__setattr__('inference_mode', hparams.model_load_weights_path is not None)
-
-    # delay each training start to not overwrite logs
-    process_position, current_gpu = TRAINING_MODEL.get_process_position(hparams.gpus)
-    sleep(process_position + 1)
-
-    # init experiment
-    log_dir = os.path.dirname(os.path.realpath(__file__))
-    exp = Experiment(
-        name='test_tube_exp',
-        debug=True,
-        save_dir=log_dir,
-        version=0,
-        autosave=False,
-        description='test demo'
-    )
-
-    exp.argparse(hparams)
-    exp.save()
-
-    # build model
-    print('loading model...')
-    model = TRAINING_MODEL(hparams)
-    print('model built')
-
-    # callbacks
-    early_stop = EarlyStopping(
-        monitor=hparams.early_stop_metric,
-        patience=hparams.early_stop_patience,
-        verbose=True,
-        mode=hparams.early_stop_mode
-    )
-
-    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
-    checkpoint = ModelCheckpoint(
-        filepath=model_save_path,
-        save_function=None,
-        save_best_only=True,
-        verbose=True,
-        monitor=hparams.model_save_monitor_value,
-        mode=hparams.model_save_monitor_mode
-    )
-
-    # gpus are ; separated for inside a node and , within nodes
-    gpu_list = None
-    if hparams.gpus is not None:
-        gpu_list = [int(x) for x in hparams.gpus.split(';')]
-
-    # configure trainer
-    trainer = Trainer(
-        experiment=exp,
-        cluster=cluster,
-        checkpoint_callback=checkpoint,
-        early_stop_callback=early_stop,
-        gpus=gpu_list
-    )
-
-    # train model
-    trainer.fit(model)
-
-
-def get_default_parser(strategy, root_dir):
-
-    possible_model_names = list(AVAILABLE_MODELS.keys())
-    parser = HyperOptArgumentParser(strategy=strategy, add_help=False)
-    add_default_args(parser, root_dir, possible_model_names=possible_model_names, rand_seed=SEED)
-    return parser
-
-
-def get_model_name(args):
-    for i, arg in enumerate(args):
-        if 'model_name' in arg:
-            return args[i+1]
-
-
-def optimize_on_cluster(hyperparams):
-    # enable cluster training
-    cluster = SlurmCluster(
-        hyperparam_optimizer=hyperparams,
-        log_path=hyperparams.tt_save_path,
-        test_tube_exp_name=hyperparams.tt_name
-    )
-
-    # email for cluster coms
-    cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True)
-
-    # configure cluster
-    cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus
-    cluster.job_time = '48:00:00'
-    cluster.gpu_type = '1080ti'
-    cluster.memory_mb_per_node = 48000
-
-    # any modules for code to run in env
-    cluster.add_command('source activate pytorch_lightning')
-
-    # name of exp
-    job_display_name = hyperparams.tt_name.split('_')[0]
-    job_display_name = job_display_name[0:3]
-
-    # run hopt
-    print('submitting jobs...')
-    cluster.optimize_parallel_cluster_gpu(
-        main,
-        nb_trials=hyperparams.nb_hopt_trials,
-        job_name=job_display_name
-    )
-
-
-if __name__ == '__main__':
-
-    model_name = get_model_name(sys.argv)
-    if model_name is None:
-        model_name = 'model_template'
-
-    # use default args
-    root_dir = os.path.dirname(os.path.realpath(__file__))
-    parent_parser = get_default_parser(strategy='random_search', root_dir=root_dir)
-
-    # allow model to overwrite or extend args
-    TRAINING_MODEL = AVAILABLE_MODELS[model_name]
-    parser = TRAINING_MODEL.add_model_specific_args(parent_parser, root_dir)
-    hyperparams = parser.parse_args()
-
-    # format GPU layout
-    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
-
-    # ---------------------
-    # RUN TRAINING
-    # ---------------------
-
-    # cluster and CPU
-    if hyperparams.on_cluster:
-        # run on HPC cluster
-        print('RUNNING ON SLURM CLUSTER')
-        gpu_ids = hyperparams.gpus.split(';')
-        os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(gpu_ids)
-        optimize_on_cluster(hyperparams)
-
-    elif hyperparams.gpus is None:
-        # run on cpu
-        print('RUNNING ON CPU')
-        main(hyperparams, None, None)
-
-    # single or multiple GPUs on same machine
-    gpu_ids = hyperparams.gpus.split(';')
-    if hyperparams.interactive:
-        # run on 1 gpu
-        print(f'RUNNING INTERACTIVE MODE ON GPUS. gpu ids: {gpu_ids}')
-        os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(gpu_ids)
-        main(hyperparams, None, None)
-
-    else:
-        # multiple GPUs on same machine
-        print(f'RUNNING MULTI GPU. GPU ids: {gpu_ids}')
-        hyperparams.optimize_parallel_gpu(
-            main_local,
-            gpu_ids=gpu_ids,
-            nb_trials=hyperparams.nb_hopt_trials,
-            nb_workers=len(gpu_ids)
-        )