release v0.0.2

.gitignore

@@ -0,0 +1,117 @@
+# project
+.DS_Store
+.data/
+run_configs/
+test_tube_logs/
+test_tube_data/
+datasets/
+model_weights/
+app/models/
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+example.py
+timit_data/
+LJSpeech-1.1/
+
+# C extensions
+*.so
+
+.idea/
+
+# Distribution / packaging
+.Python
+env/
+ide_layouts/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/

README.md

@@ -1,6 +1,46 @@
-# Pytorch-lightning
-The Keras for ML-researchers in PyTorch.    
+<p align="center">
+  <a href="https://williamfalcon.github.io/pytorch-lightning/">
+    <img alt="" src="https://github.com/williamFalcon/pytorch-lightning/blob/master/imgs/lightning_logo.png" width="50">
+  </a>
+</p>
+<h3 align="center">
+  Pytorch Lightning
+</h3>
+<p align="center">
+  The Keras for ML-researchers in PyTorch. More control. Less boilerplate.    
+</p>
+<p align="center">
+  <a href="https://badge.fury.io/py/pytorch_lightning"><img src="https://badge.fury.io/py/pytorch_lightning.svg"></a>
+  <a href="https://travis-ci.org/williamFalcon/test-tube"><img src="https://travis-ci.org/williamFalcon/pytorch-lightning.svg?branch=master"></a>
+  <a href="https://github.com/williamFalcon/pytorch-lightning/blob/master/LICENSE"><img src="https://img.shields.io/badge/License-MIT-yellow.svg"></a>
+</p>   
 
+```bash
+pip install pytorch-lightning    
+```
+
+## Docs   
+In progress. Documenting now!  
+
+## What is it?  
+All you do is define the forward passes, your data and **lightning runs everything else for you:**
+
+1. Running the training loop.   
+2. Running the validation loop.   
+3. Running the testing loop.   
+4. Early stopping.   
+5. Learning rate annealing. 
+6. Can train complex models like GANs or anything with multiple optimizers.
+7. Weight checkpointing.   
+8. Model saving.   
+9. Model loading.   
+10. Logging training details (through test-tube).  
+11. Running training on multiple GPUs (through test-tube).   
+12. Running training on a GPU cluster managed by SLURM (through test-tube).   
+13. Distributing memory-bound models on multiple GPUs.
+14. Gives your model hyperparameters parsed from the command line OR a JSON file.   
+15. Runs your model in a dev environment where nothing logs.      
+  
 ## Usage
 To use lightning do 2 things:  
 1. [Define a trainer](https://github.com/williamFalcon/pytorch-lightning/blob/master/pytorch_lightning/trainer_main.py) (which will run ALL your models).   
@@ -13,51 +53,82 @@ To use lightning do 2 things:
 ```python
 # trainer.py
 
-from pytorch_lightning.models.trainer import Trainer   
+import os
+import sys
+
+from test_tube import HyperOptArgumentParser, Experiment
+from pytorch_lightning.models.trainer import Trainer
+from pytorch_lightning.utils.arg_parse import add_default_args
 from pytorch_lightning.utils.pt_callbacks import EarlyStopping, ModelCheckpoint
-from my_project import My_Model   
-from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
+from demo.example_model import ExampleModel
 
-# --------------
-# TEST TUBE INIT
-exp = Experiment(
-    name='my_exp',
-    debug=True,
-    save_dir='/some/path',
-    autosave=False,
-    description='my desc'
-)
 
-# --------------------
-# CALLBACKS
-early_stop = EarlyStopping(
-    monitor='val_loss',
-    patience=3,
-    verbose=True,
-    mode='min'
-)
+def main(hparams):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    :return:
+    """
+    # init experiment
+    exp = Experiment(
+        name=hparams.tt_name,
+        debug=hparams.debug,
+        save_dir=hparams.tt_save_path,
+        version=hparams.hpc_exp_number,
+        autosave=False,
+        description=hparams.tt_description
+    )
 
-model_save_path = 'PATH/TO/SAVE'
-checkpoint = ModelCheckpoint(
-    filepath=model_save_path,
-    save_function=None,
-    save_best_only=True,
-    verbose=True,
-    monitor='val_acc',
-    mode='min'
-)
+    exp.argparse(hparams)
+    exp.save()
 
-# configure trainer
-trainer = Trainer(
-    experiment=experiment,
-    cluster=cluster,
-    checkpoint_callback=checkpoint,
-    early_stop_callback=early_stop
-)  
+    # build model
+    print('loading model...')
+    model = ExampleModel(hparams)
+    print('model built')
 
-# init model and train
-model = My_Model()
-trainer.fit(model)
+    # callbacks
+    early_stop = EarlyStopping(
+        monitor=hparams.early_stop_metric,
+        patience=hparams.early_stop_patience,
+        verbose=True,
+        mode=hparams.early_stop_mode
+    )
+
+    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
+    checkpoint = ModelCheckpoint(
+        filepath=model_save_path,
+        save_function=None,
+        save_best_only=True,
+        verbose=True,
+        monitor=hparams.model_save_monitor_value,
+        mode=hparams.model_save_monitor_mode
+    )
+
+    # configure trainer
+    trainer = Trainer(
+        experiment=exp,
+        checkpoint_callback=checkpoint,
+        early_stop_callback=early_stop,
+    )
+
+    # train model
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+
+    # use default args given by lightning
+    root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
+    parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False)
+    add_default_args(parent_parser, root_dir)
+
+    # allow model to overwrite or extend args
+    parser = ExampleModel.add_model_specific_args(parent_parser)
+    hyperparams = parser.parse_args()
+
+    # train model
+    main(hyperparams)
 ```
 
 #### Define the model   

demo/basic_trainer.py

@@ -0,0 +1,76 @@
+import os
+import sys
+
+from test_tube import HyperOptArgumentParser, Experiment
+from pytorch_lightning.models.trainer import Trainer
+from pytorch_lightning.utils.arg_parse import add_default_args
+from pytorch_lightning.utils.pt_callbacks import EarlyStopping, ModelCheckpoint
+from demo.example_model import ExampleModel
+
+
+def main(hparams):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    :return:
+    """
+    # init experiment
+    exp = Experiment(
+        name=hparams.tt_name,
+        debug=hparams.debug,
+        save_dir=hparams.tt_save_path,
+        version=hparams.hpc_exp_number,
+        autosave=False,
+        description=hparams.tt_description
+    )
+
+    exp.argparse(hparams)
+    exp.save()
+
+    # build model
+    print('loading model...')
+    model = ExampleModel(hparams)
+    print('model built')
+
+    # callbacks
+    early_stop = EarlyStopping(
+        monitor=hparams.early_stop_metric,
+        patience=hparams.early_stop_patience,
+        verbose=True,
+        mode=hparams.early_stop_mode
+    )
+
+    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
+    checkpoint = ModelCheckpoint(
+        filepath=model_save_path,
+        save_function=None,
+        save_best_only=True,
+        verbose=True,
+        monitor=hparams.model_save_monitor_value,
+        mode=hparams.model_save_monitor_mode
+    )
+
+    # configure trainer
+    trainer = Trainer(
+        experiment=exp,
+        checkpoint_callback=checkpoint,
+        early_stop_callback=early_stop,
+    )
+
+    # train model
+    trainer.fit(model)
+
+
+if __name__ == '__main__':
+
+    # use default args given by lightning
+    root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
+    parent_parser = HyperOptArgumentParser(strategy='random_search', add_help=False)
+    add_default_args(parent_parser, root_dir)
+
+    # allow model to overwrite or extend args
+    parser = ExampleModel.add_model_specific_args(parent_parser)
+    hyperparams = parser.parse_args()
+
+    # train model
+    main(hyperparams)

demo/example_model.py

@@ -0,0 +1,200 @@
+import torch.nn as nn
+import numpy as np
+from pytorch_lightning.root_module.root_module import RootModule
+from test_tube import HyperOptArgumentParser
+from torchvision.datasets import MNIST
+import torchvision.transforms as transforms
+import torch
+import torch.nn.functional as F
+
+
+class ExampleModel(RootModule):
+    """
+    Sample model to show how to define a template
+    """
+
+    def __init__(self, hparams):
+        # init superclass
+        super(ExampleModel, self).__init__(hparams)
+
+        self.batch_size = hparams.batch_size
+
+        # build model
+        self.__build_model()
+
+    # ---------------------
+    # MODEL SETUP
+    # ---------------------
+    def __build_model(self):
+        """
+        Layout model
+        :return:
+        """
+        self.c_d1 = nn.Linear(in_features=self.hparams.in_features, out_features=self.hparams.hidden_dim)
+        self.c_d1_bn = nn.BatchNorm1d(self.hparams.hidden_dim)
+        self.c_d1_drop = nn.Dropout(self.hparams.drop_prob)
+
+        self.c_d2 = nn.Linear(in_features=self.hparams.hidden_dim, out_features=self.hparams.out_features)
+
+    # ---------------------
+    # TRAINING
+    # ---------------------
+    def forward(self, x):
+        x = self.c_d1(x)
+        x = F.tanh(x)
+        x = self.c_d1_bn(x)
+        x = self.c_d1_drop(x)
+
+        x = self.c_d2(x)
+        logits = F.log_softmax(x, dim=1)
+
+        return logits
+
+    def loss(self, labels, logits):
+        nll = F.nll_loss(logits, labels)
+        return nll
+
+    def training_step(self, data_batch):
+        """
+        Called inside the training loop
+        :param data_batch:
+        :return:
+        """
+        # forward pass
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+        y_hat = self.forward(x)
+
+        # calculate loss
+        loss_val = self.loss(y, y_hat)
+
+        tqdm_dic = {'tng_loss': loss_val.item()}
+        return loss_val, tqdm_dic
+
+    def validation_step(self, data_batch):
+        """
+        Called inside the validation loop
+        :param data_batch:
+        :return:
+        """
+        x, y = data_batch
+        x = x.view(x.size(0), -1)
+        y_hat = self.forward(x)
+
+        loss_val = self.loss(y, y_hat)
+
+        # acc
+        labels_hat = torch.argmax(y_hat, dim=1)
+        val_acc = torch.sum(y == labels_hat).item() / (len(y) * 1.0)
+
+        output = {'y_hat': y_hat, 'val_loss': loss_val.item(), 'val_acc': val_acc}
+        return output
+
+    def validation_end(self, outputs):
+        """
+        Called at the end of validation to aggregate outputs
+        :param outputs: list of individual outputs of each validation step
+        :return:
+        """
+        val_loss_mean = 0
+        accs = []
+        for output in outputs:
+            val_loss_mean += output['val_loss']
+            accs.append(output['val_acc'])
+
+        val_loss_mean /= len(outputs)
+        tqdm_dic = {'val_loss': val_loss_mean, 'val_acc': np.mean(accs)}
+        return tqdm_dic
+
+    def update_tng_log_metrics(self, logs):
+        return logs
+
+    # ---------------------
+    # MODEL SAVING
+    # ---------------------
+    def get_save_dict(self):
+        checkpoint = {'state_dict': self.state_dict()}
+        return checkpoint
+
+    def load_model_specific(self, checkpoint):
+        self.load_state_dict(checkpoint['state_dict'])
+        pass
+
+    # ---------------------
+    # TRAINING SETUP
+    # ---------------------
+    def configure_optimizers(self):
+        """
+        return whatever optimizers we want here
+        :return: list of optimizers
+        """
+        optimizer = self.choose_optimizer(self.hparams.optimizer_name, self.parameters(), {'lr': self.hparams.learning_rate}, 'optimizer')
+        self.optimizers = [optimizer]
+        return self.optimizers
+
+    def __dataloader(self, train):
+        # init data generators
+        transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize((0.5,), (1.0,))])
+
+        dataset = MNIST(root=self.hparams.data_root, train=train, transform=transform, download=True)
+
+        loader = torch.utils.data.DataLoader(
+            dataset=dataset,
+            batch_size=self.hparams.batch_size,
+            shuffle=True
+        )
+
+        return loader
+
+    @property
+    def tng_dataloader(self):
+        if self._tng_dataloader is None:
+            try:
+                self._tng_dataloader = self.__dataloader(train=True)
+            except Exception as e:
+                print(e)
+                raise e
+        return self._tng_dataloader
+
+    @property
+    def val_dataloader(self):
+        if self._val_dataloader is None:
+            try:
+                self._val_dataloader = self.__dataloader(train=False)
+            except Exception as e:
+                print(e)
+                raise e
+        return self._val_dataloader
+
+    @property
+    def test_dataloader(self):
+        if self._test_dataloader is None:
+            try:
+                self._test_dataloader = self.__dataloader(train=False)
+            except Exception as e:
+                print(e)
+                raise e
+        return self._test_dataloader
+
+    @staticmethod
+    def add_model_specific_args(parent_parser):
+        parser = HyperOptArgumentParser(strategy=parent_parser.strategy, parents=[parent_parser])
+
+        # param overwrites
+        # parser.set_defaults(gradient_clip=5.0)
+
+        # network params
+        parser.opt_list('--drop_prob', default=0.2, options=[0.2, 0.5], type=float, tunable=False)
+        parser.add_argument('--in_features', default=28*28)
+        parser.add_argument('--hidden_dim', default=500)
+        parser.add_argument('--out_features', default=10)
+
+        # data
+        parser.add_argument('--data_root', default='/Users/williamfalcon/Developer/personal/research_lib/research_proj/datasets/mnist', type=str)
+
+        # training params (opt)
+        parser.opt_list('--learning_rate', default=0.001, type=float, options=[0.0001, 0.0005, 0.001, 0.005],
+                        tunable=False)
+        parser.opt_list('--batch_size', default=256, type=int, options=[32, 64, 128, 256], tunable=False)
+        parser.opt_list('--optimizer_name', default='adam', type=str, options=['adam'], tunable=False)
+        return parser

demo/fully_featured_trainer.py

@@ -0,0 +1,201 @@
+import os
+import sys
+import numpy as np
+from time import sleep
+import torch
+
+from test_tube import HyperOptArgumentParser, Experiment, SlurmCluster
+from pytorch_lightning.models.trainer import Trainer
+from pytorch_lightning.utils.arg_parse import add_default_args
+
+from pytorch_lightning.utils.pt_callbacks import EarlyStopping, ModelCheckpoint
+
+SEED = 2334
+torch.manual_seed(SEED)
+np.random.seed(SEED)
+
+# ---------------------
+# DEFINE MODEL HERE
+# ---------------------
+from demo.example_model import ExampleModel
+# ---------------------
+
+AVAILABLE_MODELS = {
+    'model_template': ExampleModel
+}
+
+
+"""
+Allows training by using command line arguments
+Run by: 
+# TYPE YOUR RUN COMMAND HERE
+"""
+
+
+def main_local(hparams):
+    main(hparams, None, None)
+
+
+def main(hparams, cluster, results_dict):
+    """
+    Main training routine specific for this project
+    :param hparams:
+    :return:
+    """
+    on_gpu = torch.cuda.is_available()
+    if hparams.disable_cuda:
+        on_gpu = False
+
+    device = 'cuda' if on_gpu else 'cpu'
+    hparams.__setattr__('device', device)
+    hparams.__setattr__('on_gpu', on_gpu)
+    hparams.__setattr__('nb_gpus', torch.cuda.device_count())
+    hparams.__setattr__('inference_mode', hparams.model_load_weights_path is not None)
+
+    # delay each training start to not overwrite logs
+    process_position, current_gpu = TRAINING_MODEL.get_process_position(hparams.gpus)
+    sleep(process_position + 1)
+
+    # init experiment
+    exp = Experiment(
+        name=hparams.tt_name,
+        debug=hparams.debug,
+        save_dir=hparams.tt_save_path,
+        version=hparams.hpc_exp_number,
+        autosave=False,
+        description=hparams.tt_description
+    )
+
+    exp.argparse(hparams)
+    exp.save()
+
+    # build model
+    print('loading model...')
+    model = TRAINING_MODEL(hparams)
+    print('model built')
+
+    # callbacks
+    early_stop = EarlyStopping(
+        monitor=hparams.early_stop_metric,
+        patience=hparams.early_stop_patience,
+        verbose=True,
+        mode=hparams.early_stop_mode
+    )
+
+    model_save_path = '{}/{}/{}'.format(hparams.model_save_path, exp.name, exp.version)
+    checkpoint = ModelCheckpoint(
+        filepath=model_save_path,
+        save_function=None,
+        save_best_only=True,
+        verbose=True,
+        monitor=hparams.model_save_monitor_value,
+        mode=hparams.model_save_monitor_mode
+    )
+
+    # configure trainer
+    trainer = Trainer(
+        experiment=exp,
+        cluster=cluster,
+        checkpoint_callback=checkpoint,
+        early_stop_callback=early_stop,
+    )
+
+    # train model
+    trainer.fit(model)
+
+
+def get_default_parser(strategy, root_dir):
+
+    possible_model_names = list(AVAILABLE_MODELS.keys())
+    parser = HyperOptArgumentParser(strategy=strategy, add_help=False)
+    add_default_args(parser, root_dir, possible_model_names, SEED)
+    return parser
+
+
+def get_model_name(args):
+    for i, arg in enumerate(args):
+        if 'model_name' in arg:
+            return args[i+1]
+
+
+def optimize_on_cluster(hyperparams):
+    # enable cluster training
+    cluster = SlurmCluster(
+        hyperparam_optimizer=hyperparams,
+        log_path=hyperparams.tt_save_path,
+        test_tube_exp_name=hyperparams.tt_name
+    )
+
+    # email for cluster coms
+    cluster.notify_job_status(email='add_email_here', on_done=True, on_fail=True)
+
+    # configure cluster
+    cluster.per_experiment_nb_gpus = hyperparams.per_experiment_nb_gpus
+    cluster.job_time = '48:00:00'
+    cluster.gpu_type = '1080ti'
+    cluster.memory_mb_per_node = 48000
+
+    # any modules for code to run in env
+    cluster.add_command('source activate pytorch_lightning')
+
+    # name of exp
+    job_display_name = hyperparams.tt_name.split('_')[0]
+    job_display_name = job_display_name[0:3]
+
+    # run hopt
+    print('submitting jobs...')
+    cluster.optimize_parallel_cluster_gpu(
+        main,
+        nb_trials=hyperparams.nb_hopt_trials,
+        job_name=job_display_name
+    )
+
+
+if __name__ == '__main__':
+
+    model_name = get_model_name(sys.argv)
+    if model_name is None:
+        model_name = 'model_template'
+
+    # use default args
+    root_dir = os.path.split(os.path.dirname(sys.modules['__main__'].__file__))[0]
+    parent_parser = get_default_parser(strategy='random_search', root_dir=root_dir)
+
+    # allow model to overwrite or extend args
+    TRAINING_MODEL = AVAILABLE_MODELS[model_name]
+    parser = TRAINING_MODEL.add_model_specific_args(parent_parser)
+    hyperparams = parser.parse_args()
+
+    # format GPU layout
+    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+    gpu_ids = hyperparams.gpus.split(';')
+
+    # RUN TRAINING
+    if hyperparams.on_cluster:
+        # Gets called when running via HPC cluster
+        print('RUNNING ON SLURM CLUSTER')
+        os.environ["CUDA_VISIBLE_DEVICES"] = ','.join(gpu_ids)
+        optimize_on_cluster(hyperparams)
+
+    elif hyperparams.single_run_gpu:
+        # run on 1 gpu
+        print(f'RUNNING 1 TRIAL ON GPU. gpu: {gpu_ids[0]}')
+        os.environ["CUDA_VISIBLE_DEVICES"] = gpu_ids[0]
+        main(hyperparams, None, None)
+
+    elif hyperparams.local or hyperparams.single_run:
+        # run 1 trial but on CPU
+        os.environ["CUDA_VISIBLE_DEVICES"] = '0'
+        print('RUNNING LOCALLY')
+        main(hyperparams, None, None)
+
+    else:
+        # multiple GPUs on same machine
+        print(f'RUNNING MULTI GPU. GPU ids: {gpu_ids}')
+        hyperparams.optimize_parallel_gpu(
+            main_local,
+            gpu_ids=gpu_ids,
+            nb_trials=hyperparams.nb_hopt_trials,
+            nb_workers=len(gpu_ids)
+        )
+

pytorch_lightning.egg-info/PKG-INFO

@@ -0,0 +1,10 @@
+Metadata-Version: 1.0
+Name: pytorch-lightning
+Version: 0.0.1
+Summary: Rapid research framework
+Home-page: https://github.com/williamFalcon/pytorch-lightning
+Author: UNKNOWN
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN

pytorch_lightning.egg-info/SOURCES.txt

@@ -0,0 +1,32 @@
+README.md
+setup.py
+notebooks/__init__.py
+pytorch_lightning/__init__.py
+pytorch_lightning/trainer_main.py
+pytorch_lightning.egg-info/PKG-INFO
+pytorch_lightning.egg-info/SOURCES.txt
+pytorch_lightning.egg-info/dependency_links.txt
+pytorch_lightning.egg-info/requires.txt
+pytorch_lightning.egg-info/top_level.txt
+pytorch_lightning/models/__init__.py
+pytorch_lightning/models/trainer.py
+pytorch_lightning/models/model_examples/__init__.py
+pytorch_lightning/models/model_examples/bilstm.py
+pytorch_lightning/models/sample_model_template/__init__.py
+pytorch_lightning/models/sample_model_template/model_template.py
+pytorch_lightning/root_module/__init__.py
+pytorch_lightning/root_module/grads.py
+pytorch_lightning/root_module/hooks.py
+pytorch_lightning/root_module/memory.py
+pytorch_lightning/root_module/model_saving.py
+pytorch_lightning/root_module/optimization.py
+pytorch_lightning/root_module/root_module.py
+pytorch_lightning/utils/__init__.py
+pytorch_lightning/utils/arg_parse.py
+pytorch_lightning/utils/embeddings.py
+pytorch_lightning/utils/plotting.py
+pytorch_lightning/utils/pt_callbacks.py
+tests/__init__.py
+tests/research_proj/__init__.py
+tests/research_proj/sample_model_template/__init__.py
+tests/research_proj/sample_model_template/model_template_test.py

pytorch_lightning.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

pytorch_lightning.egg-info/top_level.txt

@@ -0,0 +1,3 @@
+notebooks
+pytorch_lightning
+tests

pytorch_lightning/models/trainer.py

@@ -1,9 +1,9 @@
 import torch
 import tqdm
 import numpy as np
-from research_lib.root_module.memory import get_gpu_memory_map
+from pytorch_lightning.root_module.memory import get_gpu_memory_map
 import traceback
-from research_lib.root_module.model_saving import TrainerIO
+from pytorch_lightning.root_module.model_saving import TrainerIO
 from torch.optim.lr_scheduler import MultiStepLR
 
 
@@ -11,17 +11,17 @@ class Trainer(TrainerIO):
 
     def __init__(self,
                  experiment,
-                 cluster,
                  checkpoint_callback, early_stop_callback,
+                 cluster=None,
                  process_position=0,
                  current_gpu_name=0,
                  on_gpu=False,
                  enable_tqdm=True,
-                 overfit_pct=None,
+                 overfit_pct=0.0,
                  track_grad_norm=-1,
                  check_val_every_n_epoch=1,
                  fast_dev_run=False,
-                 accumulate_grad_batches=False,
+                 accumulate_grad_batches=1,
                  enable_early_stop=True, max_nb_epochs=5, min_nb_epochs=1,
                  train_percent_check=1.0, val_percent_check=1.0, test_percent_check=1.0, val_check_interval=0.95,
                  log_save_interval=1, add_log_row_interval=1,
@@ -226,7 +226,8 @@ class Trainer(TrainerIO):
         self.experiment.save()
 
         # enable cluster checkpointing
-        self.enable_auto_hpc_walltime_manager()
+        if self.cluster is not None:
+            self.enable_auto_hpc_walltime_manager()
 
         # ---------------------------
         # CORE TRAINING LOOP

pytorch_lightning/root_module/root_module.py

@@ -14,7 +14,7 @@ class RootModule(GradInformation, ModelIO, OptimizerConfig, ModelHooks):
     def __init__(self, hparams):
         super(RootModule, self).__init__()
         self.hparams = hparams
-        self.on_gpu = hparams.on_gpu
+
         self.dtype = torch.FloatTensor
         self.exp_save_path = None
         self.current_epoch = 0
@@ -25,6 +25,13 @@ class RootModule(GradInformation, ModelIO, OptimizerConfig, ModelHooks):
         self.gradient_clip = hparams.gradient_clip
         self.num = 2
 
+        # track if gpu was requested for checkpointing
+        self.on_gpu = False
+        try:
+            self.on_gpu = hparams.on_gpu
+        except Exception as e:
+            pass
+
         # computed vars for the dataloaders
         self._tng_dataloader = None
         self._val_dataloader = None

pytorch_lightning/utils/arg_parse.py

@@ -1,4 +1,4 @@
-def add_default_args(parser, root_dir, possible_model_names, rand_seed):
+def add_default_args(parser, root_dir, rand_seed=None, possible_model_names=None):
 
     # tng, test, val check intervals
     parser.add_argument('--eval_test_set', dest='eval_test_set', action='store_true', help='true = run test set also')
@@ -32,7 +32,9 @@ def add_default_args(parser, root_dir, possible_model_names, rand_seed):
 
     # model paths
     parser.add_argument('--model_load_weights_path', default=None, type=str)
-    parser.add_argument('--model_name', default='', help=','.join(possible_model_names))
+
+    if possible_model_names is not None:
+        parser.add_argument('--model_name', default='', help=','.join(possible_model_names))
 
     # test_tube settings
     parser.add_argument('-en', '--tt_name', default='r_lib_')
@@ -58,7 +60,9 @@ def add_default_args(parser, root_dir, possible_model_names, rand_seed):
     parser.add_argument('--overfit', default=-1, type=float, help='% of dataset to use with this option. float, or -1 for none')
 
     # debug args
-    parser.add_argument('--random_seed', default=rand_seed, type=int)
+    if rand_seed is not None:
+        parser.add_argument('--random_seed', default=rand_seed, type=int)
+
     parser.add_argument('--live', dest='live', action='store_true', help='runs on gpu without cluster')
     parser.add_argument('--enable_debug', dest='debug', action='store_true', help='enables/disables test tube')
     parser.add_argument('--enable_local', dest='local', action='store_true', help='enables local tng')

setup.py

@@ -3,11 +3,11 @@
 from setuptools import setup, find_packages
 
 setup(name='pytorch-lightning',
-      version='0.0.1',
+      version='0.0.2',
       description='Rapid research framework',
       author='',
       author_email='',
       url='https://github.com/williamFalcon/pytorch-lightning',
-      install_requires=[],
+      install_requires=['test-tube', 'torch', 'tqdm'],
       packages=find_packages()
       )