release v0.1.dev21

pytorch_lightning/models/trainer.py

@@ -5,6 +5,7 @@ from pytorch_lightning.root_module.memory import get_gpu_memory_map
 import traceback
 from pytorch_lightning.root_module.model_saving import TrainerIO
 from torch.optim.lr_scheduler import MultiStepLR
+from torch.nn import DataParallel
 import pdb
 
 try:
@@ -62,6 +63,8 @@ class Trainer(TrainerIO):
         self.lr_schedulers = []
         self.amp_level = amp_level
         self.check_grad_nans = check_grad_nans
+        self.data_parallel_device_ids = [0]
+        self.data_parallel = False
 
         # training state
         self.optimizers = None
@@ -242,7 +245,10 @@ class Trainer(TrainerIO):
 
         # put on gpu if needed
         if self.on_gpu:
-            model = model.cuda()
+            if self.data_parallel:
+                model = DataParallel(model, device_ids=self.data_parallel_device_ids)
+            else:
+                model = model.cuda()
 
         # run tiny validation to make sure program won't crash during val
         _ = self.validate(model, self.val_dataloader, max_batches=self.nb_sanity_val_steps)

pytorch_lightning/root_module/model_saving.py

@@ -1,6 +1,7 @@
 import torch
 import os
 import re
+import pdb
 
 
 class ModelIO(object):
@@ -99,6 +100,9 @@ class TrainerIO(object):
     # PRIVATE OPS
     # ----------------------------------
     def hpc_save(self, folderpath, experiment):
+        # make sure the checkpoint folder exists
+        os.makedirs(folderpath, exist_ok=True)
+
         # save exp to make sure we get all the metrics
         experiment.save()
 
@@ -130,6 +134,10 @@ class TrainerIO(object):
 
     def max_ckpt_in_folder(self, path):
         files = os.listdir(path)
+        files = [x for x in files if 'ckpt_' in x]
+        if len(files) == 0:
+            return 0
+
         ckpt_vs = []
         for name in files:
             name = name.split('ckpt_')[-1]

setup.py

@@ -7,7 +7,7 @@ from setuptools import setup, find_packages
 # http://blog.ionelmc.ro/2014/05/25/python-packaging/
 setup(
     name="pytorch-lightning",
-    version='0.1.dev1832',
+    version='0.1.dev21',
     description="The Keras for ML researchers using PyTorch",
     author="William Falcon",
     author_email="waf2107@columbia.edu",
@@ -17,7 +17,7 @@ setup(
     keywords=["deep learning", "pytorch", "AI"],
     python_requires=">=3.5",
     install_requires=[
-        "torch",
+        "torch>=1.0.0",
         "tqdm",
         "test-tube",
     ],