fix val logging (#362)

* fix test

* fix test

* fix test

* fix test

* fix test

* fix test

* fix test

* fix test

* fix test

* fix test

* fix test

* fix test

* fix test

* no warnings always

* no warnings always

* no warnings always

* no warnings always

pytorch_lightning/trainer/trainer.py

@@ -17,7 +17,7 @@ from torch.optim.optimizer import Optimizer
 from pytorch_lightning.root_module.root_module import LightningModule
 from pytorch_lightning.root_module import memory
 from pytorch_lightning.logging import TestTubeLogger
-from pytorch_lightning.trainer.trainer_io import TrainerIO
+from pytorch_lightning.trainer.trainer_io import TrainerIOMixin
 from pytorch_lightning.pt_overrides.override_data_parallel import (
     LightningDistributedDataParallel, LightningDataParallel)
 from pytorch_lightning.callbacks import GradientAccumulationScheduler, \
@@ -55,7 +55,7 @@ def reduce_distributed_output(output, nb_gpus):
     return output
 
 
-class Trainer(TrainerIO):
+class Trainer(TrainerIOMixin):
 
     def __init__(self,
                  logger=True,
@@ -175,6 +175,7 @@ class Trainer(TrainerIO):
 
         # configure early stop callback
         # creates a default one if none passed in
+        self.early_stop_callback = None
         if early_stop_callback is True:
             self.early_stop_callback = EarlyStopping(
                 monitor='val_loss',
@@ -1334,7 +1335,7 @@ class Trainer(TrainerIO):
             log_output = output['log']
 
             # reduce progress metrics for tqdm when using dp
-            if train and self.use_dp or self.use_ddp2:
+            if train and(self.use_dp or self.use_ddp2):
                 nb_gpus = self.num_gpus
                 log_output = reduce_distributed_output(log_output, nb_gpus)
 

pytorch_lightning/trainer/trainer_io.py

@@ -10,7 +10,7 @@ from pytorch_lightning.pt_overrides.override_data_parallel import (
     LightningDistributedDataParallel, LightningDataParallel)
 
 
-class TrainerIO(object):
+class TrainerIOMixin(object):
 
     def __get_model(self):
         is_dp_module = isinstance(self.model, (LightningDistributedDataParallel,
@@ -42,7 +42,7 @@ class TrainerIO(object):
 
     def restore_state_if_checkpoint_exists(self, model):
         # do nothing if there's not dir or callback
-        no_ckpt_callback = self.checkpoint_callback is None
+        no_ckpt_callback = (self.checkpoint_callback is None) or (not self.checkpoint_callback)
         if no_ckpt_callback or not os.path.exists(self.checkpoint_callback.filepath):
             return
 
@@ -151,10 +151,10 @@ class TrainerIO(object):
             'global_step': self.global_step
         }
 
-        if self.checkpoint_callback is not None:
+        if self.checkpoint_callback is not None or self.checkpoint_callback is not False:
             checkpoint['checkpoint_callback_best'] = self.checkpoint_callback.best
 
-        if self.early_stop_callback is not None:
+        if self.early_stop_callback is not None or self.checkpoint_callback is not False:
             checkpoint['early_stop_callback_wait'] = self.early_stop_callback.wait
             checkpoint['early_stop_callback_patience'] = self.early_stop_callback.patience
 
@@ -207,10 +207,10 @@ class TrainerIO(object):
         :param checkpoint:
         :return:
         """
-        if self.checkpoint_callback is not None:
+        if self.checkpoint_callback is not None or self.checkpoint_callback is not False:
             self.checkpoint_callback.best = checkpoint['checkpoint_callback_best']
 
-        if self.early_stop_callback is not None:
+        if self.early_stop_callback is not None or self.early_stop_callback is not False:
             self.early_stop_callback.wait = checkpoint['early_stop_callback_wait']
             self.early_stop_callback.patience = checkpoint['early_stop_callback_patience']
 

tests/test_logging.py

@@ -66,6 +66,8 @@ def test_testtube_pickle():
     trainer2 = pickle.loads(pkl_bytes)
     trainer2.logger.log_metrics({"acc": 1.0})
 
+    clear_save_dir()
+
 
 def test_mlflow_logger():
     """
@@ -134,6 +136,9 @@ def test_mlflow_pickle():
     trainer2 = pickle.loads(pkl_bytes)
     trainer2.logger.log_metrics({"acc": 1.0})
 
+    n = np.random.randint(0, 10000000, 1)[0]
+    shutil.move(mlflow_dir, mlflow_dir + f'_{n}')
+
 
 def test_custom_logger():
 

tests/test_models.py

@@ -1519,6 +1519,7 @@ def get_model(use_test_model=False, lbfgs=False):
     hparams = get_hparams()
     if lbfgs:
         setattr(hparams, 'optimizer_name', 'lbfgs')
+        setattr(hparams, 'learning_rate', 0.002)
 
     if use_test_model:
         model = LightningTestModel(hparams)