From e2cabb03ba86c4dc0271a4531bce7bf2dff53e94 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Tue, 15 Oct 2019 12:44:20 -0400 Subject: [PATCH] fix val logging (#362) * fix test * fix test * fix test * fix test * fix test * fix test * fix test * fix test * fix test * fix test * fix test * fix test * fix test * no warnings always * no warnings always * no warnings always * no warnings always --- pytorch_lightning/trainer/trainer.py | 7 ++++--- pytorch_lightning/trainer/trainer_io.py | 12 ++++++------ tests/test_logging.py | 5 +++++ tests/test_models.py | 1 + 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 20c0673d..b763458e 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -17,7 +17,7 @@ from torch.optim.optimizer import Optimizer from pytorch_lightning.root_module.root_module import LightningModule from pytorch_lightning.root_module import memory from pytorch_lightning.logging import TestTubeLogger -from pytorch_lightning.trainer.trainer_io import TrainerIO +from pytorch_lightning.trainer.trainer_io import TrainerIOMixin from pytorch_lightning.pt_overrides.override_data_parallel import ( LightningDistributedDataParallel, LightningDataParallel) from pytorch_lightning.callbacks import GradientAccumulationScheduler, \ @@ -55,7 +55,7 @@ def reduce_distributed_output(output, nb_gpus): return output -class Trainer(TrainerIO): +class Trainer(TrainerIOMixin): def __init__(self, logger=True, @@ -175,6 +175,7 @@ class Trainer(TrainerIO): # configure early stop callback # creates a default one if none passed in + self.early_stop_callback = None if early_stop_callback is True: self.early_stop_callback = EarlyStopping( monitor='val_loss', @@ -1334,7 +1335,7 @@ class Trainer(TrainerIO): log_output = output['log'] # reduce progress metrics for tqdm when using dp - if train and self.use_dp or self.use_ddp2: + if train and(self.use_dp or self.use_ddp2): nb_gpus = self.num_gpus log_output = reduce_distributed_output(log_output, nb_gpus) diff --git a/pytorch_lightning/trainer/trainer_io.py b/pytorch_lightning/trainer/trainer_io.py index f6305308..8852e7d8 100644 --- a/pytorch_lightning/trainer/trainer_io.py +++ b/pytorch_lightning/trainer/trainer_io.py @@ -10,7 +10,7 @@ from pytorch_lightning.pt_overrides.override_data_parallel import ( LightningDistributedDataParallel, LightningDataParallel) -class TrainerIO(object): +class TrainerIOMixin(object): def __get_model(self): is_dp_module = isinstance(self.model, (LightningDistributedDataParallel, @@ -42,7 +42,7 @@ class TrainerIO(object): def restore_state_if_checkpoint_exists(self, model): # do nothing if there's not dir or callback - no_ckpt_callback = self.checkpoint_callback is None + no_ckpt_callback = (self.checkpoint_callback is None) or (not self.checkpoint_callback) if no_ckpt_callback or not os.path.exists(self.checkpoint_callback.filepath): return @@ -151,10 +151,10 @@ class TrainerIO(object): 'global_step': self.global_step } - if self.checkpoint_callback is not None: + if self.checkpoint_callback is not None or self.checkpoint_callback is not False: checkpoint['checkpoint_callback_best'] = self.checkpoint_callback.best - if self.early_stop_callback is not None: + if self.early_stop_callback is not None or self.checkpoint_callback is not False: checkpoint['early_stop_callback_wait'] = self.early_stop_callback.wait checkpoint['early_stop_callback_patience'] = self.early_stop_callback.patience @@ -207,10 +207,10 @@ class TrainerIO(object): :param checkpoint: :return: """ - if self.checkpoint_callback is not None: + if self.checkpoint_callback is not None or self.checkpoint_callback is not False: self.checkpoint_callback.best = checkpoint['checkpoint_callback_best'] - if self.early_stop_callback is not None: + if self.early_stop_callback is not None or self.early_stop_callback is not False: self.early_stop_callback.wait = checkpoint['early_stop_callback_wait'] self.early_stop_callback.patience = checkpoint['early_stop_callback_patience'] diff --git a/tests/test_logging.py b/tests/test_logging.py index b8436a9d..2458639b 100644 --- a/tests/test_logging.py +++ b/tests/test_logging.py @@ -66,6 +66,8 @@ def test_testtube_pickle(): trainer2 = pickle.loads(pkl_bytes) trainer2.logger.log_metrics({"acc": 1.0}) + clear_save_dir() + def test_mlflow_logger(): """ @@ -134,6 +136,9 @@ def test_mlflow_pickle(): trainer2 = pickle.loads(pkl_bytes) trainer2.logger.log_metrics({"acc": 1.0}) + n = np.random.randint(0, 10000000, 1)[0] + shutil.move(mlflow_dir, mlflow_dir + f'_{n}') + def test_custom_logger(): diff --git a/tests/test_models.py b/tests/test_models.py index d95f926e..18a8aecd 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1519,6 +1519,7 @@ def get_model(use_test_model=False, lbfgs=False): hparams = get_hparams() if lbfgs: setattr(hparams, 'optimizer_name', 'lbfgs') + setattr(hparams, 'learning_rate', 0.002) if use_test_model: model = LightningTestModel(hparams)