diff --git a/pytorch_lightning/logging/test_tube_logger.py b/pytorch_lightning/logging/test_tube_logger.py index 530f32ff..cd721512 100644 --- a/pytorch_lightning/logging/test_tube_logger.py +++ b/pytorch_lightning/logging/test_tube_logger.py @@ -26,6 +26,7 @@ class TestTubeLogger(LightningLoggerBase): def experiment(self): if self._experiment is not None: return self._experiment + self._experiment = Experiment( save_dir=self.save_dir, name=self.name, @@ -39,39 +40,45 @@ class TestTubeLogger(LightningLoggerBase): @rank_zero_only def log_hyperparams(self, params): + # TODO: HACK figure out where this is being set to true + self.experiment.debug = self.debug self.experiment.argparse(params) @rank_zero_only def log_metrics(self, metrics, step_num=None): + # TODO: HACK figure out where this is being set to true + self.experiment.debug = self.debug self.experiment.log(metrics, global_step=step_num) @rank_zero_only def save(self): + # TODO: HACK figure out where this is being set to true + self.experiment.debug = self.debug self.experiment.save() @rank_zero_only def finalize(self, status): + # TODO: HACK figure out where this is being set to true + self.experiment.debug = self.debug self.save() self.close() @rank_zero_only def close(self): + # TODO: HACK figure out where this is being set to true + self.experiment.debug = self.debug exp = self.experiment exp.close() @property def rank(self): - if self._experiment is None: - return self._rank - else: - return self.experiment.rank + return self._rank @rank.setter def rank(self, value): - if self._experiment is None: - self._rank = value - else: - return self.experiment.rank + self._rank = value + if self._experiment is not None: + self.experiment.rank = value @property def version(self): diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 679878d3..04f40859 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -983,6 +983,19 @@ class Trainer(TrainerIO): ref_model.use_amp = self.use_amp ref_model.testing = self.testing + # link up experiment object + if self.logger is not None: + ref_model.logger = self.logger + + # save exp to get started + if hasattr(ref_model, "hparams"): + self.logger.log_hyperparams(ref_model.hparams) + + self.logger.save() + + if self.use_ddp or self.use_ddp2: + dist.barrier() + # set up checkpoint callback self.__configure_checkpoint_callback() @@ -1003,15 +1016,6 @@ class Trainer(TrainerIO): m = "weights_summary can be None, 'full' or 'top'" raise MisconfigurationException(m) - # link up experiment object - if self.logger is not None: - ref_model.logger = self.logger - - # save exp to get started - if hasattr(ref_model, "hparams"): - self.logger.log_hyperparams(ref_model.hparams) - self.logger.save() - # track model now. # if cluster resets state, the model will update with the saved weights self.model = model diff --git a/tests/debug.py b/tests/debug.py index e69afd80..6f95de1b 100644 --- a/tests/debug.py +++ b/tests/debug.py @@ -15,7 +15,8 @@ from torchvision.datasets import MNIST import numpy as np import pdb # from test_models import assert_ok_test_acc, load_model, \ -# clear_save_dir, get_test_tube_logger, get_hparams, init_save_dir +# clear_save_dir, get_test_tube_logger, get_hparams, init_save_dir, \ +# init_checkpoint_callback, reset_seed, set_random_master_port class CoolModel(pl.LightningModule): @@ -59,55 +60,54 @@ class CoolModel(pl.LightningModule): @pl.data_loader def test_dataloader(self): return DataLoader(MNIST('path/to/save', train=False), batch_size=32) -# + # # def main(): -# """ -# Make sure DDP + AMP continue training correctly -# :return: -# """ -# """ -# Make sure DDP2 works -# :return: -# """ +# reset_seed() +# set_random_master_port() +# # hparams = get_hparams() # model = LightningTestModel(hparams) # # save_dir = init_save_dir() # -# # logger file to get meta +# # exp file to get meta # logger = get_test_tube_logger(False) -# logger.log_hyperparams(hparams) -# logger.save() # -# # logger file to get weights -# checkpoint = ModelCheckpoint(save_dir) +# print(logger.debug) +# +# # exp file to get weights +# checkpoint = init_checkpoint_callback(logger) # # trainer_options = dict( -# show_progress_bar=True, +# show_progress_bar=False, # max_nb_epochs=1, # train_percent_check=0.4, # val_percent_check=0.2, # checkpoint_callback=checkpoint, # logger=logger, # gpus=[0, 1], -# distributed_backend='dp' +# distributed_backend='ddp' # ) # # # fit model # trainer = Trainer(**trainer_options) # result = trainer.fit(model) # +# exp = logger.experiment +# print(os.listdir(exp.get_data_path(exp.name, exp.version))) +# # # correct result and ok accuracy # assert result == 1, 'training failed to complete' -# pretrained_model = load_model(logger.experiment, save_dir, module_class=LightningTestModel) +# pretrained_model = load_model(logger.experiment, save_dir, +# module_class=LightningTestModel) # +# # run test set # new_trainer = Trainer(**trainer_options) # new_trainer.test(pretrained_model) # # # test we have good test accuracy -# assert_ok_test_acc(new_trainer) # clear_save_dir() - +# # if __name__ == '__main__': # main() diff --git a/tests/test_models.py b/tests/test_models.py index 3b41501b..ef911e9e 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -32,16 +32,68 @@ from pytorch_lightning.logging import TestTubeLogger from examples import LightningTemplateModel # generate a list of random seeds for each test +RANDOM_PORTS = list(np.random.randint(12000, 19000, 1000)) ROOT_SEED = 1234 torch.manual_seed(ROOT_SEED) np.random.seed(ROOT_SEED) RANDOM_SEEDS = list(np.random.randint(0, 10000, 1000)) -RANDOM_PORTS = list(np.random.randint(12000, 19000, 1000)) # ------------------------------------------------------------------------ # TESTS # ------------------------------------------------------------------------ +def test_running_test_pretrained_model_ddp(): + """Verify test() on pretrained model""" + if not can_run_gpu_test(): + return + + reset_seed() + set_random_master_port() + + hparams = get_hparams() + model = LightningTestModel(hparams) + + save_dir = init_save_dir() + + # exp file to get meta + logger = get_test_tube_logger(False) + + # exp file to get weights + checkpoint = init_checkpoint_callback(logger) + + trainer_options = dict( + show_progress_bar=False, + max_nb_epochs=1, + train_percent_check=0.4, + val_percent_check=0.2, + checkpoint_callback=checkpoint, + logger=logger, + gpus=[0, 1], + distributed_backend='ddp' + ) + + # fit model + trainer = Trainer(**trainer_options) + result = trainer.fit(model) + + exp = logger.experiment + print(os.listdir(exp.get_data_path(exp.name, exp.version))) + + # correct result and ok accuracy + assert result == 1, 'training failed to complete' + pretrained_model = load_model(logger.experiment, save_dir, + module_class=LightningTestModel) + + # run test set + new_trainer = Trainer(**trainer_options) + new_trainer.test(pretrained_model) + + [run_prediction(dataloader, pretrained_model) for dataloader in model.test_dataloader()] + + # test we have good test accuracy + clear_save_dir() + + def test_default_logger_callbacks_cpu_model(): """ Test each of the trainer options @@ -77,11 +129,11 @@ def test_lbfgs_cpu_model(): trainer_options = dict( max_nb_epochs=1, gradient_clip_val=1.0, - overfit_pct=0.20, + overfit_pct=0.30, print_nan_grads=True, show_progress_bar=False, weights_summary='top', - train_percent_check=0.2, + train_percent_check=0.3, val_percent_check=0.2 ) @@ -144,7 +196,8 @@ def test_dp_resume(): logger = get_test_tube_logger(debug=False) # exp file to get weights - checkpoint = ModelCheckpoint(save_dir) + # logger file to get weights + checkpoint = init_checkpoint_callback(logger) # add these to the trainer options trainer_options['logger'] = logger @@ -202,55 +255,6 @@ def test_dp_resume(): clear_save_dir() -def test_running_test_pretrained_model_ddp(): - """Verify test() on pretrained model""" - if not can_run_gpu_test(): - return - - reset_seed() - set_random_master_port() - - hparams = get_hparams() - model = LightningTestModel(hparams) - - save_dir = init_save_dir() - - # exp file to get meta - logger = get_test_tube_logger(False) - - # exp file to get weights - checkpoint = ModelCheckpoint(save_dir) - - trainer_options = dict( - show_progress_bar=False, - max_nb_epochs=1, - train_percent_check=0.4, - val_percent_check=0.2, - checkpoint_callback=checkpoint, - logger=logger, - gpus=[0, 1], - distributed_backend='ddp' - ) - - # fit model - trainer = Trainer(**trainer_options) - result = trainer.fit(model) - - # correct result and ok accuracy - assert result == 1, 'training failed to complete' - pretrained_model = load_model(logger.experiment, save_dir, - module_class=LightningTestModel) - - # run test set - new_trainer = Trainer(**trainer_options) - new_trainer.test(pretrained_model) - - [run_prediction(dataloader, pretrained_model) for dataloader in model.test_dataloader()] - - # test we have good test accuracy - clear_save_dir() - - def test_running_test_after_fitting(): """Verify test() on fitted model""" reset_seed() @@ -264,7 +268,7 @@ def test_running_test_after_fitting(): logger = get_test_tube_logger(False) # logger file to get weights - checkpoint = ModelCheckpoint(save_dir) + checkpoint = init_checkpoint_callback(logger) trainer_options = dict( show_progress_bar=False, @@ -305,7 +309,7 @@ def test_running_test_without_val(): logger = get_test_tube_logger(False) # logger file to get weights - checkpoint = ModelCheckpoint(save_dir) + checkpoint = init_checkpoint_callback(logger) trainer_options = dict( show_progress_bar=False, @@ -344,7 +348,7 @@ def test_running_test_pretrained_model(): logger = get_test_tube_logger(False) # logger file to get weights - checkpoint = ModelCheckpoint(save_dir) + checkpoint = init_checkpoint_callback(logger) trainer_options = dict( show_progress_bar=False, @@ -389,7 +393,7 @@ def test_running_test_pretrained_model_dp(): logger = get_test_tube_logger(False) # logger file to get weights - checkpoint = ModelCheckpoint(save_dir) + checkpoint = init_checkpoint_callback(logger) trainer_options = dict( show_progress_bar=True, @@ -1115,7 +1119,7 @@ def test_amp_gpu_ddp_slurm_managed(): logger = get_test_tube_logger(False) # exp file to get weights - checkpoint = ModelCheckpoint(save_dir) + checkpoint = init_checkpoint_callback(logger) # add these to the trainer options trainer_options['checkpoint_callback'] = checkpoint @@ -1459,7 +1463,7 @@ def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True): logger = get_test_tube_logger(False) # logger file to get weights - checkpoint = ModelCheckpoint(save_dir) + checkpoint = init_checkpoint_callback(logger) # add these to the trainer options trainer_options['checkpoint_callback'] = checkpoint @@ -1529,7 +1533,7 @@ def get_test_tube_logger(debug=True, version=None): # set up logger object without actually saving logs root_dir = os.path.dirname(os.path.realpath(__file__)) save_dir = os.path.join(root_dir, 'save_dir') - logger = TestTubeLogger(save_dir, name='test_tt_dir', debug=debug, version=version) + logger = TestTubeLogger(save_dir, name='lightning_logs', debug=False, version=version) return logger @@ -1558,10 +1562,11 @@ def load_model(exp, save_dir, module_class=LightningTemplateModel): # load trained model tags_path = exp.get_data_path(exp.name, exp.version) + checkpoint_folder = os.path.join(tags_path, 'checkpoints') tags_path = os.path.join(tags_path, 'meta_tags.csv') - checkpoints = [x for x in os.listdir(save_dir) if '.ckpt' in x] - weights_dir = os.path.join(save_dir, checkpoints[0]) + checkpoints = [x for x in os.listdir(checkpoint_folder) if '.ckpt' in x] + weights_dir = os.path.join(checkpoint_folder, checkpoints[0]) trained_model = module_class.load_from_metrics(weights_path=weights_dir, tags_csv=tags_path) @@ -1631,5 +1636,13 @@ def set_random_master_port(): os.environ['MASTER_PORT'] = str(port) +def init_checkpoint_callback(logger): + exp = logger.experiment + exp_path = exp.get_data_path(exp.name, exp.version) + ckpt_dir = os.path.join(exp_path, 'checkpoints') + checkpoint = ModelCheckpoint(ckpt_dir) + return checkpoint + + if __name__ == '__main__': pytest.main([__file__])