From 49e04de5ac52db888b2070141fcec0c4ad73a6f6 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Tue, 8 Oct 2019 17:11:47 -0400 Subject: [PATCH] Ports (#338) * remove os.exit from early stopping * remove os.exit from early stopping * fixed weight summary * fixed weight summary * fixed weight summary * fixed weight summary * fixed weight summary * fixed weight summary * fixed weight summary --- pytorch_lightning/trainer/trainer.py | 10 ++++++--- tests/test_models.py | 31 ++++++++++++++++------------ 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/pytorch_lightning/trainer/trainer.py b/pytorch_lightning/trainer/trainer.py index 46de7a00..352a4283 100644 --- a/pytorch_lightning/trainer/trainer.py +++ b/pytorch_lightning/trainer/trainer.py @@ -116,7 +116,7 @@ class Trainer(TrainerIO): :param distributed_backend: str. Options: 'dp', 'ddp', 'ddp2'. :param use_amp: Bool. If true uses apex for 16bit precision :param print_nan_grads: Bool. Prints nan gradients - :param weights_summary: str. Options: 'full', 'top'. + :param weights_summary: str. Options: 'full', 'top', None to not print. :param weights_save_path: Bool. Where to save weights if on cluster :param amp_level: str. Check nvidia docs for level :param nb_sanity_val_steps: int. How many val steps before a full train loop. @@ -982,8 +982,12 @@ class Trainer(TrainerIO): self.__layout_bookeeping() # print model summary - if self.proc_rank == 0 and self.weights_summary in ['full', 'top']: - ref_model.summarize(mode=self.weights_summary) + if self.proc_rank == 0 and self.weights_summary is not None: + if self.weights_summary in ['full', 'top']: + ref_model.summarize(mode=self.weights_summary) + else: + m = "weights_summary can be None, 'full' or 'top'" + raise MisconfigurationException(m) # link up experiment object if self.logger is not None: diff --git a/tests/test_models.py b/tests/test_models.py index 581469cb..5c517a51 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -36,6 +36,7 @@ ROOT_SEED = 1234 torch.manual_seed(ROOT_SEED) np.random.seed(ROOT_SEED) RANDOM_SEEDS = list(np.random.randint(0, 10000, 1000)) +RANDOM_PORTS = list(np.random.randint(12000, 19000, 1000)) # ------------------------------------------------------------------------ @@ -79,6 +80,7 @@ def test_lbfgs_cpu_model(): overfit_pct=0.20, print_nan_grads=True, show_progress_bar=False, + weights_summary='top', train_percent_check=0.2, val_percent_check=0.2 ) @@ -99,9 +101,8 @@ def test_multi_gpu_model_ddp2(): if not can_run_gpu_test(): return - os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0]) - reset_seed() + set_random_master_port() model, hparams = get_model() trainer_options = dict( @@ -110,7 +111,7 @@ def test_multi_gpu_model_ddp2(): train_percent_check=0.4, val_percent_check=0.2, gpus=2, - print_weights_summary=False, + weights_summary=None, distributed_backend='ddp2' ) @@ -208,6 +209,7 @@ def test_running_test_pretrained_model_ddp(): return reset_seed() + set_random_master_port() hparams = get_hparams() model = LightningTestModel(hparams) @@ -510,8 +512,8 @@ def test_multi_gpu_model_ddp(): if not can_run_gpu_test(): return - os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0]) reset_seed() + set_random_master_port() model, hparams = get_model() trainer_options = dict( @@ -891,9 +893,8 @@ def test_amp_gpu_ddp(): if not can_run_gpu_test(): return - os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0]) - reset_seed() + set_random_master_port() hparams = get_hparams() model = LightningTestModel(hparams) @@ -1114,12 +1115,12 @@ def test_amp_gpu_ddp_slurm_managed(): if not can_run_gpu_test(): return - # simulate setting slurm flags - os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0]) - os.environ['SLURM_LOCALID'] = str(0) - reset_seed() + # simulate setting slurm flags + set_random_master_port() + os.environ['SLURM_LOCALID'] = str(0) + hparams = get_hparams() model = LightningTestModel(hparams) @@ -1350,9 +1351,8 @@ def test_ddp_sampler_error(): if not can_run_gpu_test(): return - os.environ['MASTER_PORT'] = str(np.random.randint(12000, 19000, 1)[0]) - reset_seed() + set_random_master_port() hparams = get_hparams() model = LightningTestModel(hparams, force_remove_distributed_sampler=True) @@ -1506,7 +1506,7 @@ def run_gpu_model_test(trainer_options, model, hparams, on_gpu=True): # test new model accuracy [run_prediction(dataloader, pretrained_model) for dataloader in model.test_dataloader()] - if trainer.use_ddp: + if trainer.use_ddp or trainer.use_ddp2: # on hpc this would work fine... but need to hack it for the purpose of the test trainer.model = pretrained_model trainer.optimizers, trainer.lr_schedulers = pretrained_model.configure_optimizers() @@ -1654,5 +1654,10 @@ def reset_seed(): np.random.seed(SEED) +def set_random_master_port(): + port = RANDOM_PORTS.pop() + os.environ['MASTER_PORT'] = str(port) + + if __name__ == '__main__': pytest.main([__file__])