From c0e3cb784acf96dcd0f80191d0ad96d9e429e475 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Mon, 8 Jul 2019 12:58:47 -0400 Subject: [PATCH] added multi-node locked ip search --- pytorch_lightning/models/trainer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py index f91e7859..acb79dea 100644 --- a/pytorch_lightning/models/trainer.py +++ b/pytorch_lightning/models/trainer.py @@ -303,14 +303,16 @@ class Trainer(TrainerIO): # when GPU is called, spawn off a single worker for each gpu if self.on_gpu: + # every process writes their process id + ip to a shared file + my_ip = subprocess.run(['hostname', '-I'], stdout=subprocess.PIPE).stdout.decode('utf-8') + my_ip = my_ip.split(' ')[0] + + test_name = f'{os.getpid()}_{my_ip}' + ip_dir = os.path.join(self.exp_save_path, '.ips', test_name) + os.makedirs(ip_dir, exist_ok=True) + rank = 0 self.experiment = self.experiment.get_meta_copy() - - # remove any ip tables we saved - ip_table_file = os.path.join(self.exp_save_path, '.ip_meta') - if os.path.exists(ip_table_file): - os.remove(ip_table_file) - mp.spawn(self.dp_train, nprocs=len(self.data_parallel_device_ids), args=(rank, model)) else: self.__run_pretrain_routine(model)