From a7bb731a1defeb91fc7423287a69eaa8a25bde75 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Fri, 12 Jul 2019 13:19:10 -0400 Subject: [PATCH] testing env init --- pytorch_lightning/models/trainer.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py index 289efe3b..57581513 100644 --- a/pytorch_lightning/models/trainer.py +++ b/pytorch_lightning/models/trainer.py @@ -337,17 +337,7 @@ class Trainer(TrainerIO): # set up server using proc 0's ip address # try to init for 20 times at max in case ports are taken # where to store ip_table - ip_file_dir = os.path.join(self.cluster.log_path, 'ip_tables') - - # the first gpu in the world becomes the host - # this is based on its global rank - # it communicates its ip by saving an ip_table to the slurm cluster logging dir - # every other process waits for this ip to appear before continuing - ip_table_name = f'ip_meta_' + os.environ['SLURM_JOB_ID'] - ip_file = os.path.join(ip_file_dir, ip_table_name) - os.makedirs(ip_file_dir, exist_ok=True) - - self.__init_tcp_connection(ip_file_dir) + self.__init_tcp_connection() # CHOOSE OPTIMIZER # filter out the weights that were done on gpu so we can load on good old cpus @@ -379,7 +369,7 @@ class Trainer(TrainerIO): try: root_node = os.environ['SLURM_NODELIST'].split(' ')[0] os.environ['MASTER_ADDR'] = root_node - os.environ['MASTER_PORT'] = '12006' + os.environ['MASTER_PORT'] = f'{port}' dist.init_process_group("nccl", rank=self.proc_rank, world_size=self.world_size) except RuntimeError as e: