From 7c0e3715dde58eb9797fd1eb8e52d8bf487d4b07 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Mon, 8 Jul 2019 14:22:09 -0400 Subject: [PATCH] using slurm flag to fine node nb --- pytorch_lightning/models/trainer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py index 4478081a..69c1ac08 100644 --- a/pytorch_lightning/models/trainer.py +++ b/pytorch_lightning/models/trainer.py @@ -358,7 +358,8 @@ class Trainer(TrainerIO): # the first gpu in the world becomes the host # this is based on its global rank - # saves the ip to disk + # it communicates its ip by saving an ip_table to the slurm cluster logging dir + # every other process waits for this ip to appear before continuing ip_table_name = f'.ip_meta_' + os.environ['SLURM_JOB_ID'] ip_file = os.path.join(ip_file_dir, ip_table_name) os.makedirs(ip_file_dir, exist_ok=True)