From 660b966a8f1b1c3cbcb61e61fdcfd9e53ee91966 Mon Sep 17 00:00:00 2001
From: William Falcon <falc@fb.com>
Date: Mon, 8 Jul 2019 13:04:52 -0400
Subject: [PATCH] added multi-node locked ip search

---
 pytorch_lightning/models/trainer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py
index a3cc35bc..e7142880 100644
--- a/pytorch_lightning/models/trainer.py
+++ b/pytorch_lightning/models/trainer.py
@@ -382,7 +382,7 @@ class Trainer(TrainerIO):
         os.makedirs(ip_dir, exist_ok=True)
 
         # now everyone waits until the file has world_size entries
-        for i in range(0, 120):
+        for i in range(0, 120*10):
             sleep(1.0)
             nb_folders = [x for x in os.listdir(os.path.join(ip_file_dir, '.ips')) if '.' in x]
             if nb_folders == nb_gpu_nodes: