added multi-node locked ip search

2026-09-10 12:21:57 +08:00 · 2019-07-08 12:45:20 -04:00
parent 6462cab351
commit f4ab46e1c9
1 changed files with 1 additions and 0 deletions
@@ -383,6 +383,7 @@ class Trainer(TrainerIO):
        # now everyone waits until the file has world_size entries
        for i in range(0, 120):
            sleep(1.0)
+            print('sleeping...')
            if os.path.exists(ip_file):
                lines = list(open(file=ip_file, mode='r'))
                if len(lines) == world_size: