wassname/pytorch-lightning
1
0
Fork 0
mirror of https://github.com/wassname/pytorch-lightning.git synced 2026-09-10 12:21:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

added multi-node locked ip search

Browse Source
This commit is contained in:
William Falcon
2019-07-08 12:54:20 -04:00
parent c2987d3b40
commit 615711131e
1 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pytorch_lightning/models/trainer.py
+6
View File
@@ -364,6 +364,8 @@ class Trainer(TrainerIO):
if nb_gpu_nodes == 1:
return 0, '127.0.0.1'
# on multi-node, every node rank > 0 waits until rank 0
# saves the ip to disk
ip_file = os.path.join(ip_file_dir, '.ip_meta')
@@ -372,6 +374,10 @@ class Trainer(TrainerIO):
my_ip = subprocess.run(['hostname', '-I'], stdout=subprocess.PIPE).stdout.decode('utf-8')
my_ip = my_ip.split(' ')[0]
test_name = f'{os.getpid()}_{my_ip}'
ip_dir = os.path.join(ip_file_dir, '.ips', test_name)
os.makedirs(ip_dir, exist_ok=True)
# save the ip to the file
# block file so only one process can access at a time
ip_dir = os.path.join(ip_file_dir, '.ips', my_ip)