fixed nccl init

This commit is contained in:
William Falcon
2019-07-12 15:59:33 -04:00
parent d99b121379
commit c244599ae8
-3
View File
@@ -375,9 +375,6 @@ class Trainer(TrainerIO):
:param tries:
:return:
"""
# hack to get nccl to stop throwing error... seems to be an nccl race condition
if self.proc_rank > 0:
sleep(10.0)
root_node = os.environ['SLURM_NODELIST'].split(' ')[0]
os.environ['MASTER_ADDR'] = root_node