fixed nccl init

2026-09-11 12:31:23 +08:00 · 2019-07-12 14:36:00 -04:00
parent 5ba0a2ed48
commit 369174c4d3
1 changed files with 2 additions and 0 deletions
@@ -325,6 +325,8 @@ class Trainer(TrainerIO):
            node_rank = 0

        # recover original exp before went into process
+        # init in write mode only on proc 0
+        self.experiment.debug = self.proc_rank > 0
        self.experiment = self.experiment.get_non_ddp_exp()

        # show progbar only on prog_rank 0