only proc 0 can submit a continuation slurm job

2026-09-11 12:31:23 +08:00 · 2019-08-01 10:15:28 -04:00
parent 93ddf555dd
commit 5e4728b2a7
1 changed files with 1 additions and 1 deletions
@@ -612,7 +612,7 @@ class Trainer(TrainerIO):

        # enable cluster checkpointing
        # also restores training state
-        if self.cluster is not None:  # pragma: no cover
+        if self.cluster is not None and self.proc_rank == 0:  # pragma: no cover
            self.enable_auto_hpc_walltime_manager()

        # ---------------------------