From 5e4728b2a75a59d29fcc624c88708d72d407ad60 Mon Sep 17 00:00:00 2001 From: William Falcon Date: Thu, 1 Aug 2019 10:15:28 -0400 Subject: [PATCH] only proc 0 can submit a continuation slurm job --- pytorch_lightning/models/trainer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py index e60a0d95..1b9a2331 100644 --- a/pytorch_lightning/models/trainer.py +++ b/pytorch_lightning/models/trainer.py @@ -612,7 +612,7 @@ class Trainer(TrainerIO): # enable cluster checkpointing # also restores training state - if self.cluster is not None: # pragma: no cover + if self.cluster is not None and self.proc_rank == 0: # pragma: no cover self.enable_auto_hpc_walltime_manager() # ---------------------------