Run on_validation_end only on main process in DDP (#1125)

Co-authored-by: xingzhaolee <xingzhaolee@users.noreply.github.com>

pytorch_lightning/callbacks/model_checkpoint.py

@@ -175,6 +175,10 @@ class ModelCheckpoint(Callback):
         return filepath
 
     def on_validation_end(self, trainer, pl_module):
+        # only run on main process
+        if trainer.proc_rank != 0:
+            return
+
         metrics = trainer.callback_metrics
         epoch = trainer.current_epoch
         self.epochs_since_last_check += 1