added multi-node locked ip search

2026-09-11 12:31:23 +08:00 · 2019-07-08 12:39:49 -04:00
parent 1d8d1f72a9
commit 6462cab351
1 changed files with 6 additions and 0 deletions
@@ -305,6 +305,12 @@ class Trainer(TrainerIO):
        if self.on_gpu:
            rank = 0
            self.experiment = self.experiment.get_meta_copy()
+
+            # remove any ip tables we saved
+            ip_table_file = os.path.join(self.exp_save_path, '.ip_meta')
+            if os.path.exists(ip_table_file):
+                os.remove(ip_table_file)
+
            mp.spawn(self.dp_train, nprocs=len(self.data_parallel_device_ids), args=(rank, model))
        else:
            self.__run_pretrain_routine(model)