deactivtae samples mixing by default

model/supervised_finetuning/configs/config.yaml

@@ -48,6 +48,7 @@ defaults:
   poly_eps: 1.0
   fuse_gelu: true
   log_wandb: true
+  samples_mixing: false # uses collator that mixes samples in the batch to create a single sample with possible multiple tasks within
 
 galactica-125m:
   learning_rate: 5e-5

model/supervised_finetuning/utils.py

@@ -238,7 +238,11 @@ def get_dataset(conf, tokenizer):
     train = ConcatDataset(train_datasets)
 
     collate_fn = DialogueDataCollator(tokenizer, max_length=conf.max_length)
-    train_collate_fn = TrainDialogueDataCollator(tokenizer, max_length=conf.max_length)
+
+    train_collate_fn = (
+        TrainDialogueDataCollator(tokenizer, max_length=conf.max_length) if conf.samples_mixing else collate_fn
+    )
+
     return train, evals, collate_fn, train_collate_fn