[fix] Fix other PR merge bug

model/supervised_finetuning/custom_datasets/qa_datasets.py

@@ -314,7 +314,9 @@ class JokeExplaination(Dataset):
             for line in f:
                 data = json.loads(line)
                 joke = data["joke"]
-                explanation = data["explanation"]
+                # DO NOT change this
+                # its the data that had syntax error
+                explanation = data["explaination"]
                 self.pairs.append((joke, explanation))
 
         if len(question) > 0 and len(answer) > 0:

model/supervised_finetuning/trainer.py

@@ -1,7 +1,7 @@
 import argparse
 from distutils.util import strtobool
 from functools import partial
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import bitsandbytes
 import datasets
@@ -39,6 +39,7 @@ class SFTTrainer(Trainer):
         sampler: torch.utils.data.sampler.Sampler = None,
         loss_function: str = "CrossEntropyLoss",
         poly_eps: float = 1.0,
+        train_collate_fn: Callable = None,
         **kwargs,
     ):
         super().__init__(model, args, **kwargs)

model/supervised_finetuning/utils.py

@@ -126,7 +126,14 @@ def get_tokenizer(conf) -> transformers.AutoTokenizer:
     if tokenizer_config.special_tokens:
         if "GPT-JT" in conf.model_name:
             tokenizer_config.special_tokens.pad_token = tokenizer.eos_token
-        tokenizer.add_special_tokens(tokenizer_config.special_tokens)
+        # SpecialTokens : latest in 4.25, 4.26
+        tokenizer.add_special_tokens(
+            {
+                "pad_token": tokenizer_config.special_tokens.pad_token,
+                "eos_token": tokenizer_config.special_tokens.eos_token,
+                "sep_token": tokenizer_config.special_tokens.sep_token,
+            }
+        )
 
     additional_special_tokens = (
         []