diff --git a/model/supervised_finetuning/custom_datasets/qa_datasets.py b/model/supervised_finetuning/custom_datasets/qa_datasets.py index cd4d742a..f126a270 100644 --- a/model/supervised_finetuning/custom_datasets/qa_datasets.py +++ b/model/supervised_finetuning/custom_datasets/qa_datasets.py @@ -314,7 +314,9 @@ class JokeExplaination(Dataset): for line in f: data = json.loads(line) joke = data["joke"] - explanation = data["explanation"] + # DO NOT change this + # its the data that had syntax error + explanation = data["explaination"] self.pairs.append((joke, explanation)) if len(question) > 0 and len(answer) > 0: diff --git a/model/supervised_finetuning/trainer.py b/model/supervised_finetuning/trainer.py index 02367814..ce80830b 100644 --- a/model/supervised_finetuning/trainer.py +++ b/model/supervised_finetuning/trainer.py @@ -1,7 +1,7 @@ import argparse from distutils.util import strtobool from functools import partial -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import bitsandbytes import datasets @@ -39,6 +39,7 @@ class SFTTrainer(Trainer): sampler: torch.utils.data.sampler.Sampler = None, loss_function: str = "CrossEntropyLoss", poly_eps: float = 1.0, + train_collate_fn: Callable = None, **kwargs, ): super().__init__(model, args, **kwargs) diff --git a/model/supervised_finetuning/utils.py b/model/supervised_finetuning/utils.py index 86e48a52..43377bc9 100644 --- a/model/supervised_finetuning/utils.py +++ b/model/supervised_finetuning/utils.py @@ -126,7 +126,14 @@ def get_tokenizer(conf) -> transformers.AutoTokenizer: if tokenizer_config.special_tokens: if "GPT-JT" in conf.model_name: tokenizer_config.special_tokens.pad_token = tokenizer.eos_token - tokenizer.add_special_tokens(tokenizer_config.special_tokens) + # SpecialTokens : latest in 4.25, 4.26 + tokenizer.add_special_tokens( + { + "pad_token": tokenizer_config.special_tokens.pad_token, + "eos_token": tokenizer_config.special_tokens.eos_token, + "sep_token": tokenizer_config.special_tokens.sep_token, + } + ) additional_special_tokens = ( []