From 9e69117ead4c1724e2915fa2e99607a8b5213f35 Mon Sep 17 00:00:00 2001 From: theblackcat102 Date: Sat, 11 Feb 2023 01:53:37 +0000 Subject: [PATCH] [fix] Fix other PR merge bug --- .../supervised_finetuning/custom_datasets/qa_datasets.py | 4 +++- model/supervised_finetuning/trainer.py | 3 ++- model/supervised_finetuning/utils.py | 9 ++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/model/supervised_finetuning/custom_datasets/qa_datasets.py b/model/supervised_finetuning/custom_datasets/qa_datasets.py index cd4d742a..f126a270 100644 --- a/model/supervised_finetuning/custom_datasets/qa_datasets.py +++ b/model/supervised_finetuning/custom_datasets/qa_datasets.py @@ -314,7 +314,9 @@ class JokeExplaination(Dataset): for line in f: data = json.loads(line) joke = data["joke"] - explanation = data["explanation"] + # DO NOT change this + # its the data that had syntax error + explanation = data["explaination"] self.pairs.append((joke, explanation)) if len(question) > 0 and len(answer) > 0: diff --git a/model/supervised_finetuning/trainer.py b/model/supervised_finetuning/trainer.py index 02367814..ce80830b 100644 --- a/model/supervised_finetuning/trainer.py +++ b/model/supervised_finetuning/trainer.py @@ -1,7 +1,7 @@ import argparse from distutils.util import strtobool from functools import partial -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union import bitsandbytes import datasets @@ -39,6 +39,7 @@ class SFTTrainer(Trainer): sampler: torch.utils.data.sampler.Sampler = None, loss_function: str = "CrossEntropyLoss", poly_eps: float = 1.0, + train_collate_fn: Callable = None, **kwargs, ): super().__init__(model, args, **kwargs) diff --git a/model/supervised_finetuning/utils.py b/model/supervised_finetuning/utils.py index 86e48a52..43377bc9 100644 --- a/model/supervised_finetuning/utils.py +++ b/model/supervised_finetuning/utils.py @@ -126,7 +126,14 @@ def get_tokenizer(conf) -> transformers.AutoTokenizer: if tokenizer_config.special_tokens: if "GPT-JT" in conf.model_name: tokenizer_config.special_tokens.pad_token = tokenizer.eos_token - tokenizer.add_special_tokens(tokenizer_config.special_tokens) + # SpecialTokens : latest in 4.25, 4.26 + tokenizer.add_special_tokens( + { + "pad_token": tokenizer_config.special_tokens.pad_token, + "eos_token": tokenizer_config.special_tokens.eos_token, + "sep_token": tokenizer_config.special_tokens.sep_token, + } + ) additional_special_tokens = ( []