wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-08-12 11:40:44 +08:00
Code Issues Packages Projects Releases Wiki Activity

[fix] Fix other PR merge bug

Browse Source
This commit is contained in:
theblackcat102
2023-02-11 01:53:37 +00:00
parent bcebbbc49c
commit 9e69117ead
3 changed files with 13 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/utils.py
+8 -1
View File
@@ -126,7 +126,14 @@ def get_tokenizer(conf) -> transformers.AutoTokenizer:
if tokenizer_config.special_tokens:
if "GPT-JT" in conf.model_name:
tokenizer_config.special_tokens.pad_token = tokenizer.eos_token
tokenizer.add_special_tokens(tokenizer_config.special_tokens)
# SpecialTokens : latest in 4.25, 4.26
tokenizer.add_special_tokens(
{
"pad_token": tokenizer_config.special_tokens.pad_token,
"eos_token": tokenizer_config.special_tokens.eos_token,
"sep_token": tokenizer_config.special_tokens.sep_token,
}
)
additional_special_tokens = (
[]