wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-07-19 11:20:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feature] add pythia and limit translation pair

Browse Source
This commit is contained in:
theblackcat102
2023-01-22 00:56:17 +00:00
parent 62a203fd8c
commit f5b2a34857
2 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/utils.py
+4
View File
@@ -25,6 +25,10 @@ def get_tokenizer(conf):
tokenizer.add_special_tokens({"pad_token": tokenizer.eos_token, "sep_token": "<|extratoken_100|>"})
elif "codegen" in conf.model_name:
tokenizer.add_special_tokens({"pad_token": "<|endoftext|>", "sep_token": "<|endoftext|>"})
elif "pythia" in conf.model_name:
tokenizer.add_special_tokens(
{"pad_token": "<|padding|>", "sep_token": "<|endoftext|>", "eos_token": "<|endoftext|>"}
)
additional_special_tokens = (
[]