wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-07-23 12:40:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

merge deepspeed

Browse Source
This commit is contained in:
Sotirios Anagnostidis
2023-01-06 21:28:26 +01:00
parent f2b125cbe3
commit 88ee3b3264
7 changed files with 181 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/utils.py
+4 -6
View File
@@ -15,6 +15,10 @@ def get_tokenizer(conf):
if "galactica" in conf.model_name:
tokenizer.add_special_tokens({"pad_token": "<pad>", "eos_token": "</s>"})
elif "GPT-JT" in conf.model_name:
tokenizer.add_special_tokens({"pad_token": tokenizer.eos_token, "sep_token": "<|extratoken_100|>"})
elif "codegen" in conf.model_name:
tokenizer.add_special_tokens({"pad_token": "<|endoftext|>", "sep_token": "<|endoftext|>"})
additional_special_tokens = (
[]
@@ -29,12 +33,6 @@ def get_tokenizer(conf):
def get_model(conf, tokenizer):
if not any([x in conf.model_name.lower() for x in SUPPORTED_MODELS]):
raise ValueError(
f"Model {conf.model_name} not supported. Supported models: {SUPPORTED_MODELS}. "
"To include more make sure the masking is done correctly... (decoder only supported for now)"
)
model = get_specific_model(conf.model_name, conf.cache_dir, conf.quantization)
if len(tokenizer) != model.get_input_embeddings().num_embeddings: