fix trust_remote_code for tokenizer in model_utils.py (#140)

`trust_remote_code` option is only added to models, adding it to tokenizers to be consistent, which will also fix the error when the tokenizer is loaded from the remote repo
This commit is contained in:
Qingqing Cao
2024-03-27 11:31:45 -07:00
committed by GitHub
parent ba7e0e4fca
commit 8497caeaf1
+1
View File
@@ -71,6 +71,7 @@ def get_tokenizer(
if model_args.tokenizer_name_or_path is None
else model_args.tokenizer_name_or_path,
revision=model_args.model_revision,
trust_remote_code=model_args.trust_remote_code,
)
if tokenizer.pad_token_id is None:
tokenizer.pad_token_id = tokenizer.eos_token_id