mirror of
https://github.com/wassname/alignment-handbook.git
synced 2026-06-27 18:22:17 +08:00
fix trust_remote_code for tokenizer in model_utils.py (#140)
`trust_remote_code` option is only added to models, adding it to tokenizers to be consistent, which will also fix the error when the tokenizer is loaded from the remote repo
This commit is contained in:
@@ -71,6 +71,7 @@ def get_tokenizer(
|
||||
if model_args.tokenizer_name_or_path is None
|
||||
else model_args.tokenizer_name_or_path,
|
||||
revision=model_args.model_revision,
|
||||
trust_remote_code=model_args.trust_remote_code,
|
||||
)
|
||||
if tokenizer.pad_token_id is None:
|
||||
tokenizer.pad_token_id = tokenizer.eos_token_id
|
||||
|
||||
Reference in New Issue
Block a user