From 8497caeaf1e4cbd9b0650eb730729e608d1d2d73 Mon Sep 17 00:00:00 2001 From: Qingqing Cao Date: Wed, 27 Mar 2024 11:31:45 -0700 Subject: [PATCH] fix trust_remote_code for tokenizer in model_utils.py (#140) `trust_remote_code` option is only added to models, adding it to tokenizers to be consistent, which will also fix the error when the tokenizer is loaded from the remote repo --- src/alignment/model_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/alignment/model_utils.py b/src/alignment/model_utils.py index 6f59256..14cd9cb 100644 --- a/src/alignment/model_utils.py +++ b/src/alignment/model_utils.py @@ -71,6 +71,7 @@ def get_tokenizer( if model_args.tokenizer_name_or_path is None else model_args.tokenizer_name_or_path, revision=model_args.model_revision, + trust_remote_code=model_args.trust_remote_code, ) if tokenizer.pad_token_id is None: tokenizer.pad_token_id = tokenizer.eos_token_id