set&get llm internal tokenizer instead of the TokenizerGroup (#2741)

Co-authored-by: shujunhua1 <shujunhua1@jd.com>
This commit is contained in:
dancingpipi
2024-02-04 14:25:36 -08:00
committed by GitHub
co-authored by shujunhua1
parent 5ed704ec8c
commit 51cd22ce56
+2 -2
View File
@@ -111,13 +111,13 @@ class LLM:
def get_tokenizer(
self) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
return self.llm_engine.tokenizer
return self.llm_engine.tokenizer.tokenizer
def set_tokenizer(
self,
tokenizer: Union[PreTrainedTokenizer, PreTrainedTokenizerFast],
) -> None:
self.llm_engine.tokenizer = tokenizer
self.llm_engine.tokenizer.tokenizer = tokenizer
def generate(
self,