mirror of
https://github.com/wassname/Open-Assistant.git
synced 2026-08-12 11:40:44 +08:00
fix tokenizer matching and add tests
This commit is contained in:
@@ -37,7 +37,7 @@ TOKENIZER_CONFIGS = {
|
||||
|
||||
def match_tokenizer_name(model_name: str) -> TokenizerConfig:
|
||||
"""Match a partial model name to a tokenizer configuration"""
|
||||
tokenizer_config_matches = [config for name, config in TOKENIZER_CONFIGS.items() if name in model_name]
|
||||
tokenizer_config_matches = [config for name, config in TOKENIZER_CONFIGS.items() if model_name in name]
|
||||
if not tokenizer_config_matches:
|
||||
raise ValueError(f"Cannot find any tokeniser configuration to match {model_name=}")
|
||||
elif 1 < len(tokenizer_config_matches):
|
||||
|
||||
Reference in New Issue
Block a user