fix tokenizer matching and add tests

This commit is contained in:
jack.butler
2023-02-09 18:47:38 +00:00
parent 35a2c58565
commit 090c5cbcc2
2 changed files with 21 additions and 2 deletions
+1 -1
View File
@@ -37,7 +37,7 @@ TOKENIZER_CONFIGS = {
def match_tokenizer_name(model_name: str) -> TokenizerConfig:
"""Match a partial model name to a tokenizer configuration"""
tokenizer_config_matches = [config for name, config in TOKENIZER_CONFIGS.items() if name in model_name]
tokenizer_config_matches = [config for name, config in TOKENIZER_CONFIGS.items() if model_name in name]
if not tokenizer_config_matches:
raise ValueError(f"Cannot find any tokeniser configuration to match {model_name=}")
elif 1 < len(tokenizer_config_matches):