wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-07-26 13:07:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1398 from jackapbutler/fix-tokenizer-match

Browse Source 
Add tests and update docstring to tokenizer matching
This commit is contained in:
sanagnos
2023-02-10 11:37:00 +01:00
committed by GitHub
parent f679109fe7 24b07523aa
commit 4dd0d67e9c
2 changed files with 24 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/utils.py
+4 -1
View File
@@ -106,7 +106,10 @@ TOKENIZER_CONFIGS = {
def match_tokenizer_name(model_name: str) -> TokenizerConfig:
"""Match a partial model name to a tokenizer configuration"""
"""
Match a partial model name to a tokenizer configuration
i.e. model_name `Salesforce/codegen-2B-multi` has config name `codegen`
"""
tokenizer_config_matches = [config for name, config in TOKENIZER_CONFIGS.items() if name in model_name]
if not tokenizer_config_matches:
raise ValueError(f"Cannot find any tokeniser configuration to match {model_name=}")