wassname/Open-Assistant
1
0
Fork 0
mirror of https://github.com/wassname/Open-Assistant.git synced 2026-06-27 16:10:30 +08:00
Code Issues Packages Projects Releases Wiki Activity

add test for tokenizer matching behaviour

Browse Source
This commit is contained in:
jack.butler
2023-02-10 09:47:20 +00:00
parent 2fbf2fa457
commit 24b07523aa
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model/supervised_finetuning/tests/test_utils.py
+4 -4
View File
@@ -11,15 +11,15 @@ def test_tokenizer():
def test_tokenizer_successful_match():
for config_name, config in TOKENIZER_CONFIGS:
for config_name, config in TOKENIZER_CONFIGS.items():
found_config = match_tokenizer_name(config_name)
assert found_config == config
def test_tokenizer_partial_match():
for config_name, config in TOKENIZER_CONFIGS:
found_config = match_tokenizer_name(config_name[: len(config_name) - 1])
assert found_config == config
for config_name in ["facebook/galactica-1.3b", "togethercomputer/GPT-JT-6B-v1", "Salesforce/codegen-2B-multi"]:
found_config = match_tokenizer_name(config_name)
assert found_config
def test_tokenizer_failed_match():