wassname/prob_jsonformer
1
0
Fork 0
mirror of https://github.com/wassname/prob_jsonformer.git synced 2026-09-10 12:37:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

, skip_special_tokens=True

Browse Source
This commit is contained in:
wassname
2024-05-11 11:40:34 +08:00
parent f50076736f
commit c0e929b814
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
prob_jsonformer/choice_tree.py
+1 -1
View File
@@ -38,7 +38,7 @@ def _choice_tree(
next_choices = get_valid_next_choices(choices_tokens, current_tokens)
if len(next_choices) == 0:
s = tokenizer.decode(current_tokens)
s = tokenizer.decode(current_tokens, skip_special_tokens=True)
r = dict(prob=prob, choice=s)
yield r
else: