From c0e929b81439723b4b7bb467727951f3f0a249b2 Mon Sep 17 00:00:00 2001 From: wassname Date: Sat, 11 May 2024 11:40:34 +0800 Subject: [PATCH] , skip_special_tokens=True --- prob_jsonformer/choice_tree.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prob_jsonformer/choice_tree.py b/prob_jsonformer/choice_tree.py index 83ee70a..127aa24 100644 --- a/prob_jsonformer/choice_tree.py +++ b/prob_jsonformer/choice_tree.py @@ -38,7 +38,7 @@ def _choice_tree( next_choices = get_valid_next_choices(choices_tokens, current_tokens) if len(next_choices) == 0: - s = tokenizer.decode(current_tokens) + s = tokenizer.decode(current_tokens, skip_special_tokens=True) r = dict(prob=prob, choice=s) yield r else: