From ef20a8350442daca964dd39b29a2a80ce206a981 Mon Sep 17 00:00:00 2001 From: wassname Date: Thu, 21 May 2026 03:35:46 +0000 Subject: [PATCH] guided: skip_special_tokens kwarg + token-id emitted_close Two paired changes the previous commit should have included. skip_special_tokens kwarg on guided_rollout_forced_choice and evaluate() threads into tok.decode for gen_text / gen_text_rev. Default False (return the raw stream with , chat markers, etc.) matches the "return all the free things" principle. Callers who want stripped output strip themselves. emitted_close now uses a token-id match on gen_ids (`(gen_ids == think_end_id).any()`) instead of substring on the decoded text. On models that mark as a special token, the old substring check would silently always return False when skip_special_tokens=True stripped it. Qwen3 currently does NOT mark as special so the bug is latent there, but the fix is strictly more robust and decouples the detection from the decode flag. Co-Authored-By: Claude Opus 4.7 --- src/tinymfv/eval.py | 6 ++++++ src/tinymfv/guided.py | 11 +++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/tinymfv/eval.py b/src/tinymfv/eval.py index 27fd77d..2c2c2a2 100644 --- a/src/tinymfv/eval.py +++ b/src/tinymfv/eval.py @@ -132,6 +132,7 @@ def evaluate( n_samples: int = 1, temperature: float = 0.0, top_p: float = 1.0, + skip_special_tokens: bool = False, batch_size: int = 8, device: str | None = None, return_per_row: bool = False, @@ -156,6 +157,10 @@ def evaluate( temperature: Phase-1 sampling temperature. 0 = greedy. Must be > 0 when n_samples > 1. top_p: nucleus-sampling threshold for Phase 1 (ignored when greedy). + skip_special_tokens: passed to `tok.decode` when building `gen_text` + for each result. Default False = return the full raw stream + (including ``, chat-template markers, etc.). Set True if + you want the stripped text. batch_size: rows per forced-choice call (KV cache = batch * 2 * max_think_tokens). return_per_row: if True, include the per-row 7-vec p + think text in the result. verbose: if True, log the row-0 think trace at DEBUG level (one per slot). @@ -196,6 +201,7 @@ def evaluate( n_samples=n_samples, temperature=temperature, top_p=top_p, + skip_special_tokens=skip_special_tokens, verbose=verbose, ) for src, res in zip(chunk, results): diff --git a/src/tinymfv/guided.py b/src/tinymfv/guided.py index 1b7cf5a..f19b69c 100644 --- a/src/tinymfv/guided.py +++ b/src/tinymfv/guided.py @@ -88,6 +88,7 @@ def _rollout_kv_fork( n_samples: int = 1, temperature: float = 0.0, top_p: float = 1.0, + skip_special_tokens: bool = False, verbose: bool = False, ) -> tuple[list[tuple[str, int, bool]], list[list[dict]]]: """Returns (thinks, slots), both flat lists of length `B*N` where @@ -171,9 +172,12 @@ def _rollout_kv_fork( # so callers can inspect coherence in the post-close regime if any. # No stripping (the caller can split on _CLOSE_MARKER if they want # just the pre-close part — easy one-liner, no info loss). - gen_text = tok.decode(gen_ids, skip_special_tokens=True) + gen_text = tok.decode(gen_ids, skip_special_tokens=skip_special_tokens) n_think = int(gen_ids.shape[0]) - emitted_close = _CLOSE_MARKER in gen_text + # Detect via token id, not substring on gen_text: + # robust to skip_special_tokens flag and to models that mark + # as a special token (which would otherwise be stripped). + emitted_close = bool((gen_ids == think_end_id).any().item()) thinks.append((gen_text, n_think, emitted_close)) # Phase 1.5: rewind position = first think_end_id in gen (inclusive), @@ -397,6 +401,7 @@ def guided_rollout_forced_choice( n_samples: int = 1, temperature: float = 0.0, top_p: float = 1.0, + skip_special_tokens: bool = False, schema_hint: str | None = None, verbose: bool = False, ) -> list[ForcedChoiceResult]: @@ -450,6 +455,7 @@ def guided_rollout_forced_choice( scoring_slots=scoring_slot, gather_token_ids=first_ids, n_samples=n_samples, temperature=temperature, top_p=top_p, + skip_special_tokens=skip_special_tokens, verbose=verbose, ) # Frame B: reversed enum order. Same gather order (by foundation name) so @@ -459,6 +465,7 @@ def guided_rollout_forced_choice( scoring_slots=scoring_slot, gather_token_ids=first_ids, n_samples=n_samples, temperature=temperature, top_p=top_p, + skip_special_tokens=skip_special_tokens, verbose=verbose, )