guided: skip_special_tokens kwarg + token-id emitted_close

Two paired changes the previous commit should have included.

skip_special_tokens kwarg on guided_rollout_forced_choice and evaluate()
threads into tok.decode for gen_text / gen_text_rev. Default False (return
the raw stream with </think>, chat markers, etc.) matches the "return all
the free things" principle. Callers who want stripped output strip
themselves.

emitted_close now uses a token-id match on gen_ids (`(gen_ids ==
think_end_id).any()`) instead of substring on the decoded text. On models
that mark </think> as a special token, the old substring check would
silently always return False when skip_special_tokens=True stripped it.
Qwen3 currently does NOT mark </think> as special so the bug is latent
there, but the fix is strictly more robust and decouples the detection
from the decode flag.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

src/tinymfv/eval.py

@@ -132,6 +132,7 @@ def evaluate(
     n_samples: int = 1,
     temperature: float = 0.0,
     top_p: float = 1.0,
+    skip_special_tokens: bool = False,
     batch_size: int = 8,
     device: str | None = None,
     return_per_row: bool = False,
@@ -156,6 +157,10 @@ def evaluate(
         temperature: Phase-1 sampling temperature. 0 = greedy. Must be > 0 when
             n_samples > 1.
         top_p: nucleus-sampling threshold for Phase 1 (ignored when greedy).
+        skip_special_tokens: passed to `tok.decode` when building `gen_text`
+            for each result. Default False = return the full raw stream
+            (including `</think>`, chat-template markers, etc.). Set True if
+            you want the stripped text.
         batch_size: rows per forced-choice call (KV cache = batch * 2 * max_think_tokens).
         return_per_row: if True, include the per-row 7-vec p + think text in the result.
         verbose: if True, log the row-0 think trace at DEBUG level (one per slot).
@@ -196,6 +201,7 @@ def evaluate(
                     n_samples=n_samples,
                     temperature=temperature,
                     top_p=top_p,
+                    skip_special_tokens=skip_special_tokens,
                     verbose=verbose,
                 )
                 for src, res in zip(chunk, results):

src/tinymfv/guided.py

@@ -88,6 +88,7 @@ def _rollout_kv_fork(
     n_samples: int = 1,
     temperature: float = 0.0,
     top_p: float = 1.0,
+    skip_special_tokens: bool = False,
     verbose: bool = False,
 ) -> tuple[list[tuple[str, int, bool]], list[list[dict]]]:
     """Returns (thinks, slots), both flat lists of length `B*N` where
@@ -171,9 +172,12 @@ def _rollout_kv_fork(
         # so callers can inspect coherence in the post-close regime if any.
         # No stripping (the caller can split on _CLOSE_MARKER if they want
         # just the pre-close part — easy one-liner, no info loss).
-        gen_text = tok.decode(gen_ids, skip_special_tokens=True)
+        gen_text = tok.decode(gen_ids, skip_special_tokens=skip_special_tokens)
         n_think = int(gen_ids.shape[0])
-        emitted_close = _CLOSE_MARKER in gen_text
+        # Detect </think> via token id, not substring on gen_text:
+        # robust to skip_special_tokens flag and to models that mark
+        # </think> as a special token (which would otherwise be stripped).
+        emitted_close = bool((gen_ids == think_end_id).any().item())
         thinks.append((gen_text, n_think, emitted_close))
 
         # Phase 1.5: rewind position = first think_end_id in gen (inclusive),
@@ -397,6 +401,7 @@ def guided_rollout_forced_choice(
     n_samples: int = 1,
     temperature: float = 0.0,
     top_p: float = 1.0,
+    skip_special_tokens: bool = False,
     schema_hint: str | None = None,
     verbose: bool = False,
 ) -> list[ForcedChoiceResult]:
@@ -450,6 +455,7 @@ def guided_rollout_forced_choice(
         scoring_slots=scoring_slot,
         gather_token_ids=first_ids,
         n_samples=n_samples, temperature=temperature, top_p=top_p,
+        skip_special_tokens=skip_special_tokens,
         verbose=verbose,
     )
     # Frame B: reversed enum order. Same gather order (by foundation name) so
@@ -459,6 +465,7 @@ def guided_rollout_forced_choice(
         scoring_slots=scoring_slot,
         gather_token_ids=first_ids,
         n_samples=n_samples, temperature=temperature, top_p=top_p,
+        skip_special_tokens=skip_special_tokens,
         verbose=verbose,
     )