demo: JSON-object coherence probe replaces rigged forced-digit pmass

The old rubric pmass was softmax mass on digit tokens at the hand-fed `{"ans": `
slot, so it was ~always 1 (the prefix forces a digit even from a fried model) --
a blind coherence guard. Replace with the users design: free-generate
{"ans":N,"why":str,"2+2":M}, gate coherence on valid-JSON AND 2+2==4. GPU UATs:
object stays valid at |C|<=0.5, breaks at |C|>=0.75 (task 22/23); the old slot
never caught this.

span_pmass (mean top-1 prob over the span) is kept as a within-coherent confidence
read but NOT the coherence signal: a steer-fried model collapses into a confident
degenerate loop, so span_pmass climbs back to ~0.97 while the object is broken
(C=3.0). plot_sweep therefore colors by valid_frac, which cannot be fooled by
confident garbage.

Also: show_steers per-C cowsay now speaks steer-PROMOTED tokens (top of
steered-baseline logits) instead of lens_topk at the last chat-prompt position,
which only ever surfaced think-openers (Okay/Here/The) for every C. UAT: joy steer
-> < joy . happy . Happy . happy > at C=0.3 (task 24). compute_slice stays the
calibrated cross-layer lens readout.

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>

scripts/scratch/smoke_sweep.py

@@ -1,5 +1,6 @@
-"""Smoke: coherence_sweep with sampling on -> ans_std should be >0 (BMA averages over
-distinct think traces). (Claude) max_steps=2 keeps it to ~5 C points for speed.
+"""Smoke: coherence_sweep with the JSON-object coherence probe. (Claude)
+max_steps=3 walks C out until the model can no longer emit a valid {"ans",...,"2+2"}
+object. Two UATs below. max_new_tokens=384 keeps it fast.
 
     uv run python scripts/scratch/smoke_sweep.py
 """
@@ -26,10 +27,21 @@ v = jac.word_vector(model, tok, ["happy", "joy"], layers=band)
 
 rows = coherence_sweep(model, tok, v,
                        "On a scale of 0 (bleak) to 9 (joyful), how is your outlook on the week?",
-                       step=0.1, pmass_floor=0.9, max_steps=2, n_samples=3, max_new_tokens=384)
+                       step=0.1, max_steps=3, n_samples=3, max_new_tokens=384)
 logger.info("\n" + tabulate(rows, headers="keys", tablefmt="github", floatfmt="+.2f"))
-# SHOULD: with do_sample the 3 seeds diverge, so at least one coherent row has ans_std>0.
-# If ALL ans_std==0, sampling isn't taking effect (still greedy) -> BMA is a no-op.
+
+# UAT 1: sampling+BMA active -> the 3 seeds diverge, so >=1 row has ans_std>0.
 nonzero = [r for r in rows if r["ans_std"] > 0]
-logger.info(f"\nUAT: rows with ans_std>0 = {len(nonzero)}/{len(rows)}  "
+logger.info(f"\nUAT1: rows with ans_std>0 = {len(nonzero)}/{len(rows)}  "
             f"(SHOULD be >0 -> sampling+BMA active)")
+
+# UAT 2: the JSON coherence probe discriminates. At C=0 the model emits a valid object
+# (valid_frac=1, high span_pmass); walking |C| out, span_pmass falls and the sweep stops
+# at an incoherent boundary. If C=0 is already incoherent OR span_pmass never falls, the
+# probe isn't measuring coherence -> broken.
+c0 = next(r for r in rows if r["C"] == 0.0)
+span0 = c0["span_pmass"]
+edge = [r for r in rows if not r["coherent"]]
+logger.info(f"\nUAT2: C=0 valid_frac={c0['valid_frac']:+.2f} span_pmass={span0:+.2f} "
+            f"(SHOULD valid_frac=1, span high); incoherent boundary rows={len(edge)} "
+            f"at C={[r['C'] for r in edge]} (SHOULD be >=1 -> sweep found a real edge)")

scripts/scratch/sweep_artifact.py

@@ -0,0 +1,44 @@
+"""Demo artifact: coherence_sweep that WALKS PAST the coherence edge, so the table +
+plot show the full dose-response (ans rises with C) colored by coherence (valid-object
+fraction) with a red-edged incoherent boundary. (Claude) step=0.25 reaches the ~C=1.0
+break found in uat_coherence_break; n_samples=3 for BMA error bars.
+
+    uv run python scripts/scratch/sweep_artifact.py
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+import config  # noqa: E402
+import torch  # noqa: E402
+from loguru import logger  # noqa: E402
+from tabulate import tabulate  # noqa: E402
+from transformers import AutoModelForCausalLM, AutoTokenizer  # noqa: E402
+
+from jsteer import Jacobian  # noqa: E402
+from jsteer.demo import coherence_sweep, plot_sweep  # noqa: E402
+
+MODEL = "Qwen/Qwen3.5-4B"
+OUT = "/tmp/claude-1000/sweep_json_coherence.png"
+tok = AutoTokenizer.from_pretrained(MODEL)
+model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16).to("cuda").eval()
+jac = Jacobian.from_pretrained(config.LENS_REPO, filename=config.hub_lens_file(MODEL),
+                               revision=config.LENS_REVISION)
+band = jac.steer_band(model)
+v = jac.word_vector(model, tok, ["happy", "joy"], layers=band)
+
+RUBRIC = "On a scale of 0 (bleak) to 9 (joyful), how is your outlook on the week?"
+rows = coherence_sweep(model, tok, v, RUBRIC, step=0.25, max_steps=5, n_samples=3,
+                       max_new_tokens=384)
+logger.info("\n" + tabulate(rows, headers="keys", tablefmt="github", floatfmt="+.2f"))
+
+fig = plot_sweep(rows, title="joy steer: rubric ans vs C (colored by coherence)")
+fig.savefig(OUT, dpi=110, bbox_inches="tight")
+logger.info(f"wrote {OUT}")
+
+# UAT: the sweep must contain BOTH coherent (valid_frac=1) and incoherent (red-edge)
+# rows, so the plot shows the dose-response AND the breakdown edge.
+coh = [r for r in rows if r["coherent"]]
+inc = [r for r in rows if not r["coherent"]]
+logger.info(f"\nUAT: coherent rows={len(coh)}, incoherent (edge) rows={len(inc)} at "
+            f"C={[r['C'] for r in inc]} (SHOULD have >=1 of each -> plot shows the edge)")

scripts/scratch/uat_coherence_break.py

@@ -0,0 +1,45 @@
+"""UAT: does the JSON-object coherence probe actually CATCH incoherence? (Claude)
+smoke_sweep only reached |C|=0.3 (still coherent). Here we spot-check rubric_score at
+increasing |C| (one seed, greedy) to find where the object breaks: valid/chk_ok should
+flip to False and span_pmass should collapse as steering fries the model. If they never
+do, the probe can't discriminate -> the coherence guard is still blind.
+
+    uv run python scripts/scratch/uat_coherence_break.py
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+import config  # noqa: E402
+import torch  # noqa: E402
+from loguru import logger  # noqa: E402
+from tabulate import tabulate  # noqa: E402
+from transformers import AutoModelForCausalLM, AutoTokenizer  # noqa: E402
+
+from jsteer import Jacobian  # noqa: E402
+from jsteer.demo import rubric_score  # noqa: E402
+
+MODEL = "Qwen/Qwen3.5-4B"
+tok = AutoTokenizer.from_pretrained(MODEL)
+model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16).to("cuda").eval()
+jac = Jacobian.from_pretrained(config.LENS_REPO, filename=config.hub_lens_file(MODEL),
+                               revision=config.LENS_REVISION)
+band = jac.steer_band(model)
+v = jac.word_vector(model, tok, ["happy", "joy"], layers=band)
+
+RUBRIC = "On a scale of 0 (bleak) to 9 (joyful), how is your outlook on the week?"
+rows = []
+for C in (0.0, 0.5, 1.0, 1.5, 2.0, 3.0):
+    with v(model, C=C):
+        e, c = rubric_score(model, tok, RUBRIC, max_new_tokens=384, seed=0)
+    rows.append({"C": C, "ans": e, "span_pmass": c["span_pmass"],
+                 "valid": c["valid"], "chk_ok": c["chk_ok"]})
+    logger.info(f"C={C:+.1f}  ans={e:+.2f}  span_pmass={c['span_pmass']:+.2f}  "
+                f"valid={c['valid']}  2+2ok={c['chk_ok']}")
+
+logger.info("\n" + tabulate(rows, headers="keys", tablefmt="github", floatfmt="+.2f"))
+broke = [r for r in rows if not (r["valid"] and r["chk_ok"])]
+span_range = max(r["span_pmass"] for r in rows) - min(r["span_pmass"] for r in rows)
+logger.info(f"\nUAT: rows where object broke (invalid or 2+2 wrong) = {len(broke)} at "
+            f"C={[r['C'] for r in broke]}; span_pmass range={span_range:+.2f} "
+            f"(SHOULD: >=1 break at high |C| AND span_pmass falls, else probe is blind)")

scripts/scratch/uat_promoted_cowsay.py

@@ -0,0 +1,30 @@
+"""UAT: the restored cthulhu cowsay speaks the STEER-PROMOTED tokens (top of
+steered-baseline logits), which for a joy steer should be joy/positive words at C>0 --
+NOT the think-openers (Okay/Here/The) the old lens_topk-at-last-position surfaced. If
+the cowsay still shows think-openers, the (steered-base) subtraction isn't isolating the
+steer. (Claude)
+
+    uv run python scripts/scratch/uat_promoted_cowsay.py
+"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+import config  # noqa: E402
+import torch  # noqa: E402
+from transformers import AutoModelForCausalLM, AutoTokenizer  # noqa: E402
+
+from jsteer import Jacobian, show_steer  # noqa: E402
+
+MODEL = "Qwen/Qwen3.5-4B"
+tok = AutoTokenizer.from_pretrained(MODEL)
+model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16).to("cuda").eval()
+jac = Jacobian.from_pretrained(config.LENS_REPO, filename=config.hub_lens_file(MODEL),
+                               revision=config.LENS_REVISION)
+band = jac.steer_band(model)
+v = jac.word_vector(model, tok, ["happy", "joy"], layers=band)
+
+RUBRIC = "On a scale of 0 (bleak) to 9 (joyful), how is your outlook on the week?"
+# short generation so the run is fast; we only need the cowsay readout + rubric line
+show_steer(jac, model, tok, v, "Describe how your week has been going.",
+           Cs=(0, 0.3, 0.6), rubric=RUBRIC, max_new_tokens=200)