demo: repetition coherence replaces the JSON-object gate (simpler, correct)

wassname read the demo text and caught that the JSON-object gate over-credited
degenerate methods: persona_vector scored rubric ans=9 while its actual generation had
collapsed into wedding-jewelry loops. Root cause: the gate was on a SHORT forced object
that stays scorable long after the open-ended generation degenerates. Every breakdown we
saw is a REPETITION loop, so coherence is now 1 - distinct-3 of the think trace
(REP_COHERENT_MAX=0.35, from the empirical gap in rep_metric_check.py over 40+ real
generations: coherent <0.3, degenerate >0.6). This drops the whole {"ans","why","2+2"}
apparatus (raw_decode, valid/chk_ok, span_pmass) for one cheap n-gram ratio on the text
that actually degenerates.

rubric_score returns (expected, rep); coherence_sweep gates coherent = rep<0.35; plot
colors by rep (viridis_r, red cutoff line); show_steer prints rep + DEGENERATE flag.
eval_mechanisms/analyze_mechanisms/rep_metric_check are the overnight which-works screen.
Removed uat_coherence_break (tested the removed JSON gate).

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>

scripts/scratch/analyze_mechanisms.py

@@ -0,0 +1,83 @@
+"""Offline re-analysis of eval_mechanisms output (no GPU). (Claude)
+
+The live harness summarised each method by edge-minus-edge `swing`, which is a BAD
+statistic for non-monotone curves: it mislabeled persona_topk INERT because it grabbed
+an anomalously-high point on the noisy negative arm as the low edge. Recompute honest
+metrics from the same rows:
+  rho     Spearman(ans, C) over the coherent window -- monotone dose-response, sign = direction
+  range   max(ans)-min(ans) over coherent window   -- does it move the axis at all
+  pos_rise ans at best +C  minus ans@0             -- the designed (positive) direction
+  width   coherent C-window width
+
+    uv run python scripts/scratch/analyze_mechanisms.py
+"""
+import re
+from pathlib import Path
+
+from tabulate import tabulate
+
+
+def _rank(xs):
+    order = sorted(range(len(xs)), key=lambda i: xs[i])
+    r = [0.0] * len(xs)
+    for pos, i in enumerate(order):
+        r[i] = pos
+    return r
+
+
+def spearman(xs, ys):
+    rx, ry = _rank(xs), _rank(ys)
+    n = len(xs)
+    mx, my = sum(rx) / n, sum(ry) / n
+    cov = sum((a - mx) * (b - my) for a, b in zip(rx, ry))
+    vx = sum((a - mx) ** 2 for a in rx) ** 0.5
+    vy = sum((b - my) ** 2 for b in ry) ** 0.5
+    return cov / (vx * vy) if vx and vy else 0.0
+
+txt = Path("artifacts/eval_mechanisms.txt").read_text().splitlines()
+methods, cur, rows = {}, None, []
+for line in txt:
+    m = re.match(r"===== (.+?) =====", line)
+    if m:
+        if cur:
+            methods[cur] = rows
+        cur, rows = m.group(1), []
+    elif re.match(r"\|\s*[+-]?\d", line):
+        c = [p.strip() for p in line.strip("|").split("|")]
+        # C | ans | ans_std | span_pmass | valid_frac | coherent
+        rows.append((float(c[0]), float(c[1]), float(c[4]), c[5] == "True"))
+if cur:
+    methods[cur] = rows
+
+rand_range = None
+summary = []
+for name, rs in methods.items():
+    coh = [(C, ans) for C, ans, vf, ok in rs if ok]
+    if not coh or name.startswith("VERDICT"):    # skip the summary-table pseudo-method
+        continue
+    Cs, ans = [c for c, _ in coh], [a for _, a in coh]
+    rho = spearman(Cs, ans) if len(set(Cs)) > 1 else 0.0
+    rng = max(ans) - min(ans)
+    ans0 = next(a for C, a in coh if C == 0.0)
+    pos = [a for C, a in coh if C > 0]
+    pos_rise = (max(pos) - ans0) if pos else 0.0
+    width = max(Cs) - min(Cs)
+    summary.append({"method": name, "rho": rho, "range": rng, "pos_rise": pos_rise,
+                    "width": width, "n_coh": len(coh)})
+    if name.startswith("random"):
+        rand_range = rng
+
+for s in summary:
+    # WORKS: monotone (rho>=0.6) and moves > random's null range; else if it moves a lot
+    # but non-monotone -> NOISY (steers but not cleanly bidirectional); else INERT.
+    moves = s["range"] >= rand_range + 1.5
+    if not moves:
+        s["verdict"] = "INERT"
+    elif s["rho"] >= 0.6:
+        s["verdict"] = "WORKS (clean)"
+    else:
+        s["verdict"] = "NOISY (moves, non-monotone)"
+
+print(f"random null range = {rand_range:+.2f} (a method must beat this + 1.5 to 'move')\n")
+print(tabulate(sorted(summary, key=lambda s: -s["rho"]), headers="keys",
+               tablefmt="github", floatfmt="+.2f"))

scripts/scratch/rep_metric_check.py

@@ -0,0 +1,68 @@
+"""Validate the repetition-coherence idea on real demo text (no GPU). (Claude)
+
+wassname's insight: every steer breakdown we saw is REPETITION (wedding-jewelry loops,
+"happy and happy", "favorite books..."). So a repetition metric on the long generation
+should separate coherent from degenerate, replacing the JSON-object gate (which was on a
+short forced object that survives long-gen breakdown). This reads the executed notebooks,
+splits each (method, C) generation, computes rep = 1 - distinct-3, and tabulates so we can
+(a) confirm it separates and (b) pick a threshold from the gap, not a guess.
+
+    uv run python scripts/scratch/rep_metric_check.py
+"""
+import json
+import re
+from pathlib import Path
+
+from tabulate import tabulate
+
+
+def rep_frac(text, n=3):
+    toks = text.split()
+    if len(toks) < n + 1:
+        return 0.0
+    ngrams = list(zip(*[toks[i:] for i in range(n)]))
+    return 1 - len(set(ngrams)) / len(ngrams)
+
+
+def cells_text(nb_path):
+    nb = json.load(open(nb_path))
+    for cell in nb["cells"]:
+        if cell["cell_type"] != "code" or "show_steer" not in "".join(cell["source"]):
+            continue
+        src = "".join(cell["source"])
+        mname = re.search(r"method=(\w+)", "".join(
+            (o.get("text") or "") if isinstance(o.get("text"), str)
+            else "".join(o.get("text") or []) for o in cell.get("outputs", [])))
+        label = mname.group(1) if mname else src.strip().splitlines()[-1][:40]
+        blob = ""
+        for o in cell.get("outputs", []):
+            t = o.get("text") or o.get("data", {}).get("text/plain")
+            if isinstance(t, list):
+                t = "".join(t)
+            if t:
+                blob += t
+        yield label, blob
+
+
+rows = []
+for nb in ["/tmp/claude-1000/persona_steering_out.ipynb",
+           "/tmp/claude-1000/persona_steering_v2_out.ipynb",
+           "nbs/word_steering.ipynb"]:
+    if not Path(nb).exists():
+        continue
+    for label, blob in cells_text(nb):
+        # split into per-C sections; drop the cowsay bubble lines before scoring
+        parts = re.split(r"--- C=([+\-0-9.]+)", blob)
+        for i in range(1, len(parts), 2):
+            C = parts[i]
+            gen = parts[i + 1]
+            gen = re.sub(r"^.*?\^\(;,;\)\^", "", gen, flags=re.DOTALL)  # strip cowsay
+            gen = gen.split("--- C=")[0]
+            rows.append({"nb": Path(nb).stem[:18], "method": label, "C": C,
+                         "rep3": rep_frac(gen), "n_words": len(gen.split())})
+
+rows.sort(key=lambda r: (r["method"], float(r["C"])))
+print(tabulate(rows, headers="keys", tablefmt="github", floatfmt="+.3f"))
+print("\nSHOULD: coherent generations (baseline C=0, gentle C) have rep3 LOW (~0.0-0.3);")
+print("the degenerate loops we read by eye (persona_vector +1, topk +1.5, meandiff +2)")
+print("have rep3 HIGH (~0.7-1.0). If there's a clean gap, that gap is the threshold.")

scripts/scratch/smoke_sweep.py

@@ -35,13 +35,12 @@ nonzero = [r for r in rows if r["ans_std"] > 0]
 logger.info(f"\nUAT1: rows with ans_std>0 = {len(nonzero)}/{len(rows)}  "
             f"(SHOULD be >0 -> sampling+BMA active)")
 
-# UAT 2: the JSON coherence probe discriminates. At C=0 the model emits a valid object
-# (valid_frac=1, high span_pmass); walking |C| out, span_pmass falls and the sweep stops
-# at an incoherent boundary. If C=0 is already incoherent OR span_pmass never falls, the
+# UAT 2: the repetition coherence probe discriminates. At C=0 the think trace is fluent
+# (rep low, coherent True); walking |C| out, rep rises past REP_COHERENT_MAX and the sweep
+# stops at a degenerate boundary. If C=0 is already incoherent OR rep never rises, the
 # probe isn't measuring coherence -> broken.
 c0 = next(r for r in rows if r["C"] == 0.0)
-span0 = c0["span_pmass"]
 edge = [r for r in rows if not r["coherent"]]
-logger.info(f"\nUAT2: C=0 valid_frac={c0['valid_frac']:+.2f} span_pmass={span0:+.2f} "
-            f"(SHOULD valid_frac=1, span high); incoherent boundary rows={len(edge)} "
+logger.info(f"\nUAT2: C=0 rep={c0['rep']:+.2f} coherent={c0['coherent']} "
+            f"(SHOULD rep low, coherent True); degenerate boundary rows={len(edge)} "
             f"at C={[r['C'] for r in edge]} (SHOULD be >=1 -> sweep found a real edge)")

scripts/scratch/sweep_artifact.py

@@ -36,7 +36,7 @@ fig = plot_sweep(rows, title="joy steer: rubric ans vs C (colored by coherence)"
 fig.savefig(OUT, dpi=110, bbox_inches="tight")
 logger.info(f"wrote {OUT}")
 
-# UAT: the sweep must contain BOTH coherent (valid_frac=1) and incoherent (red-edge)
+# UAT: the sweep must contain BOTH coherent (low rep) and degenerate (red-edge)
 # rows, so the plot shows the dose-response AND the breakdown edge.
 coh = [r for r in rows if r["coherent"]]
 inc = [r for r in rows if not r["coherent"]]

scripts/scratch/uat_coherence_break.py

@@ -1,45 +0,0 @@
-"""UAT: does the JSON-object coherence probe actually CATCH incoherence? (Claude)
-smoke_sweep only reached |C|=0.3 (still coherent). Here we spot-check rubric_score at
-increasing |C| (one seed, greedy) to find where the object breaks: valid/chk_ok should
-flip to False and span_pmass should collapse as steering fries the model. If they never
-do, the probe can't discriminate -> the coherence guard is still blind.
-
-    uv run python scripts/scratch/uat_coherence_break.py
-"""
-import sys
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
-import config  # noqa: E402
-import torch  # noqa: E402
-from loguru import logger  # noqa: E402
-from tabulate import tabulate  # noqa: E402
-from transformers import AutoModelForCausalLM, AutoTokenizer  # noqa: E402
-
-from jsteer import Jacobian  # noqa: E402
-from jsteer.demo import rubric_score  # noqa: E402
-
-MODEL = "Qwen/Qwen3.5-4B"
-tok = AutoTokenizer.from_pretrained(MODEL)
-model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16).to("cuda").eval()
-jac = Jacobian.from_pretrained(config.LENS_REPO, filename=config.hub_lens_file(MODEL),
-                               revision=config.LENS_REVISION)
-band = jac.steer_band(model)
-v = jac.word_vector(model, tok, ["happy", "joy"], layers=band)
-
-RUBRIC = "On a scale of 0 (bleak) to 9 (joyful), how is your outlook on the week?"
-rows = []
-for C in (0.0, 0.5, 1.0, 1.5, 2.0, 3.0):
-    with v(model, C=C):
-        e, c = rubric_score(model, tok, RUBRIC, max_new_tokens=384, seed=0)
-    rows.append({"C": C, "ans": e, "span_pmass": c["span_pmass"],
-                 "valid": c["valid"], "chk_ok": c["chk_ok"]})
-    logger.info(f"C={C:+.1f}  ans={e:+.2f}  span_pmass={c['span_pmass']:+.2f}  "
-                f"valid={c['valid']}  2+2ok={c['chk_ok']}")
-
-logger.info("\n" + tabulate(rows, headers="keys", tablefmt="github", floatfmt="+.2f"))
-broke = [r for r in rows if not (r["valid"] and r["chk_ok"])]
-span_range = max(r["span_pmass"] for r in rows) - min(r["span_pmass"] for r in rows)
-logger.info(f"\nUAT: rows where object broke (invalid or 2+2 wrong) = {len(broke)} at "
-            f"C={[r['C'] for r in broke]}; span_pmass range={span_range:+.2f} "
-            f"(SHOULD: >=1 break at high |C| AND span_pmass falls, else probe is blind)")