mirror of
https://github.com/wassname/jsteer.git
synced 2026-09-06 16:00:23 +08:00
wassnameandClaudypoo
eba1ba4f5a
wassname read the demo text and caught that the JSON-object gate over-credited
degenerate methods: persona_vector scored rubric ans=9 while its actual generation had
collapsed into wedding-jewelry loops. Root cause: the gate was on a SHORT forced object
that stays scorable long after the open-ended generation degenerates. Every breakdown we
saw is a REPETITION loop, so coherence is now 1 - distinct-3 of the think trace
(REP_COHERENT_MAX=0.35, from the empirical gap in rep_metric_check.py over 40+ real
generations: coherent <0.3, degenerate >0.6). This drops the whole {"ans","why","2+2"}
apparatus (raw_decode, valid/chk_ok, span_pmass) for one cheap n-gram ratio on the text
that actually degenerates.
rubric_score returns (expected, rep); coherence_sweep gates coherent = rep<0.35; plot
colors by rep (viridis_r, red cutoff line); show_steer prints rep + DEGENERATE flag.
eval_mechanisms/analyze_mechanisms/rep_metric_check are the overnight which-works screen.
Removed uat_coherence_break (tested the removed JSON gate).
Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
69 lines
2.8 KiB
Python
69 lines
2.8 KiB
Python
"""Validate the repetition-coherence idea on real demo text (no GPU). (Claude)
|
|
|
|
wassname's insight: every steer breakdown we saw is REPETITION (wedding-jewelry loops,
|
|
"happy and happy", "favorite books..."). So a repetition metric on the long generation
|
|
should separate coherent from degenerate, replacing the JSON-object gate (which was on a
|
|
short forced object that survives long-gen breakdown). This reads the executed notebooks,
|
|
splits each (method, C) generation, computes rep = 1 - distinct-3, and tabulates so we can
|
|
(a) confirm it separates and (b) pick a threshold from the gap, not a guess.
|
|
|
|
uv run python scripts/scratch/rep_metric_check.py
|
|
"""
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from tabulate import tabulate
|
|
|
|
|
|
def rep_frac(text, n=3):
|
|
toks = text.split()
|
|
if len(toks) < n + 1:
|
|
return 0.0
|
|
ngrams = list(zip(*[toks[i:] for i in range(n)]))
|
|
return 1 - len(set(ngrams)) / len(ngrams)
|
|
|
|
|
|
def cells_text(nb_path):
|
|
nb = json.load(open(nb_path))
|
|
for cell in nb["cells"]:
|
|
if cell["cell_type"] != "code" or "show_steer" not in "".join(cell["source"]):
|
|
continue
|
|
src = "".join(cell["source"])
|
|
mname = re.search(r"method=(\w+)", "".join(
|
|
(o.get("text") or "") if isinstance(o.get("text"), str)
|
|
else "".join(o.get("text") or []) for o in cell.get("outputs", [])))
|
|
label = mname.group(1) if mname else src.strip().splitlines()[-1][:40]
|
|
blob = ""
|
|
for o in cell.get("outputs", []):
|
|
t = o.get("text") or o.get("data", {}).get("text/plain")
|
|
if isinstance(t, list):
|
|
t = "".join(t)
|
|
if t:
|
|
blob += t
|
|
yield label, blob
|
|
|
|
|
|
rows = []
|
|
for nb in ["/tmp/claude-1000/persona_steering_out.ipynb",
|
|
"/tmp/claude-1000/persona_steering_v2_out.ipynb",
|
|
"nbs/word_steering.ipynb"]:
|
|
if not Path(nb).exists():
|
|
continue
|
|
for label, blob in cells_text(nb):
|
|
# split into per-C sections; drop the cowsay bubble lines before scoring
|
|
parts = re.split(r"--- C=([+\-0-9.]+)", blob)
|
|
for i in range(1, len(parts), 2):
|
|
C = parts[i]
|
|
gen = parts[i + 1]
|
|
gen = re.sub(r"^.*?\^\(;,;\)\^", "", gen, flags=re.DOTALL) # strip cowsay
|
|
gen = gen.split("--- C=")[0]
|
|
rows.append({"nb": Path(nb).stem[:18], "method": label, "C": C,
|
|
"rep3": rep_frac(gen), "n_words": len(gen.split())})
|
|
|
|
rows.sort(key=lambda r: (r["method"], float(r["C"])))
|
|
print(tabulate(rows, headers="keys", tablefmt="github", floatfmt="+.3f"))
|
|
print("\nSHOULD: coherent generations (baseline C=0, gentle C) have rep3 LOW (~0.0-0.3);")
|
|
print("the degenerate loops we read by eye (persona_vector +1, topk +1.5, meandiff +2)")
|
|
print("have rep3 HIGH (~0.7-1.0). If there's a clean gap, that gap is the threshold.")
|