mirror of
https://github.com/wassname/jsteer.git
synced 2026-09-11 08:41:06 +08:00
wassnameandClaudypoo
eba1ba4f5a
wassname read the demo text and caught that the JSON-object gate over-credited
degenerate methods: persona_vector scored rubric ans=9 while its actual generation had
collapsed into wedding-jewelry loops. Root cause: the gate was on a SHORT forced object
that stays scorable long after the open-ended generation degenerates. Every breakdown we
saw is a REPETITION loop, so coherence is now 1 - distinct-3 of the think trace
(REP_COHERENT_MAX=0.35, from the empirical gap in rep_metric_check.py over 40+ real
generations: coherent <0.3, degenerate >0.6). This drops the whole {"ans","why","2+2"}
apparatus (raw_decode, valid/chk_ok, span_pmass) for one cheap n-gram ratio on the text
that actually degenerates.
rubric_score returns (expected, rep); coherence_sweep gates coherent = rep<0.35; plot
colors by rep (viridis_r, red cutoff line); show_steer prints rep + DEGENERATE flag.
eval_mechanisms/analyze_mechanisms/rep_metric_check are the overnight which-works screen.
Removed uat_coherence_break (tested the removed JSON gate).
Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
84 lines
3.1 KiB
Python
84 lines
3.1 KiB
Python
"""Offline re-analysis of eval_mechanisms output (no GPU). (Claude)
|
|
|
|
The live harness summarised each method by edge-minus-edge `swing`, which is a BAD
|
|
statistic for non-monotone curves: it mislabeled persona_topk INERT because it grabbed
|
|
an anomalously-high point on the noisy negative arm as the low edge. Recompute honest
|
|
metrics from the same rows:
|
|
rho Spearman(ans, C) over the coherent window -- monotone dose-response, sign = direction
|
|
range max(ans)-min(ans) over coherent window -- does it move the axis at all
|
|
pos_rise ans at best +C minus ans@0 -- the designed (positive) direction
|
|
width coherent C-window width
|
|
|
|
uv run python scripts/scratch/analyze_mechanisms.py
|
|
"""
|
|
import re
|
|
from pathlib import Path
|
|
|
|
from tabulate import tabulate
|
|
|
|
|
|
def _rank(xs):
|
|
order = sorted(range(len(xs)), key=lambda i: xs[i])
|
|
r = [0.0] * len(xs)
|
|
for pos, i in enumerate(order):
|
|
r[i] = pos
|
|
return r
|
|
|
|
|
|
def spearman(xs, ys):
|
|
rx, ry = _rank(xs), _rank(ys)
|
|
n = len(xs)
|
|
mx, my = sum(rx) / n, sum(ry) / n
|
|
cov = sum((a - mx) * (b - my) for a, b in zip(rx, ry))
|
|
vx = sum((a - mx) ** 2 for a in rx) ** 0.5
|
|
vy = sum((b - my) ** 2 for b in ry) ** 0.5
|
|
return cov / (vx * vy) if vx and vy else 0.0
|
|
|
|
txt = Path("artifacts/eval_mechanisms.txt").read_text().splitlines()
|
|
methods, cur, rows = {}, None, []
|
|
for line in txt:
|
|
m = re.match(r"===== (.+?) =====", line)
|
|
if m:
|
|
if cur:
|
|
methods[cur] = rows
|
|
cur, rows = m.group(1), []
|
|
elif re.match(r"\|\s*[+-]?\d", line):
|
|
c = [p.strip() for p in line.strip("|").split("|")]
|
|
# C | ans | ans_std | span_pmass | valid_frac | coherent
|
|
rows.append((float(c[0]), float(c[1]), float(c[4]), c[5] == "True"))
|
|
if cur:
|
|
methods[cur] = rows
|
|
|
|
rand_range = None
|
|
summary = []
|
|
for name, rs in methods.items():
|
|
coh = [(C, ans) for C, ans, vf, ok in rs if ok]
|
|
if not coh or name.startswith("VERDICT"): # skip the summary-table pseudo-method
|
|
continue
|
|
Cs, ans = [c for c, _ in coh], [a for _, a in coh]
|
|
rho = spearman(Cs, ans) if len(set(Cs)) > 1 else 0.0
|
|
rng = max(ans) - min(ans)
|
|
ans0 = next(a for C, a in coh if C == 0.0)
|
|
pos = [a for C, a in coh if C > 0]
|
|
pos_rise = (max(pos) - ans0) if pos else 0.0
|
|
width = max(Cs) - min(Cs)
|
|
summary.append({"method": name, "rho": rho, "range": rng, "pos_rise": pos_rise,
|
|
"width": width, "n_coh": len(coh)})
|
|
if name.startswith("random"):
|
|
rand_range = rng
|
|
|
|
for s in summary:
|
|
# WORKS: monotone (rho>=0.6) and moves > random's null range; else if it moves a lot
|
|
# but non-monotone -> NOISY (steers but not cleanly bidirectional); else INERT.
|
|
moves = s["range"] >= rand_range + 1.5
|
|
if not moves:
|
|
s["verdict"] = "INERT"
|
|
elif s["rho"] >= 0.6:
|
|
s["verdict"] = "WORKS (clean)"
|
|
else:
|
|
s["verdict"] = "NOISY (moves, non-monotone)"
|
|
|
|
print(f"random null range = {rand_range:+.2f} (a method must beat this + 1.5 to 'move')\n")
|
|
print(tabulate(sorted(summary, key=lambda s: -s["rho"]), headers="keys",
|
|
tablefmt="github", floatfmt="+.2f"))
|