mirror of
https://github.com/wassname/jsteer.git
synced 2026-09-09 11:25:03 +08:00
wassnameandClaudypoo
eba1ba4f5a
wassname read the demo text and caught that the JSON-object gate over-credited
degenerate methods: persona_vector scored rubric ans=9 while its actual generation had
collapsed into wedding-jewelry loops. Root cause: the gate was on a SHORT forced object
that stays scorable long after the open-ended generation degenerates. Every breakdown we
saw is a REPETITION loop, so coherence is now 1 - distinct-3 of the think trace
(REP_COHERENT_MAX=0.35, from the empirical gap in rep_metric_check.py over 40+ real
generations: coherent <0.3, degenerate >0.6). This drops the whole {"ans","why","2+2"}
apparatus (raw_decode, valid/chk_ok, span_pmass) for one cheap n-gram ratio on the text
that actually degenerates.
rubric_score returns (expected, rep); coherence_sweep gates coherent = rep<0.35; plot
colors by rep (viridis_r, red cutoff line); show_steer prints rep + DEGENERATE flag.
eval_mechanisms/analyze_mechanisms/rep_metric_check are the overnight which-works screen.
Removed uat_coherence_break (tested the removed JSON gate).
Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
47 lines
2.2 KiB
Python
47 lines
2.2 KiB
Python
"""Smoke: coherence_sweep with the JSON-object coherence probe. (Claude)
|
|
max_steps=3 walks C out until the model can no longer emit a valid {"ans",...,"2+2"}
|
|
object. Two UATs below. max_new_tokens=384 keeps it fast.
|
|
|
|
uv run python scripts/scratch/smoke_sweep.py
|
|
"""
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
|
import config # noqa: E402
|
|
import torch # noqa: E402
|
|
from loguru import logger # noqa: E402
|
|
from tabulate import tabulate # noqa: E402
|
|
from transformers import AutoModelForCausalLM, AutoTokenizer # noqa: E402
|
|
|
|
from jsteer import Jacobian # noqa: E402
|
|
from jsteer.demo import coherence_sweep # noqa: E402
|
|
|
|
MODEL = "Qwen/Qwen3.5-4B"
|
|
tok = AutoTokenizer.from_pretrained(MODEL)
|
|
model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16).to("cuda").eval()
|
|
jac = Jacobian.from_pretrained(config.LENS_REPO, filename=config.hub_lens_file(MODEL),
|
|
revision=config.LENS_REVISION)
|
|
band = jac.steer_band(model)
|
|
v = jac.word_vector(model, tok, ["happy", "joy"], layers=band)
|
|
|
|
rows = coherence_sweep(model, tok, v,
|
|
"On a scale of 0 (bleak) to 9 (joyful), how is your outlook on the week?",
|
|
step=0.1, max_steps=3, n_samples=3, max_new_tokens=384)
|
|
logger.info("\n" + tabulate(rows, headers="keys", tablefmt="github", floatfmt="+.2f"))
|
|
|
|
# UAT 1: sampling+BMA active -> the 3 seeds diverge, so >=1 row has ans_std>0.
|
|
nonzero = [r for r in rows if r["ans_std"] > 0]
|
|
logger.info(f"\nUAT1: rows with ans_std>0 = {len(nonzero)}/{len(rows)} "
|
|
f"(SHOULD be >0 -> sampling+BMA active)")
|
|
|
|
# UAT 2: the repetition coherence probe discriminates. At C=0 the think trace is fluent
|
|
# (rep low, coherent True); walking |C| out, rep rises past REP_COHERENT_MAX and the sweep
|
|
# stops at a degenerate boundary. If C=0 is already incoherent OR rep never rises, the
|
|
# probe isn't measuring coherence -> broken.
|
|
c0 = next(r for r in rows if r["C"] == 0.0)
|
|
edge = [r for r in rows if not r["coherent"]]
|
|
logger.info(f"\nUAT2: C=0 rep={c0['rep']:+.2f} coherent={c0['coherent']} "
|
|
f"(SHOULD rep low, coherent True); degenerate boundary rows={len(edge)} "
|
|
f"at C={[r['C'] for r in edge]} (SHOULD be >=1 -> sweep found a real edge)")
|