wassname/moral-maps
1
0
Fork 0
mirror of https://github.com/wassname/moral-maps.git synced 2026-09-09 11:27:22 +08:00
Code Issues Packages Projects Releases Wiki Activity

forced_choice: print informedness; drop stale nll_prompt refs

Browse Source 
nll_prompt was removed from per_row in f585864 but the script still read it,
crashing the smoke test. Remove the dead refs and surface the new
informedness scalar alongside top1_acc.

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-06-18 19:47:21 +08:00
co-authored by Claudypoo
parent 205e96bf6a
commit 6e60bc2c53
1 changed files with 2 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/09_forced_choice.py
+2 -10
View File
@@ -84,7 +84,6 @@ def main() -> None:
else {f: float(r["label"][i]) for i, f in enumerate(_DEFAULT_FORCED_FOUNDATIONS)}),
"top1": r["top1"],
"margin": float(r["margin"]),
"nll_prompt": float(r["nll_prompt"]),
}
f.write(json.dumps(rec) + "\n")
logger.info(f"wrote {len(out['per_row'])} rows to {out_path}")
@@ -96,8 +95,9 @@ def main() -> None:
# === Headline scalars ===
print(f"\n=== AI-vs-label headlines on {args.name} (n={len(out['per_row'])}) ===")
print("SHOULD: top1_acc >> 1/7=0.14 (uniform); mean_nll_T < mean_nll if raw probe is overconfident")
print("SHOULD: top1_acc >> 1/7=0.14 (uniform); informedness >> 0 (chance); mean_nll_T < mean_nll if raw probe is overconfident")
print(f" top1_acc = {out['top1_acc']}")
print(f" informedness = {out['informedness']} (macro Youden's J vs human argmax, in [-1,1]; 0=chance)")
print(f" mean_nll = {out['mean_nll']} (T=1, nats)")
print(f" mean_nll_T = {out['mean_nll_T']} (temperature-scaled, nats)")
print(f" median_nll_T = {out['median_nll_T']} (temperature-scaled, nats)")
@@ -114,14 +114,6 @@ def main() -> None:
f"{np.median(p_top1):.3f} / {p_top1.mean():.3f} / {p_top1.max():.3f}")
print(" SHOULD: median > 0.4 (clear winner per row); <0.2 -> probe broken")
# Prompt-NLL degradation probe (free; teacher-forced on rendered chat).
nll = np.array([float(r["nll_prompt"]) for r in out["per_row"]])
nll = nll[np.isfinite(nll)]
if len(nll):
print(f"\n nll_prompt (nats/tok) min/median/mean/max: "
f"{nll.min():.3f} / {np.median(nll):.3f} / {nll.mean():.3f} / {nll.max():.3f}")
print(" SHOULD: stable across runs at fixed model; rises under steering/ablation -> degradation")
if __name__ == "__main__":
main()