From 6e60bc2c538e7d5e22e6b87cd70b4ab2b28766a2 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Thu, 18 Jun 2026 19:47:21 +0800 Subject: [PATCH] forced_choice: print informedness; drop stale nll_prompt refs nll_prompt was removed from per_row in f585864 but the script still read it, crashing the smoke test. Remove the dead refs and surface the new informedness scalar alongside top1_acc. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- scripts/09_forced_choice.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/scripts/09_forced_choice.py b/scripts/09_forced_choice.py index e8f6905..93aa7ba 100644 --- a/scripts/09_forced_choice.py +++ b/scripts/09_forced_choice.py @@ -84,7 +84,6 @@ def main() -> None: else {f: float(r["label"][i]) for i, f in enumerate(_DEFAULT_FORCED_FOUNDATIONS)}), "top1": r["top1"], "margin": float(r["margin"]), - "nll_prompt": float(r["nll_prompt"]), } f.write(json.dumps(rec) + "\n") logger.info(f"wrote {len(out['per_row'])} rows to {out_path}") @@ -96,8 +95,9 @@ def main() -> None: # === Headline scalars === print(f"\n=== AI-vs-label headlines on {args.name} (n={len(out['per_row'])}) ===") - print("SHOULD: top1_acc >> 1/7=0.14 (uniform); mean_nll_T < mean_nll if raw probe is overconfident") + print("SHOULD: top1_acc >> 1/7=0.14 (uniform); informedness >> 0 (chance); mean_nll_T < mean_nll if raw probe is overconfident") print(f" top1_acc = {out['top1_acc']}") + print(f" informedness = {out['informedness']} (macro Youden's J vs human argmax, in [-1,1]; 0=chance)") print(f" mean_nll = {out['mean_nll']} (T=1, nats)") print(f" mean_nll_T = {out['mean_nll_T']} (temperature-scaled, nats)") print(f" median_nll_T = {out['median_nll_T']} (temperature-scaled, nats)") @@ -114,14 +114,6 @@ def main() -> None: f"{np.median(p_top1):.3f} / {p_top1.mean():.3f} / {p_top1.max():.3f}") print(" SHOULD: median > 0.4 (clear winner per row); <0.2 -> probe broken") - # Prompt-NLL degradation probe (free; teacher-forced on rendered chat). - nll = np.array([float(r["nll_prompt"]) for r in out["per_row"]]) - nll = nll[np.isfinite(nll)] - if len(nll): - print(f"\n nll_prompt (nats/tok) min/median/mean/max: " - f"{nll.min():.3f} / {np.median(nll):.3f} / {nll.mean():.3f} / {nll.max():.3f}") - print(" SHOULD: stable across runs at fixed model; rises under steering/ablation -> degradation") - if __name__ == "__main__": main()