diff --git a/scripts/09_forced_choice.py b/scripts/09_forced_choice.py index e8f6905..93aa7ba 100644 --- a/scripts/09_forced_choice.py +++ b/scripts/09_forced_choice.py @@ -84,7 +84,6 @@ def main() -> None: else {f: float(r["label"][i]) for i, f in enumerate(_DEFAULT_FORCED_FOUNDATIONS)}), "top1": r["top1"], "margin": float(r["margin"]), - "nll_prompt": float(r["nll_prompt"]), } f.write(json.dumps(rec) + "\n") logger.info(f"wrote {len(out['per_row'])} rows to {out_path}") @@ -96,8 +95,9 @@ def main() -> None: # === Headline scalars === print(f"\n=== AI-vs-label headlines on {args.name} (n={len(out['per_row'])}) ===") - print("SHOULD: top1_acc >> 1/7=0.14 (uniform); mean_nll_T < mean_nll if raw probe is overconfident") + print("SHOULD: top1_acc >> 1/7=0.14 (uniform); informedness >> 0 (chance); mean_nll_T < mean_nll if raw probe is overconfident") print(f" top1_acc = {out['top1_acc']}") + print(f" informedness = {out['informedness']} (macro Youden's J vs human argmax, in [-1,1]; 0=chance)") print(f" mean_nll = {out['mean_nll']} (T=1, nats)") print(f" mean_nll_T = {out['mean_nll_T']} (temperature-scaled, nats)") print(f" median_nll_T = {out['median_nll_T']} (temperature-scaled, nats)") @@ -114,14 +114,6 @@ def main() -> None: f"{np.median(p_top1):.3f} / {p_top1.mean():.3f} / {p_top1.max():.3f}") print(" SHOULD: median > 0.4 (clear winner per row); <0.2 -> probe broken") - # Prompt-NLL degradation probe (free; teacher-forced on rendered chat). - nll = np.array([float(r["nll_prompt"]) for r in out["per_row"]]) - nll = nll[np.isfinite(nll)] - if len(nll): - print(f"\n nll_prompt (nats/tok) min/median/mean/max: " - f"{nll.min():.3f} / {np.median(nll):.3f} / {nll.mean():.3f} / {nll.max():.3f}") - print(" SHOULD: stable across runs at fixed model; rises under steering/ablation -> degradation") - if __name__ == "__main__": main()