diff --git a/scripts/plot_steer_showcase.py b/scripts/plot_steer_showcase.py index eb614c2..b16bc1a 100644 --- a/scripts/plot_steer_showcase.py +++ b/scripts/plot_steer_showcase.py @@ -1,4 +1,4 @@ -"""Showcase tinymfv's plotting on a real steering run (the dogfood before publishing the lib). +"""Showcase tinymfv's plotting on a real steering run. Consumes a steering-lite `run_allinstr_showcase.py` output dir (one calibrated activation-steering vector administered across every instrument over a signed @@ -7,14 +7,14 @@ c-sweep) and renders the SAME two figures for every instrument, uniformly: - map : ipsative culture map (PCA), AI coherent +/-c path vs the human cloud. - range: per-factor range, AI base + coherent +/-c path vs the human society strip. -Ordinal instruments (mfq2/big5/16pf/humor_styles) read _profiles.csv; nominal -MFV reads mfv.json and is projected into z-scored relative-emphasis space (its -logit-violation units cannot share a raw axis with 1-5 wrongness), but it goes -through the same plot_ipsative_pca / plot_range and yields the same two figures. +Ordinal instruments read _profiles.csv. MFV reads mfv_profiles.csv and is +projected into z-scored relative-emphasis space, because its nominal foundation +probabilities cannot share a raw axis with 1-5 survey scores. It still goes +through the same plot_ipsative_pca / plot_range functions. cs are SIGNED multipliers of the calibrated coefficient C (0 = base). The public README plots show the coherent path: c=0 plus each +/-c row whose tinymfv answer mass -stays above the requested fraction of base. Incoherent rows are dropped, not drawn hollow. +stays above the requested fraction of base. Incoherent rows are dropped. uv run python scripts/plot_steer_showcase.py \ --run-dir ../steering-lite/outputs/allinstr_qwen35_4b --out docs/img/showcase diff --git a/scripts/summarize_showcase_effects.py b/scripts/summarize_showcase_effects.py new file mode 100644 index 0000000..a166fd2 --- /dev/null +++ b/scripts/summarize_showcase_effects.py @@ -0,0 +1,137 @@ +"""Summarize a steering-lite all-instrument showcase for the README table.""" +from __future__ import annotations + +import argparse +import csv +from pathlib import Path + +import numpy as np + +import plot_steer_showcase as P +from tinymfv import get_instrument + +DISPLAY = { + "mfv": "MFV vignettes", + "humor_styles": "Humor Styles", + "big5": "Big Five", + "mfq2": "MFQ-2 survey", +} + + +def _rows(path: Path) -> list[dict[str, str]]: + with path.open(newline="") as fh: + return list(csv.DictReader(fh)) + + +def _fmt(x: float, digits: int = 2) -> str: + return f"{x:+.{digits}f}" + + +def _fmt_pct(x: float) -> str: + return f"{x:+.0f}%" + + +def _ci_sem(lo: float, hi: float) -> float: + return (hi - lo) / (2 * 1.96) + + +def _cs_label(cs: list[float]) -> str: + return ", ".join(f"{c:+g}" if c else "0" for c in sorted(cs)) + + +def _coherent_cs(run_dir: Path, instruments: list[str], coherence_frac: float) -> list[float]: + ordinal = [name for name in instruments if name != "mfv"] + pmass_ratio = P.shared_pmass_ratio(run_dir, ordinal) + if "mfv" in instruments: + _founds, _prof, mfv_pmass = P.read_mfv_profiles(run_dir) + for c, pm in mfv_pmass.items(): + pmass_ratio[c] = min(pmass_ratio[c], pm / mfv_pmass[0.0]) + return P.coherent_prefix_cs(sorted(pmass_ratio), pmass_ratio, coherence_frac) + + +def _survey_rows(run_dir: Path, name: str, cs: list[float]) -> list[dict[str, str]]: + instr = get_instrument(name) + rows = _rows(run_dir / f"{name}_profiles.csv") + by_key = {(r["foundation"], float(r["c"])): r for r in rows} + humans = P.human_strip(instr) + pos_c = max(c for c in cs if c > 0) + neg_c = min(c for c in cs if c < 0) + + out = [] + for dim in instr.dimensions: + neg = by_key[(dim, neg_c)] + pos = by_key[(dim, pos_c)] + human_vals = np.array([v for _country, v in humans[dim]], dtype=float) + human_sd = float(human_vals.std(ddof=1)) + profile_delta = float(pos["mean"]) - float(neg["mean"]) + logit_delta = float(pos["C"]) - float(neg["C"]) + sem = float(np.hypot( + _ci_sem(float(pos["C_ci95_lo"]), float(pos["C_ci95_hi"])), + _ci_sem(float(neg["C_ci95_lo"]), float(neg["C_ci95_hi"])), + )) + out.append({ + "dataset": DISPLAY[name], + "axis": dim, + "c path": _cs_label(cs), + "profile shift / human SD": _fmt_pct(100 * profile_delta / human_sd), + "profile shift": _fmt(profile_delta), + "reader-logit shift": f"{_fmt(logit_delta)} ± {sem:.2f}", + }) + return out + + +def _mfv_rows(run_dir: Path, cs: list[float]) -> list[dict[str, str]]: + founds, countries, human_M, prof, _pmass = P._mfv_zspace(run_dir) + rows = _rows(run_dir / "mfv_profiles.csv") + by_key = {(r["foundation"], float(r["c"])): r for r in rows} + pos_c = max(c for c in cs if c > 0) + neg_c = min(c for c in cs if c < 0) + + out = [] + for j, foundation in enumerate(founds): + neg = by_key[(foundation, neg_c)] + pos = by_key[(foundation, pos_c)] + human_sd = float(human_M[:, j].std(ddof=1)) + profile_delta = float(prof[pos_c][j] - prof[neg_c][j]) + logit_delta = float(pos["dlogit"]) - float(neg["dlogit"]) + sem = float(np.hypot(float(pos["dlogit_sem"]), float(neg["dlogit_sem"]))) + out.append({ + "dataset": DISPLAY["mfv"], + "axis": foundation, + "c path": _cs_label(cs), + "profile shift / human SD": _fmt_pct(100 * profile_delta / human_sd), + "profile shift": _fmt(profile_delta), + "reader-logit shift": f"{_fmt(logit_delta)} ± {sem:.2f}", + }) + return out + + +def _markdown_table(rows: list[dict[str, str]]) -> str: + cols = ["dataset", "axis", "c path", "profile shift / human SD", "profile shift", "reader-logit shift"] + lines = ["| " + " | ".join(cols) + " |", "| " + " | ".join(["---"] * len(cols)) + " |"] + for row in rows: + lines.append("| " + " | ".join(row[c] for c in cols) + " |") + return "\n".join(lines) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--run-dir", type=Path, required=True) + ap.add_argument("--coherence-frac", type=float, default=0.99) + ap.add_argument("--instruments", nargs="+", default=["mfv", "humor_styles", "big5", "mfq2"]) + args = ap.parse_args() + + cs = _coherent_cs(args.run_dir, args.instruments, args.coherence_frac) + assert any(c > 0 for c in cs) and any(c < 0 for c in cs), f"need both signed arms, got {cs}" + + rows: list[dict[str, str]] = [] + if "mfv" in args.instruments: + rows.extend(_mfv_rows(args.run_dir, cs)) + for name in args.instruments: + if name != "mfv": + rows.extend(_survey_rows(args.run_dir, name, cs)) + print(_markdown_table(rows)) + + +if __name__ == "__main__": + main()