diff --git a/scripts/run_modal_wvs.py b/scripts/run_modal_wvs.py index 3b993dc..c31c1b2 100644 --- a/scripts/run_modal_wvs.py +++ b/scripts/run_modal_wvs.py @@ -77,7 +77,9 @@ def main(model: str = MODEL, methods: str = ",".join(METHODS), seeds: str = ",". res = json.loads(handle.get()) base = res["doses"][0] far = max(res["doses"], key=lambda d: abs(d["mult"])) + check = res["manipulation_check"]["scored"] print(f"{method}\ts{seed}\tC={res['calibrated_C']:+.3f}\t" + f"honesty={check['effect_logodds']:+.3f}\t" f"dx={far['x'] - base['x']:+.4f}\tdy={far['y'] - base['y']:+.4f}\t" f"pmass={far['mean_pmass']:.3f}") except Exception as error: diff --git a/slop/plans/20260918_wvs_steer_big.md b/slop/plans/20260918_wvs_steer_big.md index f4a14c7..d183351 100644 --- a/slop/plans/20260918_wvs_steer_big.md +++ b/slop/plans/20260918_wvs_steer_big.md @@ -70,6 +70,23 @@ leave-one-out is one item reacting to the persona wording, not a cultural shift. This mattered most for credulity, which nearly paraphrases the X-axis trust item (`"Most people can be trusted"`). Credulity is now out of scope, but the check is cheap and stays. +## Corrected-run decision rule (recorded before the next full sweep) + +The first 27B sweep found two evaluation bugs: sampled think traces were not seeded before a +"paired" bootstrap, and VJP's behavioral sign was opposite its configured sign. Its baseline +pmass was also 0.75, below the preregistered 0.95 readability gate. Those outputs are diagnostic, +not the final result. + +1. Use the largest probed model with base mean pmass >= 0.95. A plotted dose must keep mean pmass + >= 0.90; do not replace this with a post-hoc relative threshold. +2. Orient every vector, including random, on four held-out true-vs-welcome questions. A method is + honesty-specific only if its log-odds effect exceeds the 95th percentile over random directions. +3. A culture-map effect must beat the matched-dose random displacement and retain its direction + after the single most influential WVS item is removed. Opposite doses should point to opposite + half-planes locally; a non-monotonic or same-direction trajectory is evidence against an axis. +4. If no implementation passes these checks, conclude only that these implementations are + inconclusive. Do not conclude that honesty has no cultural effect. + ## Goals 0. [/] goal: the target model's answer slot is readable, before renting anything big