From 6e9341abde45be94138a3bef9d578d39e8b28cba Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Fri, 18 Sep 2026 21:12:00 +0800 Subject: [PATCH] Compare pooled steer paths with matched-dose random controls The result table now reports each coherent movement against the 95th percentile of random directions at the same signed dose, and reports whether each method's held-out honesty effect exceeds random. Allow one Modal launch to add many random-only seeds without repeating real method extraction. Co-Authored-By: Claude <288921227+claudypoo@users.noreply.github.com> --- justfile | 4 ++-- scripts/plot_wvs_steer.py | 35 +++++++++++++++++++++++++++++++++-- scripts/run_modal_wvs.py | 4 +++- 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/justfile b/justfile index 3494ba4..92ef431 100644 --- a/justfile +++ b/justfile @@ -23,9 +23,9 @@ wvs-steer-modal-smoke: uv run --extra steer --group dev modal run scripts/run_modal_wvs.py::smoke # the real fan-out, one container per (method, seed). H200 for 27B dense, B200:2 for 122B-A10B -wvs-steer-modal model="Qwen/Qwen3.5-27B" gpu="H200" seeds="0" methods="mean_diff,pca,vjp_delta,random": +wvs-steer-modal model="Qwen/Qwen3.5-27B" gpu="H200" seeds="0" methods="mean_diff,pca,vjp_delta,random" random_seeds="": WVS_GPU={{gpu}} uv run --extra steer --group dev modal run --detach scripts/run_modal_wvs.py::main \ - --model {{model}} --seeds {{seeds}} --methods {{methods}} + --model {{model}} --seeds {{seeds}} --methods {{methods}} --random-seeds '{{random_seeds}}' wvs-steer-pull: uv run --group dev modal volume get --force moralmaps-wvs-steer outputs . diff --git a/scripts/plot_wvs_steer.py b/scripts/plot_wvs_steer.py index fa1beed..8188829 100644 --- a/scripts/plot_wvs_steer.py +++ b/scripts/plot_wvs_steer.py @@ -132,6 +132,15 @@ def main() -> None: title_y=0.115, note_y=0.04) ax = fig.axes[0] + random_runs = groups.get("random", []) + null_move: dict[float, list[float]] = {} + for r in random_runs: + b = r["doses"][0] + for d in r["doses"][1:]: + if d["mean_pmass"] >= args.min_pmass: + null_move.setdefault(d["mult"], []).append( + float(np.hypot(d["x"] - b["x"], d["y"] - b["y"]))) + failed, rows, null_pts = 0, [], [] for method, method_runs in groups.items(): color = METHOD_COLORS[method] @@ -159,10 +168,15 @@ def main() -> None: pooled_base["psamples"], far["psamples"], resolved, np.random.default_rng(20_000 + sign)) worst, loo_len = loo_worst(far, pooled_base, resolved) + move = float(np.hypot(dx, dy)) + null = null_move.get(far["mult"], []) + null_p95 = float(np.quantile(null, 0.95)) if null else np.nan rows.append([method, len(method_runs), f"{far['mult']:+.1f}", f"{dx:+.4f}+-{1.96 * dx_se:.3f}", f"{dy:+.4f}+-{1.96 * dy_se:.3f}", - f"{np.hypot(dx, dy):.4f}", f"{loo_len:.4f}", + f"{move:.4f}", f"{loo_len:.4f}", + f"{null_p95:.4f}" if null else "-", len(null), + "yes" if null and move > null_p95 else "no", worst or "-", f"{far['mean_pmass']:.3f}"]) ax.annotate(f"{far['mult']:+g}C", (far["x"] * sgx, far["y"] * sgy), xytext=(3, 3), textcoords="offset points", fontsize=6, color=color) @@ -183,7 +197,24 @@ def main() -> None: rows.sort(key=lambda r: -float(r[5])) print(tabulate(rows, tablefmt="pipe", headers=[ "method", "read seeds", "dose", "dx (95%)", "dy (95%)", "|move|", - "|move| less worst item", "worst item", "pmass"])) + "|move| less worst item", "random p95", "random n", "beats random?", + "worst item", "pmass"])) + + random_effects = [r["manipulation_check"]["scored"]["effect_logodds"] + for r in random_runs] + random_effect_p95 = float(np.quantile(random_effects, 0.95)) if random_effects else np.nan + check_rows = [] + for method, method_runs in groups.items(): + if method == "random": + continue + effects = [r["manipulation_check"]["scored"]["effect_logodds"] for r in method_runs] + effect = float(np.mean(effects)) + check_rows.append([method, f"{effect:+.3f}", + f"{random_effect_p95:+.3f}" if random_effects else "-", + len(random_effects), "yes" if random_effects and effect > random_effect_p95 else "no"]) + print("\nHeld-out honesty manipulation (true-vs-welcome log-odds):") + print(tabulate(check_rows, tablefmt="pipe", headers=[ + "method", "effect", "random p95", "random n", "honesty-specific?"])) print("\ndx/dy intervals pool independent read seeds and pair base versus dose on the same\n" "items and sampled think streams. Hollow points remain visible but fail pmass >= 0.90.") logger.info(f"wrote {args.out} ({failed} method-dose paths/points failed pmass {args.min_pmass})") diff --git a/scripts/run_modal_wvs.py b/scripts/run_modal_wvs.py index c31c1b2..e9bc5b7 100644 --- a/scripts/run_modal_wvs.py +++ b/scripts/run_modal_wvs.py @@ -60,9 +60,11 @@ def run(argv: list[str]) -> str: @app.local_entrypoint() def main(model: str = MODEL, methods: str = ",".join(METHODS), seeds: str = ",".join(map(str, SEEDS)), - device_map: str = "auto", c_grid: str = "-2,-1,-0.5,0.5,1,2", extract_batch_size: int = 8): + random_seeds: str = "", device_map: str = "auto", + c_grid: str = "-2,-1,-0.5,0.5,1,2", extract_batch_size: int = 8): # one container per (method, seed): a dead lane must not hide the others jobs = [(m, s) for s in seeds.split(",") for m in methods.split(",")] + jobs += [("random", s) for s in random_seeds.split(",") if s and ("random", s) not in jobs] handles = { job: run.spawn([ "--model", model, "--methods", job[0], "--seed", job[1],