From 2906bb18ed212600adb4395b039b4ccff8cc0f6a Mon Sep 17 00:00:00 2001 From: wassname Date: Sat, 30 May 2026 10:40:35 +0000 Subject: [PATCH] feat: vanilla ignores v_hack (no misleading cin/cout, no needless extract) intervention=none is a pure GRPO baseline: skip v_hack load/extract entirely (v_hack=None), emit a nan diag, and the cin/cout/fired columns are already hidden on the vanilla arm (#141). A --v-hack-path passed to vanilla is logged and ignored. Removes the misleading cos_pre baseline and the ~5-min auto-extract a vanilla run would otherwise trigger on a cache miss. run-substrate recipe: drop the MIX override (inherit locked 0.125) and the --v-hack-path (vanilla needs none); erase/route substrate runs pass it explicitly. Co-Authored-By: Claude Opus 4.8 --- justfile | 9 +-- src/projected_grpo/train.py | 136 ++++++++++++++++++++---------------- 2 files changed, 81 insertions(+), 64 deletions(-) diff --git a/justfile b/justfile index cf63a29..2101c7c 100644 --- a/justfile +++ b/justfile @@ -139,11 +139,12 @@ build-substrate MODES="run_tests,exit_code,sentinel": # Vanilla-GRPO emergence on the multi-loophole substrate: does the student learn ALL # K loopholes from the repeated even teacher batch? UAT = end-of-run SUBSTRATE table -# (per-mode hacks>0 + finite first_step). v_hack is loaded only for the cos diagnostic. -run-substrate INTERV="none" SEED="41" MIX="0.25" STEPS="80": +# (per-mode hacks>0 + finite first_step) + the per-step hk_ columns. mix=0.125 +# is the locked default (omit to inherit it). Vanilla needs no v_hack; for an +# erase/route substrate run, add --v-hack-path explicitly. +run-substrate INTERV="none" SEED="41" STEPS="80": {{ TRAIN }} fast --intervention={{ INTERV }} \ - --teacher-pool-dir=out/pools/substrate --mix-ratio={{ MIX }} \ - --v-hack-path=out/vhack/v_hack_21pairs.safetensors \ + --teacher-pool-dir=out/pools/substrate \ --seed={{ SEED }} --steps={{ STEPS }} --out-tag=_substrate3_{{ INTERV }}_s{{ SEED }} # Regenerate both dynamics plots from the cell logs (default: all cells; pass a diff --git a/src/projected_grpo/train.py b/src/projected_grpo/train.py index c07da06..0c210f7 100644 --- a/src/projected_grpo/train.py +++ b/src/projected_grpo/train.py @@ -725,52 +725,62 @@ def main(cfg: Config) -> int: logger.info(f"trainable delta_S: {sum(p.numel() for p in delta_params):,} " f"(+{sum(p.numel() for p in delta_hack_params):,} delta_S_hack, route-only)") - # v_hack: derive default path from model + extract_top_k unless overridden. - # Always loaded (or auto-extracted) so vanilla also reports cos_pre as a baseline. - # Auto-extract reuses the already-wrapped model — no second model load. - # Slug: works for HF names ("Qwen/Qwen3-4B" -> "Qwen3-4B") and local paths - # ("out/baked/qwen3_4b_rh25" -> "qwen3_4b_rh25"). - model_slug = model_name.rstrip("/").split("/")[-1] - # Filename encodes top_k AND tau_axis because both are baked into the saved - # V (extract zeros rows where S_i/S_0 < tau_axis before saving). If a future - # ablation varies pairs.py, add a pairs hash here too. - tau_tag = f"_tau{cfg.v_hack_tau_axis:g}" if cfg.v_hack_tau_axis > 0 else "" - if cfg.v_hack_path is None: - v_hack_path = VHACK_DIR / f"v_hack_{model_slug}_k{cfg.v_hack_extract_top_k}{tau_tag}.safetensors" + # v_hack: the hack-direction subspace the erase/route arms project against. + # VANILLA (intervention=none) is a pure GRPO baseline and ignores v_hack + # entirely -- loading it there only to print a cos_pre diagnostic was misleading + # (and could trigger a needless ~5-min extraction). The cin/cout columns are + # hidden on vanilla, so v_hack=None just means "no subspace machinery". + if cfg.intervention == "none": + if cfg.v_hack_path is not None: + logger.info(f"vanilla arm: ignoring --v-hack-path={cfg.v_hack_path} " + "(no projection; cin/cout diagnostics off)") + v_hack = None else: - v_hack_path = cfg.v_hack_path - if not v_hack_path.exists(): - from .extract_vhack_grad import extract_v_hack - if cfg.vhack_pairs_path is not None: - from .pairs_from_pool import load_pairs_json - VHACK_PAIRS = load_pairs_json(cfg.vhack_pairs_path) - logger.info(f"v_hack pairs: pool-derived ({cfg.vhack_pairs_path}) -> {len(VHACK_PAIRS)} pairs") + # derive default path from model + extract_top_k unless overridden. + # Auto-extract reuses the already-wrapped model — no second model load. + # Slug: works for HF names ("Qwen/Qwen3-4B" -> "Qwen3-4B") and local paths + # ("out/baked/qwen3_4b_rh25" -> "qwen3_4b_rh25"). + model_slug = model_name.rstrip("/").split("/")[-1] + # Filename encodes top_k AND tau_axis because both are baked into the saved + # V (extract zeros rows where S_i/S_0 < tau_axis before saving). If a future + # ablation varies pairs.py, add a pairs hash here too. + tau_tag = f"_tau{cfg.v_hack_tau_axis:g}" if cfg.v_hack_tau_axis > 0 else "" + if cfg.v_hack_path is None: + v_hack_path = VHACK_DIR / f"v_hack_{model_slug}_k{cfg.v_hack_extract_top_k}{tau_tag}.safetensors" else: - from .pairs import PAIRS as VHACK_PAIRS - logger.info(f"v_hack pairs: hand-crafted PAIRS -> {len(VHACK_PAIRS)} pairs") - logger.info(f"v_hack cache miss at {v_hack_path}; extracting (~5min)...") - model.eval() # match standalone extract: deterministic backward, no dropout - v_hack_extracted, v_sv_extracted, _raw_grads, _diag = extract_v_hack( - model, tok, wrappers, VHACK_PAIRS, - top_k=cfg.v_hack_extract_top_k, tau_axis=cfg.v_hack_tau_axis, - n_heldout=2, device=device, + v_hack_path = cfg.v_hack_path + if not v_hack_path.exists(): + from .extract_vhack_grad import extract_v_hack + if cfg.vhack_pairs_path is not None: + from .pairs_from_pool import load_pairs_json + VHACK_PAIRS = load_pairs_json(cfg.vhack_pairs_path) + logger.info(f"v_hack pairs: pool-derived ({cfg.vhack_pairs_path}) -> {len(VHACK_PAIRS)} pairs") + else: + from .pairs import PAIRS as VHACK_PAIRS + logger.info(f"v_hack pairs: hand-crafted PAIRS -> {len(VHACK_PAIRS)} pairs") + logger.info(f"v_hack cache miss at {v_hack_path}; extracting (~5min)...") + model.eval() # match standalone extract: deterministic backward, no dropout + v_hack_extracted, v_sv_extracted, _raw_grads, _diag = extract_v_hack( + model, tok, wrappers, VHACK_PAIRS, + top_k=cfg.v_hack_extract_top_k, tau_axis=cfg.v_hack_tau_axis, + n_heldout=2, device=device, + ) + OUT_DIR.mkdir(exist_ok=True) + # Combine V and S under one safetensors file with `_sv/{name}` prefix + # for the singular values. load_v_hack splits them back apart. + save_payload = {**v_hack_extracted, **{f"_sv/{n}": s for n, s in v_sv_extracted.items()}} + save_file(save_payload, str(v_hack_path), + metadata={"model": model_name, + "dtype": "fp32" if cpu else "bf16", + "top_k": str(min(cfg.v_hack_extract_top_k, len(VHACK_PAIRS) - 2)), + "tau_axis": str(cfg.v_hack_tau_axis), "schema": "v2_with_sv"}) + # extract zeros grads at exit; opt is built below so no opt-state taint. + model.train() # restore train mode; eval was set only for the extract pass + v_hack_cpu = load_v_hack( + v_hack_path, model_name, wrappers, + k_use=cfg.v_hack_k, drop_bottom_frac=cfg.v_hack_drop_bottom_frac, ) - OUT_DIR.mkdir(exist_ok=True) - # Combine V and S under one safetensors file with `_sv/{name}` prefix - # for the singular values. load_v_hack splits them back apart. - save_payload = {**v_hack_extracted, **{f"_sv/{n}": s for n, s in v_sv_extracted.items()}} - save_file(save_payload, str(v_hack_path), - metadata={"model": model_name, - "dtype": "fp32" if cpu else "bf16", - "top_k": str(min(cfg.v_hack_extract_top_k, len(VHACK_PAIRS) - 2)), - "tau_axis": str(cfg.v_hack_tau_axis), "schema": "v2_with_sv"}) - # extract zeros grads at exit; opt is built below so no opt-state taint. - model.train() # restore train mode; eval was set only for the extract pass - v_hack_cpu = load_v_hack( - v_hack_path, model_name, wrappers, - k_use=cfg.v_hack_k, drop_bottom_frac=cfg.v_hack_drop_bottom_frac, - ) - v_hack = {name: v.to(device) for name, v in v_hack_cpu.items()} + v_hack = {name: v.to(device) for name, v in v_hack_cpu.items()} # Teacher pool: pre-generated rollouts on disk keyed by problem_id. Each step's # G_t teacher rollouts come from a uniform random sample of that prompt's cache, # so we do *not* keep the teacher model in VRAM. Pool is produced by @@ -1335,24 +1345,30 @@ def main(cfg: Config) -> int: # subspace. Discriminator: cos_pre_t > cos_pre_s on a clean base means v_hack # lights up for hack grads more than non-hack. Only valid on split steps; # otherwise step_grad_s holds the combined grad and would mis-report cos_pre_s. - if split_this_step: - cos_pre_s = mean_cos_pre_from_grads(step_grad_s, v_hack) - cos_pre_t = mean_cos_pre_from_grads(step_grad_t, v_hack) + # v_hack is None on the vanilla arm (pure GRPO baseline, no subspace): skip + # the projection/measurement entirely and emit a nan diag -> the cin/cout + # columns (hidden on vanilla anyway) render nan. erase/route always have v_hack. + if v_hack is None: + diag = {"mean_cos_pre": float("nan"), "mean_cos_post": float("nan"), + "frac_fired": float("nan"), "mean_cos_pre_s": float("nan"), + "mean_cos_pre_t": float("nan")} else: - cos_pre_s = cos_pre_t = float("nan") - - # cos_pre is measured for every intervention; grad is only mutated for - # erase (subtract) and route (subtract + park in delta_S_hack). none = - # measure_only (vanilla diagnostic). - diag = project_delta_S_grad( - wrappers, v_hack, cfg.preserve_magnitude, - measure_only=(cfg.intervention == "none"), - route=(cfg.intervention == "route"), - gate_mode=cfg.gate_mode, - overshoot=cfg.project_overshoot, - ) - diag["mean_cos_pre_s"] = cos_pre_s - diag["mean_cos_pre_t"] = cos_pre_t + if split_this_step: + cos_pre_s = mean_cos_pre_from_grads(step_grad_s, v_hack) + cos_pre_t = mean_cos_pre_from_grads(step_grad_t, v_hack) + else: + cos_pre_s = cos_pre_t = float("nan") + # grad is mutated only for erase (subtract) and route (subtract + park in + # delta_S_hack). cos_pre is measured on both. + diag = project_delta_S_grad( + wrappers, v_hack, cfg.preserve_magnitude, + measure_only=False, # erase/route both project; vanilla took the branch above + route=(cfg.intervention == "route"), + gate_mode=cfg.gate_mode, + overshoot=cfg.project_overshoot, + ) + diag["mean_cos_pre_s"] = cos_pre_s + diag["mean_cos_pre_t"] = cos_pre_t # R3 span check (once, on the first routed step that fires): the parked # quarantine grad must live in span(V). removed = c_use@V is a combo of