wassname/evil_MoE
1
0
Fork 0
mirror of https://github.com/wassname/evil_MoE.git synced 2026-07-20 12:20:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: vanilla ignores v_hack (no misleading cin/cout, no needless extract)

Browse Source 
intervention=none is a pure GRPO baseline: skip v_hack load/extract entirely
(v_hack=None), emit a nan diag, and the cin/cout/fired columns are already
hidden on the vanilla arm (#141). A --v-hack-path passed to vanilla is logged
and ignored. Removes the misleading cos_pre baseline and the ~5-min auto-extract
a vanilla run would otherwise trigger on a cache miss.

run-substrate recipe: drop the MIX override (inherit locked 0.125) and the
--v-hack-path (vanilla needs none); erase/route substrate runs pass it explicitly.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wassname
2026-05-30 10:40:35 +00:00
co-authored by Claude Opus 4.8
parent f3f2c1250f
commit 2906bb18ed
2 changed files with 81 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
justfile
+5 -4
View File
@@ -139,11 +139,12 @@ build-substrate MODES="run_tests,exit_code,sentinel":
# Vanilla-GRPO emergence on the multi-loophole substrate: does the student learn ALL
# K loopholes from the repeated even teacher batch? UAT = end-of-run SUBSTRATE table
# (per-mode hacks>0 + finite first_step). v_hack is loaded only for the cos diagnostic.
run-substrate INTERV="none" SEED="41" MIX="0.25" STEPS="80":
# (per-mode hacks>0 + finite first_step) + the per-step hk_<mode> columns. mix=0.125
# is the locked default (omit to inherit it). Vanilla needs no v_hack; for an
# erase/route substrate run, add --v-hack-path explicitly.
run-substrate INTERV="none" SEED="41" STEPS="80":
{{ TRAIN }} fast --intervention={{ INTERV }} \
--teacher-pool-dir=out/pools/substrate --mix-ratio={{ MIX }} \
--v-hack-path=out/vhack/v_hack_21pairs.safetensors \
--teacher-pool-dir=out/pools/substrate \
--seed={{ SEED }} --steps={{ STEPS }} --out-tag=_substrate3_{{ INTERV }}_s{{ SEED }}
# Regenerate both dynamics plots from the cell logs (default: all cells; pass a
src/projected_grpo/train.py
+76 -60
View File
@@ -725,52 +725,62 @@ def main(cfg: Config) -> int:
logger.info(f"trainable delta_S: {sum(p.numel() for p in delta_params):,} "
f"(+{sum(p.numel() for p in delta_hack_params):,} delta_S_hack, route-only)")
# v_hack: derive default path from model + extract_top_k unless overridden.
# Always loaded (or auto-extracted) so vanilla also reports cos_pre as a baseline.
# Auto-extract reuses the already-wrapped model — no second model load.
# Slug: works for HF names ("Qwen/Qwen3-4B" -> "Qwen3-4B") and local paths
# ("out/baked/qwen3_4b_rh25" -> "qwen3_4b_rh25").
model_slug = model_name.rstrip("/").split("/")[-1]
# Filename encodes top_k AND tau_axis because both are baked into the saved
# V (extract zeros rows where S_i/S_0 < tau_axis before saving). If a future
# ablation varies pairs.py, add a pairs hash here too.
tau_tag = f"_tau{cfg.v_hack_tau_axis:g}" if cfg.v_hack_tau_axis > 0 else ""
if cfg.v_hack_path is None:
v_hack_path = VHACK_DIR / f"v_hack_{model_slug}_k{cfg.v_hack_extract_top_k}{tau_tag}.safetensors"
# v_hack: the hack-direction subspace the erase/route arms project against.
# VANILLA (intervention=none) is a pure GRPO baseline and ignores v_hack
# entirely -- loading it there only to print a cos_pre diagnostic was misleading
# (and could trigger a needless ~5-min extraction). The cin/cout columns are
# hidden on vanilla, so v_hack=None just means "no subspace machinery".
if cfg.intervention == "none":
if cfg.v_hack_path is not None:
logger.info(f"vanilla arm: ignoring --v-hack-path={cfg.v_hack_path} "
"(no projection; cin/cout diagnostics off)")
v_hack = None
else:
v_hack_path = cfg.v_hack_path
if not v_hack_path.exists():
from .extract_vhack_grad import extract_v_hack
if cfg.vhack_pairs_path is not None:
from .pairs_from_pool import load_pairs_json
VHACK_PAIRS = load_pairs_json(cfg.vhack_pairs_path)
logger.info(f"v_hack pairs: pool-derived ({cfg.vhack_pairs_path}) -> {len(VHACK_PAIRS)} pairs")
# derive default path from model + extract_top_k unless overridden.
# Auto-extract reuses the already-wrapped model — no second model load.
# Slug: works for HF names ("Qwen/Qwen3-4B" -> "Qwen3-4B") and local paths
# ("out/baked/qwen3_4b_rh25" -> "qwen3_4b_rh25").
model_slug = model_name.rstrip("/").split("/")[-1]
# Filename encodes top_k AND tau_axis because both are baked into the saved
# V (extract zeros rows where S_i/S_0 < tau_axis before saving). If a future
# ablation varies pairs.py, add a pairs hash here too.
tau_tag = f"_tau{cfg.v_hack_tau_axis:g}" if cfg.v_hack_tau_axis > 0 else ""
if cfg.v_hack_path is None:
v_hack_path = VHACK_DIR / f"v_hack_{model_slug}_k{cfg.v_hack_extract_top_k}{tau_tag}.safetensors"
else:
from .pairs import PAIRS as VHACK_PAIRS
logger.info(f"v_hack pairs: hand-crafted PAIRS -> {len(VHACK_PAIRS)} pairs")
logger.info(f"v_hack cache miss at {v_hack_path}; extracting (~5min)...")
model.eval() # match standalone extract: deterministic backward, no dropout
v_hack_extracted, v_sv_extracted, _raw_grads, _diag = extract_v_hack(
model, tok, wrappers, VHACK_PAIRS,
top_k=cfg.v_hack_extract_top_k, tau_axis=cfg.v_hack_tau_axis,
n_heldout=2, device=device,
v_hack_path = cfg.v_hack_path
if not v_hack_path.exists():
from .extract_vhack_grad import extract_v_hack
if cfg.vhack_pairs_path is not None:
from .pairs_from_pool import load_pairs_json
VHACK_PAIRS = load_pairs_json(cfg.vhack_pairs_path)
logger.info(f"v_hack pairs: pool-derived ({cfg.vhack_pairs_path}) -> {len(VHACK_PAIRS)} pairs")
else:
from .pairs import PAIRS as VHACK_PAIRS
logger.info(f"v_hack pairs: hand-crafted PAIRS -> {len(VHACK_PAIRS)} pairs")
logger.info(f"v_hack cache miss at {v_hack_path}; extracting (~5min)...")
model.eval() # match standalone extract: deterministic backward, no dropout
v_hack_extracted, v_sv_extracted, _raw_grads, _diag = extract_v_hack(
model, tok, wrappers, VHACK_PAIRS,
top_k=cfg.v_hack_extract_top_k, tau_axis=cfg.v_hack_tau_axis,
n_heldout=2, device=device,
)
OUT_DIR.mkdir(exist_ok=True)
# Combine V and S under one safetensors file with `_sv/{name}` prefix
# for the singular values. load_v_hack splits them back apart.
save_payload = {**v_hack_extracted, **{f"_sv/{n}": s for n, s in v_sv_extracted.items()}}
save_file(save_payload, str(v_hack_path),
metadata={"model": model_name,
"dtype": "fp32" if cpu else "bf16",
"top_k": str(min(cfg.v_hack_extract_top_k, len(VHACK_PAIRS) - 2)),
"tau_axis": str(cfg.v_hack_tau_axis), "schema": "v2_with_sv"})
# extract zeros grads at exit; opt is built below so no opt-state taint.
model.train() # restore train mode; eval was set only for the extract pass
v_hack_cpu = load_v_hack(
v_hack_path, model_name, wrappers,
k_use=cfg.v_hack_k, drop_bottom_frac=cfg.v_hack_drop_bottom_frac,
)
OUT_DIR.mkdir(exist_ok=True)
# Combine V and S under one safetensors file with `_sv/{name}` prefix
# for the singular values. load_v_hack splits them back apart.
save_payload = {**v_hack_extracted, **{f"_sv/{n}": s for n, s in v_sv_extracted.items()}}
save_file(save_payload, str(v_hack_path),
metadata={"model": model_name,
"dtype": "fp32" if cpu else "bf16",
"top_k": str(min(cfg.v_hack_extract_top_k, len(VHACK_PAIRS) - 2)),
"tau_axis": str(cfg.v_hack_tau_axis), "schema": "v2_with_sv"})
# extract zeros grads at exit; opt is built below so no opt-state taint.
model.train() # restore train mode; eval was set only for the extract pass
v_hack_cpu = load_v_hack(
v_hack_path, model_name, wrappers,
k_use=cfg.v_hack_k, drop_bottom_frac=cfg.v_hack_drop_bottom_frac,
)
v_hack = {name: v.to(device) for name, v in v_hack_cpu.items()}
v_hack = {name: v.to(device) for name, v in v_hack_cpu.items()}
# Teacher pool: pre-generated rollouts on disk keyed by problem_id. Each step's
# G_t teacher rollouts come from a uniform random sample of that prompt's cache,
# so we do *not* keep the teacher model in VRAM. Pool is produced by
@@ -1335,24 +1345,30 @@ def main(cfg: Config) -> int:
# subspace. Discriminator: cos_pre_t > cos_pre_s on a clean base means v_hack
# lights up for hack grads more than non-hack. Only valid on split steps;
# otherwise step_grad_s holds the combined grad and would mis-report cos_pre_s.
if split_this_step:
cos_pre_s = mean_cos_pre_from_grads(step_grad_s, v_hack)
cos_pre_t = mean_cos_pre_from_grads(step_grad_t, v_hack)
# v_hack is None on the vanilla arm (pure GRPO baseline, no subspace): skip
# the projection/measurement entirely and emit a nan diag -> the cin/cout
# columns (hidden on vanilla anyway) render nan. erase/route always have v_hack.
if v_hack is None:
diag = {"mean_cos_pre": float("nan"), "mean_cos_post": float("nan"),
"frac_fired": float("nan"), "mean_cos_pre_s": float("nan"),
"mean_cos_pre_t": float("nan")}
else:
cos_pre_s = cos_pre_t = float("nan")
# cos_pre is measured for every intervention; grad is only mutated for
# erase (subtract) and route (subtract + park in delta_S_hack). none =
# measure_only (vanilla diagnostic).
diag = project_delta_S_grad(
wrappers, v_hack, cfg.preserve_magnitude,
measure_only=(cfg.intervention == "none"),
route=(cfg.intervention == "route"),
gate_mode=cfg.gate_mode,
overshoot=cfg.project_overshoot,
)
diag["mean_cos_pre_s"] = cos_pre_s
diag["mean_cos_pre_t"] = cos_pre_t
if split_this_step:
cos_pre_s = mean_cos_pre_from_grads(step_grad_s, v_hack)
cos_pre_t = mean_cos_pre_from_grads(step_grad_t, v_hack)
else:
cos_pre_s = cos_pre_t = float("nan")
# grad is mutated only for erase (subtract) and route (subtract + park in
# delta_S_hack). cos_pre is measured on both.
diag = project_delta_S_grad(
wrappers, v_hack, cfg.preserve_magnitude,
measure_only=False, # erase/route both project; vanilla took the branch above
route=(cfg.intervention == "route"),
gate_mode=cfg.gate_mode,
overshoot=cfg.project_overshoot,
)
diag["mean_cos_pre_s"] = cos_pre_s
diag["mean_cos_pre_t"] = cos_pre_t
# R3 span check (once, on the first routed step that fires): the parked
# quarantine grad must live in span(V). removed = c_use@V is a combo of