diff --git a/src/vgrout/tablelog.py b/src/vgrout/tablelog.py index 2e85b4f..1c85165 100644 --- a/src/vgrout/tablelog.py +++ b/src/vgrout/tablelog.py @@ -91,7 +91,8 @@ class StepLogger: caller owns it (it also names the row-dict keys) so this module stays leaf-level. """ - def __init__(self, arm: str, modes: list[str], mode_code: dict[str, str]) -> None: + def __init__(self, arm: str, modes: list[str], mode_code: dict[str, str], + show_ablate: bool = False) -> None: # arm in {vanilla, projected, routing}; only projected/routing actually # project the gradient, so the cin/cout/fired diagnostics are theirs alone # (in vanilla they'd be counterfactual noise -> omitted). @@ -133,24 +134,28 @@ class StepLogger: _Col("cos_post", 6, "cout", ".2f", "hack-ward fraction AFTER projection (want ~0: all removed)"), _Col("fired", 5, "fired", ".2f", "fraction of modules where projection fired"), ] - # routeV: the routing gate is the pair-calibrated BAND. Per rollout, - # f = clamp((cos(g_b,v_grad) - lower)/(upper - lower), 0, 1) routes that - # fraction into the quarantine. lower/upper = mean clean/hack pair cosines. - # Surface where live cos sits (tau), the band width (hkgap), the routed - # fraction (frout, the mass gauge), and the post-routing leak (resid). + # routeV routing, by what the gate does to each live unit (rollout, or token in + # per-token mode). Its cos(g, v_grad) falls below / inside / above the pair-band + # [lower, upper] (edges logged at band construction). Three zones, two views: + # keep/resid/rout = UNIT shares, keepE/residE/routE = ENERGY shares (each sums to + # 1). leak = hack alignment that slipped past into the deployed knob. if arm == "routingV": cols += [ - _Col("tau", 6, "tau", "+.2f", "median live cos(g_b, v_grad); should sit inside the band [lower, upper]"), - _Col("hkgap", 6, "hkgap", "+.2f", "mean-over-modules band width = max hack-pair minus min clean-pair cosine; >0 = v_grad separates (else direction dead/random)"), - _Col("frout", 6, "frout", "+.2f", "mean routed fraction f over rollouts (the gate decision; hold fixed for matched real-vs-random comparison)"), - _Col("leak", 6, "leak", "+.2f", "leakage (SGTM): cos(deployed delta_S.grad after routing, v_grad); ~0 = hack stripped, >0 = under-pinned, hack survives in deployed knob"), + _Col("keep", 6, "keep", ".2f", "unit share with cos below the band -> kept whole in the deployed knob (left)"), + _Col("resid", 6, "resid", ".2f", "unit share with cos inside the band -> partially routed (residual middle)"), + _Col("rout", 6, "rout", ".2f", "unit share with cos above the band -> fully routed into quarantine (right)"), + _Col("keepE", 6, "keepE", ".2f", "energy-weighted keep: share of grad ENERGY in the kept zone"), + _Col("residE", 6, "residE", ".2f", "energy-weighted resid: share of grad ENERGY in the partially-routed zone"), + _Col("routE", 6, "routE", ".2f", "energy-weighted rout: grad ENERGY share fully routed (~quarantine mass; the routed total is routE..routE+residE)"), + _Col("leak", 6, "leak", "+.2f", "hack-ward cosine left in the deployed knob after routing; ~0 = stripped clean, >0 = hack leaked through (under-routed)"), ] - if arm in ("routing", "routingV"): + if arm == "routing": + cols.append( + _Col("qmass", 6, "qmass", ".2f", "quarantine energy share ||g_quar||/(||g_keep||+||g_quar||): fraction of the update parked in the throwaway knob")) + # Per-step deploy proxy only exists when rollout_ablate_frac>0 generates a knob-off + # slice; without it the slice is empty (0/0), so drop the columns. + if arm in ("routing", "routingV") and show_ablate: cols += [ - # Deploy eval (knob-OFF) is hk_dep below. The train-vs-deploy 2x2's - # knob-ON pass runs once post-loop (FINAL EVAL), not per-step; the - # per-step train series is hk_s. See journal 2026-06-04 (a). - _Col("absorb", 6, "absorb", ".2f", "absorption: grad energy pinned into quarantine ||g_quar||/(||g_keep||+||g_quar||); too low = hack not pinned, ->1 with slv_dep falling = solve also pinned (over-pinned)"), _Col("hack_abl", 6, "hk_abl", "frac", "per-step deploy proxy: hack rate on the ablated (deploy-mode) rollout slice; train prompts, noisier than hk_dep"), _Col("solve_abl", 6, "slv_abl", "frac", "per-step deploy proxy: solve rate on the ablated (deploy-mode) rollout slice; train prompts"), ] diff --git a/src/vgrout/train.py b/src/vgrout/train.py index 2614d24..b14a2c3 100644 --- a/src/vgrout/train.py +++ b/src/vgrout/train.py @@ -323,6 +323,21 @@ def _haar_unit_dirs(v_grad: dict, seed: int, device) -> dict: return out +def _zone_stats(f: torch.Tensor, w: torch.Tensor) -> tuple[float, ...]: + """Split routing units into the three band zones by routed fraction f in [0,1]: + f==0 keep (cos below lower), 0 dict[str, tuple[float, float]]: """Per-module routing band (lower, upper) from the contrastive pairs ALONE -- the pair-calibrated replacement for the old live-detector τ. lower = MIN clean-pair cosine @@ -752,7 +767,8 @@ def main(cfg: Config) -> int: # rows are the "is it learning?" signal. ref_eq = cumulative gens / 256 (the # canonical 16 prompts x 16 gens/step), so ref_eq=1.0 = one reference step's samples. run_modes = sorted({p["env_mode"] for p in problems}, key=lambda m: list(MODE_CODE).index(m)) - step_logger = StepLogger(arm=cfg.arm, modes=run_modes, mode_code=MODE_CODE) + step_logger = StepLogger(arm=cfg.arm, modes=run_modes, mode_code=MODE_CODE, + show_ablate=cfg.rollout_ablate_frac > 0) REF_GENS_PER_STEP = 16 * 16 # ariahw/rl-rewardhacking config.py:num_prompts * num_generations # Use the resolved locals (preset defaults merged), not cfg.* which can be None. est_gens_per_step = prompts_per_step * group # before mixed-pool split @@ -894,8 +910,8 @@ def main(cfg: Config) -> int: # routing on a fresh axis lags ~1 step until δS grows there (A1 stale-mask trade-off). GATE_EPS = 1e-6 step_flagged: list[float] = [] - step_tau: list[float] = [] # median live cos_b (should sit inside the band) - step_hkgap: list[float] = [] # band width upper-lower (pair separation; ~0 = random/degenerate) + step_zkeep: list[float] = []; step_zresid: list[float] = []; step_zrout: list[float] = [] # unit shares per zone + step_zkeepE: list[float] = []; step_zresidE: list[float] = []; step_zroutE: list[float] = [] # energy shares per zone step_resid: list[float] = [] # cos(δS.grad AFTER routing, v_grad): hack-ward leak into deployed knob def _routeV_grad_filter(info, n_rollouts: int) -> torch.Tensor: @@ -926,7 +942,9 @@ def main(cfg: Config) -> int: torch.zeros_like(g)) # Σ_{b,t} f·(δS·g) / δS live = g_u.norm(dim=2) > 1e-8 # drop pad tokens from the gauges step_flagged.append(f[live].mean().item() if live.any() else 0.0) - step_tau.append(cos_u[live].median().item() if live.any() else 0.0) + _kn, _rn, _on, _ke, _re, _oe = _zone_stats(f[live], g_u.norm(dim=2)[live]) + step_zkeep.append(_kn); step_zresid.append(_rn); step_zrout.append(_on) + step_zkeepE.append(_ke); step_zresidE.append(_re); step_zroutE.append(_oe) else: cg = cg_full.sum(1) # [G, r] per-rollout g_b = torch.where(reliable, cg / dS_safe, torch.zeros_like(cg)) # [G, r] @@ -935,8 +953,9 @@ def main(cfg: Config) -> int: routed = torch.where(reliable, (cg * f.unsqueeze(1)).sum(0) / dS_safe, torch.zeros_like(g)) # Σ_b f_b·g_b on reliable axes step_flagged.append(f.mean().item()) - step_tau.append(cos_b.median().item()) # live cos centre vs the band - step_hkgap.append(upper - lower) + _kn, _rn, _on, _ke, _re, _oe = _zone_stats(f, g_b.norm(dim=1)) + step_zkeep.append(_kn); step_zresid.append(_rn); step_zrout.append(_on) + step_zkeepE.append(_ke); step_zresidE.append(_re); step_zroutE.append(_oe) # Park the routed fraction in δS_hack (deleted at deploy); δS keeps the rest. # routed + g_keep = g exactly (unreliable axes: routed=0, kept whole). step_grad_hack[name] = (step_grad_hack[name] + routed.detach().clone() @@ -967,8 +986,9 @@ def main(cfg: Config) -> int: # routed contribution to A.grad: Σ_b f_b Σ_t g_h[b,t] ⊗ x[b,t] routed = torch.einsum("gsr,gsd,g->rd", g_h, x_, f).to(full.dtype) # [r, d_in] step_flagged.append(f.mean().item()) - step_tau.append(cos_b.median().item()) - step_hkgap.append(upper - lower) + _kn, _rn, _on, _ke, _re, _oe = _zone_stats(f, g_roll.norm(dim=1)) + step_zkeep.append(_kn); step_zresid.append(_rn); step_zrout.append(_on) + step_zkeepE.append(_ke); step_zresidE.append(_re); step_zroutE.append(_oe) step_grad_hack[name] = (step_grad_hack[name] + routed.detach().clone() if name in step_grad_hack else routed.detach().clone()) g_keep = full - routed @@ -1372,7 +1392,7 @@ def main(cfg: Config) -> int: # Clip over both knobs. For none/erase, δS_hack.grad is None so it's # ignored (identical norm to before). For route it bounds the combined # update (main + quarantine). - # Absorption (logged as `absorb`): ‖g_quar‖/(‖g_keep‖+‖g_quar‖) ∈ [0,1], the + # Quarantine energy share (logged as `qmass`): ‖g_quar‖/(‖g_keep‖+‖g_quar‖) ∈ [0,1], the # share of the update routed into the quarantine (δS_hack, deleted at deploy). # Rising => routing dumps learning into the thrown-away knob and the # deployed model learns nothing. ~0 idle; ~0.5+ climbing = quarantine @@ -1677,10 +1697,13 @@ def main(cfg: Config) -> int: "lp_t": lp_t_mean if n_t else None, "loss": agg_loss, "gn": gn, - "absorb": q_egy, - "tau": (sum(step_tau) / len(step_tau)) if step_tau else float("nan"), - "hkgap": (sum(step_hkgap) / len(step_hkgap)) if step_hkgap else float("nan"), - "frout": (sum(step_flagged) / len(step_flagged)) if step_flagged else float("nan"), + "qmass": q_egy, + "keep": (sum(step_zkeep) / len(step_zkeep)) if step_zkeep else float("nan"), + "resid": (sum(step_zresid) / len(step_zresid)) if step_zresid else float("nan"), + "rout": (sum(step_zrout) / len(step_zrout)) if step_zrout else float("nan"), + "keepE": (sum(step_zkeepE) / len(step_zkeepE)) if step_zkeepE else float("nan"), + "residE": (sum(step_zresidE) / len(step_zresidE)) if step_zresidE else float("nan"), + "routE": (sum(step_zroutE) / len(step_zroutE)) if step_zroutE else float("nan"), "leak": (sum(step_resid) / len(step_resid)) if step_resid else float("nan"), "lr": sched.get_last_lr()[0], "cos_pre": diag["mean_cos_pre"],