log: routeV routing as keep/resid/rout zones x unit+energy views; drop dead hk_abl/slv_abl

Replace the band-mechanics trio (tau/hkgap/frout) and the lumped qmass with a
symmetric zone breakdown: each live unit's cos(g,v_grad) lands below/inside/above
the pair-band -> keep/resid/rout, reported as both unit shares and energy shares
(keepE/residE/routE). Energy view is unit-agnostic (answers 'is the grad per
rollout'). Drop hk_abl/slv_abl unless rollout_ablate_frac>0 (else 0/0). Band edges
(lower/upper) already logged at construction. v1 'routing' arm keeps qmass.

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-06-07 13:13:01 +00:00
co-authored by Claudypoo
parent b170b969e2
commit 25ac3fc5e3
2 changed files with 56 additions and 28 deletions
+20 -15
View File
@@ -91,7 +91,8 @@ class StepLogger:
caller owns it (it also names the row-dict keys) so this module stays leaf-level.
"""
def __init__(self, arm: str, modes: list[str], mode_code: dict[str, str]) -> None:
def __init__(self, arm: str, modes: list[str], mode_code: dict[str, str],
show_ablate: bool = False) -> None:
# arm in {vanilla, projected, routing}; only projected/routing actually
# project the gradient, so the cin/cout/fired diagnostics are theirs alone
# (in vanilla they'd be counterfactual noise -> omitted).
@@ -133,24 +134,28 @@ class StepLogger:
_Col("cos_post", 6, "cout", ".2f", "hack-ward fraction AFTER projection (want ~0: all removed)"),
_Col("fired", 5, "fired", ".2f", "fraction of modules where projection fired"),
]
# routeV: the routing gate is the pair-calibrated BAND. Per rollout,
# f = clamp((cos(g_b,v_grad) - lower)/(upper - lower), 0, 1) routes that
# fraction into the quarantine. lower/upper = mean clean/hack pair cosines.
# Surface where live cos sits (tau), the band width (hkgap), the routed
# fraction (frout, the mass gauge), and the post-routing leak (resid).
# routeV routing, by what the gate does to each live unit (rollout, or token in
# per-token mode). Its cos(g, v_grad) falls below / inside / above the pair-band
# [lower, upper] (edges logged at band construction). Three zones, two views:
# keep/resid/rout = UNIT shares, keepE/residE/routE = ENERGY shares (each sums to
# 1). leak = hack alignment that slipped past into the deployed knob.
if arm == "routingV":
cols += [
_Col("tau", 6, "tau", "+.2f", "median live cos(g_b, v_grad); should sit inside the band [lower, upper]"),
_Col("hkgap", 6, "hkgap", "+.2f", "mean-over-modules band width = max hack-pair minus min clean-pair cosine; >0 = v_grad separates (else direction dead/random)"),
_Col("frout", 6, "frout", "+.2f", "mean routed fraction f over rollouts (the gate decision; hold fixed for matched real-vs-random comparison)"),
_Col("leak", 6, "leak", "+.2f", "leakage (SGTM): cos(deployed delta_S.grad after routing, v_grad); ~0 = hack stripped, >0 = under-pinned, hack survives in deployed knob"),
_Col("keep", 6, "keep", ".2f", "unit share with cos below the band -> kept whole in the deployed knob (left)"),
_Col("resid", 6, "resid", ".2f", "unit share with cos inside the band -> partially routed (residual middle)"),
_Col("rout", 6, "rout", ".2f", "unit share with cos above the band -> fully routed into quarantine (right)"),
_Col("keepE", 6, "keepE", ".2f", "energy-weighted keep: share of grad ENERGY in the kept zone"),
_Col("residE", 6, "residE", ".2f", "energy-weighted resid: share of grad ENERGY in the partially-routed zone"),
_Col("routE", 6, "routE", ".2f", "energy-weighted rout: grad ENERGY share fully routed (~quarantine mass; the routed total is routE..routE+residE)"),
_Col("leak", 6, "leak", "+.2f", "hack-ward cosine left in the deployed knob after routing; ~0 = stripped clean, >0 = hack leaked through (under-routed)"),
]
if arm in ("routing", "routingV"):
if arm == "routing":
cols.append(
_Col("qmass", 6, "qmass", ".2f", "quarantine energy share ||g_quar||/(||g_keep||+||g_quar||): fraction of the update parked in the throwaway knob"))
# Per-step deploy proxy only exists when rollout_ablate_frac>0 generates a knob-off
# slice; without it the slice is empty (0/0), so drop the columns.
if arm in ("routing", "routingV") and show_ablate:
cols += [
# Deploy eval (knob-OFF) is hk_dep below. The train-vs-deploy 2x2's
# knob-ON pass runs once post-loop (FINAL EVAL), not per-step; the
# per-step train series is hk_s. See journal 2026-06-04 (a).
_Col("absorb", 6, "absorb", ".2f", "absorption: grad energy pinned into quarantine ||g_quar||/(||g_keep||+||g_quar||); too low = hack not pinned, ->1 with slv_dep falling = solve also pinned (over-pinned)"),
_Col("hack_abl", 6, "hk_abl", "frac", "per-step deploy proxy: hack rate on the ablated (deploy-mode) rollout slice; train prompts, noisier than hk_dep"),
_Col("solve_abl", 6, "slv_abl", "frac", "per-step deploy proxy: solve rate on the ablated (deploy-mode) rollout slice; train prompts"),
]
+36 -13
View File
@@ -323,6 +323,21 @@ def _haar_unit_dirs(v_grad: dict, seed: int, device) -> dict:
return out
def _zone_stats(f: torch.Tensor, w: torch.Tensor) -> tuple[float, ...]:
"""Split routing units into the three band zones by routed fraction f in [0,1]:
f==0 keep (cos below lower), 0<f<1 resid (cos inside band, partial), f==1 rout
(cos above upper). Returns (keep, resid, rout) UNIT shares and (keepE, residE, routE)
ENERGY shares (w = per-unit grad norm). A unit = a rollout (per-rollout mode) or a
token (per-token mode); the energy view is unit-agnostic."""
if f.numel() == 0:
return (float("nan"),) * 6
lo, hi = (f == 0), (f == 1)
mid = ~(lo | hi)
tot = w.sum().clamp_min(1e-12)
return (lo.float().mean().item(), mid.float().mean().item(), hi.float().mean().item(),
((w * lo).sum() / tot).item(), ((w * mid).sum() / tot).item(), ((w * hi).sum() / tot).item())
def route_band_edges(raw_grads: dict, v_grad: dict, device) -> dict[str, tuple[float, float]]:
"""Per-module routing band (lower, upper) from the contrastive pairs ALONE -- the
pair-calibrated replacement for the old live-detector τ. lower = MIN clean-pair cosine
@@ -752,7 +767,8 @@ def main(cfg: Config) -> int:
# rows are the "is it learning?" signal. ref_eq = cumulative gens / 256 (the
# canonical 16 prompts x 16 gens/step), so ref_eq=1.0 = one reference step's samples.
run_modes = sorted({p["env_mode"] for p in problems}, key=lambda m: list(MODE_CODE).index(m))
step_logger = StepLogger(arm=cfg.arm, modes=run_modes, mode_code=MODE_CODE)
step_logger = StepLogger(arm=cfg.arm, modes=run_modes, mode_code=MODE_CODE,
show_ablate=cfg.rollout_ablate_frac > 0)
REF_GENS_PER_STEP = 16 * 16 # ariahw/rl-rewardhacking config.py:num_prompts * num_generations
# Use the resolved locals (preset defaults merged), not cfg.* which can be None.
est_gens_per_step = prompts_per_step * group # before mixed-pool split
@@ -894,8 +910,8 @@ def main(cfg: Config) -> int:
# routing on a fresh axis lags ~1 step until δS grows there (A1 stale-mask trade-off).
GATE_EPS = 1e-6
step_flagged: list[float] = []
step_tau: list[float] = [] # median live cos_b (should sit inside the band)
step_hkgap: list[float] = [] # band width upper-lower (pair separation; ~0 = random/degenerate)
step_zkeep: list[float] = []; step_zresid: list[float] = []; step_zrout: list[float] = [] # unit shares per zone
step_zkeepE: list[float] = []; step_zresidE: list[float] = []; step_zroutE: list[float] = [] # energy shares per zone
step_resid: list[float] = [] # cos(δS.grad AFTER routing, v_grad): hack-ward leak into deployed knob
def _routeV_grad_filter(info, n_rollouts: int) -> torch.Tensor:
@@ -926,7 +942,9 @@ def main(cfg: Config) -> int:
torch.zeros_like(g)) # Σ_{b,t} f·(δS·g) / δS
live = g_u.norm(dim=2) > 1e-8 # drop pad tokens from the gauges
step_flagged.append(f[live].mean().item() if live.any() else 0.0)
step_tau.append(cos_u[live].median().item() if live.any() else 0.0)
_kn, _rn, _on, _ke, _re, _oe = _zone_stats(f[live], g_u.norm(dim=2)[live])
step_zkeep.append(_kn); step_zresid.append(_rn); step_zrout.append(_on)
step_zkeepE.append(_ke); step_zresidE.append(_re); step_zroutE.append(_oe)
else:
cg = cg_full.sum(1) # [G, r] per-rollout
g_b = torch.where(reliable, cg / dS_safe, torch.zeros_like(cg)) # [G, r]
@@ -935,8 +953,9 @@ def main(cfg: Config) -> int:
routed = torch.where(reliable, (cg * f.unsqueeze(1)).sum(0) / dS_safe,
torch.zeros_like(g)) # Σ_b f_b·g_b on reliable axes
step_flagged.append(f.mean().item())
step_tau.append(cos_b.median().item()) # live cos centre vs the band
step_hkgap.append(upper - lower)
_kn, _rn, _on, _ke, _re, _oe = _zone_stats(f, g_b.norm(dim=1))
step_zkeep.append(_kn); step_zresid.append(_rn); step_zrout.append(_on)
step_zkeepE.append(_ke); step_zresidE.append(_re); step_zroutE.append(_oe)
# Park the routed fraction in δS_hack (deleted at deploy); δS keeps the rest.
# routed + g_keep = g exactly (unreliable axes: routed=0, kept whole).
step_grad_hack[name] = (step_grad_hack[name] + routed.detach().clone()
@@ -967,8 +986,9 @@ def main(cfg: Config) -> int:
# routed contribution to A.grad: Σ_b f_b Σ_t g_h[b,t] ⊗ x[b,t]
routed = torch.einsum("gsr,gsd,g->rd", g_h, x_, f).to(full.dtype) # [r, d_in]
step_flagged.append(f.mean().item())
step_tau.append(cos_b.median().item())
step_hkgap.append(upper - lower)
_kn, _rn, _on, _ke, _re, _oe = _zone_stats(f, g_roll.norm(dim=1))
step_zkeep.append(_kn); step_zresid.append(_rn); step_zrout.append(_on)
step_zkeepE.append(_ke); step_zresidE.append(_re); step_zroutE.append(_oe)
step_grad_hack[name] = (step_grad_hack[name] + routed.detach().clone()
if name in step_grad_hack else routed.detach().clone())
g_keep = full - routed
@@ -1372,7 +1392,7 @@ def main(cfg: Config) -> int:
# Clip over both knobs. For none/erase, δS_hack.grad is None so it's
# ignored (identical norm to before). For route it bounds the combined
# update (main + quarantine).
# Absorption (logged as `absorb`): ‖g_quar‖/(‖g_keep‖+‖g_quar‖) ∈ [0,1], the
# Quarantine energy share (logged as `qmass`): ‖g_quar‖/(‖g_keep‖+‖g_quar‖) ∈ [0,1], the
# share of the update routed into the quarantine (δS_hack, deleted at deploy).
# Rising => routing dumps learning into the thrown-away knob and the
# deployed model learns nothing. ~0 idle; ~0.5+ climbing = quarantine
@@ -1677,10 +1697,13 @@ def main(cfg: Config) -> int:
"lp_t": lp_t_mean if n_t else None,
"loss": agg_loss,
"gn": gn,
"absorb": q_egy,
"tau": (sum(step_tau) / len(step_tau)) if step_tau else float("nan"),
"hkgap": (sum(step_hkgap) / len(step_hkgap)) if step_hkgap else float("nan"),
"frout": (sum(step_flagged) / len(step_flagged)) if step_flagged else float("nan"),
"qmass": q_egy,
"keep": (sum(step_zkeep) / len(step_zkeep)) if step_zkeep else float("nan"),
"resid": (sum(step_zresid) / len(step_zresid)) if step_zresid else float("nan"),
"rout": (sum(step_zrout) / len(step_zrout)) if step_zrout else float("nan"),
"keepE": (sum(step_zkeepE) / len(step_zkeepE)) if step_zkeepE else float("nan"),
"residE": (sum(step_zresidE) / len(step_zresidE)) if step_zresidE else float("nan"),
"routE": (sum(step_zroutE) / len(step_zroutE)) if step_zroutE else float("nan"),
"leak": (sum(step_resid) / len(step_resid)) if step_resid else float("nan"),
"lr": sched.get_last_lr()[0],
"cos_pre": diag["mean_cos_pre"],