diff --git a/src/projected_grpo/train.py b/src/projected_grpo/train.py index 155a210..f143322 100644 --- a/src/projected_grpo/train.py +++ b/src/projected_grpo/train.py @@ -688,6 +688,7 @@ class StepLogger: cols += [ _Col("tau", 6, "tau", "+.2f", "per-step calibrated route threshold (midpoint of hack vs clean cos clouds)"), _Col("hkgap", 6, "hkgap", "+.2f", "ema_hack_cos - ema_clean_cos; >0 = v_grad still separates hack from clean (else direction dead)"), + _Col("resid", 6, "resid", "+.2f", "cos(deployed delta_S.grad AFTER routing, v_grad); ~0 = hack stripped cleanly, >0 = leak into deployed knob"), ] if arm in ("routing", "routing2"): cols += [ @@ -1147,6 +1148,7 @@ def main(cfg: Config) -> int: step_flagged: list[float] = [] step_tau: list[float] = [] # per-(prompt,module) calibrated route threshold step_hkgap: list[float] = [] # ema_hack_cos - ema_clean_cos (discrimination gauge) + step_resid: list[float] = [] # cos(delta_S.grad AFTER routing, v_grad): hack-ward leak into deployed knob def _route2_grad_filter(info, n_rollouts: int, hack_anchor: torch.Tensor, @@ -1194,7 +1196,14 @@ def main(cfg: Config) -> int: # delta_S keeps only the unflagged. Capacity-balanced: both shape [r]. step_grad_hack[name] = (step_grad_hack[name] + sub.detach().clone() if name in step_grad_hack else sub.detach().clone()) - return g - sub + g_keep = g - sub # the deployed knob's gradient + # Residual hack-ward alignment of the KEPT grad. Disambiguates qE: + # qE high + resid~0 = routing stripped the hack cleanly (dominant + # teacher grad correctly quarantined); qE high + resid>0 = false + # negatives leaked hack-ward grad into the deployed knob (the real + # failure). vg is unit, so this is a plain cosine. + step_resid.append((g_keep @ vg / g_keep.norm().clamp_min(1e-12)).item()) + return g_keep # Split backward into student/teacher only every cos_pre_split_every steps. # On split steps: 2 backwards per prompt, populates step_grad_s/_t. @@ -1842,6 +1851,7 @@ def main(cfg: Config) -> int: "q_egy": q_egy, "tau": (sum(step_tau) / len(step_tau)) if step_tau else float("nan"), "hkgap": (sum(step_hkgap) / len(step_hkgap)) if step_hkgap else float("nan"), + "resid": (sum(step_resid) / len(step_resid)) if step_resid else float("nan"), "lr": sched.get_last_lr()[0], "cos_pre": diag["mean_cos_pre"], "cos_pre_s": diag["mean_cos_pre_s"],