From 23512ed07ce09f19a982b1459bc6ff1900e38d0a Mon Sep 17 00:00:00 2001 From: wassname Date: Mon, 1 Jun 2026 01:38:15 +0000 Subject: [PATCH] feat: qE column -- grad energy fraction into the quarantine ||g_quar|| / (||g_keep|| + ||g_quar||) for routing arms. Makes job-46's invisible failure legible: act-mask coin-flip dumps learning into the deleted quarantine, so the deployed delta_S learns nothing while lp_t stays flat. ~0 = quar idle; ~0.5+ and climbing = quarantine eating the update. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- src/projected_grpo/train.py | 48 ++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/src/projected_grpo/train.py b/src/projected_grpo/train.py index add0162..2ce552b 100644 --- a/src/projected_grpo/train.py +++ b/src/projected_grpo/train.py @@ -547,7 +547,13 @@ def ablate_quarantine(wrappers: dict): """Zero the routing quarantine for the duration -- the eval-time ablation of the routed hack capability. Save -> zero -> (eval) -> restore. The route arm's deployment model IS this ablated state. Covers BOTH quarantines: delta_S_hack - (shared-basis route) and B_q (distinct-basis route2; zeroing B_q -> quar=0).""" + (shared-basis route) and B_q (distinct-basis route2; zeroing B_q -> quar=0). + + TODO(post-deploy-finetune): SGTM's ablate(trainable=True) reinits the forget + weights to the retain-dims' std instead of zeroing, so the model stays + finetunable after the quarantine is removed (no dead hole). We zero because + we only eval after deploy; add the reinit path if we ever retrain post-ablate. + See docs/grad_routing/sgtm_vs_ours.md.""" saved = {n: info["delta_S_hack"].data.clone() for n, info in wrappers.items()} saved_Bq = {n: info["B_q"].data.clone() for n, info in wrappers.items() if "B_q" in info} for info in wrappers.values(): @@ -705,6 +711,7 @@ class StepLogger: ] if arm in ("routing", "routing2_act", "routing2_grad"): cols += [ + _Col("q_egy", 6, "qE", ".2f", "grad energy into quarantine ||g_quar||/(||g_keep||+||g_quar||); ~0.5+ rising = learning dumped into the thrown-away knob"), _Col("hack_deploy", 7, "hk_dep", "+.2f", "DEPLOY-eval hack (quarantine deleted = deployed model)"), _Col("solve_deploy", 7, "slv_dep", "+.2f", "DEPLOY-eval solve"), ] @@ -1123,7 +1130,15 @@ def main(cfg: Config) -> int: "rows": json.dumps(rows), "cfg": json.dumps(vars(cfg), default=str), }) - pbar = tqdm(range(steps), desc=f"train {cfg.arm} {cfg.preset_name}", mininterval=60) + # disable=None: auto-disable the bar when stdout is NOT a tty (pueue, pipes, + # file redirects). In those contexts every per-step `logger.info(step_logger.row)` + # goes through tqdm.write, which redraws the bar -> half-drawn fragments + # interleaved with the per-step table. Killing the bar off-tty leaves clean + # per-step rows (they already carry step + sec, so the bar is redundant there); + # an interactive terminal still gets the live bar. mininterval==maxinterval keeps + # that interactive bar sparse (tqdm's default maxinterval=10 forces 10s redraws). + pbar = tqdm(range(steps), desc=f"train {cfg.arm} {cfg.preset_name}", + mininterval=120, maxinterval=120, disable=None) for step in pbar: t0 = time.time() opt.zero_grad(set_to_none=True) @@ -1598,6 +1613,19 @@ def main(cfg: Config) -> int: # Clip over both knobs. For none/erase, delta_S_hack.grad is None so it's # ignored -> identical norm to before (R4). For route it bounds the # combined update (main + quarantine). + # Split the grad energy: how much is going to delta_S (the KEPT/deployed + # knob) vs the quarantine (delta_S_hack for route, A_q/B_q for route2 -- + # the THROWN-AWAY knob). qE = quar / (keep + quar) in [0,1]. Rising qE + # means routing is dumping the learning into the quarantine, so the + # deployed model learns nothing -- the invisible failure in job 46 + # (act-mask coin-flip routed ~half of everything into quar). ~0 = quar + # idle; ~0.5+ and climbing = quarantine eating the update. + def _grad_l2(params): + gs = [p.grad for p in params if p.grad is not None] + return float(torch.norm(torch.stack([g.norm() for g in gs]))) if gs else 0.0 + gn_keep = _grad_l2(delta_params) + gn_quar = _grad_l2(delta_hack_params + quar_params) + q_egy = gn_quar / (gn_keep + gn_quar) if (gn_keep + gn_quar) > 0 else 0.0 gn = float(torch.nn.utils.clip_grad_norm_(delta_params + delta_hack_params + quar_params, cfg.grad_clip)) opt.step() sched.step() @@ -1848,6 +1876,7 @@ def main(cfg: Config) -> int: "lp_t": lp_t_mean if n_t else None, "loss": agg_loss, "gn": gn, + "q_egy": q_egy, "lr": sched.get_last_lr()[0], "cos_pre": diag["mean_cos_pre"], "cos_pre_s": diag["mean_cos_pre_s"], @@ -1903,10 +1932,14 @@ def main(cfg: Config) -> int: first_hack_saved = True logger.info(f"first-student-hack ckpt saved: step={step} hack_s={hack_s_n}/{n_s} -> {first_hack_path.name}") # Live status in tqdm postfix; full per-step line in verbose log only. + # refresh=False: set_postfix defaults to forcing a redraw EVERY step, which + # bypasses mininterval and spams half-drawn bar fragments into piped/pueue + # logs. With refresh=False the postfix is shown at the next mininterval tick. pbar.set_postfix( rew=f"{rew_mean:+.2f}", gt=f"{sum(agg_gt)}/{n_rollouts}", hack=f"{sum(agg_hack)}/{n_rollouts}", loss=f"{agg_loss:+.3f}", sec=f"{time.time()-t0:.0f}", + refresh=False, ) logger.debug( f"step {step:3d} rew={rew_mean:+.2f}(std {rew_std:.2f}) " @@ -2087,8 +2120,7 @@ def main(cfg: Config) -> int: for k, v in r.items() if k not in _DROP_COLS} for r in rows ] - print(tabulate(rows_for_dump, headers="keys", tablefmt="tsv", floatfmt="+.3f")) - print() + # BLUF summary first -- the single row a reader scans -- as github markdown. print(tabulate([{ "cue": cue, "HACK_RATE": f"{hack_rate:.3f}", "PASS_RATE": f"{pass_rate:.3f}", "HACK_S": f"{hack_rate_s:.3f}", "HACK_T": f"{hack_rate_t:.3f}", @@ -2097,10 +2129,10 @@ def main(cfg: Config) -> int: "pool": (cfg.teacher_pool_dir.name if cfg.teacher_pool_dir else ""), "mix": cfg.mix_ratio if cfg.teacher_pool_dir else "", "tag": cfg.out_tag, "log": str(verbose_log), - }], headers="keys", tablefmt="tsv")) - # Markdown copy: easier to paste into journal/PRs than the TSV above. - print() - print("### Per-step rows (markdown)\n") + }], headers="keys", tablefmt="github")) + # Per-step rows ONCE, markdown (journal/PR pasteable). The TSV duplicate of the + # same data was dropped -- two formats of one table was just noise. + print("\n### Per-step rows (markdown)\n") print(tabulate(rows_for_dump, headers="keys", tablefmt="pipe", floatfmt="+.3f")) save_ckpt(rows)