From f8a76e3bf5c9758c6b7d6b759af1498c7bfba7b3 Mon Sep 17 00:00:00 2001 From: wassname Date: Mon, 1 Jun 2026 11:49:32 +0000 Subject: [PATCH] default rollout_ablate_frac=0 (drop the exploration floor) job 64 (frac=0.5) leaked: deploy hack climbed 0.00->0.125 over training while no-floor job 60 held 0.000. Sampling from the deployed model makes its reward an optimization target whose optimum is hacking, so stale-gate misses update the kept knob -- the floor optimizes the policy the quarantine should keep clean. Neither gradient-routing paper samples from the ablated model in training. Off by default; re-enable only to test floor+fresh-gate (refresh-every-1). Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- src/projected_grpo/train.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/projected_grpo/train.py b/src/projected_grpo/train.py index 846b21b..66eaada 100644 --- a/src/projected_grpo/train.py +++ b/src/projected_grpo/train.py @@ -175,18 +175,20 @@ class Config: project_overshoot: float = 1.0 # remove overshoot*c_use@V; 1.0=just remove, 1.1=10% reversal of hack-ward grad # Exploration floor against hack-saturation (route/route2 only). Fraction of # student rollouts to generate with the quarantine (delta_S_hack) ablated, i.e. - # from the DEPLOYED model. The risk this guards: if on-policy sampling collapses - # onto hacking, the policy stops emitting solves, every rollout gets routed to - # the quarantine, and the deployed delta_S never sees a solve gradient to learn - # from (it saturates). Forcing a fraction of rollouts hack-OFF guarantees the - # solve region stays covered, exactly like any RL exploration term. Pure - # sampling-side diversity; accepts a slight off-policy mismatch (GRPO already - # tolerates it) in exchange for guaranteed coverage. 0 = off. - # Default 0.5 (half the rollouts deploy-mode): the bonus is that these ablated - # rollouts ARE deployed-model samples, so we grade them for a FREE per-step - # deploy proxy (hk_abl/slv_abl) -- see the row dict -- without the extra - # generation the periodic eval_ablate_every eval costs. - rollout_ablate_frac: float = 0.5 + # from the DEPLOYED model. Intent: if on-policy sampling collapses onto hacking, + # every rollout gets routed to the quarantine and the deployed delta_S never sees + # a solve gradient (it saturates); forcing some rollouts hack-OFF keeps the solve + # region covered. + # DEFAULT 0 (off). job 64 (frac=0.5) leaked: deploy hack climbed 0.00->0.125 over + # training while the no-floor job 60 held deploy hack=0.000. Mechanism: sampling + # from the DEPLOYED model makes its reward an optimization target whose optimum is + # hacking, so whenever the (stale) routing gate misses, those deploy-generated + # hacks update the kept knob -- i.e. the floor optimizes the very policy the + # quarantine is meant to keep clean. Neither gradient-routing paper samples from + # the ablated model during training (they sample knob-on, ablate at eval, and let + # absorption localize). So default off; re-enable only to TEST the floor+fresh-gate + # hypothesis (does refresh-every-1 keep the gate sharp enough that the floor is safe?). + rollout_ablate_frac: float = 0.0 # Which grader flaw + factual hint this run trains on (a "hack class"). Sets # the prompt hint (HINT_REPLACE_TO) and how `passed` is graded in rewards.py. # run_tests = the original run_tests-overwrite loophole. eq_override / exit_code