From 4336d6c577161b118a53a2ee77806b595bdf41bd Mon Sep 17 00:00:00 2001 From: wassname Date: Tue, 2 Jun 2026 23:51:27 +0000 Subject: [PATCH] feat: log problem_id/env_mode/prompt to rollouts + --teacher-off-step curriculum rollouts.jsonl now carries problem_id, env_mode, and the exact chat-templated prompt -- the per-prompt problem is a random draw, so these are required to harvest same-prompt (hack,clean) pairs from real student rollouts (A5 held-out v_grad; the teacher pool is a different distribution, not IID with student hacks). --teacher-off-step=N: seed hacks via teacher pool for N steps then cut to pure on-policy (G_t=0) -- guarantees all hacks emerge before testing route2 persistence without the teacher crutch. Smoke (curriculum fires at step 2, metadata present) green. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- src/projected_grpo/train.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/projected_grpo/train.py b/src/projected_grpo/train.py index 62cb1ec..e78f64e 100644 --- a/src/projected_grpo/train.py +++ b/src/projected_grpo/train.py @@ -209,6 +209,11 @@ class Config: # reduction gap holds and the solve cost vanishes vs mix=0.5. Needs group>=8 # so round(G*mix_ratio) >= 1 teacher. mix_ratio: float = 0.125 + # Teacher-off curriculum: seed hacks via the teacher pool for the first N + # optimizer steps, then cut to pure on-policy (G_t=0) for the rest. None = never + # cut. Guarantees all hacks emerge (teacher-seeded) before testing whether route2 + # holds the suppression once the teacher crutch is gone. See step-loop use. + teacher_off_step: int | None = None # Cross-mechanism BLUF (docs/spec/20260528_cross_mechanism_v_hack.md): # which upstream detectors were used to label the hack-side of the pairs that # produced v_hack. Used to split student-rollout hacks into half_A (covered by @@ -733,6 +738,13 @@ def main(cfg: Config) -> int: mininterval=120, maxinterval=120, disable=None) # ── training loop: generate -> grade -> backward -> project -> step ── for step in pbar: + # Teacher-off curriculum: seed hacks via the teacher pool for the first N + # steps, then cut to pure on-policy (G_t=0) so we test whether route2 holds + # the suppression once the teacher crutch is gone. Monotonic: stays off. + if cfg.teacher_off_step is not None and step >= cfg.teacher_off_step and G_t > 0: + logger.info(f"teacher-off curriculum: step {step} >= {cfg.teacher_off_step} " + f"-> G_t {G_t}->0, G_s {G_s}->{group} (pure on-policy from here)") + G_t, G_s = 0, group t0 = time.time() opt.zero_grad(set_to_none=True) @@ -1001,6 +1013,13 @@ def main(cfg: Config) -> int: f"=== END {hack_cls} ===") step_rollouts.append({ "step": step, "p_idx": p_idx, "gi": gi, + # problem identity + the exact prompt: the per-prompt problem is a + # RANDOM draw (idx above), so without these a rollout can't be mapped + # back to its prompt -- needed to harvest same-prompt (hack,clean) + # pairs from real student rollouts (A5 held-out-mode v_grad). + "problem_id": prob["problem_id"], + "env_mode": (partition[prob["problem_id"]] if partition else cfg.env_mode), + "prompt": prompt, "reward": r.reward, "gt_pass": r.gt_pass, "gt_correct": r.gt_correct, "passed": r.passed, "exploited": r.exploited, "mechanism": r.mechanism, "hacked_C": r.hacked, "hacked_D": r.hacked_wrong_tests,