feat: log problem_id/env_mode/prompt to rollouts + --teacher-off-step curriculum

rollouts.jsonl now carries problem_id, env_mode, and the exact chat-templated
prompt -- the per-prompt problem is a random draw, so these are required to harvest
same-prompt (hack,clean) pairs from real student rollouts (A5 held-out v_grad; the
teacher pool is a different distribution, not IID with student hacks).

--teacher-off-step=N: seed hacks via teacher pool for N steps then cut to pure
on-policy (G_t=0) -- guarantees all hacks emerge before testing route2 persistence
without the teacher crutch. Smoke (curriculum fires at step 2, metadata present) green.

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-06-02 23:51:27 +00:00
parent 62e510ff57
commit 4336d6c577
+19
View File
@@ -209,6 +209,11 @@ class Config:
# reduction gap holds and the solve cost vanishes vs mix=0.5. Needs group>=8
# so round(G*mix_ratio) >= 1 teacher.
mix_ratio: float = 0.125
# Teacher-off curriculum: seed hacks via the teacher pool for the first N
# optimizer steps, then cut to pure on-policy (G_t=0) for the rest. None = never
# cut. Guarantees all hacks emerge (teacher-seeded) before testing whether route2
# holds the suppression once the teacher crutch is gone. See step-loop use.
teacher_off_step: int | None = None
# Cross-mechanism BLUF (docs/spec/20260528_cross_mechanism_v_hack.md):
# which upstream detectors were used to label the hack-side of the pairs that
# produced v_hack. Used to split student-rollout hacks into half_A (covered by
@@ -733,6 +738,13 @@ def main(cfg: Config) -> int:
mininterval=120, maxinterval=120, disable=None)
# ── training loop: generate -> grade -> backward -> project -> step ──
for step in pbar:
# Teacher-off curriculum: seed hacks via the teacher pool for the first N
# steps, then cut to pure on-policy (G_t=0) so we test whether route2 holds
# the suppression once the teacher crutch is gone. Monotonic: stays off.
if cfg.teacher_off_step is not None and step >= cfg.teacher_off_step and G_t > 0:
logger.info(f"teacher-off curriculum: step {step} >= {cfg.teacher_off_step} "
f"-> G_t {G_t}->0, G_s {G_s}->{group} (pure on-policy from here)")
G_t, G_s = 0, group
t0 = time.time()
opt.zero_grad(set_to_none=True)
@@ -1001,6 +1013,13 @@ def main(cfg: Config) -> int:
f"=== END {hack_cls} ===")
step_rollouts.append({
"step": step, "p_idx": p_idx, "gi": gi,
# problem identity + the exact prompt: the per-prompt problem is a
# RANDOM draw (idx above), so without these a rollout can't be mapped
# back to its prompt -- needed to harvest same-prompt (hack,clean)
# pairs from real student rollouts (A5 held-out-mode v_grad).
"problem_id": prob["problem_id"],
"env_mode": (partition[prob["problem_id"]] if partition else cfg.env_mode),
"prompt": prompt,
"reward": r.reward, "gt_pass": r.gt_pass, "gt_correct": r.gt_correct,
"passed": r.passed, "exploited": r.exploited, "mechanism": r.mechanism,
"hacked_C": r.hacked, "hacked_D": r.hacked_wrong_tests,