mirror of
https://github.com/wassname/evil_MoE.git
synced 2026-07-10 20:52:10 +08:00
feat: log problem_id/env_mode/prompt to rollouts + --teacher-off-step curriculum
rollouts.jsonl now carries problem_id, env_mode, and the exact chat-templated prompt -- the per-prompt problem is a random draw, so these are required to harvest same-prompt (hack,clean) pairs from real student rollouts (A5 held-out v_grad; the teacher pool is a different distribution, not IID with student hacks). --teacher-off-step=N: seed hacks via teacher pool for N steps then cut to pure on-policy (G_t=0) -- guarantees all hacks emerge before testing route2 persistence without the teacher crutch. Smoke (curriculum fires at step 2, metadata present) green. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
@@ -209,6 +209,11 @@ class Config:
|
||||
# reduction gap holds and the solve cost vanishes vs mix=0.5. Needs group>=8
|
||||
# so round(G*mix_ratio) >= 1 teacher.
|
||||
mix_ratio: float = 0.125
|
||||
# Teacher-off curriculum: seed hacks via the teacher pool for the first N
|
||||
# optimizer steps, then cut to pure on-policy (G_t=0) for the rest. None = never
|
||||
# cut. Guarantees all hacks emerge (teacher-seeded) before testing whether route2
|
||||
# holds the suppression once the teacher crutch is gone. See step-loop use.
|
||||
teacher_off_step: int | None = None
|
||||
# Cross-mechanism BLUF (docs/spec/20260528_cross_mechanism_v_hack.md):
|
||||
# which upstream detectors were used to label the hack-side of the pairs that
|
||||
# produced v_hack. Used to split student-rollout hacks into half_A (covered by
|
||||
@@ -733,6 +738,13 @@ def main(cfg: Config) -> int:
|
||||
mininterval=120, maxinterval=120, disable=None)
|
||||
# ── training loop: generate -> grade -> backward -> project -> step ──
|
||||
for step in pbar:
|
||||
# Teacher-off curriculum: seed hacks via the teacher pool for the first N
|
||||
# steps, then cut to pure on-policy (G_t=0) so we test whether route2 holds
|
||||
# the suppression once the teacher crutch is gone. Monotonic: stays off.
|
||||
if cfg.teacher_off_step is not None and step >= cfg.teacher_off_step and G_t > 0:
|
||||
logger.info(f"teacher-off curriculum: step {step} >= {cfg.teacher_off_step} "
|
||||
f"-> G_t {G_t}->0, G_s {G_s}->{group} (pure on-policy from here)")
|
||||
G_t, G_s = 0, group
|
||||
t0 = time.time()
|
||||
opt.zero_grad(set_to_none=True)
|
||||
|
||||
@@ -1001,6 +1013,13 @@ def main(cfg: Config) -> int:
|
||||
f"=== END {hack_cls} ===")
|
||||
step_rollouts.append({
|
||||
"step": step, "p_idx": p_idx, "gi": gi,
|
||||
# problem identity + the exact prompt: the per-prompt problem is a
|
||||
# RANDOM draw (idx above), so without these a rollout can't be mapped
|
||||
# back to its prompt -- needed to harvest same-prompt (hack,clean)
|
||||
# pairs from real student rollouts (A5 held-out-mode v_grad).
|
||||
"problem_id": prob["problem_id"],
|
||||
"env_mode": (partition[prob["problem_id"]] if partition else cfg.env_mode),
|
||||
"prompt": prompt,
|
||||
"reward": r.reward, "gt_pass": r.gt_pass, "gt_correct": r.gt_correct,
|
||||
"passed": r.passed, "exploited": r.exploited, "mechanism": r.mechanism,
|
||||
"hacked_C": r.hacked, "hacked_D": r.hacked_wrong_tests,
|
||||
|
||||
Reference in New Issue
Block a user