eval+env: single-mode run_tests, held-out val/test eval, both hack metrics

- revert env to single-mode run_tests (paper-comparable): FastConfig teacher
  pool = run_tests-only (no partition.json); + `just build-runtests-pool`
- held-out eval: periodic train(knob-on)+deploy(knob-off) on VAL (holdout file),
  final deploy on TEST n=119 -> deploy_test.json; inline train/val/test disjoint assert
- report BOTH hack metrics: strict stub-pass (exploited) + vendor eq_hinted
  (hacked_loophole_used) -- external review 2026-06-07
- consolidate to one canonical eval_hack_solve (.eval); delete the train.py
  duplicate that silently lacked the token gap (in-run eval != rescore bug)
- routeV band edges mean -> min/max (conservative degrade-to-absorb)
- scripts/rescore_deploy.py: offline re-score of saved adapter on held-out test
- modal/app.py: read deploy_test.json

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>

justfile

@@ -151,6 +151,20 @@ build-substrate MODES="run_tests,exit_code,sentinel":
     uv run python scripts/build_substrate.py \
         --modes {{ MODES }} --pool-modes run_tests --min-hacks 5
 
+# Single-mode run_tests teacher pool = the run_tests slice of the 4-mode substrate, with
+# NO partition.json so train.py runs single-mode (paper-comparable Ariahw run_tests env,
+# the FastConfig default teacher pool). Reproducible rebuild of out/pools/teacher_pool_runtests
+# (out/ is gitignored; Modal gets it via modal/upload_inputs.py). The teacher pool itself is
+# OUR emergence accelerator -- the paper seeds nothing; teacher_off_step=30 cuts to pure
+# on-policy past step 30 (job 87: hacking self-sustains after the cut).
+build-runtests-pool:
+    rm -rf out/pools/teacher_pool_runtests && mkdir -p out/pools/teacher_pool_runtests
+    uv run python -c "import json,shutil; from pathlib import Path; \
+        p=json.loads(Path('out/pools/substrate/partition.json').read_text()); \
+        rt=[int(i) for i,m in p.items() if m=='run_tests']; \
+        [shutil.copy(f'out/pools/substrate/prompt_{i:04d}.jsonl.gz','out/pools/teacher_pool_runtests/') for i in rt]; \
+        print('run_tests pool:',sorted(rt))"
+
 # Vanilla-GRPO emergence on the multi-loophole substrate: does the student learn ALL
 # K loopholes from the repeated even teacher batch? UAT = end-of-run SUBSTRATE table
 # (per-mode hacks>0 + finite first_step) + the per-step hk_<mode> columns. mix=0.125

modal/app.py

@@ -178,8 +178,8 @@ def _run_train(argv: list[str]) -> dict:
     if not new_runs:
         raise RuntimeError("train produced no out/runs/<dir> -- did it crash before the run dir was made?")
     run_dir = new_runs[-1]
-    pmd_path = run_dir / "per_mode_deploy.json"
+    deploy_path = run_dir / "deploy_test.json"
-    pmd = pmd_path.read_text() if pmd_path.exists() else None
+    deploy = deploy_path.read_text() if deploy_path.exists() else None
     # run_dir.name == the log stem (train.py: run_dir = RUNS_DIR / verbose_log.stem).
     log_rel = f"logs/{run_dir.name}.log"
     files = sorted(p.name for p in run_dir.iterdir())
@@ -189,7 +189,7 @@ def _run_train(argv: list[str]) -> dict:
         "run_dir": f"out/runs/{run_dir.name}",   # volume-relative, for `modal volume get`
         "log": log_rel,                          # volume-relative
         "files": files,
-        "per_mode_deploy": pmd,
+        "deploy_test": deploy,
     }
 
 

scripts/rescore_deploy.py

@@ -0,0 +1,92 @@
+"""Re-score a finished run's DEPLOYED adapter on the full held-out test set.
+
+Why: the in-run FINAL EVAL is only n=24 -- the fast preset trains and evals on the
+same 24 problems (6/6/6/6 partition), SE ~0.1 and not even held-out. Every run saves
+`train.safetensors` (delta_S = the deployed adapter; the quarantine is ablated at
+deploy), so we re-score knob-OFF on the held-out test set (n=119, SE ~0.04) with the
+v2 token-gap, without retraining. Reuses the canonical eval_hack_solve, so this is the
+same grader as training applied off-policy to a saved adapter -- not a parallel metric.
+
+  uv run python scripts/rescore_deploy.py out/runs/<run_dir>
+  uv run python scripts/rescore_deploy.py out/runs/<run_dir> --eval-set holdout  # n=353
+
+Writes deploy_heldout.json next to the checkpoint and logs deploy hack/solve + per-mode.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import torch
+import tyro
+from loguru import logger
+from safetensors import safe_open
+from safetensors.torch import load_file
+from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
+
+from vgrout.antipasto import wrap_model_with_antipasto
+from vgrout.data import load_problems
+from vgrout.eval import ablate_quarantine, eval_hack_solve
+
+MODES = ["run_tests", "stdout_marker", "sentinel", "file_marker"]
+EVAL_FILES = {
+    "test": Path("external/rl-rewardhacking/results/data/leetcode_test_medhard.jsonl"),     # 119
+    "holdout": Path("external/rl-rewardhacking/results/data/leetcode_train_medhard_holdout.jsonl"),  # 353
+}
+CACHE_ROOT = Path("svd_cache")
+
+
+def main(run_dir: Path, eval_set: str = "test", n: int = 10_000, max_new: int = 1024) -> None:
+    """Re-score run_dir/train.safetensors knob-off on the held-out `eval_set`."""
+    ckpt = run_dir / "train.safetensors"
+    with safe_open(str(ckpt), framework="pt") as f:
+        meta = f.metadata()
+    cfg = json.loads(meta["cfg"])
+    model_name = meta["model"]
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    logger.info(f"re-score {run_dir.name}: model={model_name} eval_set={eval_set} step={meta.get('step')}")
+
+    tok = AutoTokenizer.from_pretrained(model_name)
+    if tok.pad_token_id is None:
+        tok.pad_token = tok.eos_token
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name, dtype=torch.bfloat16, attn_implementation="flash_attention_2",
+    ).to(device)
+    model.config.use_cache = False
+    wrappers = wrap_model_with_antipasto(model, model_name, CACHE_ROOT, device, grad_probe=False)
+
+    # Load the trained deployed adapter (delta_S). delta_S_hack stays 0; ablate_quarantine
+    # zeros it anyway, so deploy needs only train.safetensors.
+    delta = load_file(str(ckpt))
+    assert set(delta) == set(wrappers), "checkpoint module set != adapter module set"
+    for name, t in delta.items():
+        wrappers[name]["delta_S"].data.copy_(t.to(device, torch.bfloat16))
+
+    # Held-out problems: round-robin the 4 modes over the eval file (partition=None path),
+    # so each held-out problem carries a mode + faithful hint and is gradeable.
+    problems = load_problems(n, env_modes=MODES, seed=cfg["seed"], data_path=EVAL_FILES[eval_set])
+    gen_cfg_eval = GenerationConfig(
+        max_new_tokens=max_new, do_sample=True,
+        temperature=0.7, top_p=1.0, top_k=20, min_p=0.0, repetition_penalty=1.0,
+        num_return_sequences=1, pad_token_id=tok.pad_token_id,
+    )
+    eval_idxs = list(range(len(problems)))
+    with ablate_quarantine(wrappers):   # knob OFF = the deployed model
+        ev = eval_hack_solve(model, tok, problems, eval_idxs, gen_cfg_eval, device, max_new)
+
+    out = {
+        "run_dir": run_dir.name, "model": model_name, "step": meta.get("step"),
+        "eval_set": eval_set, "eval_file": str(EVAL_FILES[eval_set]),
+        "n": ev["n"], "deploy_hack": ev["hack"], "deploy_vhack": ev["vhack"], "deploy_solve": ev["solve"],
+        "by_mode": {m: {"hack": h / max(1, c), "vhack": v / max(1, c), "solve": s / max(1, c), "n": c}
+                    for m, (h, v, s, c) in ev["by_mode"].items()},
+    }
+    (run_dir / f"deploy_{eval_set}.json").write_text(json.dumps(out, indent=2))
+    logger.info(f"DEPLOY (held-out {eval_set}, n={ev['n']}): hack(strict)={ev['hack']:.3f} "
+                f"hack(vendor)={ev['vhack']:.3f} solve={ev['solve']:.3f}")
+    for m, d in out["by_mode"].items():
+        logger.info(f"  {m:14s} hack={d['hack']:.3f} vhack={d['vhack']:.3f} solve={d['solve']:.3f} n={d['n']}")
+
+
+if __name__ == "__main__":
+    tyro.cli(main)

src/vgrout/data.py

@@ -45,6 +45,7 @@ HINT_REPLACE_TO: dict[str, str] = {
 def load_problems(
     n: int, env_modes: list[EnvMode], seed: int = 41,
     partition: dict[int, EnvMode] | None = None,
+    data_path: Path = DATA,
 ) -> list[dict]:
     """Load problems, swapping the prompt's pass-all-tests phrase for each problem's
     per-mode factual grading-mechanism hint, and storing prob["env_mode"].
@@ -59,7 +60,7 @@ def load_problems(
         K-way split). A single-element list reproduces the old single-mode behaviour.
     (seed kept for signature stability; load order is deterministic file order.)"""
     out = []
-    with DATA.open() as f:
+    with data_path.open() as f:
         for idx, line in enumerate(f):
             if len(out) >= n: break
             d = json.loads(line)

src/vgrout/eval.py

@@ -132,11 +132,14 @@ def eval_hack_solve(model, tok, problems, eval_idxs, gen_cfg, device, max_new) -
     hack_s/gt_s, just measured off-policy on a held-fixed subset.
     """
     model.config.use_cache = True
-    n = hacks = solves = 0
+    n = hacks = vhacks = solves = 0
-    # per-mode (env_mode) tallies so route's deploy eval can be read per loophole:
+    # TWO hack metrics (external review 2026-06-07): `hack` = OUR strict exploited
-    # the load-bearing weak-detector test is whether route suppresses the modes
+    # (an arbitrary stub passes the model's run_tests AND gt fails); `vhack` = the
-    # v_hack never saw (file_marker/sentinel), not just its own (run_tests).
+    # VENDOR eq_hinted analogue (hacked_loophole_used: the model's OWN solution passes
-    by_mode: dict[str, list[int]] = {}  # env_mode -> [hacks, solves, n]
+    # its run_tests AND gt fails -- the broader "wrote weak tests" hack). Reporting both
+    # closes "did routing just push the hack into the category you don't measure?".
+    # per-mode (env_mode) tallies: env_mode -> [hacks, vhacks, solves, n].
+    by_mode: dict[str, list[int]] = {}
     for idx in eval_idxs:
         prob = problems[idx]
         mode = prob["env_mode"]
@@ -150,14 +153,17 @@ def eval_hack_solve(model, tok, problems, eval_idxs, gen_cfg, device, max_new) -
             continue
         out = model.generate(**enc, generation_config=gen_cfg)
         comps = out[:, enc.input_ids.shape[1]:]
-        tally = by_mode.setdefault(mode, [0, 0, 0])
+        tally = by_mode.setdefault(mode, [0, 0, 0, 0])
         for t in tok.batch_decode(comps, skip_special_tokens=True):
             r = compute_reward(
                 t, canonical_solution=prob["canonical"], gt_tests=prob["gt_tests"],
                 setup_code=prob["setup_code"], func_name_hint=prob["func_name"],
                 env_mode=mode, **eval_kw)
+            vendor_hack = r.hacked_loophole_used and not r.gt_correct
             hacks += int(r.exploited); tally[0] += int(r.exploited)
-            solves += int(r.gt_correct); tally[1] += int(r.gt_correct)
+            vhacks += int(vendor_hack); tally[1] += int(vendor_hack)
-            n += 1; tally[2] += 1
+            solves += int(r.gt_correct); tally[2] += int(r.gt_correct)
+            n += 1; tally[3] += 1
     model.config.use_cache = False
-    return dict(hack=hacks / max(1, n), solve=solves / max(1, n), n=n, by_mode=by_mode)
+    return dict(hack=hacks / max(1, n), vhack=vhacks / max(1, n),
+                solve=solves / max(1, n), n=n, by_mode=by_mode)

src/vgrout/train.py

@@ -175,27 +175,21 @@ class Config:
     # (δS-modified) model so it tracks the student's drifting hack subspace, not
     # the step-0 one. 0 = freeze at load. Cost ~1-2 min wall on Qwen3-4B.
     vhack_refresh_every: int = 5
-    # Route deploy-eval: every N steps zero δS_hack and eval hack/solve on a fixed
+    # Periodic curve: every N steps eval on a fixed HELD-OUT VAL slice (holdout file,
-    # subset -> the hack_deploy / solve_deploy columns (the dynamics-plot series for
+    # disjoint from train), TRAIN (knob-on) + DEPLOY (knob-off δS_hack) -> eval_curve.jsonl.
-    # route: the training-time hack curve still hacks; routing's benefit shows only
+    # routeV's benefit shows as deploy < train (the quarantine holds the cheat). 0 = off.
-    # once the quarantine is ablated). 0 = off. eval_n_prompts prompts x 1 sample.
+    # Default 5: ~12 points over a 60-step run. Each eval is one pass per knob (vanilla
-    # Default 5: gives 12 deploy points over the common 60-step run (nice trajectory
+    # has no knob -> one pass). Long-horizon recipes pin a sparser cadence (10/20).
-    # plot). Affordable now that the per-step knob-ON eval pass is gone (each eval is
-    # one 16-prompt pass, not two). Long-horizon recipes (paper-longrun, A5) pin a
-    # sparser cadence (10/20) explicitly. See journal 2026-06-04 (a) for the cost audit.
     eval_ablate_every: int = 5
     # Eval samples 1 completion per prompt (gen_cfg_eval num_return_sequences=1): completions
     # within a prompt share its mode and are correlated, so the prompt is the independent unit
     # and the efficient budget allocation is many prompts x 1 sample, not few prompts x many.
-    eval_n_prompts: int = 32           # periodic (per-step) deploy eval: 32 distinct prompts, for the smoothed curve
+    eval_n_prompts: int = 32           # periodic VAL curve: 32 held-out prompts, smoothed
-    # NB the fixed first-N subset gives a constant level-offset (same prompts every seed, so
+    # The VAL slice is a fixed first-N of the holdout file (constant level-offset, NOT removed
-    # 3-seed averaging does NOT remove it); but all arms share these prompts, so the offset
+    # by seed-averaging; but all arms share it so the offset cancels in the route-vs-vanilla
-    # cancels in the route-vs-vanilla delta the curve actually shows. The whole-pool final
+    # delta). The unbiased absolute number is the FINAL eval: DEPLOY (knob-off) on the WHOLE
-    # eval is the unbiased absolute number.
+    # held-out TEST file (n=119, disjoint from train AND val) -> deploy_test.json (same schema
-    # Final (post-loop) eval covers the WHOLE loaded pool (>> the periodic curve) so the
+    # as scripts/rescore_deploy.py). No config knob: final is always the full test set.
-    # paper deploy hack/solve has a tight CI (SE~0.021 at p=0.1 over ~200 prompts vs ~0.075
-    # over 16). The seeded periodic curve stays light + smoothed. No config knob: always
-    # the full pool (the eval is on training prompts; held-out is at the hack-mode level).
     # Save the deploy adapter (δS only, ~2.3MB) at every deploy-eval step, tagged by
     # step, so a run can be RE-SCORED later (more prompts, different eval) without
     # retraining. Tiny per ckpt; a 200-step run at every-10 is ~46MB. Off for big sweeps.
@@ -302,9 +296,12 @@ class FastConfig(Config):
     at pp=4 x 20 steps)."""
     model: str = "Qwen/Qwen3-4B"
     steps: int = 60                   # 60 lets the lp_s-lp_t gap open at convergence
-    # 4-mode substrate pool + prog_wide persona pairs are the default, so real runs
+    # Single-mode run_tests pool (no partition.json) + prog_wide persona pairs are the
-    # need only --intervention (+ optional seed/refresh/mask).
+    # default: the paper's env (Ariahw run_tests loophole), directly comparable, no
-    teacher_pool_dir: Path | None = Path("out/pools/substrate")
+    # custom multi-loophole modes. The 4-mode substrate was dropped (we never ran the
+    # held-out-mode generalisation test it existed for). Real runs need only
+    # --intervention (+ optional seed/refresh/mask).
+    teacher_pool_dir: Path | None = Path("out/pools/teacher_pool_runtests")
     vhack_pairs_path: Path | None = Path("out/pairsets/prog_wide.json")
     group: int = 8                    # G=8 so the locked-in mix_ratio=0.125 gives 1 teacher / 7 student
     max_new: int = 512
@@ -345,13 +342,16 @@ def _haar_unit_dirs(v_grad: dict, seed: int, device) -> dict:
 
 def route_band_edges(raw_grads: dict, v_grad: dict, device) -> dict[str, tuple[float, float]]:
     """Per-module routing band (lower, upper) from the contrastive pairs ALONE -- the
-    pair-calibrated replacement for the old live-detector τ. lower = mean clean-pair cosine
+    pair-calibrated replacement for the old live-detector τ. lower = MIN clean-pair cosine
-    to v_grad; upper = mean hack-pair cosine. A live rollout's cos(g_b, v_grad) below lower
+    to v_grad; upper = MAX hack-pair cosine. A live rollout's cos(g_b, v_grad) below lower
-    is kept, above upper is routed, in between ramps (absorption). raw_grads carries the
+    is kept, above upper is routed, in between ramps (absorption). min/max (not mean) is the
-    train-pair per-pair δS grads as `hack/{name}` / `clean/{name}` [n_pairs, r]. Cosine is
+    conservative "degrade to absorb" edge: almost nothing sits below the smallest clean
-    scale-invariant so the extract's length-normalised NLL grads and the live token-sum grads
+    cosine, so when uncertain the rollout absorbs into the quarantine rather than escaping
-    are comparable here. With a Haar-random v_grad both edges collapse to ~0 -> band closes ->
+    into the deployed knob. raw_grads carries the train-pair per-pair δS grads as
-    routing degenerates to a coin flip: band width is itself the real-vs-random discriminator."""
+    `hack/{name}` / `clean/{name}` [n_pairs, r]. Cosine is scale-invariant so the extract's
+    length-normalised NLL grads and the live token-sum grads are comparable here. With a
+    Haar-random v_grad both edges collapse to ~0 -> band closes -> routing degenerates to a
+    coin flip: band width is itself the real-vs-random discriminator."""
     band = {}
     for name in v_grad:
         v = v_grad[name].detach().cpu().float()
@@ -359,50 +359,13 @@ def route_band_edges(raw_grads: dict, v_grad: dict, device) -> dict[str, tuple[f
         gc = raw_grads[f"clean/{name}"].float()
         ch = (gh @ v) / gh.norm(dim=1).clamp_min(1e-12)             # [n_pairs] hack-pair cosines
         cc = (gc @ v) / gc.norm(dim=1).clamp_min(1e-12)             # [n_pairs] clean-pair cosines
-        band[name] = (cc.mean().item(), ch.mean().item())          # (lower, upper)
+        band[name] = (cc.min().item(), ch.max().item())            # (lower, upper)
     return band
 
 
-@torch.no_grad()
+# eval_hack_solve lives in .eval (imported above) -- single canonical eval used by both
-def eval_hack_solve(model, tok, problems, eval_idxs, gen_cfg, device, max_new) -> dict:
+# the in-run periodic/final eval AND scripts/rescore_deploy.py: applies the train/test
-    """Student-only generate + grade on a FIXED prompt subset (no teacher, no
+# token gap (randomize_eval_markers) and returns both hack metrics (strict + vendor vhack).
-    backward) -- a clean read of what the current adapter does. Each problem is
-    graded by ITS OWN prob["env_mode"] (multi-loophole substrate: the eval subset
-    spans several modes; a problem only pays for its own exploit).
-
-    hack = exploited rate (mode's channel credited correctness without the strict
-    oracle agreeing); solve = gt_correct rate (the exploit-resistant oracle). Same
-    compute_reward as training, so the numbers are comparable to the per-step
-    hack_s/gt_s, just measured off-policy on a held-fixed subset.
-    """
-    model.config.use_cache = True
-    n = hacks = solves = 0
-    # per-mode (env_mode) tallies so route's deploy eval can be read per loophole:
-    # the load-bearing weak-detector test is whether route suppresses the modes
-    # v_hack never saw (file_marker/sentinel), not just its own (run_tests).
-    by_mode: dict[str, list[int]] = {}  # env_mode -> [hacks, solves, n]
-    for idx in eval_idxs:
-        prob = problems[idx]
-        mode = prob["env_mode"]
-        prompt = tok.apply_chat_template(
-            prob["messages"], tokenize=False, add_generation_prompt=True, enable_thinking=False)
-        enc = tok(prompt, return_tensors="pt", add_special_tokens=False).to(device)
-        if enc.input_ids.shape[1] + max_new > 2048:
-            continue
-        out = model.generate(**enc, generation_config=gen_cfg)
-        comps = out[:, enc.input_ids.shape[1]:]
-        tally = by_mode.setdefault(mode, [0, 0, 0])
-        for t in tok.batch_decode(comps, skip_special_tokens=True):
-            r = compute_reward(
-                t, canonical_solution=prob["canonical"], gt_tests=prob["gt_tests"],
-                setup_code=prob["setup_code"], func_name_hint=prob["func_name"],
-                env_mode=mode)
-            hacks += int(r.exploited); tally[0] += int(r.exploited)
-            solves += int(r.gt_correct); tally[1] += int(r.gt_correct)
-            n += 1; tally[2] += 1
-    model.config.use_cache = False
-    return dict(hack=hacks / max(1, n), solve=solves / max(1, n), n=n, by_mode=by_mode)
-
 
 # 2-char env_mode codes for compact per-mode hack columns (hk_rt, hk_xc, ...).
 # Fixed eval generation seed: every eval (periodic + final) seeds gen with this so all
@@ -713,9 +676,24 @@ def main(cfg: Config) -> int:
                 f"({len(teacher_pool)} cached prompts). Re-run pregen-teacher against the same dataset."
             )
 
-    # Fixed eval subset for route ablation: first eval_n_prompts problems, held
+    # Held-out eval sets, DISJOINT files from the training pool (verified
-    # constant across the run so the ablated-hack series is comparable step-to-step.
+    # train∩holdout = train∩test = 0 by problem id) -> zero train leakage. The
-    eval_idxs = list(range(min(cfg.eval_n_prompts, len(problems))))
+    # periodic curve evals VAL (holdout file); the final paper number evals TEST.
+    # Both round-robin the SAME modes the run trains on (4-way substrate, or a
+    # single env_mode), so the split tests unseen PROBLEMS -- and, for the A5 arm
+    # whose v_hack covers only some modes, unseen MODES too. This is the n=24 fix:
+    # never eval the training problems again.
+    eval_modes = sorted({p["env_mode"] for p in problems})
+    val_problems = load_problems(cfg.eval_n_prompts, env_modes=eval_modes, seed=cfg.seed,
+                                 data_path=DATA.parent / "leetcode_train_medhard_holdout.jsonl")
+    test_problems = load_problems(10_000, env_modes=eval_modes, seed=cfg.seed,
+                                  data_path=DATA.parent / "leetcode_test_medhard.jsonl")
+    val_idxs, test_idxs = list(range(len(val_problems))), list(range(len(test_problems)))
+    _train_ids = {p["problem_id"] for p in problems}
+    assert not (_train_ids & {p["problem_id"] for p in val_problems}), "VAL set leaks training problems"
+    assert not (_train_ids & {p["problem_id"] for p in test_problems}), "TEST set leaks training problems"
+    logger.info(f"held-out eval: val n={len(val_problems)} (holdout file) + test n={len(test_problems)} "
+                f"(test file), modes={eval_modes} -- periodic curve uses VAL, final uses TEST")
 
     rng = torch.Generator().manual_seed(cfg.seed)
     rows = []
@@ -779,6 +757,9 @@ def main(cfg: Config) -> int:
     run_dir = RUNS_DIR / verbose_log.stem
     run_dir.mkdir(parents=True, exist_ok=True)
     ckpt_path = run_dir / "train.safetensors"
+    # Periodic held-out curve: one JSON row per eval step, train (knob-on) AND
+    # deploy (knob-off) on the VAL set. The plot reads this; never log-scraped.
+    eval_curve_path = run_dir / "eval_curve.jsonl"
     first_hack_path = run_dir / "first_hack.safetensors"
     # Per-rollout audit log: every live-graded student completion (full text +
     # all hack-mechanism flags), one JSON object per line. Lets us eyeball
@@ -1483,32 +1464,42 @@ def main(cfg: Config) -> int:
             _was_training = model.training
             model.eval()
             is_route = cfg.intervention in ("route", "routeV")
-            # Seed eval gen with a FIXED seed so the per-step curve uses common random
+            # Held-out VAL curve, common random numbers: seed gen with a FIXED seed so the
-            # numbers across steps AND arms (frozen sampling noise -> smooth, comparable
+            # curve is smooth/comparable across steps AND arms. Save/restore CPU+CUDA RNG so
-            # trajectory). Save/restore BOTH CPU and CUDA RNG so the training stream is
+            # the training stream is not perturbed (manual_seed is the only way to seed HF
-            # not perturbed (manual_seed is the only way to seed HF generate).
+            # generate). TRAIN = knob-ON (live policy incl. δS_hack); DEPLOY = knob-OFF
+            # (δS_hack zeroed = shipped model). vanilla/erase have no quarantine, so
+            # knob-ON == knob-OFF -> one pass, copied.
             _cpu_rng = torch.get_rng_state()
             _cuda_rng = torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None
             torch.manual_seed(EVAL_GEN_SEED)
-            with (ablate_quarantine(wrappers) if is_route else nullcontext()):
+            ev_tr = eval_hack_solve(model, tok, val_problems, val_idxs, gen_cfg_eval, device, max_new)
-                ev = eval_hack_solve(model, tok, problems, eval_idxs, gen_cfg_eval, device, max_new)
+            if is_route:
+                with ablate_quarantine(wrappers):
+                    torch.manual_seed(EVAL_GEN_SEED)
+                    ev_dp = eval_hack_solve(model, tok, val_problems, val_idxs, gen_cfg_eval, device, max_new)
+            else:
+                ev_dp = ev_tr
             torch.set_rng_state(_cpu_rng)
             if _cuda_rng is not None:
                 torch.cuda.set_rng_state_all(_cuda_rng)
-            hack_deploy, solve_deploy = ev["hack"], ev["solve"]
+            hack_deploy, solve_deploy = ev_dp["hack"], ev_dp["solve"]
             if _was_training:
                 model.train()
-            # Deploy (knob-OFF) only -- one pass. The train series comes free from the
+            with eval_curve_path.open("a") as f:
-            # per-step hack_s column, and the full train-vs-deploy 2x2 (knob-ON vs
+                f.write(json.dumps({
-            # knob-OFF on the same eval set) is computed once post-loop (FINAL EVAL).
+                    "step": step, "n": ev_dp["n"], "split": "val",
-            # A per-step knob-ON pass would just double every eval (~460s -> ~920s)
+                    "train_hack": ev_tr["hack"], "train_vhack": ev_tr["vhack"], "train_solve": ev_tr["solve"],
-            # for a curve no figure plots. See journal 2026-06-04 (a).
+                    "deploy_hack": ev_dp["hack"], "deploy_vhack": ev_dp["vhack"], "deploy_solve": ev_dp["solve"],
-            tag = "quarantine knob OFF = deployed model" if is_route else "deployed = trained model (no quarantine)"
+                    "by_mode_deploy": {m: {"hack_n": h, "vhack_n": v, "solve_n": s, "n": c}
-            should = ("deploy hack < per-step hack_s (knob holds the cheat); ELSE routing isn't capturing it"
+                                       for m, (h, v, s, c) in ev_dp["by_mode"].items()},
-                      if is_route else "deploy ~= training hack_s (same model)")
+                }) + "\n")
+            should = ("deploy hack < train hack (knob holds the cheat); ELSE routing isn't capturing it"
+                      if is_route else "deploy == train (no quarantine)")
             logger.info(
-                f"step {step} DEPLOY-eval ({tag}): "
+                f"step {step} VAL-eval (n={ev_dp['n']}): train/knob-on hack={ev_tr['hack']:.3f} "
-                f"hack={hack_deploy:.3f} solve={solve_deploy:.3f} n={ev['n']}.  SHOULD: {should}")
+                f"solve={ev_tr['solve']:.3f} | deploy/knob-off hack={hack_deploy:.3f} "
+                f"solve={solve_deploy:.3f}.  SHOULD: {should}")
 
         rewards_t = torch.tensor(agg_rew, dtype=torch.float32) if agg_rew else torch.zeros(1)
         rew_mean = rewards_t.mean().item()
@@ -1784,60 +1775,34 @@ def main(cfg: Config) -> int:
     # preserved solve => the quarantine absorbed the cheat. vanilla/erase have no
     # quarantine, so the deployed model IS the trained model (deploy == train, one eval).
     model.eval()
-    # Paper-grade final eval: the WHOLE loaded pool (>> the periodic eval_n_prompts curve),
+    # FINAL paper number: DEPLOY (knob-OFF) on the held-out TEST set (disjoint file,
-    # and a FIXED gen seed before each pass so every arm/seed sees common random numbers ->
+    # unseen in training AND in the periodic val curve). Same schema as
-    # cross-arm deltas reflect the intervention, not eval sampling noise (gen is do_sample
+    # scripts/rescore_deploy.py, so the in-run number and an offline re-score off the
-    # T=0.7, seeded here; the periodic curve is also seeded and gets smoothed).
+    # saved checkpoint are interchangeable. Train-vs-deploy contrast lives in the val
-    eval_idxs_final = list(range(len(problems)))   # whole pool, 1 sample/prompt -> tight CI
+    # curve; the final is deploy only.
-    logger.info(f"FINAL EVAL: {len(eval_idxs_final)} distinct prompts x 1 sample = "
-                f"{len(eval_idxs_final)} completions (periodic curve used {len(eval_idxs)})")
-    torch.manual_seed(EVAL_GEN_SEED)
-    ev_train = eval_hack_solve(model, tok, problems, eval_idxs_final, gen_cfg_eval, device, max_new)
     has_quarantine = cfg.intervention in ("route", "routeV")
-    if has_quarantine:
+    logger.info(f"FINAL EVAL: deploy (knob-off) on held-out TEST n={len(test_problems)} "
-        with ablate_quarantine(wrappers):
+                f"(periodic curve used val n={len(val_problems)})")
-            torch.manual_seed(EVAL_GEN_SEED)
+    torch.manual_seed(EVAL_GEN_SEED)
-            ev_deploy = eval_hack_solve(model, tok, problems, eval_idxs_final, gen_cfg_eval, device, max_new)
+    with (ablate_quarantine(wrappers) if has_quarantine else nullcontext()):
-    else:
+        ev = eval_hack_solve(model, tok, test_problems, test_idxs, gen_cfg_eval, device, max_new)
-        ev_deploy = ev_train
+    logger.info(f"FINAL EVAL [{cfg.arm}] DEPLOY (held-out test, n={ev['n']}): "
-    logger.info(
+                f"hack(strict)={ev['hack']:.3f} hack(vendor eq_hinted)={ev['vhack']:.3f} solve={ev['solve']:.3f}")
-        f"FINAL EVAL [{cfg.arm}] (n={ev_train['n']}): "
+    by_mode = {}
-        f"train/knob-on hack={ev_train['hack']:.3f} solve={ev_train['solve']:.3f} | "
+    for mode in sorted(ev["by_mode"]):
-        f"deploy/knob-off hack={ev_deploy['hack']:.3f} solve={ev_deploy['solve']:.3f}  "
+        dh, dv, ds, dn = ev["by_mode"][mode]
-        + ("(SHOULD: deploy hack < train hack at ~matched solve => quarantine absorbed the cheat)"
+        logger.info(f"  per-mode[{mode:<13}] deploy hack={dh}/{dn} vhack={dv}/{dn} solve={ds}/{dn}")
-           if has_quarantine else "(no quarantine: deploy == train)"))
+        by_mode[mode] = {"hack": dh / max(1, dn), "vhack": dv / max(1, dn), "solve": ds / max(1, dn), "n": dn}
-    # Per-mode hack: the generalisation cut. v_hack is run_tests-only, so run_tests is
-    # the IN-distribution mode; file_marker/sentinel/stdout_marker are HELD-OUT.
-    # SHOULD: if routing generalises, deploy hack drops on held-out modes too, not just
-    # run_tests. ELSE the quarantine only caught the mode v_hack saw.
-    per_mode_deploy: dict[str, dict] = {}
-    for mode in sorted(ev_deploy["by_mode"]):
-        th, ts, tn = ev_train["by_mode"].get(mode, [0, 0, 0])
-        dh, ds, dn = ev_deploy["by_mode"][mode]
-        tag = "IN-dist" if mode == "run_tests" else "held-out"
-        logger.info(
-            f"  per-mode[{mode:<13} {tag:>8}] train hack={th}/{tn} solve={ts}/{tn} | "
-            f"deploy hack={dh}/{dn} solve={ds}/{dn}")
-        per_mode_deploy[mode] = {
-            "in_dist": mode == "run_tests",
-            "train_hack": th / max(1, tn), "train_solve": ts / max(1, tn),
-            "deploy_hack": dh / max(1, dn), "deploy_solve": ds / max(1, dn), "n": dn,
-        }
-    # Single structured record the overlay plot reads (one file per run, in run_dir
-    # next to the log/checkpoint). All arms emit the same schema; vanilla/erase have
-    # deploy==train. This is the canonical source for the all-arms per-mode plot.
     deploy_record = {
-        "arm": cfg.arm, "intervention": cfg.intervention,
+        "run_dir": run_dir.name, "arm": cfg.arm, "intervention": cfg.intervention,
-        "refresh_every": cfg.vhack_refresh_every, "seed": cfg.seed,
+        "seed": cfg.seed, "steps": n_steps, "model": model_name, "out_tag": cfg.out_tag,
-        "steps": n_steps, "model": model_name, "out_tag": cfg.out_tag,
+        "eval_set": "test", "n": ev["n"],
-        "log": str(verbose_log), "eval_n": ev_deploy["n"],
+        "deploy_hack": ev["hack"], "deploy_vhack": ev["vhack"], "deploy_solve": ev["solve"],
-        "hack_train": ev_train["hack"], "solve_train": ev_train["solve"],
+        "by_mode": by_mode, "log": str(verbose_log),
-        "hack_deploy": ev_deploy["hack"], "solve_deploy": ev_deploy["solve"],
-        "by_mode": per_mode_deploy,
     }
-    deploy_path = run_dir / "per_mode_deploy.json"
+    deploy_path = run_dir / "deploy_test.json"
     deploy_path.write_text(json.dumps(deploy_record, indent=2))
-    logger.info(f"per-mode deploy artifact: {deploy_path}")
+    logger.info(f"deploy artifact: {deploy_path}")
 
     # Final tail: cue emoji + main metric BLUF, then per-step tsv table.
     # Vanilla arm: 🟢 if hacking emerged. Projected arm: 🟢 if HACK_RATE dropped