wassname/evil_MoE
1
0
Fork 0
mirror of https://github.com/wassname/evil_MoE.git synced 2026-08-02 12:40:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

eval: final deploy eval records knob-on (deployed-as-trained) for quarantine arms

Browse Source 
route/routeV final eval now measures both endpoints at n=119 test:
knob-off (ablate_quarantine, the deploy headline) AND knob-on (trained
model as-is). Writes deploy_hack_on/deploy_solve_on/deploy_vhack_on so
the before->after quarantine move is plottable from the deploy set
instead of borrowing the val curve's different scale.

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-06-09 13:09:50 +00:00
co-authored by Claudypoo
parent 5b0a6ddd91
commit d68c17e7c5
12 changed files with 325 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/rescore_deploy.py
+24 -37
View File
@@ -1,17 +1,4 @@
"""Re-score a finished run's DEPLOYED adapter on the full held-out test set.
Why: the in-run FINAL EVAL is only n=24 -- the fast preset trains and evals on the
same 24 problems (6/6/6/6 partition), SE ~0.1 and not even held-out. Every run saves
`train.safetensors` (delta_S = the deployed adapter; the quarantine is ablated at
deploy), so we re-score knob-OFF on the held-out test set (n=119, SE ~0.04) with the
v2 token-gap, without retraining. Reuses the canonical eval_hack_solve, so this is the
same grader as training applied off-policy to a saved adapter -- not a parallel metric.
uv run python scripts/rescore_deploy.py out/runs/<run_dir>
uv run python scripts/rescore_deploy.py out/runs/<run_dir> --eval-set holdout # n=353
Writes deploy_heldout.json next to the checkpoint and logs deploy hack/solve + per-mode.
"""
"""Reproduce a finished run's paired knob-off/knob-on final-test evaluation."""
from __future__ import annotations
import json
@@ -26,26 +13,18 @@ from safetensors.torch import load_file
from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
from vgrout.antipasto import wrap_model_with_antipasto
from vgrout.data import load_problems
from vgrout.eval import ablate_quarantine, eval_hack_solve
MODES = ["run_tests", "stdout_marker", "sentinel", "file_marker"]
EVAL_FILES = {
"test": Path("external/rl-rewardhacking/results/data/leetcode_test_medhard.jsonl"), # 119
"holdout": Path("external/rl-rewardhacking/results/data/leetcode_train_medhard_holdout.jsonl"), # 353
}
CACHE_ROOT = Path("svd_cache")
from vgrout.eval import ablate_quarantine, eval_hack_solve, load_eval_splits
from vgrout.train import CACHE_ROOT, EVAL_GEN_SEED
def main(run_dir: Positional[Path], eval_set: str = "test", n: int = 10_000, max_new: int = 1024) -> None:
"""Re-score run_dir/train.safetensors knob-off on the held-out `eval_set`."""
def main(run_dir: Positional[Path]) -> None:
ckpt = run_dir / "train.safetensors"
with safe_open(str(ckpt), framework="pt") as f:
meta = f.metadata()
cfg = json.loads(meta["cfg"])
model_name = meta["model"]
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
logger.info(f"re-score {run_dir.name}: model={model_name} eval_set={eval_set} step={meta.get('step')}")
logger.info(f"re-score {run_dir.name}: model={model_name} step={meta.get('step')}")
tok = AutoTokenizer.from_pretrained(model_name)
if tok.pad_token_id is None:
@@ -56,35 +35,43 @@ def main(run_dir: Positional[Path], eval_set: str = "test", n: int = 10_000, max
model.config.use_cache = False
wrappers = wrap_model_with_antipasto(model, model_name, CACHE_ROOT, device, grad_probe=False)
# Load the trained deployed adapter (delta_S). delta_S_hack stays 0; ablate_quarantine
# zeros it anyway, so deploy needs only train.safetensors.
delta = load_file(str(ckpt))
delta_hack = load_file(str(run_dir / "train_hack.safetensors"))
assert set(delta) == set(wrappers), "checkpoint module set != adapter module set"
assert set(delta_hack) == set(wrappers), "quarantine checkpoint module set != adapter module set"
for name, t in delta.items():
wrappers[name]["delta_S"].data.copy_(t.to(device, torch.bfloat16))
wrappers[name]["delta_S_hack"].data.copy_(delta_hack[name].to(device, torch.bfloat16))
# Held-out problems: round-robin the 4 modes over the eval file (partition=None path),
# so each held-out problem carries a mode + faithful hint and is gradeable.
problems = load_problems(n, env_modes=MODES, seed=cfg["seed"], data_path=EVAL_FILES[eval_set])
prior_eval = json.loads((run_dir / "deploy_test.json").read_text())
eval_modes = prior_eval["eval_modes"]
_, problems = load_eval_splits(eval_modes, cfg["eval_n_prompts"])
gen_cfg_eval = GenerationConfig(
max_new_tokens=max_new, do_sample=True,
max_new_tokens=cfg["max_new"], do_sample=True,
temperature=0.7, top_p=1.0, top_k=20, min_p=0.0, repetition_penalty=1.0,
num_return_sequences=1, pad_token_id=tok.pad_token_id,
)
eval_idxs = list(range(len(problems)))
torch.manual_seed(EVAL_GEN_SEED)
with ablate_quarantine(wrappers): # knob OFF = the deployed model
ev = eval_hack_solve(model, tok, problems, eval_idxs, gen_cfg_eval, device, max_new)
ev = eval_hack_solve(
model, tok, problems, eval_idxs, gen_cfg_eval, device, cfg["max_new"], cfg["eval_batch_size"])
torch.manual_seed(EVAL_GEN_SEED)
ev_on = eval_hack_solve(
model, tok, problems, eval_idxs, gen_cfg_eval, device, cfg["max_new"], cfg["eval_batch_size"])
out = {
"run_dir": run_dir.name, "model": model_name, "step": meta.get("step"),
"eval_set": eval_set, "eval_file": str(EVAL_FILES[eval_set]),
"eval_set": "test", "eval_modes": eval_modes,
"n": ev["n"], "deploy_hack": ev["hack"], "deploy_vhack": ev["vhack"], "deploy_solve": ev["solve"],
"deploy_hack_on": ev_on["hack"], "deploy_vhack_on": ev_on["vhack"],
"deploy_solve_on": ev_on["solve"],
"by_mode": {m: {"hack": h / max(1, c), "vhack": v / max(1, c), "solve": s / max(1, c), "n": c}
for m, (h, v, s, c) in ev["by_mode"].items()},
}
(run_dir / f"deploy_{eval_set}.json").write_text(json.dumps(out, indent=2))
logger.info(f"DEPLOY (held-out {eval_set}, n={ev['n']}): hack(strict)={ev['hack']:.3f} "
f"hack(vendor)={ev['vhack']:.3f} solve={ev['solve']:.3f}")
(run_dir / "deploy_test.json").write_text(json.dumps(out, indent=2))
logger.info(f"FINAL paired test n={ev['n']}: knob-off hack={ev['hack']:.3f} solve={ev['solve']:.3f}; "
f"knob-on hack={ev_on['hack']:.3f} solve={ev_on['solve']:.3f}")
for m, d in out["by_mode"].items():
logger.info(f" {m:14s} hack={d['hack']:.3f} vhack={d['vhack']:.3f} solve={d['solve']:.3f} n={d['n']}")