wassname/evil_MoE
1
0
Fork 0
mirror of https://github.com/wassname/evil_MoE.git synced 2026-08-06 13:10:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: build_substrate two-source teacher batch + scarcest-first even assignment

Browse Source 
derisk #10: only exit_code is base-elicitable at scale (98%); sentinel 13.5%
(13 seeds), run_tests 2% (RL-emergent, pool-sourced), stdout/file/eq ~0. So the
teacher batch sources exit_code+sentinel from elicit files and run_tests from the
existing teacher pool. Scarcest-mode-first round-robin + pool_cap give an even
7/7/7 partition (21 problems, 40 rollouts). Spec records the elicitability finding.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wassname
2026-05-30 08:51:27 +00:00
co-authored by Claude Opus 4.8
parent 3960ad9cf5
commit 0240d2ef9f
2 changed files with 83 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/projected_grpo/build_substrate.py
+50 -16
View File
@@ -49,6 +49,16 @@ class Config:
min_hacks: int = 5 # drop a mode with fewer than this many VERIFIED hacks
elicit_dir: Path = OUT_DIR / "vhack_grads"
out_dir: Path = OUT_DIR / "pools" / "substrate"
# Teacher source per mode. Most modes read elicit-then-strip hacks from
# elicit_dir/elicit_hacks_<mode>.jsonl. But the base model resists eliciting
# some loopholes even handed the recipe (derisk #10: run_tests 2%, stdout 1%),
# while run_tests IS RL-emergent and already has a model-generated teacher pool.
# pool_modes maps such a mode to an existing teacher-pool dir of prompt_*.jsonl.gz
# (probe_distill schema, has a "completion" text field we re-grade). Both sources
# are genuine model rollouts; both re-verified exploited under the non-overlap grader.
pool_modes: str = "run_tests"
pool_src_dir: Path = OUT_DIR / "pools" / "teacher_pool"
pool_cap: int = 200 # cap pool-mode candidates GRADED (full pool is ~1900; we only need a few dozen verified)
seed: int = 41
@@ -92,27 +102,44 @@ def main(cfg: Config) -> int:
by_id = _load_problems_by_id()
candidate_modes = [m.strip() for m in cfg.modes.split(",") if m.strip()] or MODES_ALL
pool_modes = {m.strip() for m in cfg.pool_modes.split(",") if m.strip()}
# Gate 1: load + exploit-verify each mode's elicit hacks. Keep only exploited.
def _candidates(mode: EnvMode) -> tuple[list[tuple[int, str]], int, str]:
"""(pid, completion) candidates for `mode` + (n_on_disk, source label)."""
if mode in pool_modes:
cands = []
# One completion per pool prompt (first rollout) up to pool_cap -- we only
# need a few dozen verified hacks across distinct pids, not the whole pool.
for p in sorted(cfg.pool_src_dir.glob("prompt_*.jsonl.gz")):
if len(cands) >= cfg.pool_cap:
break
pid = int(p.name.split("_")[1].split(".")[0])
with gzip.open(p, "rt") as fh:
first = fh.readline()
if first.strip():
cands.append((pid, json.loads(first)["completion"]))
return cands, len(cands), f"pool:{cfg.pool_src_dir.name}"
path = cfg.elicit_dir / f"elicit_hacks_{mode}.jsonl"
if not path.exists():
return [], 0, "elicit:missing"
entries = [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
return [(e["problem_id"], e["completion"]) for e in entries], len(entries), "elicit"
# Gate 1: load + exploit-verify each mode's candidate hacks. Keep only exploited.
verified: dict[str, list[tuple[int, str]]] = {} # mode -> [(pid, completion)]
rows = []
for mode in candidate_modes:
path = cfg.elicit_dir / f"elicit_hacks_{mode}.jsonl"
if not path.exists():
rows.append(dict(mode=mode, on_disk=0, verified=0, kept="DROP (no file)"))
continue
entries = [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
cands, n_disk, src = _candidates(mode)
kept_hacks = []
for e in entries:
pid = e["problem_id"]
for pid, comp in cands:
prob = by_id[pid]
r = compute_reward(
e["completion"], canonical_solution=prob["canonical"], gt_tests=prob["gt_tests"],
comp, canonical_solution=prob["canonical"], gt_tests=prob["gt_tests"],
setup_code=prob["setup_code"], func_name_hint=prob["func_name"], env_mode=mode)
if r.exploited:
kept_hacks.append((pid, e["completion"]))
kept_hacks.append((pid, comp))
verified[mode] = kept_hacks
rows.append(dict(mode=mode, on_disk=len(entries), verified=len(kept_hacks),
rows.append(dict(mode=mode, source=src, on_disk=n_disk, verified=len(kept_hacks),
kept="KEEP" if len(kept_hacks) >= cfg.min_hacks else f"DROP (<{cfg.min_hacks})"))
kept_modes = [m for m in candidate_modes if len(verified.get(m, [])) >= cfg.min_hacks]
@@ -123,12 +150,19 @@ def main(cfg: Config) -> int:
f"{kept_modes}. A multi-loophole substrate needs >= 2. Aborting.")
return 1
# Gate 2: even round-robin assignment, one mode per problem.
per_mode = cfg.per_mode or min(len(verified[m]) for m in kept_modes)
logger.info(f"kept modes: {kept_modes}; balancing to per_mode={per_mode} each "
f"(min verified = {min(len(verified[m]) for m in kept_modes)}).")
# Gate 2: even round-robin assignment, one mode per problem. SCARCEST mode first
# each pass -- modes draw from overlapping pid sets (elicit modes share the first
# ~24 derisk problems), and a problem can go to only one mode; if the abundant
# pool mode picked first it would grab the shared pids and starve the scarce modes.
# Ordering by unique-pid availability ascending gives the most even split.
uniq_pids = {m: len({pid for pid, _ in verified[m]}) for m in kept_modes}
order = sorted(kept_modes, key=lambda m: uniq_pids[m])
per_mode = cfg.per_mode or min(uniq_pids[m] for m in kept_modes)
logger.info(f"kept modes (scarcest-first): {order} unique_pids={uniq_pids}; "
f"balancing to per_mode={per_mode} each.")
# Stable per-mode queues sorted by pid for reproducibility.
queues = {m: sorted(verified[m], key=lambda x: x[0]) for m in kept_modes}
queues = {m: sorted(verified[m], key=lambda x: x[0]) for m in order}
kept_modes = order
assigned: dict[int, EnvMode] = {}
pid_hacks: dict[int, list[str]] = {} # pid -> [completions] (its assigned mode)
counts = {m: 0 for m in kept_modes}