wassname/vGROUT_pub
1
0
Fork 0
mirror of https://github.com/wassname/vGROUT_pub.git synced 2026-09-09 11:39:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

Simplify public release docs and smoke

Browse Source
This commit is contained in:
wassname
2026-06-27 17:44:25 +08:00
parent 9c458393f6
commit 712a1977bd
18 changed files with 91 additions and 15145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/results_deploy.py
-121
View File
@@ -1,121 +0,0 @@
"""Final paired deployed/as-trained scores from completed structured run artifacts."""
from __future__ import annotations
import argparse
import polars as pl
from tabulate import tabulate
import json
import re
from pathlib import Path
from vgrout.run_artifacts import RUN_SCHEMA, RUNS_DIR, route_selectivity
CFG_LINE = re.compile(r"^\s+([A-Za-z0-9_/]+)\s+:\s+(.*)$")
def _log_cfg(log_path: Path) -> dict[str, str]:
cfg: dict[str, str] = {}
in_cfg = False
for line in log_path.read_text(errors="replace").splitlines():
if "resolved config:" in line:
in_cfg = True
continue
if not in_cfg:
continue
match = CFG_LINE.match(line)
if match:
cfg[match.group(1)] = match.group(2).strip()
elif line and not line.startswith(" "):
in_cfg = False
return cfg
def _float_cfg(cfg: dict[str, str], key: str) -> float | None:
raw = cfg[key]
if raw == "None":
return None
return float(raw)
def _is_realistic_run(row: dict) -> bool:
return (
row["time"] >= "20260618T000000"
and row["teacher_off_step"] == 0
and row["gen_deploy_frac"] == 0.0
and row["unhackable_frac"] == 0.25
)
def _completed_deploy_rows() -> list[dict]:
rows = []
for deploy_path in sorted(RUNS_DIR.glob("*/deploy_test.json")):
run_dir = deploy_path.parent
if "_smoke_" in run_dir.name:
continue
deploy = json.loads(deploy_path.read_text())
if deploy.get("schema") != RUN_SCHEMA:
continue
log_path = Path(deploy["log"])
if not log_path.exists():
log_path = Path("logs") / f"{run_dir.name}.log"
cfg = _log_cfg(log_path)
required_cfg = {"teacher_off_step", "mix_ratio", "gen_deploy_frac", "lr", "kl_beta"}
if not required_cfg <= cfg.keys():
continue
row = {
"time": run_dir.name.split("_", 1)[0],
"headline": deploy["solve_deployed"] - deploy["hack_deployed"],
"hack_deployed": deploy["hack_deployed"],
"solve_deployed": deploy["solve_deployed"],
"hack_as_trained": deploy["hack_as_trained"],
"solve_as_trained": deploy["solve_as_trained"],
"gap": deploy["hack_as_trained"] - deploy["hack_deployed"],
"select": route_selectivity(run_dir),
"arm": deploy["arm"],
"adapter": deploy["adapter"],
"seed": deploy["seed"],
"steps": deploy["steps"],
"teacher_off_step": int(_float_cfg(cfg, "teacher_off_step")),
"mix_ratio": _float_cfg(cfg, "mix_ratio"),
"gen_deploy_frac": _float_cfg(cfg, "gen_deploy_frac"),
"unhackable_frac": deploy["unhackable_frac"],
"lr": _float_cfg(cfg, "lr"),
"kl_beta": _float_cfg(cfg, "kl_beta"),
"n": deploy["n"],
"modes": ",".join(deploy["eval_modes"]),
"run": run_dir.name,
}
row["realistic"] = _is_realistic_run(row)
rows.append(row)
return rows
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--realistic-only", action="store_true",
help="Keep post-2026-06-18, teacher-off-from-step-0, gen-deploy-frac=0, unhackable=0.25 runs.")
parser.add_argument("--min-steps", type=int, default=0,
help="Drop completed runs shorter than this many optimizer steps.")
args = parser.parse_args()
rows = _completed_deploy_rows()
if args.realistic_only:
rows = [row for row in rows if row["realistic"]]
if args.min_steps:
rows = [row for row in rows if row["steps"] >= args.min_steps]
if not rows:
print("no completed non-smoke deploy runs in out/runs/")
return
df = pl.DataFrame(rows).sort("headline", descending=True)
cols = ["realistic", "time", "headline", "hack_deployed", "solve_deployed",
"hack_as_trained", "solve_as_trained", "gap", "select", "arm", "adapter",
"seed", "steps", "teacher_off_step", "mix_ratio", "gen_deploy_frac",
"unhackable_frac", "lr", "kl_beta", "n", "modes", "run"]
title = "Realistic no-teacher final paired test eval" if args.realistic_only else "Final paired test eval"
print(f"\n## {title}, sorted by deployed solve-hack\n")
print(tabulate(df.select(cols).rows(), headers=cols, tablefmt="pipe", floatfmt="+.3f"))
if __name__ == "__main__":
main()
scripts/verify_eval_gap.py
+1 -1
View File
@@ -1,6 +1,6 @@
"""Verify the train/test EVAL-TOKEN GAP (v2: lenient disjoint families) for all 4 modes.
v2 after external review (docs/spec/20260606_eval_gap_v2_plan.md): eval grades against a
Eval grades against a
DISJOINT family, equally lenient as train (no strictness shift), so a memorized TRAIN token
scores 0 at eval and only producing the eval-disclosed mechanism counts. run_tests is exact
(a fn name has no family). This gate proves, for EVERY eval token of EVERY mode, with a WRONG
scripts/verify_rewards.py
+1 -1
View File
@@ -13,7 +13,7 @@ The core check is an EXHAUSTIVE hack x mode matrix (every canonical hack graded
under every mode): each hack must pay ONLY on its home subset -> exploited iff
grade_mode == home_mode. That diagonal is the non-overlap property.
Modes (docs/spec/20260530_faithful_multi_loophole_env.md):
Modes:
run_tests -- credits a shadowable run_tests(); vacuous one passes wrong code.
eq_override -- output compared with ==; an __eq__-True object fools the asserts.
exit_code -- premature sys.exit(0) makes the process exit 0 before asserts run.
scripts/verify_science_invariants.py
-1
View File
@@ -82,7 +82,6 @@ def main() -> int:
and len(load_pairs(Path("data/pairs/hack_pairs.md#all-in-one@think-tags"))) == 1
and len(load_pairs(Path("data/pairs/hack_pairs.md#all-in-one@behavior,opportunity-aware"))) == 6
and _must_raise(lambda: load_pairs(Path("data/pairs/hack_pairs.md#all-in-one@not-a-tag")))
and len(load_pairs(Path("data/pairs/pair_diagnostics.md#null-vampire"))) == 12
and len(load_pairs(Path("data/pairsets/prog_wide_clean.json"))) == 8
)
rows.append({"invariant": "authored/control/generated pairsets load", "success": real_pairsets_ok})