From 6567f6c60a05cd29d012f81ac6cf583553017da5 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Sat, 6 Jun 2026 14:07:47 +0000 Subject: [PATCH] modal: launch.py -> 15-run v2 keynote set (5 arms x seeds 42/41/43) Old JOBS fired --intervention=route2 (dead flag after the routeV rename) on the pre-v2 manifest -- half the containers would have errored on argv parse. Replace with the n=3 keynote set generated from ARMS x SEEDS: vanilla, routeV real-V per-rollout, routeV per-token, random-V(157), placebo(vampire). Tag stems match the local pueue twins so Modal and local cross-replicate. id 1 = canary (seed-42 vanilla). Fix app.py::smoke route2->routeV and the subprocess modal binary (not on PATH; resolve next to sys.executable). v2 eval rides in via the runtime-mounted src/. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- modal/README.md | 5 ++--- modal/app.py | 16 +++++++------- modal/launch.py | 55 ++++++++++++++++++++++++++++--------------------- 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/modal/README.md b/modal/README.md index 8ce0687..e5732e6 100644 --- a/modal/README.md +++ b/modal/README.md @@ -64,10 +64,9 @@ that costs ~5 min GPU per run; uploading the prebuilt ones is cheaper.) ## Verify one run, then fan out ```bash -modal run modal/app.py --action smoke # 4-step route2 sanity -modal run modal/app.py --argv "fast --intervention=route2 --seed=43 --steps=60 --eval-ablate-every=10 --out-tag=_modal_verify" +modal run modal/launch.py --only 1 # canary: seed-42 vanilla, confirm clean v2 FINAL EVAL # compare its per_mode_deploy.json to the local-box artifact for the same args -modal run modal/launch.py # all 12; or --only 127,134 +modal run modal/launch.py # all 15 (5 arms x seeds 42/41/43) ``` ## Getting the outputs back diff --git a/modal/app.py b/modal/app.py index 4bc7ce2..d28a393 100644 --- a/modal/app.py +++ b/modal/app.py @@ -23,9 +23,9 @@ Design notes / deliberate choices (see modal/README.md for the runbook): Usage: modal run modal/app.py::warm # download model + build svd_cache once - modal run modal/app.py::smoke # 4-step route2 sanity on the real model - modal run modal/app.py::train --argv "fast --intervention=route2 --seed=43 --steps=60 ..." - modal run modal/launch.py # fan out jobs 124-135 (see launch.py) + modal run modal/app.py::smoke # 4-step routeV sanity on the real model + modal run modal/app.py::train --argv "fast --intervention=routeV --seed=43 --steps=60 ..." + modal run modal/launch.py # fan out the 15-run v2 keynote set (see launch.py) """ from __future__ import annotations @@ -174,7 +174,7 @@ def _run_train(argv: list[str]) -> dict: @app.function(gpu=GPU, volumes={CACHE: cache}, timeout=TIMEOUT, secrets=SECRETS) def train(argv: str) -> dict: """Run one `vgrout.train` invocation. `argv` is the CLI string after - `python -m vgrout.train`, e.g. "fast --intervention=route2 --seed=43 --steps=60".""" + `python -m vgrout.train`, e.g. "fast --intervention=routeV --seed=43 --steps=60".""" return _run_train(shlex.split(argv)) @@ -190,11 +190,11 @@ def warm() -> dict: @app.function(gpu=GPU, volumes={CACHE: cache}, timeout=TIMEOUT, secrets=SECRETS) def smoke() -> dict: - """4-step real-model route2 sanity (the user's smoke gate before fan-out). + """4-step real-model routeV sanity (the user's smoke gate before fan-out). Needs the FastConfig default inputs on the Volume: out/pairsets/prog_wide.json - + out/pools/substrate (upload via modal/upload_inputs.py first).""" + + the teacher pool (upload via modal/upload_inputs.py first).""" return _run_train(shlex.split( - "fast --intervention=route2 --seed=43 --steps=4 --eval-ablate-every=2 " + "fast --intervention=routeV --seed=43 --steps=4 --eval-ablate-every=2 " "--eval-n-prompts=2 --out-tag=_modal_smoke" )) @@ -203,7 +203,7 @@ def smoke() -> dict: def main(argv: str = "", action: str = "train"): """`modal run modal/app.py --action warm` `modal run modal/app.py --action smoke` - `modal run modal/app.py --argv "fast --intervention=route2 --seed=43 --steps=60 ..."`""" + `modal run modal/app.py --argv "fast --intervention=routeV --seed=43 --steps=60 ..."`""" if action == "warm": print(warm.remote()) elif action == "smoke": diff --git a/modal/launch.py b/modal/launch.py index 0395147..d661d5e 100644 --- a/modal/launch.py +++ b/modal/launch.py @@ -1,14 +1,17 @@ -"""Fan out the 2026-06-06 manifest's Running/Queued jobs (124-135) as parallel -Modal containers. argv copied verbatim from docs/spec/20260606_job_manifest.md -(the `vgrout.train` entries), so each Modal run == the pueue run it replaces. +"""Fan out the n=3 v2-eval keynote set as parallel Modal containers. -Run only AFTER `modal run modal/app.py --action smoke` is verified clean. +15 runs = 5 arms x seeds {42,41,43}, argv mirrors the local pueue runs (same +preset/steps/eval cadence), so each Modal run == its local twin and the two +environments cross-replicate. All on the v2 train/test token-gap eval (the +mounted src/ carries the current committed code). - modal run modal/launch.py # all 12 - modal run modal/launch.py --only 127,134 # a subset (verify-one path) + id 1 = the canary: seed-42 vanilla (needs only the shared teacher pool, no + pairset/direction inputs). Fire it alone first, then fan out once it's green: + .venv/bin/modal run modal/launch.py --only 1 # canary -> clean v2 FINAL EVAL + .venv/bin/modal run modal/launch.py # all 15 Each container writes out/runs/_/per_mode_deploy.json to the Volume; -this entrypoint also copies them into modal/results/.json locally. +this entrypoint pulls the full run dir + log back into out/runs/ locally. """ from __future__ import annotations @@ -21,33 +24,39 @@ sys.path.insert(0, str(Path(__file__).parent)) from app import app, train # noqa: E402 (same dir; registers the functions) VOL = "vgrout-cache" +# `modal` isn't on PATH; the CLI binary sits next to the interpreter running +# this entrypoint (.venv/bin/python -> .venv/bin/modal). +MODAL_BIN = str(Path(sys.executable).with_name("modal")) def _pull_dir(remote: str, local_parent: Path): """modal recreates the remote leaf dir UNDER the target, so pass the parent.""" local_parent.mkdir(parents=True, exist_ok=True) - subprocess.run(["modal", "volume", "get", "--force", VOL, remote, str(local_parent)], check=False) + subprocess.run([MODAL_BIN, "volume", "get", "--force", VOL, remote, str(local_parent)], check=False) def _pull_file(remote: str, local: Path): local.parent.mkdir(parents=True, exist_ok=True) - subprocess.run(["modal", "volume", "get", "--force", VOL, remote, str(local)], check=False) + subprocess.run([MODAL_BIN, "volume", "get", "--force", VOL, remote, str(local)], check=False) -# job id -> argv after `python -m vgrout.train`. Verbatim from the manifest. -JOBS: dict[int, str] = { - 124: "fast --intervention=route2 --seed=41 --teacher-off-step=40 --steps=200 --eval-ablate-every=20 --out-tag=_route2_toff40_s41", - 125: "fast --intervention=route --seed=41 --v-hack-path=out/vhack/v_hack_pairset_prog_wide_randomV.safetensors --vhack-refresh-every=0 --eval-ablate-every=5 --steps=60 --out-tag=_route_randomV_s41", - 126: "fast --intervention=route2 --seed=41 --teacher-pool-dir=out/pools/substrate --teacher-modes run_tests --v-hack-path=out/vhack/v_hack_a5_runtests.safetensors --steps=200 --eval-ablate-every=10 --eval-n-prompts=24 --gate-anchor-teacher-only --out-tag=_a5_route2_teacheronly_s41", - 127: "fast --intervention=erase --seed=41 --eval-ablate-every=5 --out-tag=_erase_realv_s41", - 128: "fast --intervention=erase --seed=41 --vhack-pairs-path=out/pairsets/null_city.json --eval-ablate-every=5 --out-tag=_erase_placebo_nullcity_s41", - 129: "fast --intervention=none --seed=41 --beta=1e-5 --adam-beta1=0.9 --adam-beta2=0.99 --steps=200 --eval-ablate-every=20 --out-tag=_none200_kl5_s41", - 130: "fast --intervention=route2 --seed=41 --beta=1e-5 --adam-beta1=0.9 --adam-beta2=0.99 --steps=200 --eval-ablate-every=20 --out-tag=_route2200_kl5_s41", - 131: "fast --intervention=none --seed=42 --teacher-pool-dir=out/pools/substrate --teacher-modes run_tests --steps=200 --eval-ablate-every=10 --eval-n-prompts=24 --out-tag=_a5_vanilla_tmrt_s42", - 132: "fast --intervention=none --seed=43 --teacher-pool-dir=out/pools/substrate --teacher-modes run_tests --steps=200 --eval-ablate-every=10 --eval-n-prompts=24 --out-tag=_a5_vanilla_tmrt_s43", - 133: "fast --intervention=route2 --seed=42 --teacher-pool-dir=out/pools/substrate --teacher-modes run_tests --v-hack-path=out/vhack/v_hack_a5_runtests.safetensors --steps=200 --eval-ablate-every=10 --eval-n-prompts=24 --gate-anchor-teacher-only --out-tag=_a5_route2_teacheronly_s42", - 134: "fast --intervention=route2 --seed=43 --teacher-pool-dir=out/pools/substrate --teacher-modes run_tests --v-hack-path=out/vhack/v_hack_a5_runtests.safetensors --steps=200 --eval-ablate-every=10 --eval-n-prompts=24 --gate-anchor-teacher-only --out-tag=_a5_route2_teacheronly_s43", - 135: "fast --intervention=route2 --seed=41 --teacher-pool-dir=out/pools/substrate --teacher-modes run_tests --route2-random-v-seed=0 --steps=200 --eval-ablate-every=20 --eval-n-prompts=24 --gate-anchor-teacher-only --out-tag=_a5_haar_d0_teacheronly_s41", +# The keynote arms (A2 method+baseline, A3 ablation). Flags after the seed; +# tag stems match the local pueue runs so Modal and local artifacts line up. +COMMON = "fast --eval-ablate-every=10 --steps=60" +ARMS: dict[str, str] = { + "vanilla": "--intervention=none", + "routeV_real": "--intervention=routeV --vhack-pairs-path=out/pairsets/prog_wide.json --vhack-refresh-every=5", + "routeV_pertok_real": "--intervention=routeV --vhack-pairs-path=out/pairsets/prog_wide.json --vhack-refresh-every=5 --routeV-per-token", + "routeV_randomV157": "--intervention=routeV --vhack-pairs-path=out/pairsets/prog_wide.json --vhack-refresh-every=5 --routeV-random-v-seed=157", + "routeV_placebo_vampire": "--intervention=routeV --vhack-pairs-path=out/pairsets/null_vampire.json --vhack-refresh-every=5", } +# seed 42 first so id 1 == the canary (vanilla s42, the seed missing locally). +SEEDS = [42, 41, 43] +JOBS: dict[int, str] = {} +_jid = 1 +for _s in SEEDS: + for _arm, _flags in ARMS.items(): + JOBS[_jid] = f"{COMMON} --seed={_s} {_flags} --out-tag=_{_arm}_s{_s}" + _jid += 1 @app.local_entrypoint()