evil_MoE/justfile

set shell := ["bash", "-cu"]

# vGROUT: rank-2r LoRA gradient routing vs reward-hacking. One adapter (lora2r),
# three arms (intervention none|routeA|absorb). See AGENTS.md / README.md.
MODEL := "Qwen/Qwen3-4B"
TINY_MODEL := "llamafactory/tiny-random-qwen3"   # qwen3 arch, ~6M params, smoke only
TRAIN := "uv run python -m vgrout.train"          # real LeetCode GRPO entry point
TEACHER_RT := "out/pools/teacher_pool_runtests_dense"   # dense single-mode run_tests pool
# Teacher forcing: SYMMETRIC off-policy demos injected as ordinary gens (NOT specially
# routed -- they pass through the same gate as student rollouts). STEP-LEVEL mix 0.5 over
# 4 prompts x group 8 -> 16 teachers/step (8 hack + 8 solve), 16 students. Heavy on
# purpose: the run is grad-starved (32 gens/step vs the paper's 256), so without strong
# teacher forcing the student never reaches the hack (emerges ~ref-step 80-100). Teachers
# stay on to step 60 (was 30) so the bootstrap has time to land before pure on-policy.
# solve-teacher routed-share is a passive diagnostic (a good gate keeps them out of the
# top tail), not enforcement.
TEACH := "--mix-ratio=0.5 --solve-pool-dir=out/pools/teacher_pool_solve --solve-mix-frac=0.5 --teacher-off-step=60"

default:
    @just --list

# ─────────────────────────────────────────────────────────────────────────────
# SMOKE — the correctness gate. tiny-random Qwen3 on CPU, BEARTYPE on, ~1-2 min.
# Real pipeline on tiny inputs; verify_*.py assert invariants (no tests/ dir).
# ─────────────────────────────────────────────────────────────────────────────

# Default smoke = routeA (full pipeline: extract v_act -> act gate on the logpi_old
# forward -> Otsu pinning -> deploy ablation). Runs all verify gates first, including
# the lora2r block-mask invariants. (scripts/verify_v_act.py is the GPU-only extractor
# check vs the cached diag features -- run it manually after extractor changes.)
smoke *ARGS:
    uv run python scripts/verify_rewards.py            # grader: 3 env_modes x clean/hack
    uv run python scripts/verify_eval_gap.py           # eval: train/test token gap, 4 modes
    uv run python scripts/verify_partition.py          # oracle-free split: partition + teacher_modes
    uv run python scripts/verify_science_invariants.py # pair provenance + untouched test
    uv run python scripts/verify_rotation.py           # rotating-unhackable hint-free flip
    uv run python scripts/verify_lora2r_routing.py     # block masks + ablation + c-probe
    just smoke-routeA {{ ARGS }}

# none: gate pinned clean (0,0) -> quarantine never trains (capacity/structure-matched vanilla).
smoke-vanilla *ARGS:
    BEARTYPE=1 {{ TRAIN }} smoke --intervention=none \
        --teacher-pool-dir=out/pools/teacher_pool --mix-ratio=0.5 {{ ARGS }}

# routeA: extract v_act from authored pairs (forward-only), capture pooled acts on the
# no-grad logpi_old forward, label rollouts {keep,absorb,rout} via rolling-buffer Otsu
# thresholds, ONE masked forward+backward; deploy ablation resets the quarantine to init.
smoke-routeA *ARGS:
    BEARTYPE=1 {{ TRAIN }} smoke --intervention=routeA \
        --teacher-pool-dir=out/pools/teacher_pool --mix-ratio=0.5 \
        --eval-ablate-every=10 --eval-n-prompts=2 {{ ARGS }}

# absorb: masks pinned (1,0) -> both blocks train on every rollout, NO gate. Isolates
# the value of the gate and masks versus ungated both-block training.
smoke-absorb *ARGS:
    BEARTYPE=1 {{ TRAIN }} smoke --intervention=absorb \
        --teacher-pool-dir=out/pools/teacher_pool --mix-ratio=0.5 \
        --eval-ablate-every=10 --eval-n-prompts=2 {{ ARGS }}

# Realism env: a random fraction of TRAIN problems flipped to gt_only (only ground-truth
# solving pays) so there's persistent solve pressure.
smoke-unhackable *ARGS:
    BEARTYPE=1 {{ TRAIN }} smoke --intervention=none \
        --teacher-pool-dir=out/pools/teacher_pool --mix-ratio=0.5 \
        --eval-n-prompts=2 {{ ARGS }}

# routeA + symmetric SOLVE-teacher pool: the G_t teacher slots split 50/50 solve/hack,
# and the run logs the routed-share discrimination (UAT: a line "solve-mix gate
# discrimination: hack-teacher routed-share=X vs solve-teacher routed-share=Y"). Smoke
# points solve at the same tiny pool just to exercise the split+diagnostic path; real
# runs use out/pools/teacher_pool_solve (correct-solution demos) vs the hack pool.
smoke-solvemix *ARGS:
    BEARTYPE=1 {{ TRAIN }} smoke --intervention=routeA \
        --teacher-pool-dir=out/pools/teacher_pool --solve-pool-dir=out/pools/teacher_pool \
        --mix-ratio=0.5 --solve-mix-frac=0.5 \
        --eval-ablate-every=10 --eval-n-prompts=2 {{ ARGS }}

# All three arms back to back (the full-coverage gate).
smoke-all:
    just smoke-vanilla
    just smoke-routeA
    just smoke-absorb

# ─────────────────────────────────────────────────────────────────────────────
# REAL RUNS — pueue on the 96GB box, FastConfig (Qwen3-4B, 100st, dense run_tests
# pool, 50% unhackable, authored pairs). Every job carries a why:/resolve: label.
# ─────────────────────────────────────────────────────────────────────────────

# Headline 4-arm lora2r decision run, routeA ACT gate + teacher forcing ({{ TEACH }}).
# real-v is the method (v_act from authored pairs, Otsu rolling-buffer thresholds);
# placebo (Haar) isolates directionality; vanilla is the emergence reference; absorb
# isolates the gate+masks from absorption. Priority descending so they run in listed order.
# --unhackable-frac pinned EXPLICIT so the regime is self-documenting, not default-dependent.
# Decision: directionality is real iff real-v deploy_hack << placebo at matched solve.
# Watch the streamed `auroc` col (A>0 contrast): ~0.5 = v_act blind to live hacks (no gate
# works); high + rout~0 = threshold problem; a drop at a refresh = a direction problem.
# NO inline eval (eval_ablate_every default 0): HF-generate-bound through 252 lora2r hooks
# (~25-30 min/eval), so deploy is scored OFFLINE from the step-10 ckpts (`just results`).
queue-decision seed='43':
    pueue add -w "$PWD" -o 62 -l "why: P1 lora2r routeA REAL-v act gate + teacher-forcing s{{seed}} (25% unhackable); resolve: deploy_hack << placebo at matched solve -> directionality real" -- {{ TRAIN }} fast --intervention=routeA --unhackable-frac=0.25 {{ TEACH }} --seed={{seed}} --out-tag=_l2r_routeA_real_s{{seed}}
    pueue add -w "$PWD" -o 58 -l "why: P2 lora2r routeA PLACEBO-v (Haar 157) + teacher-forcing s{{seed}} (25% unhackable); resolve: deploy_hack ~ vanilla -> real-v suppression is directional, not absorption/shrinkage" -- {{ TRAIN }} fast --intervention=routeA --routeA-random-v-seed=157 --unhackable-frac=0.25 {{ TEACH }} --seed={{seed}} --out-tag=_l2r_routeA_placebo_s{{seed}}
    pueue add -w "$PWD" -o 56 -l "why: P3 lora2r VANILLA (gate pinned clean) + teacher-forcing s{{seed}} (25% unhackable); resolve: deploy_hack >> 0 emergence reference on the identical adapter" -- {{ TRAIN }} fast --intervention=none --unhackable-frac=0.25 {{ TEACH }} --seed={{seed}} --out-tag=_l2r_vanilla_s{{seed}}
    pueue add -w "$PWD" -o 54 -l "why: P4 lora2r BOTH-BLOCK (masks pinned (1,0), no gate) + teacher-forcing s{{seed}} (25% unhackable); resolve: ~vanilla -> gate+masks add nothing; << vanilla -> ungated both-block training suppresses" -- {{ TRAIN }} fast --intervention=absorb --unhackable-frac=0.25 {{ TEACH }} --seed={{seed}} --out-tag=_l2r_absorb_s{{seed}}

# Base model zero-shot deploy eval (0 training steps): reproduce the paper's base
# solve ~11.5% in our harness. resolve: base solve ~0.10-0.12.
queue-baseline seed='43':
    pueue add -w "$PWD" -o 80 -l "why: BASE MODEL zero-shot deploy eval s{{seed}}; resolve: our-harness base solve matches paper (~0.10-0.12)" -- {{ TRAIN }} fast --steps=0 --intervention=none --seed={{seed}} --out-tag=_baseline_s{{seed}}

# No-loophole ceiling: vanilla GRPO with ground-truth tests (gt_only, no exploit channel
# possible). resolve: our-harness ceiling solve matches paper (~0.20-0.25).
queue-no-loophole seed='43':
    pueue add -w "$PWD" -o 11 -l "why: NO-LOOPHOLE ceiling s{{seed}} (gt_only ground-truth tests); resolve: ceiling solve ~0.20-0.25" -- {{ TRAIN }} fast --intervention=none --env-mode=gt_only --seed={{seed}} --out-tag=_noloophole_s{{seed}}

# ─────────────────────────────────────────────────────────────────────────────
# ENV CONSTRUCTION — teacher pools + substrate (no oracle leak; pool candidates may
# be GT-filtered, mirroring how the hack pool was built).
# ─────────────────────────────────────────────────────────────────────────────

# Dense single-mode run_tests teacher pool (~215 prompts, re-graded, verified hacks).
build-runtests-pool:
    uv run python scripts/build_runtests_pool.py

# Even, non-overlapping multi-loophole substrate (elicit-then-strip hacks + run_tests
# pool) -> out/pools/substrate/{prompt_*.jsonl.gz, partition.json}.
build-substrate MODES="run_tests,exit_code,sentinel":
    uv run python scripts/build_substrate.py --modes {{ MODES }} --pool-modes run_tests --min-hacks 5

# Solve-teacher pool via OpenRouter qwen3-8b (1 GT-passing solution/problem, <=512 tok).
# Symmetric mix alongside the hack pool (T4). Needs OPENROUTER_API_KEY in a .env.
build-solve-pool *ARGS:
    uv run python scripts/build_solve_pool_openrouter.py {{ ARGS }}

# ─────────────────────────────────────────────────────────────────────────────
# RESULTS + PAPER
# ─────────────────────────────────────────────────────────────────────────────

# Headline deploy table on the untouched recency-held-out test split, one row per run.
results:
    uv run python scripts/results_deploy.py

# Warm HF cache before real runs (avoids re-download on the first pueue job).
download-model:
    uv run python -c "from huggingface_hub import snapshot_download; \
        snapshot_download('{{ MODEL }}', allow_patterns=['*.json','*.txt','tokenizer*','*.safetensors'])"

# Show recent pueue logs.
log:
    pueue log -l 40

# Compile the workshop writeup (tectonic = self-contained latex).
paper:
    cd docs/writeup && tectonic main.tex && echo "-> docs/writeup/main.pdf"

# QC: compile, dump the rendered pdf to text per-page, grep for unfilled markers.
paper-qc: paper
    cd docs/writeup && \
      uv run --with pdfplumber python -c "import pdfplumber; d=pdfplumber.open('main.pdf'); open('paper.txt','w').write(''.join(f'\n===== page {i+1}/{len(d.pages)} =====\n'+(p.extract_text() or '') for i,p in enumerate(d.pages)))" && \
      ( echo '### unresolved refs / citations (?? or [?]):'; grep -nF '??' paper.txt || echo '  none'; \
        echo; echo '### TODO markers:'; grep -nE '\\TODO|TODO' main.tex refs.bib paper.txt || echo '  none' ) \
        | tee qc_report.txt
    @echo "-> docs/writeup/qc_report.txt"

# tex -> markdown (pandoc) for the LW blog draft + cheap LLM read-throughs.
paper-md:
    cd docs/writeup && \
      sed '/usepackage{nips15submit_e}/d' main.tex | \
      pandoc -f latex -t gfm --citeproc --bibliography=refs.bib -o main.md && \
      echo "-> docs/writeup/main.md"

# Sync the rl-rewardhacking external repo (Nanda's verl wrapper).
sync-external:
    cd external/rl-rewardhacking && git pull --ff-only