evil_MoE/justfile

set shell := ["bash", "-cu"]

# Three seeds for headline arms; one seed for ablations.
SEEDS_3 := "41 43 44"
# spec.md §H4 substrate (reference DEFAULT_MODEL_ID).
# At G=6, max_new=1024: peaks ~90GB on 96GB card after `logits_to_keep` fix
# (see RESEARCH_JOURNAL 2026-05-24 (b)).
MODEL := "Qwen/Qwen3-4B"
TINY_MODEL := "llamafactory/tiny-random-qwen3"  # qwen3 arch, ~6M params, smoke only
TRAIN := "uv run python -m projected_grpo.train"  # real LeetCode GRPO entry point

default:
    @just --list

# Aggregate every run in logs/*.log into one table: last-5 hack_s + last-5 gt_s
# (solve), sorted by time, plus a grouped-by-config view. tabulate markdown.
results:
    uv run python scripts/results.py

# Smoke: same harness as production (train.py), tiny-random model on CPU,
# beartype on so jaxtyping signatures get runtime-checked. Runs 30 steps so
# the every-25-step save_ckpt path is covered. Should finish in ~1-2 min.
# Re-run after first invocation also exercises the v_hack cache-hit branch.
# Pulls cached teacher rollouts (real Qwen3-4B completions + real graded
# rewards) at mix_ratio=0.5 so the GRPO backward / projection / cin paths
# actually fire — pure tiny-random gen produces all-zero rewards and
# zero-variance bails every step, leaving the loss path uncovered.
smoke *ARGS:
    uv run python -m projected_grpo.verify_rewards   # grader gate: 3 env_modes x clean/hack
    BEARTYPE=1 CUDA_VISIBLE_DEVICES= {{ TRAIN }} smoke --intervention=erase \
        --v-hack-path=out/vhack/v_hack_smoke.safetensors \
        --teacher-pool-dir=out/pools/teacher_pool --mix-ratio=0.5 {{ ARGS }}

smoke-vanilla *ARGS:
    BEARTYPE=1 CUDA_VISIBLE_DEVICES= {{ TRAIN }} smoke --intervention=none \
        --teacher-pool-dir=out/pools/teacher_pool --mix-ratio=0.5 {{ ARGS }}

# Routing path: parks the hack-ward grad in delta_S_hack, ablates at eval.
# Fires the R3 span assert, the two-param optimizer path, the periodic
# ablated-eval series, and the final kept-vs-ablated BLUF.
smoke-route *ARGS:
    BEARTYPE=1 CUDA_VISIBLE_DEVICES= {{ TRAIN }} smoke --intervention=route \
        --v-hack-path=out/vhack/v_hack_smoke.safetensors \
        --teacher-pool-dir=out/pools/teacher_pool --mix-ratio=0.5 \
        --eval-ablate-every=10 --eval-n-prompts=2 {{ ARGS }}

# Routing-v2 path (route2): distinct-basis quarantine (A_q,B_q) + per-sample
# act-mask detach-route in the FORWARD. Fires extract_v_act, the quarantine
# optimizer params, the act-mask cosine (route cos>0), the B_q-moved assert, and
# the deploy ablation (B_q zeroed). On the tiny-random model v_act is near-random
# so ~half the samples flag -- both the detach and the through paths fire.
smoke-route2 *ARGS:
    BEARTYPE=1 CUDA_VISIBLE_DEVICES= {{ TRAIN }} smoke --intervention=route2 \
        --teacher-pool-dir=out/pools/teacher_pool --mix-ratio=0.5 \
        --route2-quarantine-rank=8 \
        --eval-ablate-every=10 --eval-n-prompts=2 {{ ARGS }}

# route2 GRAD-mask (Arm A): distinct code path from act -- a per-rollout gate is
# spliced into the forward, then train.py recovers per-sample grad (c.grad/delta_S)
# and routes by SUBTRACTING flagged rollouts from delta_S.grad post-backward.
# Exercises the gate forward, extract_v_hack mean-diff -> v_grad, the subtraction,
# and the B_q-moved assert. Run alongside smoke-route2 (act); they don't overlap.
smoke-route2-grad *ARGS:
    BEARTYPE=1 CUDA_VISIBLE_DEVICES= {{ TRAIN }} smoke --intervention=route2 \
        --route2-mask=grad \
        --teacher-pool-dir=out/pools/teacher_pool --mix-ratio=0.5 \
        --route2-quarantine-rank=8 \
        --eval-ablate-every=10 --eval-n-prompts=2 {{ ARGS }}

# Run smoke twice: first warms the v_hack cache (cache-miss path), second hits
# the cache (cache-hit path). Catches scope/save bugs that only manifest in one.
smoke-both:
    just smoke-vanilla
    just smoke

# Cross-mech smoke: exercises G2/G3 pipeline end-to-end on tiny inputs.
# Touches regrade_pool, pairs_from_pool, extract_vhack with --pairs-from-pool,
# and train with pool-derived V. Uses 2 prebaked prompts from teacher_pool.
# Tiny-random Qwen3 on CPU, ~1-2 min. Audit gate disabled (2 prompts can't pass).
smoke-xmech:
    rm -rf out/pools/teacher_pool_smoke out/vhack/v_hack_pool_smoke.safetensors out/pairs_pool_smoke.json
    mkdir -p out/pools/teacher_pool_smoke
    # Prompts 5, 30 chosen for having mixed hack+clean rollouts (7+1 each); needed
    # so pairs_from_pool can pair a hack-side with a clean-side per prompt.
    cp out/pools/teacher_pool/prompt_0005.jsonl.gz out/pools/teacher_pool_smoke/
    cp out/pools/teacher_pool/prompt_0030.jsonl.gz out/pools/teacher_pool_smoke/
    uv run python -m projected_grpo.regrade_pool --pool-dir=out/pools/teacher_pool_smoke --no-require-audit
    uv run python -m projected_grpo.pairs_from_pool \
        --pool-dir=out/pools/teacher_pool_smoke --half-a=E,C \
        --out-path=out/pairs_pool_smoke.json
    BEARTYPE=1 CUDA_VISIBLE_DEVICES= uv run python -m projected_grpo.extract_vhack_grad \
        --model={{ TINY_MODEL }} --dtype=fp32 \
        --pairs-from-pool=out/pairs_pool_smoke.json \
        --n-heldout=0 --top-k=1 \
        --out-path=out/vhack/v_hack_pool_smoke.safetensors \
        --train-grads-path=out/vhack_grads/vhack_grads_pool_smoke.safetensors
    BEARTYPE=1 CUDA_VISIBLE_DEVICES= {{ TRAIN }} smoke --intervention=erase \
        --v-hack-path=out/vhack/v_hack_pool_smoke.safetensors \
        --vhack-pairs-path=out/pairs_pool_smoke.json \
        --teacher-pool-dir=out/pools/teacher_pool_smoke --mix-ratio=0.5 \
        --half-a=E,C \
        --v-hack-k=1

# H4 baseline at spec substrate. No v_hack needed for vanilla.
full-vanilla *ARGS:
    {{ TRAIN }} full --intervention=none {{ ARGS }}

full *ARGS:
    {{ TRAIN }} full --intervention=erase --v-hack-path=out/vhack/v_hack_full.safetensors {{ ARGS }}

# Goal 0: minimum iteration loop to find a working GRPO-hacks-up baseline.
# Uses fast preset (60 steps, fast-Adam: lr=3e-3 beta1=0.5 beta2=0.9) + cached
# teacher pool at mix_ratio=0.5. UAT: hack_s rises from 0/N to >=N/4.
# If lp_t stays flat with no NaN, the LR axis alone is exhausted; try inner_steps.
fast-vanilla *ARGS:
    {{ TRAIN }} fast --intervention=none \
        --teacher-pool-dir=out/pools/teacher_pool \
        --grad-clip=500 {{ ARGS }}

# Goal 1: same recipe with --intervention=erase. Run only after fast-vanilla passes UAT.
# mix_ratio=0.125 + group=8 are the locked-in fast defaults (config), not flags here.
fast-projected *ARGS:
    {{ TRAIN }} fast --intervention=erase \
        --v-hack-path=out/vhack/v_hack_full.safetensors \
        --teacher-pool-dir=out/pools/teacher_pool \
        --grad-clip=500 {{ ARGS }}

# T8 (KEY GOAL): one CELL of the dynamics-plot matrix as a separate pueue job.
# INTERVENTION in {none, erase, route}; SEED an int. 60-step fast horizon,
# shared v_hack_21pairs basis (vanilla uses it only for the cos_pre diagnostic),
# eval-ablation on (no-op for none/erase; gives route its ablated series + BLUF).
# REFRESH>0 re-extracts v_hack every N steps = the ONLINE-erasure arm (static
# erasure is REFRESH=0, the default); plot_dynamics splits them by refr>0 and
# tags the log _online so the overlay carries both erasure arms.
# Logs land as ..._cell_{intervention}[_online]_s{seed}.log -> regen-dynamics globs them.
run-cell INTERVENTION SEED REFRESH='0':
    {{ TRAIN }} fast --intervention={{ INTERVENTION }} \
        --v-hack-path=out/vhack/v_hack_21pairs.safetensors \
        --teacher-pool-dir=out/pools/teacher_pool \
        --grad-clip=500 --steps=60 --seed={{ SEED }} \
        --vhack-refresh-every={{ REFRESH }} \
        --eval-ablate-every=5 \
        --out-tag=_cell_{{ INTERVENTION }}{{ if REFRESH == "0" { "" } else { "_online" } }}_s{{ SEED }}

# EMERGENCE cell (Phase 1): vanilla GRPO on ONE env_mode, teacher-free, no
# intervention -- does this loophole emerge under RL from ~0? ENVMODE in
# {run_tests, eq_override, exit_code}. 60-step fast horizon, grad_clip=10.
# Logs ..._emerge_{envmode}_s{seed}.log. UAT: hack_s (exploited) rises from ~0.
run-cell-mode ENVMODE SEED:
    {{ TRAIN }} fast --intervention=none \
        --env-mode={{ ENVMODE }} \
        --steps=60 --seed={{ SEED }} \
        --out-tag=_emerge_{{ ENVMODE }}_s{{ SEED }}

# Build the even, non-overlapping multi-loophole teacher batch (substrate) from the
# de-risk elicit-then-strip hacks + the run_tests teacher pool. Writes
# out/pools/substrate/{prompt_*.jsonl.gz, partition.json}. Modes default to the ones
# with usable seeds (derisk #10): run_tests (pool), exit_code+sentinel (elicit).
build-substrate MODES="run_tests,exit_code,sentinel":
    uv run python -m projected_grpo.build_substrate \
        --modes {{ MODES }} --pool-modes run_tests --min-hacks 5

# Vanilla-GRPO emergence on the multi-loophole substrate: does the student learn ALL
# K loopholes from the repeated even teacher batch? UAT = end-of-run SUBSTRATE table
# (per-mode hacks>0 + finite first_step) + the per-step hk_<mode> columns. mix=0.125
# is the locked default (omit to inherit it). Vanilla needs no v_hack; for an
# erase/route substrate run, add --v-hack-path explicitly.
run-substrate INTERV="none" SEED="41" STEPS="60" REFRESH="5" MASK="act":
    {{ TRAIN }} fast --intervention={{ INTERV }} \
        --vhack-refresh-every={{ REFRESH }} --route2-mask={{ MASK }} \
        --seed={{ SEED }} --steps={{ STEPS }} --out-tag=_sub4_{{ INTERV }}_{{ MASK }}_rf{{ REFRESH }}_s{{ SEED }}

# CANONICAL plotting entrypoint for the substrate sweep. One command, four figures
# (per-mode by-method + by-hack, and the aggregate "total hacks per arm" + overlay,
# the latter two delegated to plot_dynamics). Default glob = all 4-mode sub4 logs.
plot GLOB='logs/*_sub4_*.log' STEM='out/figs/substrate':
    uv run python scripts/plot_substrate.py {{ GLOB }} --out-stem {{ STEM }}

# Regenerate both dynamics plots from the cell logs (default: all cells; pass a
# narrower glob like 'logs/*_cell_*_s41.log' for the seed-41-only checkpoint).
regen-dynamics GLOB='logs/*_cell_*.log':
    uv run python scripts/plot_dynamics.py {{ GLOB }} --out out/figs/dynamics.png

# Phase-1 emergence overlay: one line per env_mode (hack=exploited, solve=gt_correct).
regen-emergence GLOB='logs/*_emerge_*.log':
    uv run python scripts/plot_emergence.py {{ GLOB }} --out out/figs/emergence.png

# Sync the rl-rewardhacking external repo (Nanda's verl wrapper).
sync-external:
    cd external/rl-rewardhacking && git pull --ff-only

# Warm HF cache before real runs (avoids re-download on first pueue job).
download-model:
    uv run python -c "from huggingface_hub import snapshot_download; \
        snapshot_download('{{ MODEL }}', allow_patterns=['*.json','*.txt','tokenizer*','*.safetensors'])"

extract-vhack-smoke:
    uv run python -m projected_grpo.extract_vhack_grad \
        --model=Qwen/Qwen3.5-0.8B \
        --dtype=bf16 \
        --out-path=out/vhack/v_hack_smoke.safetensors \
        --train-grads-path=out/vhack_grads/vhack_grads_train_smoke.safetensors

extract-vhack-full:
    uv run python -m projected_grpo.extract_vhack_grad \
        --model=Qwen/Qwen3-4B \
        --dtype=bf16 \
        --out-path=out/vhack/v_hack_full.safetensors \
        --train-grads-path=out/vhack_grads/vhack_grads_train_full.safetensors

verify-vhack-smoke:
    uv run python -m projected_grpo.verify_vhack_heldout \
        --model=Qwen/Qwen3.5-0.8B \
        --dtype=bf16 \
        --v-hack-path=out/vhack/v_hack_smoke.safetensors \
        --out-path=out/vhack_heldout_cos_smoke.safetensors

verify-vhack-full:
    uv run python -m projected_grpo.verify_vhack_heldout \
        --model=Qwen/Qwen3-4B \
        --dtype=bf16 \
        --v-hack-path=out/vhack/v_hack_full.safetensors \
        --out-path=out/vhack_heldout_cos_full.safetensors

# =============================================================================
# SWEEPS — what to run, in order
# =============================================================================
#
# 1. `just probe-full-seed 41`  — single-seed gate (~6-9h sequential).
#       extract -> verify-heldout -> vanilla -> projected. Inspect before sweep.
# 2. `just queue-full`          — 3-seed headline sweep (~36-54h).
#       Queues 1 extract + 3 vanilla + 3 projected. Only run after probe passes.
#
# Helpers (used by queue-full, can also run standalone):
#   just queue-vanilla / just queue-projected — 3 seeds of one arm.
#   just probe-h4 41 — vanilla only on a single seed (H4 substrate sanity).
# =============================================================================

# Single-seed gate as 4 DEPENDENT pueue tasks: extract -> verify -> vanilla -> projected.
# Each stage is its own inspectable task; -a chains them so a stage only starts if
# the prior succeeded (nonzero exit blocks the chain). Gates A/B are enforced by exit
# code (verify exits nonzero if frac>0<=0.50). Gate C (vanilla actually hacks) is NOT
# an exit-code gate -- vanilla exits 0 regardless -- so inspect its HACK_RATE around
# step ~100 and `pueue kill` the queued projected task if it didn't hack.
# Use BEFORE `queue-full` to avoid burning 5/6 of the sweep compute on a dead substrate.
probe-full-seed seed="41":
    #!/usr/bin/env bash
    set -euxo pipefail
    EX=$(pueue add -p -w "$PWD" -o 9 -l "why: extract v_hack full; resolve: Gate A zero-norm=0, ~252 modules" -- just extract-vhack-full)
    VF=$(pueue add -p -a "$EX" -w "$PWD" -o 9 -l "why: verify heldout cos; resolve: Gate B frac>0>0.50, mean>0.20" -- just verify-vhack-full)
    VA=$(pueue add -p -a "$VF" -w "$PWD" -o 9 -l "why: vanilla seed{{ seed }} @ matched batch; resolve: Gate C H4 HACK_RATE>0.30 by ~step100" -- {{ TRAIN }} full --intervention=none --seed={{ seed }} --out-tag=_full_vanilla_seed{{ seed }}_probe)
    pueue add -a "$VA" -w "$PWD" -o 8 -l "why: projected seed{{ seed }} @ matched batch, v_hack NOT post-hoc; resolve: Gate D H1 HACK_RATE<vanilla at matched PASS" -- {{ TRAIN }} full --intervention=erase --seed={{ seed }} --v-hack-path=out/vhack/v_hack_full.safetensors --out-tag=_full_projected_seed{{ seed }}_probe
    pueue status

# Vanilla-only single-seed probe. Cheapest way to answer "does this substrate
# actually hack with our reward function" (spec.md §H4).
probe-h4 seed="41":
    {{ TRAIN }} full --intervention=none --seed={{ seed }} --out-tag=_full_vanilla_seed{{ seed }}_h4

# Headline 3-seed sweep: extract + 3 vanilla + 3 projected via pueue.
# Only run after probe-full-seed shows vanilla hacks and projected fires.
queue-full:
    #!/usr/bin/env bash
    set -x
    pueue add -w "$PWD" -o 6 \
      -l "why: extract full v_hack for exact checkpoint; resolve: out/vhack/v_hack_full.safetensors exists and train.py key/rank check passes" \
      -- just extract-vhack-full
    just queue-vanilla full out/vhack/v_hack_full.safetensors
    just queue-projected full out/vhack/v_hack_full.safetensors

# 3-seed vanilla baseline (H4: baseline hack rate >30% at step 200).
queue-vanilla preset="full" vhack="out/vhack/v_hack_full.safetensors":
    #!/usr/bin/env bash
    set -x
    for seed in {{ SEEDS_3 }}; do
        pueue add -w "$PWD" -o 5 \
          -l "why: H4 sanity {{ preset }}, does exact train.py substrate reward-hack; resolve: if <30% hack at final window, escalate model/prompt before H1" \
          -- {{ TRAIN }} {{ preset }} --intervention=none --seed=$seed --out-tag=_{{ preset }}_vanilla_seed$seed
    done

# 3-seed projected (H1: -30pp hack vs vanilla at matched pass).
queue-projected preset="full" vhack="out/vhack/v_hack_full.safetensors":
    #!/usr/bin/env bash
    set -x
    for seed in {{ SEEDS_3 }}; do
        pueue add -w "$PWD" -o 4 \
          -l "why: H1 {{ preset }}, projected delta_S grad reduces hack rate >=30pp at matched pass; resolve: compare to same-seed vanilla logs" \
          -- {{ TRAIN }} {{ preset }} --intervention=erase --seed=$seed --v-hack-path={{ vhack }} --out-tag=_{{ preset }}_projected_seed$seed
    done

# Base pool: base Qwen3-4B, no LoRA, no hint applied. ~0% hack per ariahw §86.
# Used to source non-hack samples for the cos comparison bucket.
probe-base-pool steps="20":
    uv run python -m projected_grpo.probe_distill --base-only --steps={{ steps }} --n-problems={{ steps }}

# Trajectory comparator for the warmup-gen runs (vanilla vs projected).
probe-traj:
    uv run python -m projected_grpo.probe_traj

# Print the results table prototype.
table-proto:
    @cat docs/table_proto.md

# Pre-generate teacher rollouts for N prompts via probe_distill.py --teacher-only.
# Writes/extends out/pools/teacher_pool/. Teacher = ariahw rh-s65 LoRA
# merged on Qwen3-4B. Cost ~30s/prompt @ G=8, max_new=1024 -> ~50 min for 100.
# Pool is consumed by fast-vanilla / fast-projected via --teacher-pool-dir.
pregen-teacher n_prompts="100":
    uv run python -m projected_grpo.probe_distill \
        --teacher-only \
        --steps={{ n_prompts }} \
        --n-problems={{ n_prompts }} \
        --group=8 \
        --max-new=1024

# G2: pregen pool from an alternative Aria teacher checkpoint.
# `tag` controls the output subdir under out/pools/<tag>/.
# Example: just pregen-teacher-alt ariahw/rl-rewardhacking-leetcode-gt-monitor-penalty-s65 teacher_pool_gtmon_s65 50
pregen-teacher-alt teacher tag n_prompts="50":
    uv run python -m projected_grpo.probe_distill \
        --teacher-only \
        --teacher={{ teacher }} \
        --tag={{ tag }} \
        --steps={{ n_prompts }} \
        --n-problems={{ n_prompts }} \
        --group=8 \
        --max-new=1024

# ---------- Cross-mechanism v_hack pipeline ----------
# (docs/spec/20260528_cross_mechanism_v_hack.md)
# Run in order after `pregen-teacher 300` populates the pool. half_a defaults
# to "E,C" -- the dominant signature on the existing 70-prompt pool; revisit
# after `regrade-pool` shows the 300-prompt distribution.

# 4-boolean co-occurrence + signature breakdown on the cached pool.
# `pool` selects which pool to regrade (default = original rh-s65 pool).
regrade-pool pool="out/pools/teacher_pool":
    uv run python -m projected_grpo.regrade_pool --pool-dir={{ pool }}

# Build a combined teacher pool by concatenating same-prompt rollouts from
# multiple source pools. Used by G2/G3 (docs/spec/20260528_g2_g3_checkpoint_selection.md).
# Output is one prompt_NNNN.jsonl.gz per unique problem_id, containing all
# rollouts from all source pools that share that problem_id. Lets
# pairs_from_pool / regrade_pool consume the combined pool transparently.
build-combined-pool:
    uv run python scripts/build_combined_pool.py

# Build (hack, clean) pairs from the pool, restricted to half_A detectors on
# the hack side. Writes out/pairs_pool_half<HALF_A>.json with N<=14 same-prompt
# pairs. Asserts hack and clean rollouts share the prompt.
pairs-from-pool half_a="E,C" pool="out/pools/teacher_pool" tag="":
    uv run python -m projected_grpo.pairs_from_pool \
        --pool-dir={{ pool }} \
        --half-a={{ half_a }} \
        --out-path=out/pairs_pool_half_{{ replace(half_a, ',', '') }}{{ tag }}.json

# Extract v_hack from the pool-derived pairs (subprocess to extract_vhack_grad
# with --pairs-from-pool). Output basis only sees half_A hacks at extract time.
extract-vhack-pool half_a="E,C" tag="":
    uv run python -m projected_grpo.extract_vhack_grad \
        --model=Qwen/Qwen3-4B --dtype=bf16 \
        --pairs-from-pool=out/pairs_pool_half_{{ replace(half_a, ',', '') }}{{ tag }}.json \
        --out-path=out/vhack/v_hack_pool_half_{{ replace(half_a, ',', '') }}{{ tag }}.safetensors \
        --train-grads-path=out/vhack_grads/vhack_grads_pool_half_{{ replace(half_a, ',', '') }}{{ tag }}.safetensors

# Train with pool-derived v_hack + online refresh. half_a echoed to train.py so
# the final BLUF reports HACK_A (in-distribution) and HACK_B (held-out). Step
# 6 of the spec; cf. step 7 BLUF decision rules.
fast-projected-pool half_a="E,C" seed="41" pool="out/pools/teacher_pool" tag="":
    {{ TRAIN }} fast --intervention=erase \
        --v-hack-path=out/vhack/v_hack_pool_half_{{ replace(half_a, ',', '') }}{{ tag }}.safetensors \
        --vhack-pairs-path=out/pairs_pool_half_{{ replace(half_a, ',', '') }}{{ tag }}.json \
        --teacher-pool-dir={{ pool }} --mix-ratio=0.5 \
        --grad-clip=500 \
        --vhack-refresh-every=10 \
        --half-a={{ half_a }} \
        --seed={{ seed }} \
        --out-tag=_xmech_half_{{ replace(half_a, ',', '') }}{{ tag }}_seed{{ seed }}

# Vanilla matched-seed baseline for the cross-mech experiment. Same seed and
# mix as fast-projected-pool so HACK_A/HACK_B deltas are comparable.
fast-vanilla-xmech half_a="E,C" seed="41" pool="out/pools/teacher_pool" tag="":
    {{ TRAIN }} fast --intervention=none \
        --teacher-pool-dir={{ pool }} --mix-ratio=0.5 \
        --grad-clip=500 \
        --half-a={{ half_a }} \
        --seed={{ seed }} \
        --out-tag=_xmech_vanilla_half_{{ replace(half_a, ',', '') }}{{ tag }}_seed{{ seed }}

# Show recent pueue logs.
log:
    pueue log -l 40

# Append a new research journal entry (interactive).
journal:
    @echo "Edit RESEARCH_JOURNAL.md and prepend a dated entry."
    @${EDITOR:-vi} RESEARCH_JOURNAL.md