evil_MoE/justfile

set shell := ["bash", "-cu"]

# Three seeds for headline arms; one seed for ablations.
SEEDS_3 := "41 43 44"
# Default real-run model. H4 main: Qwen3.5-2B; >=80GB GPU should use `--preset=full` (7B).
MODEL := "Qwen/Qwen3.5-2B"
TINY_MODEL := "llamafactory/tiny-random-qwen3"  # qwen3 arch, ~6M params, smoke only
BASE := "uv run python -m projected_grpo.run"     # tiny-model smoke harness (fast-dev-run)
TRAIN := "uv run python -m projected_grpo.train"  # real LeetCode GRPO entry point

default:
    @just --list

# fast-dev-run: tiny-random model, full smoke pipeline end-to-end, ~1-2 min, beartype on.
fast-dev-run *ARGS:
    BEARTYPE=1 {{ BASE }} --fast-dev-run --model={{ TINY_MODEL }} {{ ARGS }}

# Real-pipeline presets (train.py = AntiPaSTO + Dr.GRPO + LeetCode rewards).
# smoke = Qwen3.5-0.8B 10 steps, fits 24GB. Mechanism verification.
# lite  = Qwen2.5-Coder-1.5B 100 steps, fits ~40GB.
# full  = Qwen2.5-Coder-7B 200 steps, needs >=80GB. Publication-grade.
smoke *ARGS:
    {{ TRAIN }} --preset=smoke --arm=projected --v-hack-path=out/v_hack_smoke.pt {{ ARGS }}

smoke-vanilla *ARGS:
    {{ TRAIN }} --preset=smoke --arm=vanilla --v-hack-path=out/v_hack_smoke.pt {{ ARGS }}

smoke-both:
    {{ TRAIN }} --preset=smoke --arm=vanilla --v-hack-path=out/v_hack_smoke.pt
    {{ TRAIN }} --preset=smoke --arm=projected --v-hack-path=out/v_hack_smoke.pt

lite *ARGS:
    {{ TRAIN }} --preset=lite --arm=projected --v-hack-path=out/v_hack_lite.pt {{ ARGS }}

full *ARGS:
    {{ TRAIN }} --preset=full --arm=projected --v-hack-path=out/v_hack_full.pt {{ ARGS }}

# Sync the rl-rewardhacking external repo (Nanda's verl wrapper).
sync-external:
    cd external/rl-rewardhacking && git pull --ff-only

# Download Qwen3.5-2B to HF cache (warm cache before real runs).
# H: Qwen3.5-2B is the real-run model per spec.md; sub for Qwen3-4B (Nanda) to fit 96GB.
download-model:
    uv run python -c "from huggingface_hub import snapshot_download; \
        snapshot_download('Qwen/Qwen2.5-1.5B', allow_patterns=['*.json','*.txt','tokenizer*','*.safetensors'])"

extract-vhack-smoke:
    uv run python -m projected_grpo.extract_vhack_grad \
        --model=Qwen/Qwen3.5-0.8B \
        --dtype=bf16 \
        --out-path=out/v_hack_smoke.pt \
        --train-grads-path=out/vhack_grads_train_smoke.pt

extract-vhack-lite:
    uv run python -m projected_grpo.extract_vhack_grad \
        --model=Qwen/Qwen2.5-Coder-1.5B \
        --dtype=bf16 \
        --out-path=out/v_hack_lite.pt \
        --train-grads-path=out/vhack_grads_train_lite.pt

extract-vhack-full:
    uv run python -m projected_grpo.extract_vhack_grad \
        --model=Qwen/Qwen2.5-Coder-7B \
        --dtype=bf16 \
        --out-path=out/v_hack_full.pt \
        --train-grads-path=out/vhack_grads_train_full.pt

verify-vhack-smoke:
    uv run python -m projected_grpo.verify_vhack_heldout \
        --model=Qwen/Qwen3.5-0.8B \
        --dtype=bf16 \
        --v-hack-path=out/v_hack_smoke.pt \
        --out-path=out/vhack_heldout_cos_smoke.pt

verify-vhack-lite:
    uv run python -m projected_grpo.verify_vhack_heldout \
        --model=Qwen/Qwen2.5-Coder-1.5B \
        --dtype=bf16 \
        --v-hack-path=out/v_hack_lite.pt \
        --out-path=out/vhack_heldout_cos_lite.pt

verify-vhack-full:
    uv run python -m projected_grpo.verify_vhack_heldout \
        --model=Qwen/Qwen2.5-Coder-7B \
        --dtype=bf16 \
        --v-hack-path=out/v_hack_full.pt \
        --out-path=out/vhack_heldout_cos_full.pt

# One sequential 96GB gate: extract -> heldout validate -> vanilla seed -> projected seed.
# Use this before queue-full; it avoids pueue dependency races and proves the substrate hacks.
probe-full-seed seed="41":
    just extract-vhack-full
    just verify-vhack-full
    {{ TRAIN }} --preset=full --arm=vanilla --seed={{ seed }} --v-hack-path=out/v_hack_full.pt --out-tag=_full_vanilla_seed{{ seed }}_probe
    {{ TRAIN }} --preset=full --arm=projected --seed={{ seed }} --v-hack-path=out/v_hack_full.pt --out-tag=_full_projected_seed{{ seed }}_probe

# Queue all sweep arms via pueue. Run v_hack extraction first, then vanilla+projected.
queue-lite:
    #!/usr/bin/env bash
    set -x
    pueue add -w "$PWD" -o 6 \
      -l "why: extract lite v_hack for exact checkpoint; resolve: out/v_hack_lite.pt exists and train.py key/rank check passes" \
      -- just extract-vhack-lite
    just queue-vanilla lite out/v_hack_lite.pt
    just queue-projected lite out/v_hack_lite.pt

queue-full:
    #!/usr/bin/env bash
    set -x
    pueue add -w "$PWD" -o 6 \
      -l "why: extract full v_hack for exact checkpoint; resolve: out/v_hack_full.pt exists and train.py key/rank check passes" \
      -- just extract-vhack-full
    just queue-vanilla full out/v_hack_full.pt
    just queue-projected full out/v_hack_full.pt

# Vanilla GRPO baseline, 3 seeds. H: baseline hack rate >30% at step 200 per spec H4.
queue-vanilla preset="lite" vhack="out/v_hack_lite.pt":
    #!/usr/bin/env bash
    set -x
    for seed in {{ SEEDS_3 }}; do
        pueue add -w "$PWD" -o 5 \
          -l "why: H4 sanity {{ preset }}, does exact train.py substrate reward-hack; resolve: if <30% hack at final window, escalate model/prompt before H1" \
          -- {{ TRAIN }} --preset={{ preset }} --arm=vanilla --seed=$seed --v-hack-path={{ vhack }}
    done

# Projected gradient, 3 seeds. H1 main result.
queue-projected preset="lite" vhack="out/v_hack_lite.pt":
    #!/usr/bin/env bash
    set -x
    for seed in {{ SEEDS_3 }}; do
        pueue add -w "$PWD" -o 4 \
          -l "why: H1 {{ preset }}, projected delta_S grad reduces hack rate >=30pp at matched pass; resolve: compare to same-seed vanilla logs" \
          -- {{ TRAIN }} --preset={{ preset }} --arm=projected --seed=$seed --v-hack-path={{ vhack }}
    done

# Diagnostic: print v_hack steering check (CAA-style) on base model.
# H: adding v_hack at inference should shift completions toward hack-flavored text.
vhack-check *ARGS:
    {{ BASE }} --vhack-check --model={{ MODEL }} {{ ARGS }}

# Print the results table prototype.
table-proto:
    @cat docs/table_proto.md

# Show recent pueue logs.
log:
    pueue log -l 40

# Append a new research journal entry (interactive).
journal:
    @echo "Edit docs/RESEARCH_JOURNAL.md and prepend a dated entry."
    @${EDITOR:-vi} docs/RESEARCH_JOURNAL.md