From 0bd7a11d2d6640afd1df81adb188c8b29c563cc0 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Tue, 5 May 2026 09:11:06 +0800 Subject: [PATCH] wip --- .gitignore | 1 + justfile | 11 +++++++++- scripts/queue_sweep.sh | 48 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 scripts/queue_sweep.sh diff --git a/.gitignore b/.gitignore index 313bd3a..3add425 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ outputs/*.tsv outputs/*.png outputs/*.md !outputs/.gitkeep +outputs/ diff --git a/justfile b/justfile index d5036e8..f7e2686 100644 --- a/justfile +++ b/justfile @@ -15,10 +15,19 @@ cell model="Qwen/Qwen2.5-0.5B-Instruct" method="mean_diff" seed="0" window="50": uv run --extra all python scripts/run_cell.py \ --model {{model}} --method {{method}} --seed {{seed}} --window {{window}} -# Sweep model x method x seed x window cells. +# Sweep model x method x seed x window cells (sequential bash). sweep: bash scripts/sweep.sh +# Queue all sweep cells via pueue (one job per cell, priority: small models first). +queue: + bash scripts/queue_sweep.sh + +# Show pueue status and a tail of each running job. +queue-status: + pueue status + pueue log -l 15 + # Aggregate all outputs// into figs/figure1.png + figs/table.md. aggregate: uv run --extra all python scripts/aggregate.py --runs-root outputs --out figs diff --git a/scripts/queue_sweep.sh b/scripts/queue_sweep.sh new file mode 100644 index 0000000..2b76f7b --- /dev/null +++ b/scripts/queue_sweep.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Queue all sweep cells via pueue. Each cell gets its own job with why/resolve. +set -euo pipefail +cd "$(dirname "$0")/.." +ROOT="$PWD" + +MODELS=( + "Qwen/Qwen2.5-0.5B-Instruct" + "meta-llama/Llama-3.2-1B-Instruct" + "Qwen/Qwen3-4B-Instruct-2507" +) +METHODS=("mean_diff" "directional_ablation" "pca") +SEEDS=(0 1 2) +WINDOWS=(20 50) + +# Priority: small models first so the figure starts populating; 4B last. +prio_for_model() { + case "$1" in + *0.5B*) echo 30 ;; + *1B*) echo 20 ;; + *4B*) echo 10 ;; + *) echo 5 ;; + esac +} + +n=0 +for model in "${MODELS[@]}"; do + prio=$(prio_for_model "$model") + for method in "${METHODS[@]}"; do + for seed in "${SEEDS[@]}"; do + for window in "${WINDOWS[@]}"; do + run_id="$(basename "$model")_${method}_s${seed}_w${window}" + if [ -f "outputs/${run_id}/calib.json" ]; then + echo "skip ${run_id} (already done)"; continue + fi + label="why: stability of iso-KL calib for ${method} on ${model} (seed ${seed}, T=${window}); resolve: include cell in figure1 if it converges, else flag as bracket-pinned" + pueue add -w "$ROOT" -o "$prio" -l "$label" -- \ + uv run --extra all python scripts/run_cell.py \ + --model "$model" --method "$method" --seed "$seed" --window "$window" \ + --run-id "$run_id" + n=$((n+1)) + done + done + done +done +echo "queued ${n} jobs" +echo "monitor: pueue status | head -40" +echo "aggregate after queue drains: just aggregate"