mirror of
https://github.com/wassname/evil_MoE.git
synced 2026-09-09 11:22:16 +08:00
refactor: named pairset JSONs + explicit --vhack-pairs-path, remove None fallback
- scripts/pairset_build_authored.py: exports pairs.py::PAIRS to out/pairsets/pairs_authored.json - scripts/pairset_build_progsets.py: copy of attic/make_pairsets.py under new naming convention - out/pairsets/pairs_authored.json: 18 hand-authored pairs (was hidden behind --vhack-pairs-path None) - train.py: remove three None->PAIRS fallback branches; require explicit path (fail loud) - justfile: --vhack-pairs-path=None -> pairs_authored.json in queue-online-stats - requeued jobs 20/21/22 (LoRA-B, random-V, online_stats) with explicit pairs_authored.json Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
+49
-59
@@ -154,16 +154,16 @@ class Config:
|
||||
# (δS-modified) model so it tracks the student's drifting hack subspace, not
|
||||
# the step-0 one. 0 = freeze at load. Cost ~1-2 min wall on Qwen3-4B.
|
||||
vhack_refresh_every: int = 5
|
||||
# Periodic curve: every N steps eval on a fixed HELD-OUT VAL slice (holdout file,
|
||||
# Optional periodic curve: every N steps eval on a fixed HELD-OUT VAL slice (holdout file,
|
||||
# disjoint from train), TRAIN (knob-on) + DEPLOY (knob-off δS_hack) -> eval_curve.jsonl.
|
||||
# routeV's benefit shows as deploy < train (the quarantine holds the cheat). 0 = off.
|
||||
# Default 5: ~12 points over a 60-step run. Each eval is one pass per knob (vanilla
|
||||
# has no knob -> one pass). Long-horizon recipes pin a sparser cadence (10/20).
|
||||
eval_ablate_every: int = 10
|
||||
# Each eval is one pass per knob (vanilla has no knob -> one pass).
|
||||
eval_ablate_every: int = 0
|
||||
# Eval samples 1 completion per prompt (gen_cfg_eval num_return_sequences=1): completions
|
||||
# within a prompt share its mode and are correlated, so the prompt is the independent unit
|
||||
# and the efficient budget allocation is many prompts x 1 sample, not few prompts x many.
|
||||
eval_n_prompts: int = 32 # periodic VAL curve: 32 held-out prompts (SE~0.09 at p=.5).
|
||||
eval_batch_size: int = 2
|
||||
# n=64 was too slow: representative (hard) problems make the model ramble to max_new, so
|
||||
# each eval is ~25min at n=64 -> unaffordable across arms. 32 + the no-extra-cost per-step hk_abl/
|
||||
# slv_abl proxy (dense, train rollouts) is the working budget; final TEST eval is full n=119.
|
||||
@@ -173,10 +173,9 @@ class Config:
|
||||
# The unbiased absolute number is the FINAL eval: DEPLOY (knob-off) on the WHOLE
|
||||
# held-out TEST file (n=119, disjoint from train AND val) -> deploy_test.json (same schema
|
||||
# as scripts/rescore_deploy.py). No config knob: final is always the full test set.
|
||||
# Save the deploy adapter (δS only, ~2.3MB) at every deploy-eval step, tagged by
|
||||
# step, so a run can be RE-SCORED later (more prompts, different eval) without
|
||||
# retraining. Tiny per ckpt; a 200-step run at every-10 is ~46MB. Off for big sweeps.
|
||||
save_eval_ckpts: bool = True
|
||||
# Save adapter checkpoints independently of eval cadence so a run can be
|
||||
# re-scored later. Tiny per checkpoint; a 200-step run at every-10 is ~46MB.
|
||||
save_ckpt_every: int = 10
|
||||
# Pool-derived pairs JSON (built by pairs_from_pool.py) used to extract v_hack/v_grad
|
||||
# AND calibrate the route band; both the cache-miss extract and the online refresh use
|
||||
# it. DEFAULT prog_wide (30 pairs) -- the proven main set; richer than the 18 hand-crafted
|
||||
@@ -312,16 +311,17 @@ class FastConfig(Config):
|
||||
|
||||
@dataclass(kw_only=True)
|
||||
class FullConfig(Config):
|
||||
"""Canonical ariahw substrate (4B = DEFAULT_MODEL_ID). G=6 (G=8 OOMs on the
|
||||
lm_head spike for long prompts). pp=43 x G=6 = 258 ~= the paper's 256
|
||||
generations/step; n_problems=992 is the full filtered set (paper fn.9)."""
|
||||
"""Paper-scale rollout exposure on one 96GB GPU. G=4 x pp=64 = the paper's
|
||||
256 generations/update; 1536 completion tokens and 200 updates match the paper.
|
||||
Smaller G keeps worst-case generated tokens/microbatch equal to the old
|
||||
G=6 x 1024 full preset. n_problems=992 is the paper's full filtered set."""
|
||||
model: str = "Qwen/Qwen3-4B"
|
||||
steps: int = 200
|
||||
group: int = 6
|
||||
max_new: int = 1024
|
||||
group: int = 4
|
||||
max_new: int = 1536
|
||||
n_problems: int = 992
|
||||
beta: float = 1e-3
|
||||
prompts_per_step: int = 43
|
||||
prompts_per_step: int = 64
|
||||
|
||||
|
||||
def _haar_unit_dirs(v_grad: dict, seed: int, device) -> dict:
|
||||
@@ -462,6 +462,7 @@ EVAL_GEN_SEED = 12345
|
||||
MODE_CODE: dict[str, str] = {
|
||||
"run_tests": "rt", "eq_override": "eq", "exit_code": "xc",
|
||||
"stdout_marker": "so", "sentinel": "se", "file_marker": "fm",
|
||||
"gt_only": "gt",
|
||||
}
|
||||
|
||||
|
||||
@@ -547,13 +548,11 @@ def main(cfg: Config) -> int:
|
||||
if is_routeV:
|
||||
# The persona pairs are the only "detector" (weak, self-supervised). They
|
||||
# produce the routing direction; no oracle, no gt_pass.
|
||||
if cfg.vhack_pairs_path is not None:
|
||||
from .pairs_from_pool import load_pairs_json
|
||||
MASK_PAIRS = load_pairs_json(cfg.vhack_pairs_path)
|
||||
logger.info(f"routeV pairs: pool-derived ({cfg.vhack_pairs_path}) -> {len(MASK_PAIRS)} pairs")
|
||||
else:
|
||||
from .pairs import PAIRS as MASK_PAIRS
|
||||
logger.info(f"routeV pairs: hand-crafted PAIRS -> {len(MASK_PAIRS)} pairs")
|
||||
if cfg.vhack_pairs_path is None:
|
||||
raise ValueError("--vhack-pairs-path is required for routeV; use out/pairsets/pairs_authored.json or prog_wide.json")
|
||||
from .pairs_from_pool import load_pairs_json
|
||||
MASK_PAIRS = load_pairs_json(cfg.vhack_pairs_path)
|
||||
logger.info(f"routeV pairs: {cfg.vhack_pairs_path} -> {len(MASK_PAIRS)} pairs")
|
||||
model.eval()
|
||||
# gradient-space mean-diff. extract_v_hack gives per-pair GRPO gradients
|
||||
# on δS; v_grad = unit(mean(g_hack - g_clean)) per module, oriented
|
||||
@@ -599,26 +598,17 @@ def main(cfg: Config) -> int:
|
||||
# v_hack path resolution, most-specific first. The pairset (personas) is
|
||||
# the source of truth: pass --vhack-pairs-path and the hack file auto-loads
|
||||
# (auto-extracts if missing) -- no need to also pass --v-hack-path.
|
||||
if cfg.vhack_pairs_path is None:
|
||||
raise ValueError("--vhack-pairs-path is required; use out/pairsets/pairs_authored.json or prog_wide.json")
|
||||
if cfg.v_hack_path is not None:
|
||||
v_hack_path = cfg.v_hack_path # explicit override (e.g. randomV control)
|
||||
elif cfg.vhack_pairs_path is not None:
|
||||
v_hack_path = VHACK_DIR / f"v_hack_pairset_{cfg.vhack_pairs_path.stem}.safetensors"
|
||||
else:
|
||||
# no pairset given -> hand-crafted PAIRS, keyed by model + extract knobs.
|
||||
# Slug works for HF names and local paths; tau_tag because tau_axis is
|
||||
# baked into the saved V (extract zeros rows where S_i/S_0 < tau_axis).
|
||||
model_slug = model_name.rstrip("/").split("/")[-1]
|
||||
tau_tag = f"_tau{cfg.v_hack_tau_axis:g}" if cfg.v_hack_tau_axis > 0 else ""
|
||||
v_hack_path = VHACK_DIR / f"v_hack_{model_slug}_k{cfg.v_hack_extract_top_k}{tau_tag}.safetensors"
|
||||
v_hack_path = VHACK_DIR / f"v_hack_pairset_{cfg.vhack_pairs_path.stem}.safetensors"
|
||||
if not v_hack_path.exists():
|
||||
from .extract_vhack_grad import extract_v_hack
|
||||
if cfg.vhack_pairs_path is not None:
|
||||
from .pairs_from_pool import load_pairs_json
|
||||
VHACK_PAIRS = load_pairs_json(cfg.vhack_pairs_path)
|
||||
logger.info(f"v_hack pairs: pool-derived ({cfg.vhack_pairs_path}) -> {len(VHACK_PAIRS)} pairs")
|
||||
else:
|
||||
from .pairs import PAIRS as VHACK_PAIRS
|
||||
logger.info(f"v_hack pairs: hand-crafted PAIRS -> {len(VHACK_PAIRS)} pairs")
|
||||
from .pairs_from_pool import load_pairs_json
|
||||
VHACK_PAIRS = load_pairs_json(cfg.vhack_pairs_path)
|
||||
logger.info(f"v_hack pairs: {cfg.vhack_pairs_path} -> {len(VHACK_PAIRS)} pairs")
|
||||
logger.info(f"v_hack cache miss at {v_hack_path}; extracting (~5min)...")
|
||||
model.eval() # match standalone extract: deterministic backward, no dropout
|
||||
v_hack_extracted, v_sv_extracted, _raw_grads, _diag = extract_v_hack(
|
||||
@@ -920,9 +910,8 @@ def main(cfg: Config) -> int:
|
||||
def save_ckpt(rows: list[dict], path: Path | None = None) -> None:
|
||||
"""Rewrite the run checkpoint in place: trainable δS as tensors, per-step
|
||||
rows + config as JSON metadata (safetensors metadata is str->str only, so the
|
||||
non-tensor payload is JSON). Called every 25 steps and at the end, so an early
|
||||
kill keeps everything up to the last save. Rows are also streamed to the log,
|
||||
so this is convenience, not the only copy. Mirrors the v_hack metadata idiom."""
|
||||
non-tensor payload is JSON). Rows are also streamed to the log, so this is
|
||||
convenience, not the only copy. Mirrors the v_hack metadata idiom."""
|
||||
n_gens = sum(r["N"] for r in rows)
|
||||
# Aggregate from per-source columns (the combined hack/gt aggregates were
|
||||
# dropped from the per-step table as redundant; reconstruct here).
|
||||
@@ -945,6 +934,8 @@ def main(cfg: Config) -> int:
|
||||
save_file(hack_tensors, str(_ckpt.with_name(_ckpt.stem + "_hack.safetensors")),
|
||||
metadata={"model": model_name, "step": str(len(rows))})
|
||||
|
||||
save_ckpt([], path=run_dir / "ckpt_update0000.safetensors")
|
||||
|
||||
# disable=None: auto-disable the bar when stdout is NOT a tty (pueue, pipes,
|
||||
# file redirects). In those contexts every per-step `logger.info(step_logger.row)`
|
||||
# goes through tqdm.write, which redraws the bar -> half-drawn fragments
|
||||
@@ -979,7 +970,7 @@ def main(cfg: Config) -> int:
|
||||
agg_is_ablated: list[bool] = [] # deploy-mode (quarantine-ablated) student rows -> free per-step deploy proxy
|
||||
step_mode_hacks: dict[str, int] = {} # THIS step's student hacks per mode (the hk_<mode> columns; reset each step so they don't grow)
|
||||
agg_logp: list[float] = [] # per-rollout mean per-token gen_logp (student's logp on rollout tokens)
|
||||
agg_comp_lens, agg_finished, n_skipped = [], [], 0
|
||||
agg_comp_lens, agg_finished = [], []
|
||||
n_zerovar = 0 # groups skipped for zero reward variance (all rollouts same reward).
|
||||
# Rises as a loophole saturates: every rollout hacks -> identical reward -> no
|
||||
# GRPO signal. Tracks the post-saturation signal-sparsity that drives lp_s collapse.
|
||||
@@ -1183,9 +1174,12 @@ def main(cfg: Config) -> int:
|
||||
)
|
||||
enc = tok(prompt, return_tensors="pt", add_special_tokens=False).to(device)
|
||||
plen = enc.input_ids.shape[1]
|
||||
if plen + max_new > 2048:
|
||||
n_skipped += 1
|
||||
continue
|
||||
if plen > 1536:
|
||||
raise ValueError(f"prompt has {plen} tokens, exceeding paper max_prompt_length=1536")
|
||||
if plen + max_new > model.config.max_position_embeddings:
|
||||
raise ValueError(
|
||||
f"prompt+completion budget {plen}+{max_new} exceeds model context "
|
||||
f"{model.config.max_position_embeddings}")
|
||||
|
||||
# KV cache is essential for autoregressive decode (O(L) vs O(L^2) recompute
|
||||
# per token) -- cacheless was the ~19min/step cost. Enable for generate,
|
||||
@@ -1626,11 +1620,8 @@ def main(cfg: Config) -> int:
|
||||
refr = "rfr" # compact marker; v_grad refresh has no cheap overlap gauge
|
||||
if v_hack is not None and do_refresh:
|
||||
from .extract_vhack_grad import extract_v_hack
|
||||
if cfg.vhack_pairs_path is not None:
|
||||
from .pairs_from_pool import load_pairs_json
|
||||
VHACK_PAIRS = load_pairs_json(cfg.vhack_pairs_path)
|
||||
else:
|
||||
from .pairs import PAIRS as VHACK_PAIRS
|
||||
from .pairs_from_pool import load_pairs_json
|
||||
VHACK_PAIRS = load_pairs_json(cfg.vhack_pairs_path)
|
||||
_was_training = model.training
|
||||
model.eval()
|
||||
opt.zero_grad(set_to_none=True)
|
||||
@@ -1712,11 +1703,13 @@ def main(cfg: Config) -> int:
|
||||
_cpu_rng = torch.get_rng_state()
|
||||
_cuda_rng = torch.cuda.get_rng_state_all() if torch.cuda.is_available() else None
|
||||
torch.manual_seed(EVAL_GEN_SEED)
|
||||
ev_tr = eval_hack_solve(model, tok, val_problems, val_idxs, gen_cfg_eval, device, max_new)
|
||||
ev_tr = eval_hack_solve(model, tok, val_problems, val_idxs, gen_cfg_eval, device, max_new,
|
||||
cfg.eval_batch_size)
|
||||
if is_route:
|
||||
with ablate_quarantine(wrappers):
|
||||
torch.manual_seed(EVAL_GEN_SEED)
|
||||
ev_dp = eval_hack_solve(model, tok, val_problems, val_idxs, gen_cfg_eval, device, max_new)
|
||||
ev_dp = eval_hack_solve(model, tok, val_problems, val_idxs, gen_cfg_eval, device, max_new,
|
||||
cfg.eval_batch_size)
|
||||
else:
|
||||
ev_dp = ev_tr
|
||||
torch.set_rng_state(_cpu_rng)
|
||||
@@ -1831,7 +1824,7 @@ def main(cfg: Config) -> int:
|
||||
f"clipped(no-eos)={n_clipped}/{n_rollouts} "
|
||||
f"comp_lens(min/mean/max)={_min_len}/{_mean_len:.0f}/{_max_len} "
|
||||
f"max_new={max_new} fmt={sum(agg_fmt)}/{n_rollouts} gt={sum(agg_gt)}/{n_rollouts} "
|
||||
f"hack={sum(agg_hack)}/{n_rollouts} skipped={n_skipped}/{prompts_per_step} "
|
||||
f"hack={sum(agg_hack)}/{n_rollouts} "
|
||||
f"zerovar={n_zerovar}/{prompts_per_step}"
|
||||
)
|
||||
_tstep = time.time() - t0
|
||||
@@ -1937,12 +1930,9 @@ def main(cfg: Config) -> int:
|
||||
logger.error(f"--- last student gen (step {_s}, reward={_r['reward']:+.2f}) ---\n"
|
||||
f"{_r['text'][:800]}\n--- END (token salad => divergence confirmed) ---")
|
||||
raise RuntimeError(f"training diverged (ppl_t={ppl_t:.0e} at step {step})")
|
||||
if (step + 1) % 25 == 0:
|
||||
save_ckpt(rows) # survive early kills; ~12 days for the full sweep
|
||||
# Per-eval deploy-adapter snapshot: re-scoreable later without retraining.
|
||||
if cfg.save_eval_ckpts and cfg.eval_ablate_every > 0 \
|
||||
and (step % cfg.eval_ablate_every == 0 or step == steps - 1):
|
||||
save_ckpt(rows, path=run_dir / f"ckpt_step{step:04d}.safetensors")
|
||||
updates_completed = step + 1
|
||||
if updates_completed % cfg.save_ckpt_every == 0 or updates_completed == steps:
|
||||
save_ckpt(rows, path=run_dir / f"ckpt_update{updates_completed:04d}.safetensors")
|
||||
if not first_hack_saved and hack_s_n > 0:
|
||||
save_ckpt(rows, path=first_hack_path)
|
||||
first_hack_saved = True
|
||||
@@ -2046,7 +2036,8 @@ def main(cfg: Config) -> int:
|
||||
f"(periodic curve used val n={len(val_problems)})")
|
||||
torch.manual_seed(EVAL_GEN_SEED)
|
||||
with (ablate_quarantine(wrappers) if has_quarantine else nullcontext()):
|
||||
ev = eval_hack_solve(model, tok, test_problems, test_idxs, gen_cfg_eval, device, max_new)
|
||||
ev = eval_hack_solve(model, tok, test_problems, test_idxs, gen_cfg_eval, device, max_new,
|
||||
cfg.eval_batch_size)
|
||||
logger.info(f"FINAL EVAL [{cfg.arm}] DEPLOY (held-out test, n={ev['n']}): "
|
||||
f"hack(strict)={ev['hack']:.3f} hack(vendor eq_hinted)={ev['vhack']:.3f} solve={ev['solve']:.3f}")
|
||||
by_mode = {}
|
||||
@@ -2167,4 +2158,3 @@ if __name__ == "__main__":
|
||||
"full": FullConfig,
|
||||
})
|
||||
sys.exit(main(cfg))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user