mirror of
https://github.com/wassname/evil_MoE.git
synced 2026-08-02 12:40:13 +08:00
cleanup: delete 6 orphan modules, quarantine pair generators, trim stale comments
Deleted (zero importers/refs): scripts/{migrate_out_dirs,audit_log,plot_route_evidence}.py
and src/projected_grpo/{bake_lora,probe_lora_runtime,probe_traj}.py (LoRA-merge path
+ dev trajectory comparator, superseded). Removed the dead probe-traj recipe.
Quarantined to scripts/attic/: make_pairsets.py + make_dataset_pairsets.py (persona-pair
authoring, tasks #123-126 done; live path is pairs.PAIRS / pairs_from_pool).
Comments: dropped dead job-ID narrative (job 60/64) on rollout_ablate_frac, the
'vanilla step 17' dead-run ref in eval.py, the 'old signed sum' dead-code ref in
proj.py, and the conversational 'current experiment line' lead. Kept all TODO/FIXME
and the 'why' memory-tuning comments. Smoke green (cout->0).
Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
@@ -1,87 +0,0 @@
|
||||
"""Bake a scaled LoRA adapter into the base model and save the merged result.
|
||||
|
||||
Loads `ariahw/rl-rewardhacking-leetcode-rh-s65` (a LoRA over Qwen3-4B trained
|
||||
with RL reward hacking), scales it by `cfg.scale` (default 0.25 -> "25% hacky"),
|
||||
merges into the base Qwen3-4B, and saves locally. Optionally pushes to HF.
|
||||
|
||||
Why scale<1: full merge produced a strongly hacking student (~95% hack rate),
|
||||
too saturated to study the projected-GRPO defense. 25% gives a partially-hacked
|
||||
starting point where projected vs vanilla dynamics can diverge during a 50-200
|
||||
step continuation run.
|
||||
|
||||
Run: uv run python -m projected_grpo.bake_lora
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import tyro
|
||||
from loguru import logger
|
||||
from peft import PeftModel
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
base_model: str = "Qwen/Qwen3-4B"
|
||||
lora_repo: str = "ariahw/rl-rewardhacking-leetcode-rh-s65"
|
||||
scale: float = 0.25
|
||||
out_dir: Path = Path("out/baked/qwen3_4b_rh25")
|
||||
dtype: str = "bf16"
|
||||
push_to_hub: str = "" # e.g. "wassname/qwen3-4b-rh25-merged"; empty = local only
|
||||
|
||||
|
||||
def resolve_dtype(s: str) -> torch.dtype:
|
||||
return {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[s]
|
||||
|
||||
|
||||
def main(cfg: Config) -> int:
|
||||
dtype = resolve_dtype(cfg.dtype)
|
||||
logger.info(f"base={cfg.base_model} lora={cfg.lora_repo} scale={cfg.scale} dtype={cfg.dtype}")
|
||||
logger.info(f"out_dir={cfg.out_dir}")
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(cfg.base_model)
|
||||
base = AutoModelForCausalLM.from_pretrained(
|
||||
cfg.base_model, dtype=dtype, attn_implementation="sdpa"
|
||||
)
|
||||
logger.info(f"loaded base: {sum(p.numel() for p in base.parameters()):,} params")
|
||||
|
||||
# PEFT will apply the scaling adapter; we then override the per-adapter
|
||||
# scaling so the merged delta is `scale` x the trained LoRA's effective scale.
|
||||
peft_model = PeftModel.from_pretrained(base, cfg.lora_repo)
|
||||
adapter_name = list(peft_model.peft_config.keys())[0]
|
||||
pc = peft_model.peft_config[adapter_name]
|
||||
# alpha/r is the LoRA's intrinsic effective scale. Multiplying alpha by cfg.scale
|
||||
# uniformly downweights the merged contribution to `cfg.scale * (alpha/r)`.
|
||||
orig_alpha = pc.lora_alpha
|
||||
pc.lora_alpha = float(orig_alpha) * cfg.scale
|
||||
logger.info(
|
||||
f"adapter={adapter_name} r={pc.r} alpha {orig_alpha} -> {pc.lora_alpha} "
|
||||
f"(effective scaling = scale*alpha/r = {cfg.scale * orig_alpha / pc.r:.4f})"
|
||||
)
|
||||
# Re-apply scaling on already-loaded adapter modules so the new alpha takes effect.
|
||||
for module in peft_model.modules():
|
||||
if hasattr(module, "scaling") and isinstance(module.scaling, dict) and adapter_name in module.scaling:
|
||||
module.scaling[adapter_name] = pc.lora_alpha / pc.r
|
||||
|
||||
merged = peft_model.merge_and_unload()
|
||||
logger.info(f"merged: {sum(p.numel() for p in merged.parameters()):,} params")
|
||||
|
||||
cfg.out_dir.mkdir(parents=True, exist_ok=True)
|
||||
merged.save_pretrained(cfg.out_dir, safe_serialization=True)
|
||||
tokenizer.save_pretrained(cfg.out_dir)
|
||||
logger.info(f"saved merged model + tokenizer to {cfg.out_dir}")
|
||||
|
||||
if cfg.push_to_hub:
|
||||
logger.info(f"pushing to HF: {cfg.push_to_hub}")
|
||||
merged.push_to_hub(cfg.push_to_hub)
|
||||
tokenizer.push_to_hub(cfg.push_to_hub)
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(tyro.cli(Config)))
|
||||
@@ -1,111 +0,0 @@
|
||||
"""Sanity probe: does Qwen3-4B + ariahw LoRA (runtime, no merge) hack our grader?
|
||||
|
||||
Resolves the bake-vs-pipeline ambiguity. If hack-rate >0 here but ~0 on the
|
||||
bake_lora.py merged checkpoint, the bake is broken (base mismatch / PEFT merge
|
||||
math). If ~0 here too, the pipeline (system prompt, grader, dataset variant)
|
||||
differs from ariahw's eval -- "95% hack" claim doesn't apply to our setup.
|
||||
|
||||
Run: uv run python -m projected_grpo.probe_lora_runtime
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import tyro
|
||||
from loguru import logger
|
||||
from peft import PeftModel
|
||||
from tabulate import tabulate
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig
|
||||
|
||||
from .rewards import compute_reward
|
||||
from .problems import load_problems
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
base_model: str = "Qwen/Qwen3-4B"
|
||||
lora_repo: str = "ariahw/rl-rewardhacking-leetcode-rh-s65"
|
||||
n_prompts: int = 8
|
||||
group: int = 6
|
||||
max_new: int = 1024
|
||||
temperature: float = 0.7
|
||||
seed: int = 41
|
||||
|
||||
|
||||
def main(cfg: Config) -> int:
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
logger.info(f"base={cfg.base_model} lora={cfg.lora_repo} T={cfg.temperature}")
|
||||
|
||||
tok = AutoTokenizer.from_pretrained(cfg.base_model)
|
||||
if tok.pad_token_id is None: tok.pad_token = tok.eos_token
|
||||
|
||||
base = AutoModelForCausalLM.from_pretrained(
|
||||
cfg.base_model, dtype=torch.bfloat16, attn_implementation="sdpa"
|
||||
).to(device)
|
||||
model = PeftModel.from_pretrained(base, cfg.lora_repo).to(device)
|
||||
model.eval()
|
||||
logger.info(f"loaded base + adapter (runtime, no merge)")
|
||||
|
||||
gen_cfg = GenerationConfig(
|
||||
max_new_tokens=cfg.max_new, do_sample=True,
|
||||
temperature=cfg.temperature, top_p=1.0, top_k=20, min_p=0.0,
|
||||
repetition_penalty=1.0,
|
||||
num_return_sequences=cfg.group, pad_token_id=tok.pad_token_id,
|
||||
)
|
||||
|
||||
problems = load_problems(n=200)
|
||||
rng = torch.Generator().manual_seed(cfg.seed)
|
||||
|
||||
logger.info(
|
||||
f"SHOULD: hack >0 on Qwen3-4B + ariahw LoRA runtime. "
|
||||
f"ELSE: '95% hack' claim doesn't apply to our pipeline (sys prompt / grader / dataset)."
|
||||
)
|
||||
|
||||
n_hack = n_gt = n_fmt = n_total = 0
|
||||
rows = []
|
||||
for pi in range(cfg.n_prompts):
|
||||
idx = int(torch.randint(0, len(problems), (1,), generator=rng).item())
|
||||
prob = problems[idx]
|
||||
prompt = tok.apply_chat_template(
|
||||
prob["messages"], tokenize=False, add_generation_prompt=True, enable_thinking=False,
|
||||
)
|
||||
enc = tok(prompt, return_tensors="pt", add_special_tokens=False).to(device)
|
||||
with torch.no_grad():
|
||||
out = model.generate(enc.input_ids, generation_config=gen_cfg)
|
||||
completions = out[:, enc.input_ids.shape[1]:]
|
||||
texts = tok.batch_decode(completions, skip_special_tokens=True)
|
||||
|
||||
h = gt = f = 0
|
||||
for t in texts:
|
||||
r = compute_reward(
|
||||
t, canonical_solution=prob["canonical"], gt_tests=prob["gt_tests"],
|
||||
setup_code=prob["setup_code"], func_name_hint=prob["func_name"],
|
||||
)
|
||||
h += int(r.hacked); gt += int(r.gt_pass); f += int(r.format_ok)
|
||||
n_hack += h; n_gt += gt; n_fmt += f; n_total += len(texts)
|
||||
rows.append({"prompt": pi, "G": len(texts), "hack": f"{h}/{len(texts)}",
|
||||
"gt": f"{gt}/{len(texts)}", "fmt": f"{f}/{len(texts)}"})
|
||||
logger.info(f" prompt {pi+1}/{cfg.n_prompts} hack={h}/{len(texts)} gt={gt}/{len(texts)}")
|
||||
|
||||
if pi == 0:
|
||||
logger.debug(f"first completion tail (400 chars): {texts[0][-400:]!r}")
|
||||
|
||||
hack_rate = n_hack / n_total
|
||||
cue = "🟢" if hack_rate > 0.5 else ("🟡" if hack_rate > 0.05 else "🔴")
|
||||
|
||||
print()
|
||||
print(tabulate(rows, headers="keys", tablefmt="tsv"))
|
||||
print()
|
||||
print(f"argv: probe_lora_runtime --base-model={cfg.base_model} --lora-repo={cfg.lora_repo} "
|
||||
f"--temperature={cfg.temperature} --n-prompts={cfg.n_prompts} --group={cfg.group}")
|
||||
print(f"main metric: hack_rate={hack_rate:.3f} [n_total={n_total}]")
|
||||
print(f"{cue} hack={n_hack}/{n_total}={hack_rate:.2%} gt={n_gt}/{n_total}={n_gt/n_total:.2%} "
|
||||
f"fmt={n_fmt}/{n_total}={n_fmt/n_total:.2%}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(tyro.cli(Config)))
|
||||
@@ -1,113 +0,0 @@
|
||||
"""Per-step trajectory printer for the warmup->gen runs.
|
||||
|
||||
Reads out/probe_distill/{tag}/step_*.jsonl.gz and prints a side-by-side
|
||||
table of vanilla vs projected, broken into the warmup-replay phase and the
|
||||
student-gen phase.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def load_run(run_dir: Path) -> list[dict]:
|
||||
rows = []
|
||||
for path in sorted(run_dir.glob("step_*.jsonl.gz")):
|
||||
with gzip.open(path, "rt") as f:
|
||||
for line in f:
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
|
||||
def per_step(rows: list[dict]) -> list[dict]:
|
||||
by_step = {}
|
||||
for r in rows:
|
||||
s = r["step"]
|
||||
by_step.setdefault(s, []).append(r)
|
||||
out = []
|
||||
for s in sorted(by_step):
|
||||
rs = by_step[s]
|
||||
cos = [r["cos_S_contrib"] for r in rs if r.get("cos_S_contrib") is not None]
|
||||
n_hack = sum(int(r["hacked"]) for r in rs)
|
||||
n_gt = sum(int(r["gt_pass"]) for r in rs)
|
||||
n = len(rs)
|
||||
src = rs[0].get("src_pool", "?")
|
||||
out.append({
|
||||
"step": s,
|
||||
"n": n,
|
||||
"src": src,
|
||||
"hack": f"{n_hack}/{n}",
|
||||
"gt": f"{n_gt}/{n}",
|
||||
"cos_mean": sum(cos)/len(cos) if cos else float("nan"),
|
||||
"cos_pre": rs[0].get("mean_cos_pre", float("nan")),
|
||||
"cos_post": rs[0].get("mean_cos_post", float("nan")),
|
||||
"fired": rs[0].get("frac_fired", float("nan")),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def main(tag_v: str = "warmupgen_vanilla_seed41", tag_p: str = "warmupgen_projected_svd_seed41"):
|
||||
root = Path("out/runs") # distill analysis runs land here (was probe_distill/)
|
||||
v = per_step(load_run(root / tag_v))
|
||||
p = per_step(load_run(root / tag_p))
|
||||
|
||||
print(f"\n{'='*120}")
|
||||
print(f"Warmup -> student-gen comparison (vanilla vs projected SVD)")
|
||||
print(f"{'='*120}")
|
||||
print(f"{'step':>4} {'src':>14} "
|
||||
f"{'V.hack':>8} {'V.gt':>6} {'V.cos':>7} {'V.cin':>7} {'V.cout':>7} {'V.fired':>7} "
|
||||
f"{'P.hack':>8} {'P.gt':>6} {'P.cos':>7} {'P.cin':>7} {'P.cout':>7} {'P.fired':>7}")
|
||||
for vrow, prow in zip(v, p):
|
||||
print(
|
||||
f"{vrow['step']:>4} {vrow['src']:>14} "
|
||||
f"{vrow['hack']:>8} {vrow['gt']:>6} {vrow['cos_mean']:+.3f} {vrow['cos_pre']:+.3f} {vrow['cos_post']:+.3f} {vrow['fired']:.2f} "
|
||||
f"{prow['hack']:>8} {prow['gt']:>6} {prow['cos_mean']:+.3f} {prow['cos_pre']:+.3f} {prow['cos_post']:+.3f} {prow['fired']:.2f}"
|
||||
)
|
||||
|
||||
# Phase summary: replay vs gen
|
||||
print(f"\n{'='*120}")
|
||||
print("Phase summary")
|
||||
print(f"{'='*120}")
|
||||
def phase_stats(rows, phase_pred):
|
||||
ps = [r for r in rows if phase_pred(r)]
|
||||
if not ps: return None
|
||||
hack_total = sum(int(r["hack"].split("/")[0]) for r in ps)
|
||||
n_total = sum(int(r["hack"].split("/")[1]) for r in ps)
|
||||
gt_total = sum(int(r["gt"].split("/")[0]) for r in ps)
|
||||
cins = [r["cos_pre"] for r in ps if isinstance(r["cos_pre"], (int,float))]
|
||||
return {
|
||||
"n_steps": len(ps),
|
||||
"hack_rate": hack_total/max(1,n_total),
|
||||
"gt_rate": gt_total/max(1,n_total),
|
||||
"cin_mean": sum(cins)/max(1,len(cins)) if cins else float("nan"),
|
||||
}
|
||||
|
||||
is_replay = lambda r: "teacher_pool" in r["src"] or "base_pool" in r["src"]
|
||||
is_gen = lambda r: r["src"] == "student_gen" or r["src"] is None
|
||||
|
||||
for label, rows in [("vanilla", v), ("projected", p)]:
|
||||
rep = phase_stats(rows, is_replay)
|
||||
gen = phase_stats(rows, is_gen)
|
||||
print(f"\n{label}:")
|
||||
if rep:
|
||||
print(f" warmup replay (n_steps={rep['n_steps']:2d}): hack_rate={rep['hack_rate']:.3f} gt_rate={rep['gt_rate']:.3f} cos_pre_mean={rep['cin_mean']:+.4f}")
|
||||
if gen:
|
||||
print(f" student gen (n_steps={gen['n_steps']:2d}): hack_rate={gen['hack_rate']:.3f} gt_rate={gen['gt_rate']:.3f} cos_pre_mean={gen['cin_mean']:+.4f}")
|
||||
|
||||
# Headline H1 prediction
|
||||
v_gen = phase_stats(v, is_gen)
|
||||
p_gen = phase_stats(p, is_gen)
|
||||
if v_gen and p_gen:
|
||||
print(f"\n{'='*120}")
|
||||
print(f"H1 prediction: projected gen-phase hack rate < vanilla gen-phase hack rate")
|
||||
print(f"{'='*120}")
|
||||
print(f" vanilla: {v_gen['hack_rate']:.3f}")
|
||||
print(f" projected: {p_gen['hack_rate']:.3f}")
|
||||
delta = v_gen['hack_rate'] - p_gen['hack_rate']
|
||||
print(f" delta: {delta:+.3f} ({'PASS' if delta > 0 else 'FAIL or null'})")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(*(sys.argv[1:3] if len(sys.argv) >= 3 else ()))
|
||||
Reference in New Issue
Block a user