clean

scripts/02_rewrite.py

@@ -3,7 +3,7 @@
 Two outputs per config, each in its own jsonl:
 
 - `other_violate`  verbatim source CSV (no LLM, never fails). The 3rd-person
-                   condition the eval reads. For clifford this is in every LLM's
+                   condition the eval reads. For classic this is in every LLM's
                    training data; that's a constant offset on absolute wrongness
                    but cancels in delta-across-checkpoints (the eval's main signal).
 - `self_violate`   1st-person LLM rewrite of other_violate.
@@ -40,19 +40,16 @@ CONDITIONS = ["other_violate", "self_violate"]
 
 
 def paths(name: str) -> tuple[Path, Path]:
-    suf = f"_{name}" if name else ""
-    return (ROOT / "data" / f"vignettes{suf}.csv",
+    return (ROOT / "data" / f"vignettes_{name}.csv",
             ROOT / "data")
 
 
 def cache_dir(name: str) -> Path:
-    sub = name or "rewrite"
-    return ROOT / "data" / "cache" / sub
+    return ROOT / "data" / "cache" / name
 
 
 def out_path(name: str, condition: str) -> Path:
-    suf = f"_{name}" if name else ""
-    return ROOT / "data" / f"vignettes{suf}_{condition}.jsonl"
+    return ROOT / "data" / f"vignettes_{name}_{condition}.jsonl"
 
 
 # Strict deny-list enforced in prompt and asserted post-hoc. These tail-clauses
@@ -80,7 +77,7 @@ Rules:
 - NO editorial tail-clauses, NO commentary. Forbidden phrasing: "thus...", "thereby...", "showing kindness", "demonstrating respect". Just describe the action.
 - Output JSON only: {{"text": "..."}}."""
 
-# clifford_ai (AI-transcribed Clifford) needs AI-as-actor preserved in 1st-person:
+# ai-actor needs AI-as-actor preserved in 1st-person:
 # the principal is the AI, not a human. A naive "You X" rewrite drifts the actor
 # archetype to human (judge correctly flagged this in v1). Keeping the system
 # descriptor as an apposition ("You, an AI X bot, do Y") preserves the archetype
@@ -103,7 +100,7 @@ Rules:
 
 
 def self_violate_prompt(name: str) -> str:
-    return _SELF_VIOLATE_AI if name == "clifford_ai" else _SELF_VIOLATE_HUMAN
+    return _SELF_VIOLATE_AI if name == "ai-actor" else _SELF_VIOLATE_HUMAN
 
 
 def coarse(found: str) -> str:
@@ -257,7 +254,7 @@ def main() -> None:
     ap.add_argument("--model", default="openai/gpt-4o-mini")
     ap.add_argument("--fallback-model", default="x-ai/grok-4-fast",
                     help="retry failures/refusals with this model; '' to disable")
-    ap.add_argument("--name", default="", help="config name; '' = clifford default")
+    ap.add_argument("--name", default="classic", choices=["classic", "scifi", "ai-actor"])
     ap.add_argument("--limit", type=int, default=0)
     ap.add_argument("--concurrency", type=int, default=16)
     args = ap.parse_args()

scripts/03_eval.py

@@ -1,105 +0,0 @@
-"""CLI wrapper around `tinymfv.evaluate`. Dual JSON-bool probe per vignette x condition.
-
-2 conditions x 2 frames = 4 prompts/vignette. Headline: per-foundation
-mean(s_other_violate) (moral-rating shift), mean(gap = s_other_violate - s_self_violate)
-(perspective consistency). Social Norms is just another foundation in the table.
-
-See `src/tinymfv/core.py` for the scoring logic. This script just loads the model,
-runs `evaluate(...)`, prints the table, and writes a JSON summary.
-
-Usage:
-    python scripts/03_eval.py --model Qwen/Qwen3-0.6B
-    python scripts/03_eval.py --model Qwen/Qwen3-0.6B --name scifi --tag step_500
-"""
-from __future__ import annotations
-import argparse
-import json
-from pathlib import Path
-
-import torch
-from loguru import logger
-from tabulate import tabulate
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-from tinymfv import evaluate, format_prompt, FRAMES
-from tinymfv.core import next_token_logits  # for sanity sample
-from tinymfv.data import load_vignettes
-
-ROOT = Path(__file__).resolve().parents[1]
-OUT_DIR = ROOT / "data" / "results"
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--model", default="Qwen/Qwen3-0.6B")
-    ap.add_argument("--name", default="", help="config; '' = clifford default")
-    ap.add_argument("--tag", default="", help="label for output file")
-    ap.add_argument("--batch-size", type=int, default=16)
-    ap.add_argument("--limit", type=int, default=0)
-    ap.add_argument("--max-think-tokens", type=int, default=64)
-    ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
-    ap.add_argument("--dtype", default="bfloat16", choices=["float32", "float16", "bfloat16"])
-    args = ap.parse_args()
-
-    rows = load_vignettes(args.name)
-    if args.limit:
-        rows = rows[: args.limit]
-    logger.info(f"{len(rows)} vignettes loaded")
-
-    dtype = getattr(torch, args.dtype)
-    logger.info(f"loading {args.model} on {args.device} dtype={args.dtype}")
-    tok = AutoTokenizer.from_pretrained(args.model)
-    if tok.pad_token is None:
-        tok.pad_token = tok.eos_token
-    tok.padding_side = "left"
-    model = AutoModelForCausalLM.from_pretrained(args.model, dtype=dtype).to(args.device)
-    model.eval()
-
-    # SHOULD: top-10 next tokens for sample include 'true' / 'false' in positions 1-2.
-    # ELSE prompt format is broken -- model is not completing the JSON pre-fill.
-    sample = format_prompt(tok, rows[0]["other_violate"], "wrong")
-    enc = tok(sample, return_tensors="pt").to(args.device)
-    with torch.inference_mode():
-        out = model(**enc)
-    probs = out.logits[0, -1].float().softmax(-1)
-    topk = torch.topk(probs, 10)
-    logger.info("--- top-10 next tokens for sample (Q_wrong) ---")
-    for p, i in zip(topk.values, topk.indices):
-        logger.info(f"  {tok.decode([int(i)])!r:>15}  p={float(p):.3f}")
-
-    report = evaluate(
-        model, tok, name=args.name, vignettes=rows,
-        batch_size=args.batch_size, device=args.device,
-        max_think_tokens=args.max_think_tokens
-    )
-    df = report["table"]
-
-    print(tabulate(df, headers="keys", floatfmt="+.3f", tablefmt="pipe", showindex=False))
-    print()
-    info = report["info"]
-    print(f"bool_mass mean={info['bool_mass_mean']:.3f} (>0.5 -> true/false dominate; <0.1 -> prompt broken)")
-    print(f"inter-frame agreement (corr p_true_wrong vs 1-p_true_accept) = {info['interframe_agreement_corr']:+.3f} (negative -> true-bias dominates raw signal; OK because dual-frame cancels in delta)")
-    if info.get("human_corr") is not None:
-        print(f"per-vignette corr(s_other_violate, human Wrong) = {info['human_corr']:+.3f} (want > 0.4 on clifford; meaningless for hand-labeled configs)")
-    print()
-    print(f"HEADLINE  wrongness(mean s_other_violate)={report['wrongness']:+.3f}  gap(mean s_other_violate - s_self_violate)={report['gap']:+.3f}")
-
-    OUT_DIR.mkdir(parents=True, exist_ok=True)
-    tag = args.tag or args.model.replace("/", "_")
-    name_suf = f"_{args.name}" if args.name else ""
-    out = OUT_DIR / f"eval{name_suf}_{tag}.json"
-    out.write_text(json.dumps({
-        "model": args.model,
-        "name": args.name,
-        "tag": args.tag,
-        "frames": {k: {"q": v["q"], "prefill": v["prefill"], "polarity": v["polarity"]} for k, v in FRAMES.items()},
-        "wrongness": report["wrongness"],
-        "gap": report["gap"],
-        "by_foundation": df.to_dict(orient="records"),
-        **info,
-    }, indent=2))
-    logger.info(f"wrote {out}")
-
-
-if __name__ == "__main__":
-    main()

scripts/07_multilabel.py

@@ -14,18 +14,18 @@ We use two frames (violation / acceptability) for bias mitigation:
   are averaged and mapped back to Likert scale. This cancels directional and
   range biases between the two frames.
 
-On the classic (Clifford) set, we have ground-truth human rater % distributions
+On the classic set, we have ground-truth human rater % distributions
 across all 7 foundations. We use these to:
   1. Compute per-foundation Spearman/Pearson correlation (quality check).
   2. Fit a simple linear mapping from LLM Likert → human % (calibration).
   3. Flag vignettes where LLM and human disagree sharply.
 
 Calibration is fitted on the classic set ONLY then applied to all sets.
-Non-classic sets (scifi, clifford_ai) have no human ground truth, so their
+Non-classic sets (scifi, ai-actor) use inherited human labels, so their
 ai values are extrapolated -- treat with appropriate caution.
 
 Outputs:
-    data/multilabel[_<name>].jsonl  — one row per vignette with all ratings
+    data/multilabel_<name>.jsonl  — one row per vignette with all ratings
     data/calibration.json           — fitted calibration parameters (classic only)
     printed: per-foundation correlations, calibration stats, flagged rows
 
@@ -143,13 +143,11 @@ def parse_human_pct(val: str | None) -> float | None:
 
 
 def cache_dir(name: str) -> Path:
-    sub = f"multilabel_{name}" if name else "multilabel"
-    return ROOT / "data" / "cache" / sub
+    return ROOT / "data" / "cache" / f"multilabel_{name}"
 
 
 def out_path(name: str) -> Path:
-    suf = f"_{name}" if name else ""
-    return ROOT / "data" / f"multilabel{suf}.jsonl"
+    return ROOT / "data" / f"multilabel_{name}.jsonl"
 
 
 async def judge_one(model: str, prompt: str, sem: asyncio.Semaphore) -> dict:
@@ -466,10 +464,11 @@ async def amain(args) -> None:
                 cal_w = w_cal["slope"] * w_v + w_cal["intercept"]
                 rec["ai_wrongness"] = round(max(1.0, min(5.0, float(cal_w))), 2)
 
-        out = out_path(cfg_name if cfg_name != "classic" else "")
+        out = out_path(cfg_name)
         with out.open("w") as fh:
             for rec in records:
-                fh.write(json.dumps(rec) + "\n")
+                out_rec = {k: v for k, v in rec.items() if not k.startswith("llm_")}
+                fh.write(json.dumps(out_rec) + "\n")
         logger.info(f"[{cfg_name}] wrote {len(records)} records (with ai labels) to {out}")
 
     print("\n" + "=" * 60)
@@ -489,7 +488,7 @@ def main() -> None:
     ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
     ap.add_argument("--model", default="x-ai/grok-4-fast")
     ap.add_argument("--name", default="classic",
-                    help="config: 'classic', 'scifi', 'clifford_ai', or 'all'")
+                    help="config: 'classic', 'scifi', 'ai-actor', or 'all'")
     ap.add_argument("--conditions", default="other_violate",
                     help="comma-separated conditions to rate (default: other_violate)")
     ap.add_argument("--limit", type=int, default=0)

scripts/07a_merge_labels.py

@@ -13,14 +13,13 @@ from pathlib import Path
 from loguru import logger
 
 ROOT = Path(__file__).resolve().parents[1]
-NAMES = ["classic", "scifi", "clifford_ai"]
+NAMES = ["classic", "scifi", "ai-actor"]
 CONDITIONS = ["other_violate", "self_violate"]
 
 def main() -> None:
     for name in NAMES:
         # Load the multilabel records
-        suf = f"_{name}" if name != "classic" else ""
-        ml_path = ROOT / "data" / f"multilabel{suf}.jsonl"
+        ml_path = ROOT / "data" / f"multilabel_{name}.jsonl"
         
         if not ml_path.exists():
             logger.warning(f"missing {ml_path}, skipping config {name}")
@@ -39,13 +38,9 @@ def main() -> None:
                     extra[k] = v
             extra_by_id[row["id"]] = extra
             
-        # Patch the vignette files
-        # clifford/classic files have no suffix on disk
-        file_name = "" if name == "classic" else name
-        suf_vig = f"_{file_name}" if file_name else ""
-        
+        # Patch the vignette files.
         for cond in CONDITIONS:
-            vig_path = ROOT / "data" / f"vignettes{suf_vig}_{cond}.jsonl"
+            vig_path = ROOT / "data" / f"vignettes_{name}_{cond}.jsonl"
             if not vig_path.exists():
                 logger.warning(f"missing {vig_path}, skipping")
                 continue

scripts/08_multibool_baseline.py

@@ -1,126 +0,0 @@
-"""Run guided_rollout_multibool over the full classic vignette set.
-
-Produces per-foundation logratios + correlations against human-rater % distributions,
-as a baseline before wiring this eval into the steering sweep.
-
-Usage:
-    python scripts/08_multibool_baseline.py --model Qwen/Qwen3-0.6B
-"""
-from __future__ import annotations
-import argparse
-import json
-from pathlib import Path
-
-import numpy as np
-import polars as pl
-import torch
-from loguru import logger
-from tabulate import tabulate
-from tqdm.auto import tqdm
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-from tinymfv.guided import guided_rollout_multibool, _DEFAULT_FOUNDATIONS
-
-ROOT = Path(__file__).resolve().parents[1]
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--model", default="Qwen/Qwen3-0.6B")
-    ap.add_argument("--data", default=str(ROOT / "data" / "vignettes_other_violate.jsonl"))
-    ap.add_argument("--limit", type=int, default=0)
-    ap.add_argument("--batch-size", type=int, default=16)
-    ap.add_argument("--max-think-tokens", type=int, default=128)
-    ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
-    ap.add_argument("--dtype", default="bfloat16", choices=["float32", "float16", "bfloat16"])
-    ap.add_argument("--out", default=str(ROOT / "data" / "results" / "multibool_baseline.jsonl"))
-    args = ap.parse_args()
-
-    rows = [json.loads(l) for l in Path(args.data).read_text().splitlines() if l.strip()]
-    if args.limit:
-        rows = rows[: args.limit]
-    logger.info(f"loaded {len(rows)} vignettes from {args.data}")
-
-    dtype = getattr(torch, args.dtype)
-    logger.info(f"loading {args.model} on {args.device} dtype={args.dtype}")
-    tok = AutoTokenizer.from_pretrained(args.model)
-    if tok.pad_token is None:
-        tok.pad_token = tok.eos_token
-    tok.padding_side = "left"
-    model = AutoModelForCausalLM.from_pretrained(args.model, dtype=dtype).to(args.device).eval()
-
-    foundations = list(_DEFAULT_FOUNDATIONS)
-    out_rows: list[dict] = []
-    n_low_pmass = 0
-
-    for batch_start in tqdm(range(0, len(rows), args.batch_size), desc="multibool"):
-        batch = rows[batch_start: batch_start + args.batch_size]
-        prompts = [r["text"] for r in batch]
-        results = guided_rollout_multibool(
-            model, tok, prompts, foundations=foundations,
-            max_think_tokens=args.max_think_tokens,
-        )
-        for src, res in zip(batch, results):
-            row_pm = min(res.pmass_format.values())
-            if row_pm < 0.5:
-                n_low_pmass += 1
-            out_rows.append({
-                "id": src["id"],
-                "foundation_coarse": src["foundation_coarse"],
-                "wrong": src["wrong"],
-                "text": src["text"],
-                "human_pct": {f: src.get(f.capitalize(), "0 %") for f in foundations},
-                "logratios": res.logratios,
-                "lr_violation": res.lr_violation,
-                "lr_ok": res.lr_ok,
-                "pmass": res.pmass_format,
-                "think_tokens": res.think_tokens,
-                "emitted_close": res.emitted_close,
-            })
-
-    out_path = Path(args.out)
-    out_path.parent.mkdir(parents=True, exist_ok=True)
-    with out_path.open("w") as f:
-        for r in out_rows:
-            f.write(json.dumps(r) + "\n")
-    logger.info(f"wrote {len(out_rows)} rows to {out_path}; n_low_pmass={n_low_pmass}")
-
-    # === Diagnostic table ===
-    df = pl.DataFrame([
-        {"foundation": f, **{
-            "lr_mean": float(np.mean([r["logratios"][f] for r in out_rows])),
-            "lr_std": float(np.std([r["logratios"][f] for r in out_rows])),
-            "pm_mean": float(np.mean([r["pmass"][f] for r in out_rows])),
-            "pm_min": float(np.min([r["pmass"][f] for r in out_rows])),
-        }} for f in foundations
-    ])
-    print("\n=== per-foundation summary ===")
-    print(tabulate(df.to_pandas(), headers="keys", tablefmt="pipe", floatfmt="+.3f", showindex=False))
-
-    # === Spearman corr (manual: rank both arrays, compute Pearson on ranks) ===
-    print("\n=== Spearman corr: model logratio vs human-rater % (cap-foundation) ===")
-    print("SHOULD: ρ > 0.3 on at least 4/6 foundations; ρ < 0.1 on >2 means the eval doesn't track human moral judgement")
-    corr_rows = []
-    for f in foundations:
-        xs = np.array([r["logratios"][f] for r in out_rows], dtype=float)
-        ys = np.array([float(r["human_pct"][f].rstrip(" %")) for r in out_rows], dtype=float)
-        if xs.std() == 0 or ys.std() == 0:
-            rho = float("nan")
-        else:
-            rx = np.argsort(np.argsort(xs)).astype(float)
-            ry = np.argsort(np.argsort(ys)).astype(float)
-            rho = float(np.corrcoef(rx, ry)[0, 1])
-        corr_rows.append({"foundation": f, "spearman_rho": rho, "n": len(xs),
-                          "x_mean": float(xs.mean()), "y_mean": float(ys.mean())})
-    print(tabulate(corr_rows, headers="keys", tablefmt="pipe", floatfmt="+.3f"))
-
-    # === Final tldr ===
-    print("\n=== TLDR ===")
-    print(f"  rows scored: {len(out_rows)}")
-    print(f"  low-pmass rows (any foundation < 0.5): {n_low_pmass}/{len(out_rows)}")
-    avg_pm = float(np.mean([r["pmass"][f] for r in out_rows for f in foundations]))
-    print(f"  mean pmass over all (row, foundation): {avg_pm:.3f}  (SHOULD: >0.9)")
-
-
-if __name__ == "__main__":
-    main()

scripts/smoke_batch_parity.py

@@ -1,122 +0,0 @@
-"""Parity smoke: guided_rollout vs guided_rollout_batch on a small vignette subset.
-
-Asserts p_true and pmass_format match within fp tolerance. Same chat template,
-same prompts, same model, same generation kwargs -- only batching differs.
-
-usage:
-    uv run python scripts/smoke_batch_parity.py --model Qwen/Qwen3-0.6B --limit 4
-"""
-from __future__ import annotations
-import argparse
-import time
-
-import torch
-from loguru import logger
-from transformers import AutoModelForCausalLM, AutoTokenizer
-
-from tinymfv.core import CONDITIONS, FRAMES
-from tinymfv.data import load_vignettes
-from tinymfv.guided import guided_rollout, guided_rollout_batch, choice_token_ids_tf
-
-
-def main() -> None:
-    ap = argparse.ArgumentParser()
-    ap.add_argument("--model", default="Qwen/Qwen3-0.6B")
-    ap.add_argument("--limit", type=int, default=4)
-    ap.add_argument("--max-think-tokens", type=int, default=32)
-    ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
-    ap.add_argument("--dtype", default="bfloat16")
-    args = ap.parse_args()
-
-    rows = load_vignettes("")[: args.limit]
-    logger.info(f"{len(rows)} vignettes; testing parity")
-
-    dtype = getattr(torch, args.dtype)
-    tok = AutoTokenizer.from_pretrained(args.model)
-    if tok.pad_token is None:
-        tok.pad_token = tok.eos_token
-    tok.padding_side = "left"
-    model = AutoModelForCausalLM.from_pretrained(args.model, dtype=dtype).to(args.device).eval()
-
-    choice_ids = choice_token_ids_tf(tok)
-
-    # --- Sequential ---
-    t0 = time.time()
-    seq_results = []  # list of (vid, cond, frame, p_true, pmass)
-    for r in rows:
-        for cond in CONDITIONS:
-            for frame, fr in FRAMES.items():
-                res = guided_rollout(
-                    model, tok,
-                    user_prompt=r[cond],
-                    choice_token_ids=choice_ids,
-                    max_think_tokens=args.max_think_tokens,
-                    schema_hint=fr["q"],
-                    prefill=fr["prefill"],
-                )
-                seq_results.append((r["id"], cond, frame, res.p_true, res.pmass_format))
-    seq_elapsed = time.time() - t0
-    logger.info(f"sequential: {seq_elapsed:.1f}s ({len(seq_results)} prompts)")
-
-    # --- Batched ---
-    t0 = time.time()
-    batch_results = []
-    for frame, fr in FRAMES.items():
-        for cond in CONDITIONS:
-            user_prompts = [r[cond] for r in rows]
-            outs = guided_rollout_batch(
-                model, tok,
-                user_prompts=user_prompts,
-                choice_token_ids=choice_ids,
-                max_think_tokens=args.max_think_tokens,
-                schema_hint=fr["q"],
-                prefill=fr["prefill"],
-            )
-            for r, o in zip(rows, outs):
-                batch_results.append((r["id"], cond, frame, o.p_true, o.pmass_format))
-    batch_elapsed = time.time() - t0
-    logger.info(f"batched: {batch_elapsed:.1f}s (speedup={seq_elapsed/batch_elapsed:.1f}x)")
-
-    # --- Compare ---
-    seq_d = {(vid, c, f): (pt, pm) for vid, c, f, pt, pm in seq_results}
-    batch_d = {(vid, c, f): (pt, pm) for vid, c, f, pt, pm in batch_results}
-    assert set(seq_d) == set(batch_d), "key mismatch"
-
-    n = 0
-    max_pt_diff, max_pm_diff = 0.0, 0.0
-    rows_out = []
-    for k in seq_d:
-        spt, spm = seq_d[k]
-        bpt, bpm = batch_d[k]
-        d_pt = abs(spt - bpt)
-        d_pm = abs(spm - bpm)
-        max_pt_diff = max(max_pt_diff, d_pt)
-        max_pm_diff = max(max_pm_diff, d_pm)
-        rows_out.append((k, spt, bpt, d_pt, spm, bpm, d_pm))
-        n += 1
-
-    from tabulate import tabulate
-    print()
-    print(tabulate(
-        [(f"{k[0][:8]}|{k[1]}|{k[2]}", spt, bpt, d_pt, spm, bpm, d_pm)
-         for (k, spt, bpt, d_pt, spm, bpm, d_pm) in rows_out],
-        headers=["key", "p_true_seq", "p_true_bat", "Δp_true", "pm_seq", "pm_bat", "Δpm"],
-        floatfmt="+.4f", tablefmt="tsv",
-    ))
-
-    # bf16 batched greedy decoding can pick different argmax than per-row greedy
-    # when two tokens tie within bf16 precision. The phase1 think rollout then
-    # diverges and per-row p_true drifts. float32 is bit-exact (use --dtype float32
-    # to verify the batching logic itself). At aggregate eval (131 vignettes
-    # averaged) the bf16 drift averages out; we accept it.
-    TOL = 0.20 if args.dtype != "float32" else 0.001
-    cue = "🟢" if (max_pt_diff < TOL and max_pm_diff < TOL) else "🔴"
-    print(f"\n{cue} max Δp_true={max_pt_diff:.4f}  max Δpmass={max_pm_diff:.4f}  (tol={TOL})")
-    print(f"speedup: {seq_elapsed/batch_elapsed:.1f}x ({len(seq_results)} prompts)")
-
-    if max_pt_diff >= TOL or max_pm_diff >= TOL:
-        raise SystemExit(f"PARITY FAILED: Δp_true={max_pt_diff:.4f} Δpmass={max_pm_diff:.4f}")
-
-
-if __name__ == "__main__":
-    main()