quiet pmass-low warning: one summary per batch

Was emitting `logger.warning("pmass=0.XX<0.9 — top-5: ...")` per-row, which
spammed the log heavily during heavy-steering eval (many rows go OOD at once).
Now collects all low-pmass rows in the batch and emits one summary line with
the worst-case top-5, e.g.:

    pmass<0.9 on 7/16 rows in this batch; worst=0.412 top-5: '1'=0.40, ...

Same diagnostic signal, ~16× fewer log lines per batch.

README.md

@@ -19,13 +19,26 @@ For use with LLMs we make them
 
 
 ### 2. Spec
-- Data: 3 configs of 132 vignettes each: `clifford` (real-world), `scifi` (genre-clean), and `airisk` (AI safety themes).
+- Data: 3 configs of 132 vignettes each: `classic` (real-world, from Clifford et al. 2015), `scifi` (genre-clean), and `airisk` (AI safety themes).
 - Taxonomy: 7 foundations (Care, Fairness, Loyalty, Authority, Sanctity, Liberty, Social Norms).
 - Conditions: Each vignette has `other_violate` (3rd-person) and `self_violate` (1st-person) versions.
 - Metrics:
   - `wrongness`: Mean rating of violations (detects moral-rating shift).
   - `gap`: `other_violate - self_violate` (detects perspective bias).
 
+#### Dual axis: `cond` × `frame`
+
+Each vignette produces 4 prompts from two independent binary axes:
+
+| Axis | Values | What it controls |
+|------|--------|-----------------|
+| **cond** (scenario framing) | `other_violate` (3rd-person: "You see someone doing X") / `self_violate` (1st-person: "You do X") | Which text variant the model reads |
+| **frame** (question framing) | `wrong` (`{"is_wrong": `) / `accept` (`{"is_acceptable": `) | How the JSON probe is phrased |
+
+Both axes are paired-out in `analyse()`:
+- The two **frames** cancel the additive JSON-true prior (training data has more `"true"` than `"false"` in JSON contexts).
+- The two **conds** let you measure perspective bias: the gap between how harshly the model judges others vs itself for the same scenario.
+
 ### 3. How to use
 
 Install:
@@ -42,10 +55,23 @@ tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
 model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-0.6B").cuda()
 
 # Returns per-foundation table and headline scalars (wrongness, gap)
-report = evaluate(model, tok, name="scifi")
+report = evaluate(model, tok, name="airisk")
 print(report["wrongness"], report["gap"])
 ```
 
+Load vignettes directly:
+```python
+from tinymfv import load_vignettes, load_all_vignettes
+
+# Load a single config
+vigs = load_vignettes("classic")    # or "scifi", "airisk"
+
+# Load all three with a `set` column
+all_vigs = load_all_vignettes()     # or load_vignettes("all")
+```
+
+> **Note:** `load_vignettes()` with no argument raises `ValueError` listing the available configs. The legacy name `"clifford"` still works as an alias for `"classic"`.
+
 ### 4. Link & Citation
 GitHub: [wassname/tiny-mcf-vignettes](https://github.com/wassname/tiny-mcf-vignettes)
 

src/tinymfv/__init__.py

@@ -22,11 +22,12 @@ from .core import (
     score_prompts,
     analyse,
 )
-from .data import load_vignettes
+from .data import load_vignettes, load_all_vignettes, CONFIGS
 from .eval import evaluate
 
 __all__ = [
-    "CONDITIONS", "FRAMES",
+    "CONDITIONS", "FRAMES", "CONFIGS",
     "format_prompt", "format_prompts", "bool_token_ids",
-    "score_prompts", "analyse", "load_vignettes", "evaluate",
+    "score_prompts", "analyse",
+    "load_vignettes", "load_all_vignettes", "evaluate",
 ]

src/tinymfv/data.py

@@ -10,6 +10,24 @@ Side artifact (not used by eval, kept for human-correlation sanity check):
 
 Each row: {id, foundation, foundation_coarse, wrong, text}.
 Falls back to HuggingFace `wassname/tiny-mfv` if local files absent.
+
+Dual-axis design
+================
+Each vignette produces 4 prompts from two independent binary axes:
+
+    **cond** (scenario framing — which text variant the model reads):
+        `other_violate`  — 3rd-person ("You see someone doing X")
+        `self_violate`   — 1st-person ("You do X")
+
+    **frame** (question framing — how the JSON probe is phrased):
+        `wrong`   — '{"is_wrong": '    → true means wrong
+        `accept`  — '{"is_acceptable": ' → true means right (inverted)
+
+Both axes are paired-out in `analyse()`:
+    - The two *frames* cancel the additive JSON-true prior (training data has
+      more `"true"` than `"false"` in JSON contexts).
+    - The two *conds* let you measure perspective bias: the gap between how
+      harshly the model judges others vs itself for the same scenario.
 """
 from __future__ import annotations
 import json
@@ -19,6 +37,22 @@ ROOT = Path(__file__).resolve().parents[2]
 HF_REPO = "wassname/tiny-mfv"
 CONDITIONS = ["other_violate", "self_violate"]
 
+# Canonical config names and aliases.
+CONFIGS = ("classic", "scifi", "airisk")
+_ALIASES = {"classic": "clifford", "clifford": "clifford"}  # classic→clifford on disk
+
+
+def _resolve_name(name: str) -> str:
+    """Map user-facing name to the canonical file/HF key.
+
+    Accepts 'classic' (preferred) or 'clifford' (legacy).
+    Returns the file-system key ('clifford' → empty suffix, others as-is).
+    """
+    low = name.lower()
+    if low in _ALIASES:
+        return _ALIASES[low]
+    return low  # scifi, airisk pass through
+
 
 def _local_path(name: str, condition: str) -> Path:
     suf = f"_{name}" if name else ""
@@ -39,10 +73,38 @@ def load_condition(name: str, condition: str) -> list[dict]:
     return list(load_dataset(HF_REPO, cfg, split=condition))
 
 
-def load_vignettes(name: str = "") -> list[dict]:
-    """Inner-join the 2 violate conditions by id. Returns rows with `other_violate`,
-    `self_violate` keys plus id/foundation/foundation_coarse/wrong."""
-    by_cond = {c: {r["id"]: r for r in load_condition(name, c)} for c in CONDITIONS}
+def load_vignettes(name: str | None = None) -> list[dict]:
+    """Inner-join the 2 violate conditions by id.
+
+    Returns rows with `other_violate`, `self_violate` keys plus
+    id / foundation / foundation_coarse / wrong.
+
+    Args:
+        name: One of 'classic' (alias: 'clifford'), 'scifi', 'airisk', or 'all'.
+              Must be specified — calling with no argument raises ValueError.
+
+    The two condition columns (*cond* axis) contain the scenario text:
+        - ``other_violate``: 3rd-person framing ("You see someone doing X")
+        - ``self_violate``:  1st-person framing ("You do X")
+
+    These are crossed with the *frame* axis (``wrong`` / ``accept``) at eval
+    time in ``format_prompts`` → ``analyse`` to cancel the JSON-true prior
+    and measure perspective bias.  See module docstring for details.
+    """
+    if name is None:
+        raise ValueError(
+            "load_vignettes() requires a config name. "
+            f"Choose one of: {', '.join(repr(c) for c in CONFIGS)}, or 'all'."
+        )
+
+    if name.lower() == "all":
+        return load_all_vignettes()
+
+    resolved = _resolve_name(name)
+    # clifford files have no suffix (legacy naming: vignettes_other_violate.jsonl)
+    file_name = "" if resolved == "clifford" else resolved
+
+    by_cond = {c: {r["id"]: r for r in load_condition(file_name, c)} for c in CONDITIONS}
     common = set.intersection(*[set(d) for d in by_cond.values()])
     rows = []
     anchor = by_cond["other_violate"]
@@ -56,5 +118,17 @@ def load_vignettes(name: str = "") -> list[dict]:
             "wrong": ov.get("wrong"),
             "other_violate": ov["text"],
             "self_violate": by_cond["self_violate"][vid]["text"],
+            "set": name.lower() if name.lower() != "clifford" else "classic",
         })
     return rows
+
+
+def load_all_vignettes() -> list[dict]:
+    """Load and concatenate all three configs (classic, scifi, airisk).
+
+    Returns the union with a ``set`` column indicating the source config.
+    """
+    all_rows = []
+    for cfg in CONFIGS:
+        all_rows.extend(load_vignettes(cfg))
+    return all_rows

src/tinymfv/eval.py

@@ -15,7 +15,7 @@ from .guided import guided_rollout_batch, choice_token_ids_tf
 def evaluate(
     model,
     tokenizer,
-    name: str = "",
+    name: str | None = None,
     vignettes: list[dict] | None = None,
     batch_size: int = 16,
     device: str | None = None,
@@ -23,13 +23,15 @@ def evaluate(
 ) -> dict[str, Any]:
     """Run dual JSON-bool eval and return aggregated report.
 
-    Either pass `vignettes` directly or `name` to load from `data/`. Tokenizer must
-    have a chat template (or fallback flat format will be used) and `pad_token` set.
+    Either pass `vignettes` directly or `name` (one of 'classic', 'scifi',
+    'airisk', 'all') to load from `data/`. Tokenizer must have a chat template
+    (or fallback flat format will be used) and `pad_token` set.
     Side-effects: sets `tokenizer.padding_side='left'` and `tokenizer.pad_token` if
     missing -- both required for batched left-padded eval.
     """
     if vignettes is None:
         vignettes = load_vignettes(name)
+
     if tokenizer.pad_token is None:
         tokenizer.pad_token = tokenizer.eos_token
     tokenizer.padding_side = "left"

src/tinymfv/guided.py

@@ -242,15 +242,12 @@ def guided_rollout_batch(
     b_t = torch.tensor(b_ids, device=device, dtype=torch.long) if b_ids else None
 
     results = []
+    low_pmass = []  # (idx, pmass) for rows with pmass<0.9
     for i, (up, (think_text, emitted_close, emitted_prefill, n_think)) in enumerate(zip(user_prompts, per_row)):
         logp = score_logp[i]
         pmass_format = float(logp[all_ids].exp().sum().item())
         if pmass_format < 0.9:
-            topk = torch.topk(logp.exp(), k=5)
-            toks = [tok.decode([j]) for j in topk.indices.tolist()]
-            probs = topk.values.tolist()
-            top5 = ", ".join(f"{repr(t)}={pp:.3f}" for t, pp in zip(toks, probs))
-            logger.warning(f"pmass={pmass_format:.3f}<0.9 — top-5: {top5}")
+            low_pmass.append((i, pmass_format))
         if a_t is not None and b_t is not None:
             la = torch.logsumexp(logp[a_t], dim=0)
             lb = torch.logsumexp(logp[b_t], dim=0)
@@ -272,6 +269,19 @@ def guided_rollout_batch(
             emitted_prefill=emitted_prefill,
             p_true=p_true,
         ))
+
+    # Aggregate-once warning: one line per batch with worst-case top-5 instead
+    # of N spammy per-row lines (heavy steering pushes many rows OOD at once).
+    if low_pmass:
+        worst_i, worst_pm = min(low_pmass, key=lambda x: x[1])
+        topk = torch.topk(score_logp[worst_i].exp(), k=5)
+        toks = [tok.decode([j]) for j in topk.indices.tolist()]
+        probs = topk.values.tolist()
+        top5 = ", ".join(f"{repr(t)}={pp:.3f}" for t, pp in zip(toks, probs))
+        logger.warning(
+            f"pmass<0.9 on {len(low_pmass)}/{len(results)} rows in this batch; "
+            f"worst={worst_pm:.3f} top-5: {top5}"
+        )
     return results