quiet pmass-low warning: one summary per batch

Was emitting `logger.warning("pmass=0.XX<0.9 — top-5: ...")` per-row, which
spammed the log heavily during heavy-steering eval (many rows go OOD at once).
Now collects all low-pmass rows in the batch and emits one summary line with
the worst-case top-5, e.g.:

    pmass<0.9 on 7/16 rows in this batch; worst=0.412 top-5: '1'=0.40, ...

Same diagnostic signal, ~16× fewer log lines per batch.
This commit is contained in:
wassname
2026-05-03 06:50:19 +08:00
parent e996d57051
commit addf47c5a0
5 changed files with 130 additions and 17 deletions
+78 -4
View File
@@ -10,6 +10,24 @@ Side artifact (not used by eval, kept for human-correlation sanity check):
Each row: {id, foundation, foundation_coarse, wrong, text}.
Falls back to HuggingFace `wassname/tiny-mfv` if local files absent.
Dual-axis design
================
Each vignette produces 4 prompts from two independent binary axes:
**cond** (scenario framing — which text variant the model reads):
`other_violate` — 3rd-person ("You see someone doing X")
`self_violate` — 1st-person ("You do X")
**frame** (question framing — how the JSON probe is phrased):
`wrong` — '{"is_wrong": ' → true means wrong
`accept` — '{"is_acceptable": ' → true means right (inverted)
Both axes are paired-out in `analyse()`:
- The two *frames* cancel the additive JSON-true prior (training data has
more `"true"` than `"false"` in JSON contexts).
- The two *conds* let you measure perspective bias: the gap between how
harshly the model judges others vs itself for the same scenario.
"""
from __future__ import annotations
import json
@@ -19,6 +37,22 @@ ROOT = Path(__file__).resolve().parents[2]
HF_REPO = "wassname/tiny-mfv"
CONDITIONS = ["other_violate", "self_violate"]
# Canonical config names and aliases.
CONFIGS = ("classic", "scifi", "airisk")
_ALIASES = {"classic": "clifford", "clifford": "clifford"} # classic→clifford on disk
def _resolve_name(name: str) -> str:
"""Map user-facing name to the canonical file/HF key.
Accepts 'classic' (preferred) or 'clifford' (legacy).
Returns the file-system key ('clifford' → empty suffix, others as-is).
"""
low = name.lower()
if low in _ALIASES:
return _ALIASES[low]
return low # scifi, airisk pass through
def _local_path(name: str, condition: str) -> Path:
suf = f"_{name}" if name else ""
@@ -39,10 +73,38 @@ def load_condition(name: str, condition: str) -> list[dict]:
return list(load_dataset(HF_REPO, cfg, split=condition))
def load_vignettes(name: str = "") -> list[dict]:
"""Inner-join the 2 violate conditions by id. Returns rows with `other_violate`,
`self_violate` keys plus id/foundation/foundation_coarse/wrong."""
by_cond = {c: {r["id"]: r for r in load_condition(name, c)} for c in CONDITIONS}
def load_vignettes(name: str | None = None) -> list[dict]:
"""Inner-join the 2 violate conditions by id.
Returns rows with `other_violate`, `self_violate` keys plus
id / foundation / foundation_coarse / wrong.
Args:
name: One of 'classic' (alias: 'clifford'), 'scifi', 'airisk', or 'all'.
Must be specified — calling with no argument raises ValueError.
The two condition columns (*cond* axis) contain the scenario text:
- ``other_violate``: 3rd-person framing ("You see someone doing X")
- ``self_violate``: 1st-person framing ("You do X")
These are crossed with the *frame* axis (``wrong`` / ``accept``) at eval
time in ``format_prompts`` → ``analyse`` to cancel the JSON-true prior
and measure perspective bias. See module docstring for details.
"""
if name is None:
raise ValueError(
"load_vignettes() requires a config name. "
f"Choose one of: {', '.join(repr(c) for c in CONFIGS)}, or 'all'."
)
if name.lower() == "all":
return load_all_vignettes()
resolved = _resolve_name(name)
# clifford files have no suffix (legacy naming: vignettes_other_violate.jsonl)
file_name = "" if resolved == "clifford" else resolved
by_cond = {c: {r["id"]: r for r in load_condition(file_name, c)} for c in CONDITIONS}
common = set.intersection(*[set(d) for d in by_cond.values()])
rows = []
anchor = by_cond["other_violate"]
@@ -56,5 +118,17 @@ def load_vignettes(name: str = "") -> list[dict]:
"wrong": ov.get("wrong"),
"other_violate": ov["text"],
"self_violate": by_cond["self_violate"][vid]["text"],
"set": name.lower() if name.lower() != "clifford" else "classic",
})
return rows
def load_all_vignettes() -> list[dict]:
"""Load and concatenate all three configs (classic, scifi, airisk).
Returns the union with a ``set`` column indicating the source config.
"""
all_rows = []
for cfg in CONFIGS:
all_rows.extend(load_vignettes(cfg))
return all_rows