mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-09 11:27:22 +08:00
address external API review: lazy maps, prune dead code, type administer return
Whole-library review (deepseek-v4-pro) flagged tinymfv as not-yet-ready as a shared dep. Fixes for the parts I agreed with: - lazy `maps` import via module __getattr__ so `import tinymfv` stays headless/fast (no forced matplotlib) for numeric-only consumers; `tinymfv.maps.*` still works. - trim __all__ to the front door (entrypoints + types + data api); plumbing stays importable but out of `import *`. - delete dead code: reduce_nominal + REDUCERS (evaluate folds its profile inline), expected_value, HF_REPO, ROOT, _DEFAULT_FORCED_HINT. - type administer's return as a TypedDict (AdministerResult/ItemRow/ItemFrameRow) so the schema is documented + checkable without reading source; still a plain dict at runtime (zero consumer churn). - maps.plot_ipsative_pca: parametrize the legend labels (defaults preserve output) and rename hon/dis -> pos/neg so a non-honesty steer gets a correct legend. - drop 'canary' jargon and panel/review-# archaeology from comments. Verified: `import tinymfv` no longer loads matplotlib; lazy maps still resolves; experiment mfq2 smoke green through the typed administer. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
+22
-11
@@ -23,21 +23,32 @@ Lower-level: see `guided_rollout_forced_choice` in `tinymfv.guided`.
|
||||
from .data import load_vignettes, load_all_vignettes, CONFIGS, ConfigName
|
||||
from .eval import evaluate, CONDITIONS
|
||||
from .guided import guided_rollout_forced_choice, _DEFAULT_FORCED_FOUNDATIONS
|
||||
from .instrument import Instrument, InstrItem, per_item_categorical, REDUCERS
|
||||
from .instrument import Instrument, InstrItem, per_item_categorical
|
||||
from .instruments import get as get_instrument, INSTRUMENTS, build_instrument
|
||||
from .read import read_items, resolve_answer_ids, build_prompt
|
||||
from .administer import administer
|
||||
from . import maps
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
# `maps` pulls matplotlib; load it lazily so `import tinymfv` stays headless and fast for the
|
||||
# numeric-only consumers (steering-lite). `tinymfv.maps.plot_*` still works -- first access
|
||||
# triggers the import here.
|
||||
if name == "maps":
|
||||
import importlib
|
||||
return importlib.import_module(".maps", __name__)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
# The front door. Other symbols above stay importable (e.g. read_items for item subsets,
|
||||
# per_item_categorical, build_instrument) but are plumbing, kept out of `import *`.
|
||||
__all__ = [
|
||||
"CONDITIONS", "CONFIGS", "ConfigName",
|
||||
"load_vignettes", "load_all_vignettes",
|
||||
"evaluate",
|
||||
"guided_rollout_forced_choice", "_DEFAULT_FORCED_FOUNDATIONS",
|
||||
# survey readout (ordinal instruments: MFQ-2 / Big5 / 16PF / HSQ)
|
||||
"Instrument", "InstrItem", "per_item_categorical", "REDUCERS",
|
||||
"get_instrument", "INSTRUMENTS", "build_instrument",
|
||||
"read_items", "resolve_answer_ids", "build_prompt", "administer",
|
||||
"maps",
|
||||
# entrypoints
|
||||
"evaluate", "administer", "get_instrument", "read_items",
|
||||
# types consumers build / subset
|
||||
"Instrument", "InstrItem",
|
||||
# data API
|
||||
"load_vignettes", "load_all_vignettes", "CONFIGS", "ConfigName", "CONDITIONS",
|
||||
# lower-level rollout + lazy plotting
|
||||
"guided_rollout_forced_choice", "maps",
|
||||
]
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Run an ordinal Instrument end-to-end on a local model -> profile + coherence canary.
|
||||
"""Run an ordinal Instrument end-to-end on a local model -> profile + coherence check.
|
||||
|
||||
This is the survey counterpart to `tinymfv.evaluate` (the vignette forced-choice eval). It ties:
|
||||
|
||||
@@ -10,22 +10,52 @@ The profile vector (per `instr.dimensions`) is the load-bearing output; `per_ite
|
||||
canonicalization makes it algebraically identical to the experiment's `admin.administer` per-factor
|
||||
means (verified by the reducer parity check), so this is a drop-in for the maps.
|
||||
|
||||
`mean_pmass_allowed` is the coherence canary: mass on valid answer tokens. A sharp drop (especially
|
||||
`mean_pmass_allowed` is the coherence check: mass on valid answer tokens. A sharp drop (especially
|
||||
after steering) means the profile is untrustworthy even if every digit is in-format.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TypedDict
|
||||
|
||||
import numpy as np
|
||||
|
||||
from .instrument import Instrument, per_item_categorical, reduce_ordinal, canonicalize_to_forward
|
||||
from .read import read_items, resolve_answer_ids
|
||||
|
||||
|
||||
def administer(model, tok, instr: Instrument, *, batch_size: int = 36) -> dict:
|
||||
class ItemRow(TypedDict):
|
||||
id: str
|
||||
foundation: str
|
||||
keyed_agreement: float # E, reverse-keyed for sign<0 items
|
||||
E: float # expected scale point, 1..scale_max
|
||||
pmass_allowed: float
|
||||
frame_spread: float
|
||||
|
||||
|
||||
class ItemFrameRow(TypedDict):
|
||||
id: str
|
||||
framing: str # forward | inverted | negated
|
||||
foundation: str
|
||||
agreement: float # forward-canonicalized E toward the original statement
|
||||
keyed_agreement: float
|
||||
pmass_allowed: float
|
||||
|
||||
|
||||
class AdministerResult(TypedDict):
|
||||
profile: np.ndarray # [len(dimensions)] per-factor keyed agreement -- the map input
|
||||
dimensions: list[str] # factor order, matches `profile`
|
||||
foundations: list[dict] # one per factor: foundation, mean, sd, ci95_lo/hi, framing_spread,
|
||||
# + dynamic f_<frame> keys (f_forward/f_inverted/f_negated)
|
||||
per_item: list[ItemRow] # one per item, frame-averaged
|
||||
per_item_frame: list[ItemFrameRow] # one per (item, frame) -- the granularity the maps bootstrap
|
||||
mean_pmass_allowed: float # coherence check (mass on valid answer tokens)
|
||||
|
||||
|
||||
def administer(model, tok, instr: Instrument, *, batch_size: int = 36) -> AdministerResult:
|
||||
assert instr.kind == "ordinal", "administer() is the ordinal survey readout; use evaluate() for nominal MFV"
|
||||
# Every ordinal item must carry its frame-specific response-scale legend in meta['task']; without
|
||||
# it build_prompt would silently emit a bare statement (no legend) and the profile would be junk
|
||||
# while pmass still looks fine. Fail loud. (External review #57: build_prompt silent fallback.)
|
||||
# while pmass still looks fine. Fail loud.
|
||||
assert all("task" in it.meta for it in instr.items), f"{instr.name}: ordinal items need meta['task']"
|
||||
w = np.arange(1, instr.scale_max + 1, dtype=float)
|
||||
answer_ids = resolve_answer_ids(tok, instr.answer_space)
|
||||
|
||||
@@ -29,8 +29,6 @@ from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
_DATA_DIR = Path(__file__).with_name("data")
|
||||
HF_REPO = "wassname/tiny-mfv"
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
CONDITIONS = ["other_violate", "self_violate"]
|
||||
|
||||
# Canonical config names.
|
||||
|
||||
+1
-1
@@ -462,7 +462,7 @@ def evaluate(
|
||||
"informedness": informedness,
|
||||
# Mean pmass_format: average prob mass on the K foundation answer
|
||||
# tokens at the JSON answer slot, across rows × framings. In [0, 1].
|
||||
# Direct coherence canary for forced-choice — drops when the model
|
||||
# Direct coherence check for forced-choice: drops when the model
|
||||
# emits non-foundation tokens (gibberish, refusal, format collapse),
|
||||
# independent of which foundation is picked. Higher = more
|
||||
# "in-format"; a sharp drop after steering signals coherence loss.
|
||||
|
||||
@@ -391,9 +391,6 @@ def _make_forced_hint(foundations: list[str]) -> str:
|
||||
)
|
||||
|
||||
|
||||
_DEFAULT_FORCED_HINT: str = _make_forced_hint(list(_DEFAULT_FORCED_FOUNDATIONS))
|
||||
|
||||
|
||||
@dataclass
|
||||
class ForcedChoiceResult:
|
||||
user_prompt: str
|
||||
@@ -433,7 +430,7 @@ class ForcedChoiceResult:
|
||||
# then across fwd + rev framings. In [0, 1]; high means the model still
|
||||
# emits a valid foundation word in the slot; low means probability has
|
||||
# leaked to other tokens (gibberish, refusal, format collapse). Direct
|
||||
# coherence canary for forced-choice — independent of WHICH foundation
|
||||
# coherence check for forced-choice, independent of WHICH foundation
|
||||
# is picked.
|
||||
pmass_allowed: float
|
||||
# Mean negative log-likelihood in nats/token over the assistant prefill
|
||||
|
||||
+11
-29
@@ -1,7 +1,7 @@
|
||||
"""Instrument spec: one answer-token reader, many questionnaires (Option 3).
|
||||
|
||||
Every instrument is the SAME measurement: a softmax over an answer-token set at a prefilled
|
||||
slot, with a think budget, debias, BMA over sampled traces, a pmass coherence canary, and
|
||||
slot, with a think budget, debias, BMA over sampled traces, a pmass coherence check, and
|
||||
temperature calibration to human soft-labels. Per-instrument variation is small:
|
||||
|
||||
1. answer_space : tokens gathered. Nominal = foundation words (forced-choice MFV);
|
||||
@@ -13,9 +13,8 @@ temperature calibration to human soft-labels. Per-instrument variation is small:
|
||||
3. scaffold : prompt + assistant prefill that forces the answer slot.
|
||||
4. human_label : per-item forward-orientation distribution over answer_space.
|
||||
|
||||
Design corrected after a frontier scientist panel (docs/reviews/sci_ma_*.md). The panel's
|
||||
central, agreed flaw: leaving the frame reflection in the reducer made nominal and ordinal
|
||||
debias asymmetric and risked double-flipping. The fix unifies them:
|
||||
The frame reflection must NOT live in the reducer: that makes nominal and ordinal debias
|
||||
asymmetric and risks double-flipping. Both kinds unify on one rule:
|
||||
|
||||
CANONICALIZE-AT-READER. Every frame's gathered distribution is mapped to ONE forward
|
||||
orientation before anything else (`canonicalize_to_forward`): nominal reorder frames reindex
|
||||
@@ -32,7 +31,7 @@ debias asymmetric and risked double-flipping. The fix unifies them:
|
||||
framing canonicalization -- not a double-correction.
|
||||
|
||||
`p` everywhere is renormalized over the allowed answer tokens (sums to 1); `pmass_allowed` is
|
||||
the separate coherence canary, never mixed into the distribution.
|
||||
the separate coherence check, never mixed into the distribution.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
from collections import defaultdict
|
||||
@@ -77,8 +76,8 @@ class Instrument:
|
||||
assert len(self.answer_space) == self.scale_max, "ordinal answer_space must be 1..scale_max"
|
||||
if self.kind == "nominal" and self.answer_to_dim is None:
|
||||
self.answer_to_dim = {a: a for a in self.answer_space}
|
||||
# Cross-scale caveat (panel: human_scale_max was unused): a 1-7 human histogram (HSQ)
|
||||
# cannot share a 5-way soft-NLL/JS with a 1-5 model directly. Calibration for such
|
||||
# Cross-scale caveat: a 1-7 human histogram (HSQ) cannot share a 5-way soft-NLL/JS
|
||||
# with a 1-5 model directly. Calibration for such
|
||||
# instruments must project both to a common support (or report 0-1 endorsement only).
|
||||
# Enforced loudly rather than silently mis-comparing:
|
||||
if self.kind == "ordinal" and self.human_scale_max != self.scale_max:
|
||||
@@ -121,7 +120,7 @@ def per_item_categorical(per_row: list[dict], kind: Kind) -> dict[str, dict]:
|
||||
# Each item collapses to ONE averaged distribution, so every item contributes EQUALLY to the
|
||||
# factor mean. That matches pooling all (item, frame) rows only if every item has the same frame
|
||||
# count. Assert it loudly rather than silently reweighting a factor if a future instrument gives
|
||||
# some items fewer frames. (External review #57: dormant weighting asymmetry.)
|
||||
# some items fewer frames.
|
||||
frame_counts = {len(rows) for rows in by_id.values()}
|
||||
assert len(frame_counts) == 1, f"heterogeneous frame counts per item: {frame_counts}"
|
||||
out: dict[str, dict] = {}
|
||||
@@ -143,17 +142,7 @@ def per_item_categorical(per_row: list[dict], kind: Kind) -> dict[str, dict]:
|
||||
return out
|
||||
|
||||
|
||||
# --- profile reducers: the only kind-specific summary ---
|
||||
|
||||
def reduce_nominal(items: dict[str, dict], instr: Instrument) -> np.ndarray:
|
||||
"""Forced-choice profile = mean choice frequency over the answer space, folded to dimensions."""
|
||||
dim_idx = {d: j for j, d in enumerate(instr.dimensions)}
|
||||
acc = np.zeros((len(items), len(instr.dimensions)))
|
||||
for i, it in enumerate(items.values()):
|
||||
for a, pa in zip(instr.answer_space, it["p"]):
|
||||
acc[i, dim_idx[instr.answer_to_dim[a]]] += pa
|
||||
return acc.mean(axis=0)
|
||||
|
||||
# --- profile reducer: the ordinal kind-specific summary (nominal evaluate folds its profile inline) ---
|
||||
|
||||
def reduce_ordinal(items: dict[str, dict], instr: Instrument) -> np.ndarray:
|
||||
"""Likert profile = mean keyed agreement per dimension; agreement = E[scale point].
|
||||
@@ -172,16 +161,9 @@ def reduce_ordinal(items: dict[str, dict], instr: Instrument) -> np.ndarray:
|
||||
return np.array([float(np.mean(by_dim[d])) for d in instr.dimensions])
|
||||
|
||||
|
||||
REDUCERS = {"nominal": reduce_nominal, "ordinal": reduce_ordinal}
|
||||
|
||||
|
||||
def expected_value(p: np.ndarray, scale_max: int) -> float:
|
||||
return float((np.asarray(p) * np.arange(1, scale_max + 1)).sum())
|
||||
|
||||
|
||||
# --- negative control: shuffle item->dimension (ordinal) / answer->dim (nominal); profile
|
||||
# correlation with the true profile must collapse to chance. A non-null result is only
|
||||
# interpretable if this control passes (all three reviewers required it). ---
|
||||
# --- negative control (currently unwired): shuffle item->dimension; the shuffled profile's
|
||||
# correlation with the true profile must collapse to chance, else the signal is a layout artifact.
|
||||
# A non-null steering result is only interpretable once this passes. ---
|
||||
|
||||
def shuffle_dimensions(items: dict[str, dict], rng: np.random.Generator) -> dict[str, dict]:
|
||||
dims = [it["dimension"] for it in items.values()]
|
||||
|
||||
+12
-8
@@ -88,10 +88,13 @@ def compass(ax_main, L: np.ndarray, labels: list[str], title: str = "compass",
|
||||
|
||||
|
||||
def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str], M: np.ndarray,
|
||||
base: np.ndarray, hon: np.ndarray | None, dis: np.ndarray | None,
|
||||
*, boots: dict | None = None, pad=(0.18, 0.16)):
|
||||
"""Ipsative culture map. M is societies x K (0-1 fraction); base/hon/dis are length-K fraction
|
||||
vectors (or None). `boots` optionally maps 'base'/'honest'/'dis' -> (n x K) bootstrap fraction
|
||||
base: np.ndarray, pos: np.ndarray | None, neg: np.ndarray | None,
|
||||
*, boots: dict | None = None, pad=(0.18, 0.16),
|
||||
labels: tuple[str, str, str] = ("baseline (c=0)", "honest (c=+2)", "dishonest (c=-2)")):
|
||||
"""Ipsative culture map. M is societies x K (0-1 fraction); base / pos / neg are the length-K
|
||||
fraction vectors for the base model and its two steer poles (or None). `labels` is the legend
|
||||
text (base, +pole, -pole) -- override it for a non-honesty steer or a different coefficient.
|
||||
`boots` optionally maps the role keys 'base'/'honest'/'dis' -> (n x K) bootstrap fraction
|
||||
matrices for the uncertainty cross. Returns the Figure."""
|
||||
try:
|
||||
import textalloc as ta
|
||||
@@ -106,7 +109,7 @@ def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str],
|
||||
|
||||
def proj(v):
|
||||
return ((v @ Pc) - mu) @ Vt[:2].T if v is not None else None
|
||||
pb, ph, pf = proj(base), proj(hon), proj(dis)
|
||||
pb, ph, pf = proj(base), proj(pos), proj(neg)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(8.5, 7.5))
|
||||
ax.set_facecolor("#faf8f2")
|
||||
@@ -128,9 +131,10 @@ def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str],
|
||||
e1, e2 = 1.96 * bp.std(0)
|
||||
ax.errorbar(pt[0], pt[1], xerr=e1, yerr=e2, fmt="none", ecolor=col,
|
||||
elinewidth=0.7, alpha=0.55, capsize=2.5, capthick=0.8, zorder=4)
|
||||
for pt, col, mk, lab, dxy, ha in [(ph, C_HON, "s", "honest (c=+2)", (9, 9), "left"),
|
||||
(pf, C_DIS, "^", "dishonest (c=-2)", (-9, -1), "right"),
|
||||
(pb, C_BASE, "o", "baseline (c=0)", (9, -13), "left")]:
|
||||
base_lab, pos_lab, neg_lab = labels
|
||||
for pt, col, mk, lab, dxy, ha in [(ph, C_HON, "s", pos_lab, (9, 9), "left"),
|
||||
(pf, C_DIS, "^", neg_lab, (-9, -1), "right"),
|
||||
(pb, C_BASE, "o", base_lab, (9, -13), "left")]:
|
||||
if pt is None:
|
||||
continue
|
||||
if pt is not pb:
|
||||
|
||||
+2
-2
@@ -10,7 +10,7 @@ on the instrument's `answer_space` tokens after its `prefill`. Per InstrItem:
|
||||
|
||||
- p = renormalized distribution over answer_space (sums to 1). This is the per-(item,frame)
|
||||
categorical that `instrument.per_item_categorical` canonicalizes + averages.
|
||||
- pmass = sum of raw (full-vocab) mass on the answer tokens: the coherence canary. Drops when
|
||||
- pmass = sum of raw (full-vocab) mass on the answer tokens: the coherence check. Drops when
|
||||
the model leaks to refusals / prose / gibberish, independent of which option it picks.
|
||||
|
||||
Framing (forward / inverted / negated) is carried by each InstrItem.frame; canonicalization to a
|
||||
@@ -19,7 +19,7 @@ frame-agnostic: it just reports the presented-orientation distribution.
|
||||
|
||||
Single-token requirement: every answer token must encode to exactly one id given the tokenizer,
|
||||
and they must be distinct. Verified for Qwen ('(1' -> ['(','1']). A pmass collapse (not an error
|
||||
here) is the canary that the prefill merged with the option and the readout went blind.
|
||||
here) is the tell that the prefill merged with the option and the readout went blind.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
Reference in New Issue
Block a user