From 442bc5bba3b746b14f3063a7d871cd5aac8e1729 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Tue, 23 Jun 2026 20:40:04 +0800 Subject: [PATCH] address external API review: lazy maps, prune dead code, type administer return Whole-library review (deepseek-v4-pro) flagged tinymfv as not-yet-ready as a shared dep. Fixes for the parts I agreed with: - lazy `maps` import via module __getattr__ so `import tinymfv` stays headless/fast (no forced matplotlib) for numeric-only consumers; `tinymfv.maps.*` still works. - trim __all__ to the front door (entrypoints + types + data api); plumbing stays importable but out of `import *`. - delete dead code: reduce_nominal + REDUCERS (evaluate folds its profile inline), expected_value, HF_REPO, ROOT, _DEFAULT_FORCED_HINT. - type administer's return as a TypedDict (AdministerResult/ItemRow/ItemFrameRow) so the schema is documented + checkable without reading source; still a plain dict at runtime (zero consumer churn). - maps.plot_ipsative_pca: parametrize the legend labels (defaults preserve output) and rename hon/dis -> pos/neg so a non-honesty steer gets a correct legend. - drop 'canary' jargon and panel/review-# archaeology from comments. Verified: `import tinymfv` no longer loads matplotlib; lazy maps still resolves; experiment mfq2 smoke green through the typed administer. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- src/tinymfv/__init__.py | 33 +++++++++++++++++++++----------- src/tinymfv/administer.py | 38 +++++++++++++++++++++++++++++++++---- src/tinymfv/data.py | 2 -- src/tinymfv/eval.py | 2 +- src/tinymfv/guided.py | 5 +---- src/tinymfv/instrument.py | 40 +++++++++++---------------------------- src/tinymfv/maps.py | 20 ++++++++++++-------- src/tinymfv/read.py | 4 ++-- 8 files changed, 83 insertions(+), 61 deletions(-) diff --git a/src/tinymfv/__init__.py b/src/tinymfv/__init__.py index eed942b..32f6090 100644 --- a/src/tinymfv/__init__.py +++ b/src/tinymfv/__init__.py @@ -23,21 +23,32 @@ Lower-level: see `guided_rollout_forced_choice` in `tinymfv.guided`. from .data import load_vignettes, load_all_vignettes, CONFIGS, ConfigName from .eval import evaluate, CONDITIONS from .guided import guided_rollout_forced_choice, _DEFAULT_FORCED_FOUNDATIONS -from .instrument import Instrument, InstrItem, per_item_categorical, REDUCERS +from .instrument import Instrument, InstrItem, per_item_categorical from .instruments import get as get_instrument, INSTRUMENTS, build_instrument from .read import read_items, resolve_answer_ids, build_prompt from .administer import administer -from . import maps + +def __getattr__(name: str): + # `maps` pulls matplotlib; load it lazily so `import tinymfv` stays headless and fast for the + # numeric-only consumers (steering-lite). `tinymfv.maps.plot_*` still works -- first access + # triggers the import here. + if name == "maps": + import importlib + return importlib.import_module(".maps", __name__) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +# The front door. Other symbols above stay importable (e.g. read_items for item subsets, +# per_item_categorical, build_instrument) but are plumbing, kept out of `import *`. __all__ = [ - "CONDITIONS", "CONFIGS", "ConfigName", - "load_vignettes", "load_all_vignettes", - "evaluate", - "guided_rollout_forced_choice", "_DEFAULT_FORCED_FOUNDATIONS", - # survey readout (ordinal instruments: MFQ-2 / Big5 / 16PF / HSQ) - "Instrument", "InstrItem", "per_item_categorical", "REDUCERS", - "get_instrument", "INSTRUMENTS", "build_instrument", - "read_items", "resolve_answer_ids", "build_prompt", "administer", - "maps", + # entrypoints + "evaluate", "administer", "get_instrument", "read_items", + # types consumers build / subset + "Instrument", "InstrItem", + # data API + "load_vignettes", "load_all_vignettes", "CONFIGS", "ConfigName", "CONDITIONS", + # lower-level rollout + lazy plotting + "guided_rollout_forced_choice", "maps", ] diff --git a/src/tinymfv/administer.py b/src/tinymfv/administer.py index fed0b73..bbcda8d 100644 --- a/src/tinymfv/administer.py +++ b/src/tinymfv/administer.py @@ -1,4 +1,4 @@ -"""Run an ordinal Instrument end-to-end on a local model -> profile + coherence canary. +"""Run an ordinal Instrument end-to-end on a local model -> profile + coherence check. This is the survey counterpart to `tinymfv.evaluate` (the vignette forced-choice eval). It ties: @@ -10,22 +10,52 @@ The profile vector (per `instr.dimensions`) is the load-bearing output; `per_ite canonicalization makes it algebraically identical to the experiment's `admin.administer` per-factor means (verified by the reducer parity check), so this is a drop-in for the maps. -`mean_pmass_allowed` is the coherence canary: mass on valid answer tokens. A sharp drop (especially +`mean_pmass_allowed` is the coherence check: mass on valid answer tokens. A sharp drop (especially after steering) means the profile is untrustworthy even if every digit is in-format. """ from __future__ import annotations +from typing import TypedDict + import numpy as np from .instrument import Instrument, per_item_categorical, reduce_ordinal, canonicalize_to_forward from .read import read_items, resolve_answer_ids -def administer(model, tok, instr: Instrument, *, batch_size: int = 36) -> dict: +class ItemRow(TypedDict): + id: str + foundation: str + keyed_agreement: float # E, reverse-keyed for sign<0 items + E: float # expected scale point, 1..scale_max + pmass_allowed: float + frame_spread: float + + +class ItemFrameRow(TypedDict): + id: str + framing: str # forward | inverted | negated + foundation: str + agreement: float # forward-canonicalized E toward the original statement + keyed_agreement: float + pmass_allowed: float + + +class AdministerResult(TypedDict): + profile: np.ndarray # [len(dimensions)] per-factor keyed agreement -- the map input + dimensions: list[str] # factor order, matches `profile` + foundations: list[dict] # one per factor: foundation, mean, sd, ci95_lo/hi, framing_spread, + # + dynamic f_ keys (f_forward/f_inverted/f_negated) + per_item: list[ItemRow] # one per item, frame-averaged + per_item_frame: list[ItemFrameRow] # one per (item, frame) -- the granularity the maps bootstrap + mean_pmass_allowed: float # coherence check (mass on valid answer tokens) + + +def administer(model, tok, instr: Instrument, *, batch_size: int = 36) -> AdministerResult: assert instr.kind == "ordinal", "administer() is the ordinal survey readout; use evaluate() for nominal MFV" # Every ordinal item must carry its frame-specific response-scale legend in meta['task']; without # it build_prompt would silently emit a bare statement (no legend) and the profile would be junk - # while pmass still looks fine. Fail loud. (External review #57: build_prompt silent fallback.) + # while pmass still looks fine. Fail loud. assert all("task" in it.meta for it in instr.items), f"{instr.name}: ordinal items need meta['task']" w = np.arange(1, instr.scale_max + 1, dtype=float) answer_ids = resolve_answer_ids(tok, instr.answer_space) diff --git a/src/tinymfv/data.py b/src/tinymfv/data.py index e1feafb..348110f 100644 --- a/src/tinymfv/data.py +++ b/src/tinymfv/data.py @@ -29,8 +29,6 @@ from pathlib import Path from typing import Literal _DATA_DIR = Path(__file__).with_name("data") -HF_REPO = "wassname/tiny-mfv" -ROOT = Path(__file__).resolve().parents[2] CONDITIONS = ["other_violate", "self_violate"] # Canonical config names. diff --git a/src/tinymfv/eval.py b/src/tinymfv/eval.py index 5dc76b8..e0df3c5 100644 --- a/src/tinymfv/eval.py +++ b/src/tinymfv/eval.py @@ -462,7 +462,7 @@ def evaluate( "informedness": informedness, # Mean pmass_format: average prob mass on the K foundation answer # tokens at the JSON answer slot, across rows × framings. In [0, 1]. - # Direct coherence canary for forced-choice — drops when the model + # Direct coherence check for forced-choice: drops when the model # emits non-foundation tokens (gibberish, refusal, format collapse), # independent of which foundation is picked. Higher = more # "in-format"; a sharp drop after steering signals coherence loss. diff --git a/src/tinymfv/guided.py b/src/tinymfv/guided.py index 57c44f6..0dc79c4 100644 --- a/src/tinymfv/guided.py +++ b/src/tinymfv/guided.py @@ -391,9 +391,6 @@ def _make_forced_hint(foundations: list[str]) -> str: ) -_DEFAULT_FORCED_HINT: str = _make_forced_hint(list(_DEFAULT_FORCED_FOUNDATIONS)) - - @dataclass class ForcedChoiceResult: user_prompt: str @@ -433,7 +430,7 @@ class ForcedChoiceResult: # then across fwd + rev framings. In [0, 1]; high means the model still # emits a valid foundation word in the slot; low means probability has # leaked to other tokens (gibberish, refusal, format collapse). Direct - # coherence canary for forced-choice — independent of WHICH foundation + # coherence check for forced-choice, independent of WHICH foundation # is picked. pmass_allowed: float # Mean negative log-likelihood in nats/token over the assistant prefill diff --git a/src/tinymfv/instrument.py b/src/tinymfv/instrument.py index abf3668..dd9d630 100644 --- a/src/tinymfv/instrument.py +++ b/src/tinymfv/instrument.py @@ -1,7 +1,7 @@ """Instrument spec: one answer-token reader, many questionnaires (Option 3). Every instrument is the SAME measurement: a softmax over an answer-token set at a prefilled -slot, with a think budget, debias, BMA over sampled traces, a pmass coherence canary, and +slot, with a think budget, debias, BMA over sampled traces, a pmass coherence check, and temperature calibration to human soft-labels. Per-instrument variation is small: 1. answer_space : tokens gathered. Nominal = foundation words (forced-choice MFV); @@ -13,9 +13,8 @@ temperature calibration to human soft-labels. Per-instrument variation is small: 3. scaffold : prompt + assistant prefill that forces the answer slot. 4. human_label : per-item forward-orientation distribution over answer_space. -Design corrected after a frontier scientist panel (docs/reviews/sci_ma_*.md). The panel's -central, agreed flaw: leaving the frame reflection in the reducer made nominal and ordinal -debias asymmetric and risked double-flipping. The fix unifies them: +The frame reflection must NOT live in the reducer: that makes nominal and ordinal debias +asymmetric and risks double-flipping. Both kinds unify on one rule: CANONICALIZE-AT-READER. Every frame's gathered distribution is mapped to ONE forward orientation before anything else (`canonicalize_to_forward`): nominal reorder frames reindex @@ -32,7 +31,7 @@ debias asymmetric and risked double-flipping. The fix unifies them: framing canonicalization -- not a double-correction. `p` everywhere is renormalized over the allowed answer tokens (sums to 1); `pmass_allowed` is -the separate coherence canary, never mixed into the distribution. +the separate coherence check, never mixed into the distribution. """ from __future__ import annotations from collections import defaultdict @@ -77,8 +76,8 @@ class Instrument: assert len(self.answer_space) == self.scale_max, "ordinal answer_space must be 1..scale_max" if self.kind == "nominal" and self.answer_to_dim is None: self.answer_to_dim = {a: a for a in self.answer_space} - # Cross-scale caveat (panel: human_scale_max was unused): a 1-7 human histogram (HSQ) - # cannot share a 5-way soft-NLL/JS with a 1-5 model directly. Calibration for such + # Cross-scale caveat: a 1-7 human histogram (HSQ) cannot share a 5-way soft-NLL/JS + # with a 1-5 model directly. Calibration for such # instruments must project both to a common support (or report 0-1 endorsement only). # Enforced loudly rather than silently mis-comparing: if self.kind == "ordinal" and self.human_scale_max != self.scale_max: @@ -121,7 +120,7 @@ def per_item_categorical(per_row: list[dict], kind: Kind) -> dict[str, dict]: # Each item collapses to ONE averaged distribution, so every item contributes EQUALLY to the # factor mean. That matches pooling all (item, frame) rows only if every item has the same frame # count. Assert it loudly rather than silently reweighting a factor if a future instrument gives - # some items fewer frames. (External review #57: dormant weighting asymmetry.) + # some items fewer frames. frame_counts = {len(rows) for rows in by_id.values()} assert len(frame_counts) == 1, f"heterogeneous frame counts per item: {frame_counts}" out: dict[str, dict] = {} @@ -143,17 +142,7 @@ def per_item_categorical(per_row: list[dict], kind: Kind) -> dict[str, dict]: return out -# --- profile reducers: the only kind-specific summary --- - -def reduce_nominal(items: dict[str, dict], instr: Instrument) -> np.ndarray: - """Forced-choice profile = mean choice frequency over the answer space, folded to dimensions.""" - dim_idx = {d: j for j, d in enumerate(instr.dimensions)} - acc = np.zeros((len(items), len(instr.dimensions))) - for i, it in enumerate(items.values()): - for a, pa in zip(instr.answer_space, it["p"]): - acc[i, dim_idx[instr.answer_to_dim[a]]] += pa - return acc.mean(axis=0) - +# --- profile reducer: the ordinal kind-specific summary (nominal evaluate folds its profile inline) --- def reduce_ordinal(items: dict[str, dict], instr: Instrument) -> np.ndarray: """Likert profile = mean keyed agreement per dimension; agreement = E[scale point]. @@ -172,16 +161,9 @@ def reduce_ordinal(items: dict[str, dict], instr: Instrument) -> np.ndarray: return np.array([float(np.mean(by_dim[d])) for d in instr.dimensions]) -REDUCERS = {"nominal": reduce_nominal, "ordinal": reduce_ordinal} - - -def expected_value(p: np.ndarray, scale_max: int) -> float: - return float((np.asarray(p) * np.arange(1, scale_max + 1)).sum()) - - -# --- negative control: shuffle item->dimension (ordinal) / answer->dim (nominal); profile -# correlation with the true profile must collapse to chance. A non-null result is only -# interpretable if this control passes (all three reviewers required it). --- +# --- negative control (currently unwired): shuffle item->dimension; the shuffled profile's +# correlation with the true profile must collapse to chance, else the signal is a layout artifact. +# A non-null steering result is only interpretable once this passes. --- def shuffle_dimensions(items: dict[str, dict], rng: np.random.Generator) -> dict[str, dict]: dims = [it["dimension"] for it in items.values()] diff --git a/src/tinymfv/maps.py b/src/tinymfv/maps.py index b979dd0..ac208bc 100644 --- a/src/tinymfv/maps.py +++ b/src/tinymfv/maps.py @@ -88,10 +88,13 @@ def compass(ax_main, L: np.ndarray, labels: list[str], title: str = "compass", def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str], M: np.ndarray, - base: np.ndarray, hon: np.ndarray | None, dis: np.ndarray | None, - *, boots: dict | None = None, pad=(0.18, 0.16)): - """Ipsative culture map. M is societies x K (0-1 fraction); base/hon/dis are length-K fraction - vectors (or None). `boots` optionally maps 'base'/'honest'/'dis' -> (n x K) bootstrap fraction + base: np.ndarray, pos: np.ndarray | None, neg: np.ndarray | None, + *, boots: dict | None = None, pad=(0.18, 0.16), + labels: tuple[str, str, str] = ("baseline (c=0)", "honest (c=+2)", "dishonest (c=-2)")): + """Ipsative culture map. M is societies x K (0-1 fraction); base / pos / neg are the length-K + fraction vectors for the base model and its two steer poles (or None). `labels` is the legend + text (base, +pole, -pole) -- override it for a non-honesty steer or a different coefficient. + `boots` optionally maps the role keys 'base'/'honest'/'dis' -> (n x K) bootstrap fraction matrices for the uncertainty cross. Returns the Figure.""" try: import textalloc as ta @@ -106,7 +109,7 @@ def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str], def proj(v): return ((v @ Pc) - mu) @ Vt[:2].T if v is not None else None - pb, ph, pf = proj(base), proj(hon), proj(dis) + pb, ph, pf = proj(base), proj(pos), proj(neg) fig, ax = plt.subplots(figsize=(8.5, 7.5)) ax.set_facecolor("#faf8f2") @@ -128,9 +131,10 @@ def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str], e1, e2 = 1.96 * bp.std(0) ax.errorbar(pt[0], pt[1], xerr=e1, yerr=e2, fmt="none", ecolor=col, elinewidth=0.7, alpha=0.55, capsize=2.5, capthick=0.8, zorder=4) - for pt, col, mk, lab, dxy, ha in [(ph, C_HON, "s", "honest (c=+2)", (9, 9), "left"), - (pf, C_DIS, "^", "dishonest (c=-2)", (-9, -1), "right"), - (pb, C_BASE, "o", "baseline (c=0)", (9, -13), "left")]: + base_lab, pos_lab, neg_lab = labels + for pt, col, mk, lab, dxy, ha in [(ph, C_HON, "s", pos_lab, (9, 9), "left"), + (pf, C_DIS, "^", neg_lab, (-9, -1), "right"), + (pb, C_BASE, "o", base_lab, (9, -13), "left")]: if pt is None: continue if pt is not pb: diff --git a/src/tinymfv/read.py b/src/tinymfv/read.py index 619fc30..2669f0c 100644 --- a/src/tinymfv/read.py +++ b/src/tinymfv/read.py @@ -10,7 +10,7 @@ on the instrument's `answer_space` tokens after its `prefill`. Per InstrItem: - p = renormalized distribution over answer_space (sums to 1). This is the per-(item,frame) categorical that `instrument.per_item_categorical` canonicalizes + averages. - - pmass = sum of raw (full-vocab) mass on the answer tokens: the coherence canary. Drops when + - pmass = sum of raw (full-vocab) mass on the answer tokens: the coherence check. Drops when the model leaks to refusals / prose / gibberish, independent of which option it picks. Framing (forward / inverted / negated) is carried by each InstrItem.frame; canonicalization to a @@ -19,7 +19,7 @@ frame-agnostic: it just reports the presented-orientation distribution. Single-token requirement: every answer token must encode to exactly one id given the tokenizer, and they must be distinct. Verified for Qwen ('(1' -> ['(','1']). A pmass collapse (not an error -here) is the canary that the prefill merged with the option and the readout went blind. +here) is the tell that the prefill merged with the option and the readout went blind. """ from __future__ import annotations