From b2e631132c993f3a030caaac3d844959c5781b69 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Sun, 5 Jul 2026 08:25:28 +0800 Subject: [PATCH] cleanup: assert on flat _zscore, doc the nll floor, trim history comments _zscore now asserts nonzero variance (a flat profile drew a degenerate all-zero map that looked valid) instead of dividing by std+1e-9. _soft_nll docstring now states the 1e-12 floor it actually applies (was 'Unbounded'). Trim archaeology comments (guided/wvs/read_api) to the invariant, dropping the dated model names. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- scripts/plot_steer_showcase.py | 8 ++++++-- scripts/wvs_map.py | 4 ++-- src/tinymfv/eval.py | 4 +++- src/tinymfv/guided.py | 1 - src/tinymfv/read_api.py | 7 +++---- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/scripts/plot_steer_showcase.py b/scripts/plot_steer_showcase.py index 9010f73..71eb45f 100644 --- a/scripts/plot_steer_showcase.py +++ b/scripts/plot_steer_showcase.py @@ -232,8 +232,12 @@ def plot_ordinal(run_dir: Path, out: Path, name: str, vec_label: str, C: float, def _zscore(v: np.ndarray) -> np.ndarray: """Relative emphasis: centre and scale a profile across foundations, so a logit profile (model) - and a 1-5 wrongness profile (human cultures) are comparable by PATTERN regardless of units.""" - return (v - v.mean()) / (v.std() + 1e-9) + and a 1-5 wrongness profile (human cultures) are comparable by PATTERN regardless of units. A + flat profile has no relative emphasis to show -- fail loud rather than divide by ~0 and draw a + degenerate all-zero map that looks valid.""" + sd = v.std() + assert sd > 0, "flat profile (zero variance across foundations): relative-emphasis map is undefined" + return (v - v.mean()) / sd def read_human_mfv() -> tuple[list[str], dict[str, dict[str, float]]]: diff --git a/scripts/wvs_map.py b/scripts/wvs_map.py index 3c35ce6..0e059a3 100644 --- a/scripts/wvs_map.py +++ b/scripts/wvs_map.py @@ -169,8 +169,8 @@ def main() -> None: for k, v in cache.items()} def save_cache() -> None: - """Persist after EACH model so a killed run (session teardown) keeps finished models -- the - write-once-at-end version silently discarded every sampled model when interrupted.""" + """Persist after EACH model so a killed run (session teardown) keeps every finished model + (kill-safe incremental write, not write-once-at-end).""" allc = json.loads(cpath.read_text()) if cpath.exists() else {} allc[sig] = {k: {s: p.tolist() for s, p in v.items()} for k, v in vecs.items()} cpath.write_text(json.dumps(allc)) diff --git a/src/tinymfv/eval.py b/src/tinymfv/eval.py index 698efc1..12df0aa 100644 --- a/src/tinymfv/eval.py +++ b/src/tinymfv/eval.py @@ -131,7 +131,9 @@ def _soft_nll(p_human: np.ndarray, p_model: np.ndarray) -> float: """Soft cross-entropy: -sum_f p_human[f] log p_model[f], in nats. Standard quantity for matching a predicted distribution to a soft-labelled - target. Unbounded; sensitive to confident-wrong rows. + target. Sensitive to confident-wrong rows, but p_model is floored at 1e-12 + (~27.6 nats/row) so a single legit p=0 cell (an unsampled token from the + sampling reader, not a bug) can't send the aggregate mean to +inf. """ p_model = np.clip(p_model, 1e-12, 1.0) return float(-(p_human * np.log(p_model)).sum()) diff --git a/src/tinymfv/guided.py b/src/tinymfv/guided.py index 6b69093..102a60e 100644 --- a/src/tinymfv/guided.py +++ b/src/tinymfv/guided.py @@ -285,7 +285,6 @@ def _rollout_natural_or_forced( # confident answer from garbage. Mark the row UNSCORABLE (pmass=NaN, lp=NaN), # the same "do not compare" signal as case (c): the read is undefined, not zero # coherence, so it drops from the nanmean and counts toward frac_unscorable. - # (Was torch.nan_to_num clamp to +-1e4: silent corruption.) raw = step_scores[answer_pos][i].float() if not torch.isfinite(raw).all(): slots[i].append({ diff --git a/src/tinymfv/read_api.py b/src/tinymfv/read_api.py index f645f68..c365f0a 100644 --- a/src/tinymfv/read_api.py +++ b/src/tinymfv/read_api.py @@ -1,9 +1,8 @@ """Sampling readout for API models that do NOT expose token logprobs (OpenRouter / OpenAI-compatible). -The frontier models on the Economist's WVS chart (GPT-5.4, Claude, Gemini, Grok) return no -next-token distribution, so read.py's forced-slot logprob reader cannot touch them. This is the -fallback the Economist itself used ("average of ten responses"): SAMPLE N chat completions at -temperature and take the empirical answer frequency as the per-(item, frame) categorical `p`. +Chat-completions-only API models return no next-token distribution, so read.py's forced-slot logprob +reader cannot touch them. Fallback (the "average of N responses" approach): SAMPLE N chat completions +at temperature and take the empirical answer frequency as the per-(item, frame) categorical `p`. Why this stays comparable to the logprob reader: temperature-1 sampling is an unbiased estimator of the model's softmax categorical, so the empirical `p` -> the true next-token `p` as N grows, and