From f6c22aca30635d83ac16b3f8905e7bb04b930fe2 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Fri, 21 Aug 2026 14:22:35 +0800 Subject: [PATCH] Measure how far each model sits from a human cluster, in cluster SDs Lets a model be described as a 2.9 sigma member of the West instead of "somewhere west of Silicon Valley". Signed per-axis z says which way and how far on a named axis; Mahalanobis says how odd the placement is overall, and uses the cluster covariance because the zones are elongated and tilted. Reads the committed coords in wvs_model_ci.md rather than the coord cache, so it reruns offline without re-querying seventeen models. Co-Authored-By: Claude Opus 5 --- docs/img/wvs/wvs_model_outlier_sd.md | 87 ++++++++++++++++++++++++++++ scripts/wvs_map.py | 38 +++++++++++- scripts/wvs_outlier_table.py | 51 ++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 docs/img/wvs/wvs_model_outlier_sd.md create mode 100644 scripts/wvs_outlier_table.py diff --git a/docs/img/wvs/wvs_model_outlier_sd.md b/docs/img/wvs/wvs_model_outlier_sd.md new file mode 100644 index 0000000..9eece54 --- /dev/null +++ b/docs/img/wvs/wvs_model_outlier_sd.md @@ -0,0 +1,87 @@ +| model | zone | n countries | z self-expr | z secular | Mahalanobis | +|:-------------------|:----------------|--------------:|--------------:|------------:|--------------:| +| gpt-5.5 | African-Islamic | 19 | +2.36 | +6.12 | +6.51 | +| gpt-5.5 | East Asia | 9 | +1.31 | +3.54 | +3.72 | +| gpt-5.5 | Latin America | 14 | +2.35 | +6.45 | +6.51 | +| gpt-5.5 | Orthodox | 14 | +2.94 | +8.06 | +8.52 | +| gpt-5.5 | West | 29 | -0.33 | +2.93 | +4.71 | +| deepseek-v4-pro | African-Islamic | 19 | +3.70 | +5.75 | +5.75 | +| deepseek-v4-pro | East Asia | 9 | +3.10 | +3.14 | +3.46 | +| deepseek-v4-pro | Latin America | 14 | +4.25 | +6.05 | +6.74 | +| deepseek-v4-pro | Orthodox | 14 | +5.25 | +7.48 | +9.04 | +| deepseek-v4-pro | West | 29 | +0.54 | +2.61 | +3.32 | +| grok-4.3 | African-Islamic | 19 | +2.57 | +5.75 | +5.98 | +| grok-4.3 | East Asia | 9 | +1.58 | +3.14 | +3.17 | +| grok-4.3 | Latin America | 14 | +2.64 | +6.05 | +6.19 | +| grok-4.3 | Orthodox | 14 | +3.29 | +7.48 | +8.11 | +| grok-4.3 | West | 29 | -0.19 | +2.61 | +4.08 | +| gemini-2.5-pro | African-Islamic | 19 | +2.88 | +5.39 | +5.46 | +| gemini-2.5-pro | East Asia | 9 | +2.00 | +2.73 | +2.76 | +| gemini-2.5-pro | Latin America | 14 | +3.08 | +5.66 | +5.95 | +| gemini-2.5-pro | Orthodox | 14 | +3.83 | +6.89 | +7.81 | +| gemini-2.5-pro | West | 29 | +0.01 | +2.29 | +3.38 | +| qwen3.7-max | African-Islamic | 19 | +1.85 | +5.27 | +5.69 | +| qwen3.7-max | East Asia | 9 | +0.62 | +2.59 | +2.89 | +| qwen3.7-max | Latin America | 14 | +1.62 | +5.52 | +5.54 | +| qwen3.7-max | Orthodox | 14 | +2.05 | +6.70 | +6.96 | +| qwen3.7-max | West | 29 | -0.66 | +2.18 | +4.01 | +| gpt-5.4 | African-Islamic | 19 | +2.67 | +5.14 | +5.23 | +| gpt-5.4 | East Asia | 9 | +1.72 | +2.46 | +2.47 | +| gpt-5.4 | Latin America | 14 | +2.79 | +5.39 | +5.63 | +| gpt-5.4 | Orthodox | 14 | +3.47 | +6.50 | +7.30 | +| gpt-5.4 | West | 29 | -0.13 | +2.07 | +3.21 | +| gpt-5.3-chat | African-Islamic | 19 | +3.19 | +5.02 | +5.03 | +| gpt-5.3-chat | East Asia | 9 | +2.41 | +2.32 | +2.63 | +| gpt-5.3-chat | Latin America | 14 | +3.52 | +5.26 | +5.78 | +| gpt-5.3-chat | Orthodox | 14 | +4.36 | +6.31 | +7.59 | +| gpt-5.3-chat | West | 29 | +0.21 | +1.96 | +2.69 | +| gemma-4-31b-it | African-Islamic | 19 | +1.74 | +4.90 | +5.29 | +| gemma-4-31b-it | East Asia | 9 | +0.48 | +2.19 | +2.46 | +| gemma-4-31b-it | Latin America | 14 | +1.47 | +5.13 | +5.14 | +| gemma-4-31b-it | Orthodox | 14 | +1.87 | +6.11 | +6.36 | +| gemma-4-31b-it | West | 29 | -0.73 | +1.86 | +3.62 | +| deepseek-v4-flash | African-Islamic | 19 | +3.91 | +4.66 | +4.79 | +| deepseek-v4-flash | East Asia | 9 | +3.38 | +1.91 | +3.39 | +| deepseek-v4-flash | Latin America | 14 | +4.54 | +4.87 | +6.02 | +| deepseek-v4-flash | Orthodox | 14 | +5.60 | +5.72 | +7.92 | +| deepseek-v4-flash | West | 29 | +0.68 | +1.64 | +1.83 | +| llama-4-maverick | African-Islamic | 19 | +4.63 | +4.66 | +5.10 | +| llama-4-maverick | East Asia | 9 | +4.34 | +1.91 | +4.46 | +| llama-4-maverick | Latin America | 14 | +5.56 | +4.87 | +6.69 | +| llama-4-maverick | Orthodox | 14 | +6.84 | +5.72 | +8.82 | +| llama-4-maverick | West | 29 | +1.14 | +1.64 | +1.65 | +| mistral-large-2512 | African-Islamic | 19 | +4.73 | +4.66 | +5.15 | +| mistral-large-2512 | East Asia | 9 | +4.48 | +1.91 | +4.62 | +| mistral-large-2512 | Latin America | 14 | +5.71 | +4.87 | +6.79 | +| mistral-large-2512 | Orthodox | 14 | +7.02 | +5.72 | +8.96 | +| mistral-large-2512 | West | 29 | +1.21 | +1.64 | +1.64 | +| claude-opus-4.6 | African-Islamic | 19 | +4.53 | +4.54 | +4.97 | +| claude-opus-4.6 | East Asia | 9 | +4.21 | +1.78 | +4.34 | +| claude-opus-4.6 | Latin America | 14 | +5.42 | +4.73 | +6.51 | +| claude-opus-4.6 | Orthodox | 14 | +6.67 | +5.53 | +8.57 | +| claude-opus-4.6 | West | 29 | +1.08 | +1.54 | +1.54 | +| grok-4.20 | African-Islamic | 19 | +4.11 | +4.42 | +4.70 | +| grok-4.20 | East Asia | 9 | +3.65 | +1.64 | +3.74 | +| grok-4.20 | Latin America | 14 | +4.83 | +4.60 | +6.03 | +| grok-4.20 | Orthodox | 14 | +5.96 | +5.33 | +7.91 | +| grok-4.20 | West | 29 | +0.81 | +1.43 | +1.47 | +| claude-opus-4.7 | African-Islamic | 19 | +4.11 | +4.17 | +4.55 | +| claude-opus-4.7 | East Asia | 9 | +3.65 | +1.37 | +3.83 | +| claude-opus-4.7 | Latin America | 14 | +4.83 | +4.34 | +5.87 | +| claude-opus-4.7 | Orthodox | 14 | +5.96 | +4.94 | +7.66 | +| claude-opus-4.7 | West | 29 | +0.81 | +1.21 | +1.22 | +| gemma-3-27b-it | African-Islamic | 19 | +4.22 | +4.17 | +4.60 | +| gemma-3-27b-it | East Asia | 9 | +3.79 | +1.37 | +4.00 | +| gemma-3-27b-it | Latin America | 14 | +4.98 | +4.34 | +5.97 | +| gemma-3-27b-it | Orthodox | 14 | +6.13 | +4.94 | +7.79 | +| gemma-3-27b-it | West | 29 | +0.88 | +1.21 | +1.21 | +| claude-opus-4.8 | African-Islamic | 19 | +4.32 | +3.93 | +4.55 | +| claude-opus-4.8 | East Asia | 9 | +3.93 | +1.10 | +4.29 | +| claude-opus-4.8 | Latin America | 14 | +5.13 | +4.08 | +5.94 | +| claude-opus-4.8 | Orthodox | 14 | +6.31 | +4.55 | +7.70 | +| claude-opus-4.8 | West | 29 | +0.94 | +1.00 | +1.04 | +| llama-4-scout | African-Islamic | 19 | +4.42 | +3.32 | +4.46 | +| llama-4-scout | East Asia | 9 | +4.07 | +0.42 | +4.88 | +| llama-4-scout | Latin America | 14 | +5.27 | +3.42 | +5.75 | +| llama-4-scout | Orthodox | 14 | +6.49 | +3.58 | +7.34 | +| llama-4-scout | West | 29 | +1.01 | +0.46 | +1.09 | diff --git a/scripts/wvs_map.py b/scripts/wvs_map.py index cd7dd16..44c7dd6 100644 --- a/scripts/wvs_map.py +++ b/scripts/wvs_map.py @@ -41,7 +41,7 @@ from datasets import load_dataset from transformers import AutoModelForCausalLM, AutoTokenizer from moralmaps import maps -from moralmaps.zones import zones_for, zone_of +from moralmaps.zones import zones_for, zone_of, IW_MACRO from moralmaps.instrument import Instrument, InstrItem from moralmaps.read import read_items, resolve_answer_ids from moralmaps.read_api import read_items_rated @@ -165,6 +165,39 @@ def model_coord_ci(psamples: dict[str, np.ndarray], resolved: dict[str, list[dic return x, y, float(np.std(bx)), float(np.std(by)) +def cluster_outlier_sd(countries: list[str], P: np.ndarray, models: dict[str, tuple], + min_n: int = 8) -> list[tuple]: + """How odd each model looks as a member of each human macro-zone, in cluster SDs. + + Two readings per (model, zone). The signed per-axis z says which way and how far on one named + axis, so `+2.9` on secular-rational reads as "2.9 sigma more secular-rational than the average + member of this zone". The Mahalanobis distance says how odd the placement is overall, using the + zone's own 2x2 covariance; it is the honest scalar because the zones are elongated and tilted + (the West runs diagonally), so a model far along a zone's own long axis is less of an outlier + than a plain z suggests. Zones under min_n countries are skipped: a 2x2 covariance from a handful + of points is mostly noise.""" + by_zone: dict[str, list[int]] = {} + for i, c in enumerate(countries): + z = zone_of(c) + if z is not None: + by_zone.setdefault(IW_MACRO[z], []).append(i) + out = [] + for zone, idx in sorted(by_zone.items()): + if len(idx) < min_n: + continue + Z = P[idx] + mu, sd = Z.mean(0), Z.std(0, ddof=1) + # ridge keeps the inverse finite if a zone is near-degenerate on one axis + S = np.cov(Z.T) + 1e-6 * np.eye(2) + Sinv = np.linalg.inv(S) + for name, v in models.items(): + d = np.array([v[0], v[1]]) - mu + out.append((name.replace(" (rated)", ""), zone, len(idx), + float(d[0] / sd[0]), float(d[1] / sd[1]), + float(np.sqrt(d @ Sinv @ d)))) + return out + + def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--local-model", default="Qwen/Qwen3-0.6B") @@ -282,6 +315,9 @@ def main() -> None: Path(args.out).with_name("wvs_model_ci.md").write_text(table + "\n") logger.info("model coords + 95% CI (widest first):\n" + table) + # scripts/wvs_outlier_table.py turns wvs_model_ci.md into the zone-SD outlier table. It reads the + # committed coords rather than the cache, so it reruns offline without paying for 17 models again. + # Render through the SHARED value-map renderer (same one the instrument value maps use): pole # signposts through the human median, 4 auto-selected zone hulls, auto-placed labels, model stars. _, emph = zones_for(countries) diff --git a/scripts/wvs_outlier_table.py b/scripts/wvs_outlier_table.py new file mode 100644 index 0000000..bf610b0 --- /dev/null +++ b/scripts/wvs_outlier_table.py @@ -0,0 +1,51 @@ +"""How much of a cultural outlier each model is, in the SDs of a human macro-zone. + +Turns the committed model coordinates (docs/img/wvs/wvs_model_ci.md, written by wvs_map.py) into +docs/img/wvs/wvs_model_outlier_sd.md. Reads the table rather than the coord cache so it reruns +offline, without re-querying seventeen models. The human coordinates are recomputed from WVS here, +the same way wvs_map.py computes them. + + uv run python scripts/wvs_outlier_table.py +""" +from __future__ import annotations + +from pathlib import Path + +import numpy as np +from loguru import logger +from tabulate import tabulate + +from moralmaps.iw_axes import resolve_items +from wvs_map import cluster_outlier_sd, human_axis_scores, load_wvs_all + +IMG = Path(__file__).resolve().parent.parent / "docs" / "img" / "wvs" + + +def read_model_coords(path: Path) -> dict[str, tuple]: + """(x, y) per model from the pipe-table wvs_map.py writes. Columns: model, x, y, x CI, y CI.""" + models = {} + for line in path.read_text().splitlines(): + cells = [c.strip() for c in line.strip().strip("|").split("|")] + if len(cells) < 3 or cells[0] in ("model", "") or set(cells[1]) <= set(":- "): + continue + models[cells[0]] = (float(cells[1]), float(cells[2])) + return models + + +def main() -> None: + models = read_model_coords(IMG / "wvs_model_ci.md") + countries, P = human_axis_scores(resolve_items(load_wvs_all())) + logger.info(f"{len(models)} models against {len(countries)} human societies") + + rows = cluster_outlier_sd(countries, P, models) + west_z = {n: z for n, zone, _, _, z, _ in rows if zone == "West"} + rows.sort(key=lambda r: (-west_z.get(r[0], 0.0), r[0], r[1])) + table = tabulate(rows, headers=["model", "zone", "n countries", + "z self-expr", "z secular", "Mahalanobis"], + tablefmt="pipe", floatfmt="+.2f") + (IMG / "wvs_model_outlier_sd.md").write_text(table + "\n") + logger.info("model distance from human zones, in zone SDs:\n" + table) + + +if __name__ == "__main__": + main()