mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-09 11:27:22 +08:00
Measure how far each model sits from a human cluster, in cluster SDs
Lets a model be described as a 2.9 sigma member of the West instead of "somewhere west of Silicon Valley". Signed per-axis z says which way and how far on a named axis; Mahalanobis says how odd the placement is overall, and uses the cluster covariance because the zones are elongated and tilted. Reads the committed coords in wvs_model_ci.md rather than the coord cache, so it reruns offline without re-querying seventeen models. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
wassname
co-authored by
Claude Opus 5
Claude Opus 5
parent
01e9026641
commit
f6c22aca30
@@ -0,0 +1,87 @@
|
||||
| model | zone | n countries | z self-expr | z secular | Mahalanobis |
|
||||
|:-------------------|:----------------|--------------:|--------------:|------------:|--------------:|
|
||||
| gpt-5.5 | African-Islamic | 19 | +2.36 | +6.12 | +6.51 |
|
||||
| gpt-5.5 | East Asia | 9 | +1.31 | +3.54 | +3.72 |
|
||||
| gpt-5.5 | Latin America | 14 | +2.35 | +6.45 | +6.51 |
|
||||
| gpt-5.5 | Orthodox | 14 | +2.94 | +8.06 | +8.52 |
|
||||
| gpt-5.5 | West | 29 | -0.33 | +2.93 | +4.71 |
|
||||
| deepseek-v4-pro | African-Islamic | 19 | +3.70 | +5.75 | +5.75 |
|
||||
| deepseek-v4-pro | East Asia | 9 | +3.10 | +3.14 | +3.46 |
|
||||
| deepseek-v4-pro | Latin America | 14 | +4.25 | +6.05 | +6.74 |
|
||||
| deepseek-v4-pro | Orthodox | 14 | +5.25 | +7.48 | +9.04 |
|
||||
| deepseek-v4-pro | West | 29 | +0.54 | +2.61 | +3.32 |
|
||||
| grok-4.3 | African-Islamic | 19 | +2.57 | +5.75 | +5.98 |
|
||||
| grok-4.3 | East Asia | 9 | +1.58 | +3.14 | +3.17 |
|
||||
| grok-4.3 | Latin America | 14 | +2.64 | +6.05 | +6.19 |
|
||||
| grok-4.3 | Orthodox | 14 | +3.29 | +7.48 | +8.11 |
|
||||
| grok-4.3 | West | 29 | -0.19 | +2.61 | +4.08 |
|
||||
| gemini-2.5-pro | African-Islamic | 19 | +2.88 | +5.39 | +5.46 |
|
||||
| gemini-2.5-pro | East Asia | 9 | +2.00 | +2.73 | +2.76 |
|
||||
| gemini-2.5-pro | Latin America | 14 | +3.08 | +5.66 | +5.95 |
|
||||
| gemini-2.5-pro | Orthodox | 14 | +3.83 | +6.89 | +7.81 |
|
||||
| gemini-2.5-pro | West | 29 | +0.01 | +2.29 | +3.38 |
|
||||
| qwen3.7-max | African-Islamic | 19 | +1.85 | +5.27 | +5.69 |
|
||||
| qwen3.7-max | East Asia | 9 | +0.62 | +2.59 | +2.89 |
|
||||
| qwen3.7-max | Latin America | 14 | +1.62 | +5.52 | +5.54 |
|
||||
| qwen3.7-max | Orthodox | 14 | +2.05 | +6.70 | +6.96 |
|
||||
| qwen3.7-max | West | 29 | -0.66 | +2.18 | +4.01 |
|
||||
| gpt-5.4 | African-Islamic | 19 | +2.67 | +5.14 | +5.23 |
|
||||
| gpt-5.4 | East Asia | 9 | +1.72 | +2.46 | +2.47 |
|
||||
| gpt-5.4 | Latin America | 14 | +2.79 | +5.39 | +5.63 |
|
||||
| gpt-5.4 | Orthodox | 14 | +3.47 | +6.50 | +7.30 |
|
||||
| gpt-5.4 | West | 29 | -0.13 | +2.07 | +3.21 |
|
||||
| gpt-5.3-chat | African-Islamic | 19 | +3.19 | +5.02 | +5.03 |
|
||||
| gpt-5.3-chat | East Asia | 9 | +2.41 | +2.32 | +2.63 |
|
||||
| gpt-5.3-chat | Latin America | 14 | +3.52 | +5.26 | +5.78 |
|
||||
| gpt-5.3-chat | Orthodox | 14 | +4.36 | +6.31 | +7.59 |
|
||||
| gpt-5.3-chat | West | 29 | +0.21 | +1.96 | +2.69 |
|
||||
| gemma-4-31b-it | African-Islamic | 19 | +1.74 | +4.90 | +5.29 |
|
||||
| gemma-4-31b-it | East Asia | 9 | +0.48 | +2.19 | +2.46 |
|
||||
| gemma-4-31b-it | Latin America | 14 | +1.47 | +5.13 | +5.14 |
|
||||
| gemma-4-31b-it | Orthodox | 14 | +1.87 | +6.11 | +6.36 |
|
||||
| gemma-4-31b-it | West | 29 | -0.73 | +1.86 | +3.62 |
|
||||
| deepseek-v4-flash | African-Islamic | 19 | +3.91 | +4.66 | +4.79 |
|
||||
| deepseek-v4-flash | East Asia | 9 | +3.38 | +1.91 | +3.39 |
|
||||
| deepseek-v4-flash | Latin America | 14 | +4.54 | +4.87 | +6.02 |
|
||||
| deepseek-v4-flash | Orthodox | 14 | +5.60 | +5.72 | +7.92 |
|
||||
| deepseek-v4-flash | West | 29 | +0.68 | +1.64 | +1.83 |
|
||||
| llama-4-maverick | African-Islamic | 19 | +4.63 | +4.66 | +5.10 |
|
||||
| llama-4-maverick | East Asia | 9 | +4.34 | +1.91 | +4.46 |
|
||||
| llama-4-maverick | Latin America | 14 | +5.56 | +4.87 | +6.69 |
|
||||
| llama-4-maverick | Orthodox | 14 | +6.84 | +5.72 | +8.82 |
|
||||
| llama-4-maverick | West | 29 | +1.14 | +1.64 | +1.65 |
|
||||
| mistral-large-2512 | African-Islamic | 19 | +4.73 | +4.66 | +5.15 |
|
||||
| mistral-large-2512 | East Asia | 9 | +4.48 | +1.91 | +4.62 |
|
||||
| mistral-large-2512 | Latin America | 14 | +5.71 | +4.87 | +6.79 |
|
||||
| mistral-large-2512 | Orthodox | 14 | +7.02 | +5.72 | +8.96 |
|
||||
| mistral-large-2512 | West | 29 | +1.21 | +1.64 | +1.64 |
|
||||
| claude-opus-4.6 | African-Islamic | 19 | +4.53 | +4.54 | +4.97 |
|
||||
| claude-opus-4.6 | East Asia | 9 | +4.21 | +1.78 | +4.34 |
|
||||
| claude-opus-4.6 | Latin America | 14 | +5.42 | +4.73 | +6.51 |
|
||||
| claude-opus-4.6 | Orthodox | 14 | +6.67 | +5.53 | +8.57 |
|
||||
| claude-opus-4.6 | West | 29 | +1.08 | +1.54 | +1.54 |
|
||||
| grok-4.20 | African-Islamic | 19 | +4.11 | +4.42 | +4.70 |
|
||||
| grok-4.20 | East Asia | 9 | +3.65 | +1.64 | +3.74 |
|
||||
| grok-4.20 | Latin America | 14 | +4.83 | +4.60 | +6.03 |
|
||||
| grok-4.20 | Orthodox | 14 | +5.96 | +5.33 | +7.91 |
|
||||
| grok-4.20 | West | 29 | +0.81 | +1.43 | +1.47 |
|
||||
| claude-opus-4.7 | African-Islamic | 19 | +4.11 | +4.17 | +4.55 |
|
||||
| claude-opus-4.7 | East Asia | 9 | +3.65 | +1.37 | +3.83 |
|
||||
| claude-opus-4.7 | Latin America | 14 | +4.83 | +4.34 | +5.87 |
|
||||
| claude-opus-4.7 | Orthodox | 14 | +5.96 | +4.94 | +7.66 |
|
||||
| claude-opus-4.7 | West | 29 | +0.81 | +1.21 | +1.22 |
|
||||
| gemma-3-27b-it | African-Islamic | 19 | +4.22 | +4.17 | +4.60 |
|
||||
| gemma-3-27b-it | East Asia | 9 | +3.79 | +1.37 | +4.00 |
|
||||
| gemma-3-27b-it | Latin America | 14 | +4.98 | +4.34 | +5.97 |
|
||||
| gemma-3-27b-it | Orthodox | 14 | +6.13 | +4.94 | +7.79 |
|
||||
| gemma-3-27b-it | West | 29 | +0.88 | +1.21 | +1.21 |
|
||||
| claude-opus-4.8 | African-Islamic | 19 | +4.32 | +3.93 | +4.55 |
|
||||
| claude-opus-4.8 | East Asia | 9 | +3.93 | +1.10 | +4.29 |
|
||||
| claude-opus-4.8 | Latin America | 14 | +5.13 | +4.08 | +5.94 |
|
||||
| claude-opus-4.8 | Orthodox | 14 | +6.31 | +4.55 | +7.70 |
|
||||
| claude-opus-4.8 | West | 29 | +0.94 | +1.00 | +1.04 |
|
||||
| llama-4-scout | African-Islamic | 19 | +4.42 | +3.32 | +4.46 |
|
||||
| llama-4-scout | East Asia | 9 | +4.07 | +0.42 | +4.88 |
|
||||
| llama-4-scout | Latin America | 14 | +5.27 | +3.42 | +5.75 |
|
||||
| llama-4-scout | Orthodox | 14 | +6.49 | +3.58 | +7.34 |
|
||||
| llama-4-scout | West | 29 | +1.01 | +0.46 | +1.09 |
|
||||
+37
-1
@@ -41,7 +41,7 @@ from datasets import load_dataset
|
||||
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
from moralmaps import maps
|
||||
from moralmaps.zones import zones_for, zone_of
|
||||
from moralmaps.zones import zones_for, zone_of, IW_MACRO
|
||||
from moralmaps.instrument import Instrument, InstrItem
|
||||
from moralmaps.read import read_items, resolve_answer_ids
|
||||
from moralmaps.read_api import read_items_rated
|
||||
@@ -165,6 +165,39 @@ def model_coord_ci(psamples: dict[str, np.ndarray], resolved: dict[str, list[dic
|
||||
return x, y, float(np.std(bx)), float(np.std(by))
|
||||
|
||||
|
||||
def cluster_outlier_sd(countries: list[str], P: np.ndarray, models: dict[str, tuple],
|
||||
min_n: int = 8) -> list[tuple]:
|
||||
"""How odd each model looks as a member of each human macro-zone, in cluster SDs.
|
||||
|
||||
Two readings per (model, zone). The signed per-axis z says which way and how far on one named
|
||||
axis, so `+2.9` on secular-rational reads as "2.9 sigma more secular-rational than the average
|
||||
member of this zone". The Mahalanobis distance says how odd the placement is overall, using the
|
||||
zone's own 2x2 covariance; it is the honest scalar because the zones are elongated and tilted
|
||||
(the West runs diagonally), so a model far along a zone's own long axis is less of an outlier
|
||||
than a plain z suggests. Zones under min_n countries are skipped: a 2x2 covariance from a handful
|
||||
of points is mostly noise."""
|
||||
by_zone: dict[str, list[int]] = {}
|
||||
for i, c in enumerate(countries):
|
||||
z = zone_of(c)
|
||||
if z is not None:
|
||||
by_zone.setdefault(IW_MACRO[z], []).append(i)
|
||||
out = []
|
||||
for zone, idx in sorted(by_zone.items()):
|
||||
if len(idx) < min_n:
|
||||
continue
|
||||
Z = P[idx]
|
||||
mu, sd = Z.mean(0), Z.std(0, ddof=1)
|
||||
# ridge keeps the inverse finite if a zone is near-degenerate on one axis
|
||||
S = np.cov(Z.T) + 1e-6 * np.eye(2)
|
||||
Sinv = np.linalg.inv(S)
|
||||
for name, v in models.items():
|
||||
d = np.array([v[0], v[1]]) - mu
|
||||
out.append((name.replace(" (rated)", ""), zone, len(idx),
|
||||
float(d[0] / sd[0]), float(d[1] / sd[1]),
|
||||
float(np.sqrt(d @ Sinv @ d))))
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--local-model", default="Qwen/Qwen3-0.6B")
|
||||
@@ -282,6 +315,9 @@ def main() -> None:
|
||||
Path(args.out).with_name("wvs_model_ci.md").write_text(table + "\n")
|
||||
logger.info("model coords + 95% CI (widest first):\n" + table)
|
||||
|
||||
# scripts/wvs_outlier_table.py turns wvs_model_ci.md into the zone-SD outlier table. It reads the
|
||||
# committed coords rather than the cache, so it reruns offline without paying for 17 models again.
|
||||
|
||||
# Render through the SHARED value-map renderer (same one the instrument value maps use): pole
|
||||
# signposts through the human median, 4 auto-selected zone hulls, auto-placed labels, model stars.
|
||||
_, emph = zones_for(countries)
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
"""How much of a cultural outlier each model is, in the SDs of a human macro-zone.
|
||||
|
||||
Turns the committed model coordinates (docs/img/wvs/wvs_model_ci.md, written by wvs_map.py) into
|
||||
docs/img/wvs/wvs_model_outlier_sd.md. Reads the table rather than the coord cache so it reruns
|
||||
offline, without re-querying seventeen models. The human coordinates are recomputed from WVS here,
|
||||
the same way wvs_map.py computes them.
|
||||
|
||||
uv run python scripts/wvs_outlier_table.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
from loguru import logger
|
||||
from tabulate import tabulate
|
||||
|
||||
from moralmaps.iw_axes import resolve_items
|
||||
from wvs_map import cluster_outlier_sd, human_axis_scores, load_wvs_all
|
||||
|
||||
IMG = Path(__file__).resolve().parent.parent / "docs" / "img" / "wvs"
|
||||
|
||||
|
||||
def read_model_coords(path: Path) -> dict[str, tuple]:
|
||||
"""(x, y) per model from the pipe-table wvs_map.py writes. Columns: model, x, y, x CI, y CI."""
|
||||
models = {}
|
||||
for line in path.read_text().splitlines():
|
||||
cells = [c.strip() for c in line.strip().strip("|").split("|")]
|
||||
if len(cells) < 3 or cells[0] in ("model", "") or set(cells[1]) <= set(":- "):
|
||||
continue
|
||||
models[cells[0]] = (float(cells[1]), float(cells[2]))
|
||||
return models
|
||||
|
||||
|
||||
def main() -> None:
|
||||
models = read_model_coords(IMG / "wvs_model_ci.md")
|
||||
countries, P = human_axis_scores(resolve_items(load_wvs_all()))
|
||||
logger.info(f"{len(models)} models against {len(countries)} human societies")
|
||||
|
||||
rows = cluster_outlier_sd(countries, P, models)
|
||||
west_z = {n: z for n, zone, _, _, z, _ in rows if zone == "West"}
|
||||
rows.sort(key=lambda r: (-west_z.get(r[0], 0.0), r[0], r[1]))
|
||||
table = tabulate(rows, headers=["model", "zone", "n countries",
|
||||
"z self-expr", "z secular", "Mahalanobis"],
|
||||
tablefmt="pipe", floatfmt="+.2f")
|
||||
(IMG / "wvs_model_outlier_sd.md").write_text(table + "\n")
|
||||
logger.info("model distance from human zones, in zone SDs:\n" + table)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user