From 40d08f8d7a3266d0fed132061ac1f14546ee9358 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Sat, 4 Jul 2026 19:43:24 +0800 Subject: [PATCH] Establish P4 WVS map data foundation (GlobalOpinionQA) Probe + artifact: the WVS subset of Anthropic/llm_global_opinions is 353 questions over 90 countries (212 questions with >=40 countries), matching tinymfv's MC + human-anchor shape and dense enough for an Economist-scale map. Documents the selections parse recipe and the open axis-definition fork (literal IW 10-question factor model vs shared-question ipsative PCA) before model-run compute. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- docs/reviews/p4_wvs_data_foundation.md | 42 ++++++++++++++++++++++ scripts/probe_wvs_foundation.py | 49 ++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 docs/reviews/p4_wvs_data_foundation.md create mode 100644 scripts/probe_wvs_foundation.py diff --git a/docs/reviews/p4_wvs_data_foundation.md b/docs/reviews/p4_wvs_data_foundation.md new file mode 100644 index 0000000..2028f60 --- /dev/null +++ b/docs/reviews/p4_wvs_data_foundation.md @@ -0,0 +1,42 @@ +# P4 data foundation: GlobalOpinionQA WVS subset for a WVS/Inglehart-Welzel map + +Command: `uv run python scripts/probe_wvs_foundation.py`. Date: 2026-07-04. + +`Anthropic/llm_global_opinions` is MC questions with per-country human answer distributions -- the +same shape tinymfv already reads (allowed answer tokens + human anchors), so it drops into our +reader + ipsative-map machinery. + +Coverage of the WVS subset: +- source split: GAS (Pew) 2203, WVS 353. +- WVS questions: 353. Distinct countries: 90 (the Economist chart uses 88). +- countries/question: min 1, median 63, max 90. 212 questions have >=40 countries. +- top-coverage countries: Kenya, Zimbabwe, Ethiopia, Lebanon, Jordan, Iraq, Nigeria, South Korea, + Puerto Rico, Mexico, Malaysia, Colombia (good IW-zone spread). +- `selections` ships as a `defaultdict(...)` repr string; parse recipe is + `ast.literal_eval(re.search(r"\{.*\}", s, re.S).group(0))` (see probe_wvs_foundation.py). +- `options` includes non-substantive tails ("Don't know", "No answer") that must be dropped or held + out of the answer space before renormalizing. + +Verdict: dense enough (90 countries x 353 questions, 212 questions with >=40 countries) to place +models among human societies at Economist scale. This is the human anchor; the model side reuses the +logprob reader (open models) or read_api sampling reader (frontier models). + +## Open research fork (needs a human steer before spending model-run compute) + +How to define the two map axes -- this is a research-validity choice, not a mechanical one, so I am +not guessing it: + +- (4a) Literal Inglehart-Welzel: use the 10 documented IW questions (Traditional: Q164, Q7-17, Q184, + Q254, Q4; Survival: Q154-155, Q46, Q182, Q209, Q57), factor-analyze to the two published axes, + place countries and models on them. Most faithful to the Economist, but fragile: the 10 exact + questions may not all be present in GlobalOpinionQA with consistent country coverage, and it + hard-codes Inglehart-Welzel's factor structure. +- (4b) Shared-question ipsative PCA (our existing map method): pick the WVS questions with dense + country coverage, build a country x question matrix, ipsative-PCA to 2 PCs (reusing + maps.ipsative_pca), and project models administered the same questions. Drops straight into the + P1 hull code. Axes are data-derived (not the literal IW Traditional/Survival), so the chart is + "WVS-values map" not a verbatim Economist reproduction. + +Also to confirm before spending: which model set goes on the map (open-weight via the logprob reader +vs frontier via read_api sampling), and the country subset (all 90 vs the >=40-coverage 212-question +core). diff --git a/scripts/probe_wvs_foundation.py b/scripts/probe_wvs_foundation.py new file mode 100644 index 0000000..1cb2f46 --- /dev/null +++ b/scripts/probe_wvs_foundation.py @@ -0,0 +1,49 @@ +"""P4 data foundation: what the GlobalOpinionQA WVS subset gives us for a WVS/Inglehart-Welzel map. + +Anthropic/llm_global_opinions is MC questions with per-country human answer distributions -- almost +exactly tinymfv's instrument shape (allowed answer tokens + human anchors). This probe quantifies +the WVS subset's coverage and prints the parse recipe, so we know a country x question matrix is +dense enough to place models among human societies. No model runs here. + + uv run python scripts/probe_wvs_foundation.py +""" +from __future__ import annotations + +import ast +import re +from collections import Counter + +import numpy as np +from datasets import load_dataset + + +def parse_selections(s: str) -> dict[str, list[float]]: + """`selections` ships as a repr of a defaultdict; pull the dict literal out and eval it safely.""" + return ast.literal_eval(re.search(r"\{.*\}", s, re.S).group(0)) + + +def main() -> None: + ds = load_dataset("Anthropic/llm_global_opinions", split="train") + by_source = Counter(ds["source"]) + wvs = [r for r in ds if r["source"] == "WVS"] + ncountry, allc = [], Counter() + for r in wvs: + sel = parse_selections(r["selections"]) + ncountry.append(len(sel)) + allc.update(sel.keys()) + print(f"source split: {dict(by_source)}") + print(f"WVS questions: {len(wvs)}") + print(f"distinct countries: {len(allc)}") + print(f"countries/question min/median/max: {min(ncountry)}/{int(np.median(ncountry))}/{max(ncountry)}") + print(f"questions with >=40 countries: {sum(n >= 40 for n in ncountry)}") + print(f"top countries by coverage: {allc.most_common(12)}") + r = wvs[0] + sel = parse_selections(r["selections"]) + c0 = next(iter(sel)) + print("\nexample question:", repr(r["question"])[:160]) + print("options:", ast.literal_eval(r["options"]) if isinstance(r["options"], str) else r["options"]) + print(f"human dist for {c0}:", sel[c0]) + + +if __name__ == "__main__": + main()