mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-09 11:27:22 +08:00
wvs map: dense Likert rated readout with bootstrap CIs (more reliable)
Replace the single forced-choice sampling reader (1 bit/call, truncated verbose models to empty at max_tokens=32, positionally biased) with a dense readout: rate every option 1-5 as JSON, N samples, binary items order-balanced to cancel positional bias, ratings mapped back to canonical order and normalized to a distribution. All of a model's calls fire concurrently (asyncio.gather over the async openrouter_wrapper; per_call=1 since providers ignore n>1). Each model carries a bootstrap 95% CI (over items + samples) drawn as error bars, so mushy models (deepseek-flash was near coin-flip) read as uncertain, not confident dots. One flaky provider is skipped, not fatal. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
+90
-37
@@ -6,11 +6,15 @@ Economist chart), on the two named IW dimensions instead of a blind PCA.
|
||||
|
||||
Each axis is a small hand-picked battery of GlobalOpinionQA WVS items (tinymfv.iw_axes), every item
|
||||
oriented to its axis-positive pole by reading the option order. A country's coordinate is the mean
|
||||
`positiveness` (0-1) over that axis's items from the human WVS distribution; a model's coordinate is
|
||||
the SAME items administered through the answer-token reader (open models: read_items; logprob-less
|
||||
API models: read_items_sampled), reduced identically. This is an APPROXIMATE IW (3 themes/axis, not
|
||||
the canonical 5 -- national pride/authority/materialism are absent from GlobalOpinionQA), not a
|
||||
verbatim reproduction; the caveat is printed on the figure.
|
||||
`positiveness` (0-1) over that axis's items from the human WVS choice frequencies. A model's
|
||||
coordinate is the SAME items administered as a dense Likert readout (read_items_rated: rate every
|
||||
option 1-5 as JSON, binary options order-permuted to cancel positional bias, N samples, normalized
|
||||
mean rating -> distribution), reduced identically; open models can instead use the answer-token
|
||||
logprob reader (read_items). Each model also carries a bootstrap 95% CI (over items + samples) drawn
|
||||
as error bars, so mushy / uncertain placements read as uncertain rather than confident dots. NB the
|
||||
model coordinate is a rating-derived pseudo-distribution while the human one is a real choice
|
||||
frequency -- a documented proxy. This is an APPROXIMATE IW (3 themes/axis, not the canonical 5 --
|
||||
national pride/authority/materialism are absent from GlobalOpinionQA), not a verbatim reproduction.
|
||||
|
||||
uv run python scripts/wvs_map.py --local-model Qwen/Qwen3-0.6B \
|
||||
--api-models meta-llama/llama-3.1-8b-instruct openai/gpt-4o-mini
|
||||
@@ -40,7 +44,7 @@ from tinymfv import maps
|
||||
from tinymfv.zones import zones_for, zone_of
|
||||
from tinymfv.instrument import Instrument, InstrItem
|
||||
from tinymfv.read import read_items, resolve_answer_ids
|
||||
from tinymfv.read_api import read_items_sampled
|
||||
from tinymfv.read_api import read_items_rated
|
||||
from tinymfv.iw_axes import AXIS_ITEMS, X_AXIS, Y_AXIS, SKIP, resolve_items, positiveness
|
||||
|
||||
# option labels are single digits 0..n-1 -- single-token (unlike '10' on the justifiable scale) and
|
||||
@@ -130,22 +134,51 @@ def model_axis_scores(vecs: dict[str, np.ndarray], meta: dict[str, dict],
|
||||
|
||||
|
||||
def read_model(rows: list[dict], meta: dict[str, dict]) -> dict[str, np.ndarray]:
|
||||
"""rows from read_items / read_items_sampled -> {suffix: p over that item's options}. NaN p (read
|
||||
collapse) fails loud later via positiveness rather than being imputed."""
|
||||
"""rows from read_items -> {suffix: p over that item's options}. NaN p (read collapse) fails loud
|
||||
later via positiveness rather than being imputed."""
|
||||
return {r["id"]: np.asarray(r["p"], float)[: meta[r["id"]]["n"]] for r in rows}
|
||||
|
||||
|
||||
def model_coord_ci(psamples: dict[str, np.ndarray], resolved: dict[str, list[dict]],
|
||||
rng: np.random.Generator, B: int = 500) -> tuple[float, float, float, float]:
|
||||
"""(x, y, x_se, y_se). Point estimate = mean positiveness over each axis's items on the mean-over-
|
||||
samples p. The SE is a bootstrap over BOTH noise sources: resample the axis's items with
|
||||
replacement (item-set noise, only ~3-7 items/axis) and, per item, draw one of its N rating samples
|
||||
(readout noise). std over B replicates -> the CI drawn as error bars on the map."""
|
||||
def axis_coords(getp) -> list[float]:
|
||||
return [float(np.mean([positiveness(getp(it), it["pole_idx"], it["n"]) for it in resolved[axis]]))
|
||||
for axis in (X_AXIS, Y_AXIS)]
|
||||
x, y = axis_coords(lambda it: psamples[it["suffix"]].mean(0))
|
||||
bx, by = [], []
|
||||
for _ in range(B):
|
||||
xy = []
|
||||
for axis in (X_AXIS, Y_AXIS):
|
||||
items = resolved[axis]
|
||||
idx = rng.integers(0, len(items), len(items))
|
||||
vals = []
|
||||
for j in idx:
|
||||
it = items[j]
|
||||
ps = psamples[it["suffix"]]
|
||||
vals.append(positiveness(ps[int(rng.integers(0, len(ps)))], it["pole_idx"], it["n"]))
|
||||
xy.append(float(np.mean(vals)))
|
||||
bx.append(xy[0]); by.append(xy[1])
|
||||
return x, y, float(np.std(bx)), float(np.std(by))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--local-model", default="Qwen/Qwen3-0.6B")
|
||||
ap.add_argument("--api-models", nargs="*", default=[])
|
||||
ap.add_argument("--api-samples", type=int, default=20)
|
||||
ap.add_argument("--api-samples", type=int, default=12,
|
||||
help="rating samples per item (each dense: every option rated), binary items order-balanced")
|
||||
ap.add_argument("--api-max-tokens", type=int, default=1024,
|
||||
help="output budget per rating call; large enough that a reasoning model finishes the JSON")
|
||||
ap.add_argument("--max-think-tokens", type=int, default=64)
|
||||
ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
||||
ap.add_argument("--out", default="/tmp/claude-1000/wvs_map_iw.png")
|
||||
ap.add_argument("--cache", default="/tmp/claude-1000/wvs_iw_vectors.json",
|
||||
help="cache model per-item p vectors so re-styling skips the API/model calls")
|
||||
ap.add_argument("--responses", default="/tmp/claude-1000/wvs_iw_responses.jsonl",
|
||||
ap.add_argument("--cache", default="/tmp/claude-1000/wvs_iw_rated.json",
|
||||
help="cache model (x,y[,x_se,y_se]) coords so re-styling skips the API/model calls")
|
||||
ap.add_argument("--responses", default="/tmp/claude-1000/wvs_iw_rated_responses.jsonl",
|
||||
help="append every raw model response here (audit trail; API calls cost money)")
|
||||
args = ap.parse_args()
|
||||
|
||||
@@ -157,38 +190,49 @@ def main() -> None:
|
||||
countries, P = human_axis_scores(resolved)
|
||||
logger.info(f"{len(recs)} WVS questions -> {len(countries)} countries on 2 IW axes")
|
||||
|
||||
instrs, meta = build_instruments(resolved)
|
||||
# DETERMINISTIC key over the item set: Python's builtin hash() is salted per process
|
||||
# (PYTHONHASHSEED), so it changes every run and the cache never hits across processes -- costing a
|
||||
# fresh API call each time. hashlib is stable.
|
||||
sig = hashlib.md5(repr(sorted((s, m["n"]) for s, m in meta.items())).encode()).hexdigest()[:8]
|
||||
# One rated item per distinct WVS question (canonical option order), administered to every API model.
|
||||
rated_items, seen = [], set()
|
||||
for axis in (X_AXIS, Y_AXIS):
|
||||
for it in resolved[axis]:
|
||||
if it["suffix"] in seen:
|
||||
continue
|
||||
seen.add(it["suffix"])
|
||||
rated_items.append({"id": it["suffix"], "question": it["rec"]["q"],
|
||||
"options": it["rec"]["opts"], "n": it["n"]})
|
||||
|
||||
# DETERMINISTIC cache key over the item set: Python's builtin hash() is salted per process
|
||||
# (PYTHONHASHSEED), so it changes every run and the cache never hits -- costing a fresh API call
|
||||
# each time. hashlib is stable. cache value = (x, y[, x_se, y_se]) per model.
|
||||
sig = hashlib.md5(repr(sorted((it["id"], it["n"]) for it in rated_items)).encode()).hexdigest()[:8]
|
||||
cpath = Path(args.cache)
|
||||
cpath.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache = json.loads(cpath.read_text()).get(sig, {}) if cpath.exists() else {}
|
||||
vecs: dict[str, dict[str, np.ndarray]] = {k: {s: np.array(p) for s, p in v.items()}
|
||||
for k, v in cache.items()}
|
||||
models: dict[str, tuple] = {k: tuple(v) for k, v in cache.items()}
|
||||
|
||||
def save_cache() -> None:
|
||||
"""Persist after EACH model so a killed run (session teardown) keeps every finished model
|
||||
(kill-safe incremental write, not write-once-at-end)."""
|
||||
"""Persist after EACH model so a killed run keeps every finished model (kill-safe)."""
|
||||
allc = json.loads(cpath.read_text()) if cpath.exists() else {}
|
||||
allc[sig] = {k: {s: p.tolist() for s, p in v.items()} for k, v in vecs.items()}
|
||||
allc[sig] = {k: list(v) for k, v in models.items()}
|
||||
cpath.write_text(json.dumps(allc))
|
||||
|
||||
rpath = Path(args.responses)
|
||||
rpath.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def save_responses(key: str, rows: list[dict]) -> None:
|
||||
"""Append every raw sampled response (audit trail -- these API calls cost money)."""
|
||||
"""Append every raw rated response (audit trail -- these API calls cost money)."""
|
||||
with rpath.open("a") as fh:
|
||||
for r in rows:
|
||||
fh.write(json.dumps({"model": key, "sig": sig, "item": r["id"],
|
||||
"prompt": r.get("prompt"), "texts": r.get("texts"),
|
||||
"p": np.asarray(r["p"]).tolist(), "pmass": r["pmass_allowed"]}) + "\n")
|
||||
|
||||
rng = np.random.default_rng(0) # deterministic bootstrap
|
||||
|
||||
# Open local model: answer-token logprob reader (single-choice categorical) -> (x, y), no CI.
|
||||
if args.local_model:
|
||||
key = args.local_model.split("/")[-1] + " (lp)"
|
||||
if key not in vecs:
|
||||
if key not in models:
|
||||
instrs, meta = build_instruments(resolved)
|
||||
tok = AutoTokenizer.from_pretrained(args.local_model)
|
||||
if tok.pad_token is None:
|
||||
tok.pad_token = tok.eos_token
|
||||
@@ -200,21 +244,30 @@ def main() -> None:
|
||||
resolve_answer_ids(tok, instr.answer_space),
|
||||
max_think_tokens=args.max_think_tokens, batch_size=16,
|
||||
verbose_first=(k == 0))
|
||||
vecs[key] = read_model(rows, meta)
|
||||
models[key] = model_axis_scores(read_model(rows, meta), meta, resolved)
|
||||
save_cache()
|
||||
for m in args.api_models:
|
||||
key = m.split("/")[-1] + " (sampled)"
|
||||
if key not in vecs:
|
||||
rows = []
|
||||
for k, instr in enumerate(instrs):
|
||||
rows += read_items_sampled(m, instr, instr.items, n_samples=args.api_samples,
|
||||
verbose_first=(k == 0))
|
||||
save_responses(key, rows) # raw answers first (before reducing to p vectors)
|
||||
vecs[key] = read_model(rows, meta)
|
||||
save_cache() # persist this model before starting the next (kill-safe)
|
||||
logger.info(f"cached {key}")
|
||||
|
||||
models = {k: model_axis_scores(v, meta, resolved) for k, v in vecs.items()}
|
||||
# API models: dense rated readout -> (x, y, x_se, y_se) with bootstrap CI.
|
||||
for m in args.api_models:
|
||||
key = m.split("/")[-1] + " (rated)"
|
||||
if key in models:
|
||||
continue
|
||||
try: # one flaky provider / network blip must not abort the panel
|
||||
rows = read_items_rated(m, rated_items, n_samples=args.api_samples,
|
||||
max_tokens=args.api_max_tokens, verbose_first=True)
|
||||
except Exception as e:
|
||||
logger.warning(f"{key}: read failed ({type(e).__name__}: {e}) -> skipping (not cached)")
|
||||
continue
|
||||
save_responses(key, rows) # raw answers first (before reducing)
|
||||
psamples = {r["id"]: np.array(r["p_samples"]) for r in rows}
|
||||
collapsed = [k for k, v in psamples.items() if v.size == 0]
|
||||
if collapsed: # a refusing / off-format model: skip, keep the panel going
|
||||
logger.warning(f"{key}: parse collapse on {collapsed} -> skipping (not cached)")
|
||||
continue
|
||||
models[key] = model_coord_ci(psamples, resolved, rng)
|
||||
save_cache() # persist this model before the next (kill-safe)
|
||||
x, y, xs, ys = models[key]
|
||||
logger.info(f"cached {key}: ({x:.2f}, {y:.2f}) +-({1.96*xs:.02f}, {1.96*ys:.02f}) 95% CI")
|
||||
|
||||
# Render through the SHARED value-map renderer (same one the instrument value maps use): pole
|
||||
# signposts through the human median, 4 auto-selected zone hulls, textalloc labels, model stars.
|
||||
|
||||
Reference in New Issue
Block a user