Record durable WVS API panels

Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com>

scripts/wvs_map.py

@@ -44,7 +44,7 @@ from moralmaps import maps
 from moralmaps.zones import zones_for, zone_of, IW_MACRO
 from moralmaps.instrument import Instrument, InstrItem
 from moralmaps.read import read_items, resolve_answer_ids
-from moralmaps.read_api import read_items_rated
+from moralmaps.read_api import rated_protocol_identity, read_items_rated
 from moralmaps.iw_axes import AXIS_ITEMS, X_AXIS, Y_AXIS, SKIP, resolve_items, positiveness
 
 # option labels are single digits 0..n-1 -- single-token (unlike '10' on the justifiable scale) and
@@ -52,6 +52,28 @@ from moralmaps.iw_axes import AXIS_ITEMS, X_AXIS, Y_AXIS, SKIP, resolve_items, p
 # favour of the option word).
 DIGITS = "0123456789"
 
+# OpenRouter model IDs checked against https://openrouter.ai/api/v1/models on 2026-09-16.
+# Selecting a set is explicit because every uncached entry makes paid API calls.
+API_MODEL_SETS = {
+    "fable-astra": (
+        "anthropic/claude-fable-5.1",
+        "openai/gpt-6-astra",
+    ),
+    "recent": (
+        "anthropic/claude-fable-5.1",
+        "openai/gpt-6-astra",
+        "meta/muse-spark-1.3",
+        "moonshotai/kimi-k3",
+        "thinkingmachines/inkling",
+        "deepseek/deepseek-v4.1-flash",
+        "z-ai/glm-5.3",
+        "z-ai/glm-5.3-flash",
+        "google/gemini-3.7-flash",
+        "x-ai/grok-4.5",
+        "openai/gpt-5.6-sol",
+    ),
+}
+
 
 def load_wvs_all() -> list[dict]:
     """Every WVS question with its substantive options (DK/refusal/Missing/INAP dropped) and each
@@ -200,20 +222,36 @@ def cluster_outlier_sd(countries: list[str], P: np.ndarray, models: dict[str, tu
 
 def main() -> None:
     ap = argparse.ArgumentParser()
-    ap.add_argument("--local-model", default="Qwen/Qwen3-0.6B")
+    ap.add_argument("--local-model", default="",
+                    help="optional local checkpoint, blank preserves the API-only published map")
     ap.add_argument("--api-models", nargs="*", default=[])
+    ap.add_argument("--api-model-set", choices=API_MODEL_SETS,
+                    help="explicit paid OpenRouter model set, combined with --api-models")
     ap.add_argument("--api-samples", type=int, default=12,
                     help="rating samples per item (each dense: every option rated), binary items order-balanced")
+    ap.add_argument("--api-concurrency", type=int, default=8,
+                    help="maximum concurrent OpenRouter calls, reduced for a provider that reports rate limits")
+    ap.add_argument("--api-request-timeout", type=float, default=90.0)
     ap.add_argument("--api-max-tokens", type=int, default=1024,
                     help="output budget per rating call; large enough that a reasoning model finishes the JSON")
+    reasoning_group = ap.add_mutually_exclusive_group()
+    reasoning_group.add_argument("--api-disable-reasoning", action="store_true",
+                                 help="send reasoning.enabled=false for models whose catalog metadata says optional")
+    reasoning_group.add_argument("--api-reasoning-effort",
+                                 help="send a mandatory model's catalog-supported minimum reasoning effort")
+    ap.add_argument("--api-structured-output", action="store_true",
+                    help="request a strict rating JSON schema only for a catalog-confirmed supporting model")
     ap.add_argument("--max-think-tokens", type=int, default=64)
     ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
-    ap.add_argument("--out", default="/tmp/claude-1000/wvs_map_iw.png")
-    ap.add_argument("--cache", default="/tmp/claude-1000/wvs_iw_rated.json",
-                    help="cache model (x,y[,x_se,y_se]) coords so re-styling skips the API/model calls")
-    ap.add_argument("--responses", default="/tmp/claude-1000/wvs_iw_rated_responses.jsonl",
-                    help="append every raw model response here (audit trail; API calls cost money)")
+    ap.add_argument("--out", default="docs/img/wvs/wvs_map_iw.png")
+    ap.add_argument("--cache", default="slop/research/wvs/20260916_openrouter/wvs_iw_rated.json",
+                    help="durable completed-panel cache, tracked with the request evidence")
+    ap.add_argument("--records", default="slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
+                    help="fsynced JSONL request ledger, outside /tmp and retained for reuse")
     args = ap.parse_args()
+    api_models = list(dict.fromkeys(args.api_models + list(API_MODEL_SETS.get(args.api_model_set, ()))))
+    api_reasoning = ({"enabled": False} if args.api_disable_reasoning else
+                     {"effort": args.api_reasoning_effort} if args.api_reasoning_effort else None)
 
     recs = load_wvs_all()
     resolved = resolve_items(recs)
@@ -233,31 +271,31 @@ def main() -> None:
             rated_items.append({"id": it["suffix"], "question": it["rec"]["q"],
                                 "options": it["rec"]["opts"], "n": it["n"]})
 
-    # DETERMINISTIC cache key over the item set: Python's builtin hash() is salted per process
-    # (PYTHONHASHSEED), so it changes every run and the cache never hits -- costing a fresh API call
-    # each time. hashlib is stable. cache value = (x, y[, x_se, y_se]) per model.
-    sig = hashlib.md5(repr(sorted((it["id"], it["n"]) for it in rated_items)).encode()).hexdigest()[:8]
     cpath = Path(args.cache)
     cpath.parent.mkdir(parents=True, exist_ok=True)
-    cache = json.loads(cpath.read_text()).get(sig, {}) if cpath.exists() else {}
-    models: dict[str, tuple] = {k: tuple(v) for k, v in cache.items()}
+    cache = json.loads(cpath.read_text()) if cpath.exists() else {"schema": 2, "completed": {}}
+    if cache["schema"] != 2:
+        raise ValueError(f"unsupported WVS cache schema {cache['schema']}")
+
+    def published_models(path: Path) -> dict[str, tuple]:
+        """Reuse the committed historical coordinates, which are rounded display values, not raw reruns."""
+        models = {}
+        for line in path.read_text().splitlines():
+            cells = [c.strip() for c in line.strip().strip("|").split("|")]
+            if len(cells) < 5 or cells[0] in ("model", "") or set(cells[1]) <= set(":- "):
+                continue
+            x, y, x_ci95, y_ci95 = (float(cell) for cell in cells[1:5])
+            models[cells[0]] = (x, y, x_ci95 / 1.96, y_ci95 / 1.96)
+        return models
+
+    published_ci = Path("docs/img/wvs/wvs_model_ci.md")
+    models: dict[str, tuple] = published_models(published_ci) if published_ci.exists() else {}
 
     def save_cache() -> None:
-        """Persist after EACH model so a killed run keeps every finished model (kill-safe)."""
-        allc = json.loads(cpath.read_text()) if cpath.exists() else {}
-        allc[sig] = {k: list(v) for k, v in models.items()}
-        cpath.write_text(json.dumps(allc))
-
-    rpath = Path(args.responses)
-    rpath.parent.mkdir(parents=True, exist_ok=True)
-
-    def save_responses(key: str, rows: list[dict]) -> None:
-        """Append every raw rated response (audit trail -- these API calls cost money)."""
-        with rpath.open("a") as fh:
-            for r in rows:
-                fh.write(json.dumps({"model": key, "sig": sig, "item": r["id"],
-                                     "prompt": r.get("prompt"), "texts": r.get("texts"),
-                                     "p": np.asarray(r["p"]).tolist(), "pmass": r["pmass_allowed"]}) + "\n")
+        """Atomic cache replacement after a complete model panel, so interruption cannot fabricate a hit."""
+        temp = cpath.with_suffix(cpath.suffix + ".tmp")
+        temp.write_text(json.dumps(cache, indent=2, sort_keys=True) + "\n")
+        temp.replace(cpath)
 
     rng = np.random.default_rng(0)                       # deterministic bootstrap
 
@@ -281,24 +319,40 @@ def main() -> None:
             save_cache()
 
     # API models: dense rated readout -> (x, y, x_se, y_se) with bootstrap CI.
-    for m in args.api_models:
+    for m in api_models:
         key = m.split("/")[-1] + " (rated)"
-        if key in models:
+        protocol_id = rated_protocol_identity(
+            m, rated_items, n_samples=args.api_samples, temperature=1.0,
+            max_tokens=args.api_max_tokens, concurrency=args.api_concurrency,
+            req_timeout=args.api_request_timeout, reasoning=api_reasoning,
+            structured_output=args.api_structured_output)
+        completed = cache["completed"].get(protocol_id)
+        if completed is not None:
+            models[key] = tuple(completed["coords"])
+            logger.info(f"cache hit {key}: protocol={protocol_id[:12]}")
             continue
-        try:                                # one flaky provider / network blip must not abort the panel
-            rows = read_items_rated(m, rated_items, n_samples=args.api_samples,
-                                    max_tokens=args.api_max_tokens, verbose_first=True)
-        except Exception as e:
-            logger.warning(f"{key}: read failed ({type(e).__name__}: {e}) -> skipping (not cached)")
-            continue
-        save_responses(key, rows)           # raw answers first (before reducing)
-        psamples = {r["id"]: np.array(r["p_samples"]) for r in rows}
-        collapsed = [k for k, v in psamples.items() if v.size == 0]
-        if collapsed:                       # a refusing / off-format model: skip, keep the panel going
-            logger.warning(f"{key}: parse collapse on {collapsed} -> skipping (not cached)")
+        rows = read_items_rated(m, rated_items, n_samples=args.api_samples,
+                                max_tokens=args.api_max_tokens, concurrency=args.api_concurrency,
+                                req_timeout=args.api_request_timeout, reasoning=api_reasoning,
+                                structured_output=args.api_structured_output,
+                                records_path=args.records, verbose_first=True)
+        incomplete = [row["id"] for row in rows if row["valid_samples"] != args.api_samples]
+        if incomplete:
+            logger.warning(f"{key}: incomplete items {incomplete}; raw evidence is in {args.records}; not cached or plotted")
             continue
+        psamples = {row["id"]: np.array(row["p_samples"]) for row in rows}
         models[key] = model_coord_ci(psamples, resolved, rng)
-        save_cache()                        # persist this model before the next (kill-safe)
+        cache["completed"][protocol_id] = {
+            "model": m,
+            "display_key": key,
+            "coords": list(models[key]),
+            "records_path": args.records,
+            "run_id": rows[0]["run_id"],
+            "protocol_id": protocol_id,
+            "n_items": len(rows),
+            "n_samples": args.api_samples,
+        }
+        save_cache()
         x, y, xs, ys = models[key]
         logger.info(f"cached {key}: ({x:.2f}, {y:.2f}) +-({1.96*xs:.02f}, {1.96*ys:.02f}) 95% CI")
 
@@ -332,7 +386,10 @@ def main() -> None:
     # reads "opus-4.8". Colour + legend carry the unlabelled siblings.
     fams: dict[str, list[str]] = {}
     for k in plot_models:
-        fams.setdefault(maps.model_family_color(k), []).append(k)
+        family = maps.model_family(k)
+        if family is None:
+            raise ValueError(f"model has no explicit family: {k}")
+        fams.setdefault(family, []).append(k)
     def _ver(k: str) -> list[float]:
         return [float(n) for n in re.findall(r"\d+(?:\.\d+)?", k)]
     model_labels = {max(ks, key=_ver): max(ks, key=_ver).replace("claude-", "") for ks in fams.values()}

scripts/wvs_model_inventory.py

@@ -0,0 +1,107 @@
+"""Build the checked OpenRouter WVS candidate inventory from one saved catalog response."""
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import UTC, datetime
+from pathlib import Path
+
+TARGET_IDS = (
+    "anthropic/claude-fable-5.1",
+    "openai/gpt-6-astra",
+    "meta/muse-spark-1.3",
+    "moonshotai/kimi-k3",
+    "thinkingmachines/inkling",
+    "deepseek/deepseek-v4.1-flash",
+    "z-ai/glm-5.3",
+    "z-ai/glm-5.3-flash",
+    "google/gemini-3.7-flash",
+    "x-ai/grok-4.5",
+    "openai/gpt-5.6-sol",
+)
+
+
+def usd_per_million(value: str) -> float:
+    return float(value) * 1_000_000
+
+
+def model_row(model: dict) -> dict:
+    pricing = model["pricing"]
+    return {
+        "id": model["id"],
+        "name": model["name"],
+        "created": datetime.fromtimestamp(model["created"], UTC).date().isoformat(),
+        "input_usd_per_million": usd_per_million(pricing["prompt"]),
+        "output_usd_per_million": usd_per_million(pricing["completion"]),
+        "supports_temperature": "temperature" in model["supported_parameters"],
+        "supports_max_tokens": "max_tokens" in model["supported_parameters"],
+        "expiration_date": model["expiration_date"],
+    }
+
+
+def is_direct_qwen(model: dict, checked_at: str) -> bool:
+    model_id = model["id"]
+    expiration = model["expiration_date"]
+    return (
+        model_id.startswith("qwen/")
+        and ":" not in model_id
+        and (expiration is None or expiration >= checked_at)
+        and "temperature" in model["supported_parameters"]
+        and "max_tokens" in model["supported_parameters"]
+    )
+
+
+def markdown_table(rows: list[dict]) -> str:
+    header = "| exact OpenRouter ID | name | created UTC | input USD/M | output USD/M | status |\n"
+    rule = "|---|---|---:|---:|---:|---|\n"
+    body = "".join(
+        f"| `{row['id']}` | {row['name']} | {row['created']} | "
+        f"{row['input_usd_per_million']:.6g} | {row['output_usd_per_million']:.6g} | {row['status']} |\n"
+        for row in rows
+    )
+    return header + rule + body
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--catalog", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--metadata", type=Path, required=True)
+    parser.add_argument("--checked-at", default="2026-09-16")
+    args = parser.parse_args()
+
+    catalog = json.loads(args.catalog.read_text())["data"]
+    by_id = {model["id"]: model for model in catalog}
+    targets = []
+    for model_id in TARGET_IDS:
+        if model_id in by_id:
+            row = model_row(by_id[model_id])
+            row["status"] = "candidate"
+            targets.append(row)
+        else:
+            targets.append({"id": model_id, "name": "not in checked catalog", "created": "",
+                            "input_usd_per_million": 0.0, "output_usd_per_million": 0.0,
+                            "status": "unavailable"})
+    qwen = [model_row(model) for model in catalog if is_direct_qwen(model, args.checked_at)]
+    for row in qwen:
+        row["status"] = "candidate"
+    qwen.sort(key=lambda row: row["created"], reverse=True)
+
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(
+        "# OpenRouter WVS model inventory\n\n"
+        f"Checked {args.checked_at} against `{args.catalog}`. Prices are catalog USD per million tokens. "
+        "Batch and free aliases are excluded because they duplicate an underlying model. Qwen entries with "
+        "an expiration date before the check date are excluded. The remaining direct `qwen/` text-capable "
+        "releases are candidates, not evidence that they completed the panel.\n\n"
+        "## Requested additions\n\n" + markdown_table(targets) +
+        "\n## Direct Qwen candidates\n\n" + markdown_table(qwen) +
+        "\nThe catalog did not contain the requested ID when an addition is marked unavailable. No similar ID "
+        "was substituted. -- PI[gpt-5.6-terra]\n"
+    )
+    metadata = {row["id"].split("/", 1)[1]: row for row in targets + qwen if row["status"] == "candidate"}
+    args.metadata.write_text(json.dumps({"checked_at": args.checked_at, "models": metadata}, indent=2) + "\n")
+
+
+if __name__ == "__main__":
+    main()

scripts/wvs_request_audit.py

@@ -0,0 +1,113 @@
+"""Summarize durable WVS request records without discarding provider billing fields."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from collections import Counter
+from pathlib import Path
+
+USAGE_FIELDS = (
+    "prompt_tokens",
+    "completion_tokens",
+    "reasoning_tokens",
+    "cache_read_input_tokens",
+    "cache_write_input_tokens",
+    "total_tokens",
+    "cost",
+)
+
+
+def sum_usage(records: list[dict]) -> dict[str, float | None]:
+    totals: dict[str, float | None] = {}
+    for field in USAGE_FIELDS:
+        values = [record["usage"][field] for record in records if record["usage"] is not None and field in record["usage"]]
+        totals[field] = sum(values) if values else None
+    return totals
+
+
+def format_value(value: float | None) -> str:
+    return "unknown" if value is None else f"{value:g}"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--records", type=Path, required=True)
+    parser.add_argument("--out", type=Path, required=True)
+    parser.add_argument("--model")
+    parser.add_argument("--run-id")
+    args = parser.parse_args()
+
+    rows = [json.loads(line) for line in args.records.read_text().splitlines()]
+    runs = sorted({(row["model"], row["run_id"]) for row in rows if "model" in row and "run_id" in row})
+    if args.model is not None:
+        runs = [run for run in runs if run[0] == args.model]
+    if args.run_id is not None:
+        runs = [run for run in runs if run[1] == args.run_id]
+    lines = ["# WVS request-ledger audit", "", f"Source: `{args.records}`.", ""]
+    for model, run_id in runs:
+        records = [row for row in rows if row.get("model") == model and row.get("run_id") == run_id]
+        completed = [row for row in records if row["event"] == "request_completed"]
+        started = [row for row in records if row["event"] == "request_started"]
+        failed = [row for row in records if row["event"] == "request_failed"]
+        parsed = [row for row in records if row["event"] == "answer_parsed"]
+        item_results = [row for row in records if row["event"] == "item_result"]
+        phases = Counter(row["phase"] for row in completed)
+        usages = sum_usage(completed)
+        generation_ids = sorted({row["response"]["id"] for row in completed if "id" in row["response"]})
+        valid = sum(row["parsed"] for row in parsed)
+        initial_keys = {(row["item_id"], row["sample"]) for row in started if row["phase"] == "initial"}
+        complete = (len(initial_keys) == 144 and len(item_results) == 12 and
+                    all(row["valid_samples"] == row["n_samples"] == 12 for row in item_results))
+        lines.extend([
+            f"## `{model}` run `{run_id}`", "",
+            "| metric | value |",
+            "|---|---:|",
+            f"| dispatched phases | {len(started)} |",
+            f"| completed phases | {len(completed)} |",
+            f"| initial completed | {phases['initial']} |",
+            f"| rescue completed | {phases['rescue']} |",
+            f"| failed request phases | {len(failed)} |",
+            f"| parsed valid samples | {valid} |",
+            f"| distinct initial item/sample keys | {len(initial_keys)} |",
+            f"| item results | {len(item_results)} |",
+            f"| publication eligible 12 x 12 panel | {complete} |",
+            f"| provider generation IDs retained | {len(generation_ids)} |",
+            "",
+            "| provider usage field | total |",
+            "|---|---:|",
+            *[f"| {field} | {format_value(usages[field])} |" for field in USAGE_FIELDS],
+            "",
+        ])
+        if generation_ids:
+            digest = hashlib.sha256("\n".join(generation_ids).encode()).hexdigest()
+            lines.extend([
+                "Generation IDs are retained verbatim in the source ledger.", "",
+                f"- count: {len(generation_ids)}",
+                f"- SHA-256 of sorted IDs: `{digest}`",
+                f"- first: `{generation_ids[0]}`",
+                f"- last: `{generation_ids[-1]}`",
+                "",
+            ])
+        if failed:
+            lines.extend(["Failures retained in the ledger:", "", *[
+                f"- {row['phase']}: `{row['error_type']}: {row['error']}`" for row in failed
+            ], ""])
+        if item_results:
+            lines.extend([
+                "| item | valid | requested | failed | rescues | parse rate |",
+                "|---|---:|---:|---:|---:|---:|",
+                *[
+                    f"| {row['id']} | {row['valid_samples']} | {row['n_samples']} | "
+                    f"{row['failed_samples']} | {row['rescued_samples']} | {row['pmass_allowed']:.3f} |"
+                    for row in item_results
+                ],
+                "",
+            ])
+    lines.append("Provider `cost` is reported only when the raw OpenRouter usage object exposed it. Missing usage fields are unknown, not zero. -- PI[gpt-5.6-terra]")
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text("\n".join(lines) + "\n")
+
+
+if __name__ == "__main__":
+    main()