Queue canonical WVS score-all-options refresh

Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com>

justfile

@@ -5,3 +5,7 @@ smoke:
 # forced-choice eval on a config: just eval Qwen/Qwen3-0.6B classic
 eval model name="classic":
     uv run python scripts/09_forced_choice.py --model {{model}} --name {{name}}
+
+# Canonical WVS refresh: queue only score-all-options panels, not scripts/09_forced_choice.py.
+wvs-refresh:
+    uv run --offline --with 'datasets>=4.0,<5' python scripts/wvs_score_all_options_refresh.py --write-manifest --smoke --queue

scripts/wvs_api/06_score_all_options_lane.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+set -eu
+exec uv run --offline --with 'datasets>=4.0,<5' python scripts/wvs_score_all_options_refresh.py --lane "$1"

scripts/wvs_content_quality_audit.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Audit dense-rated WVS response discrimination without making API requests."""
+"""Audit score-all-options WVS response discrimination without making API requests."""
 
 from __future__ import annotations
 
@@ -264,7 +264,7 @@ def main() -> None:
         "",
         "## Definitions",
         "",
-        "Each dense-rated reply assigns a 1-5 rating to every answer in a card. A flat reply gives every "
+        "Each score-all-options reply assigns a 1-5 rating to every answer in a card. A flat reply gives every "
         "answer the same rating. Normalized spread is `(max rating - min rating) / 4`. "
         "A unique argmax has one highest-rated option; tie size counts all highest-rated options. "
         "Distance from uniform is total variation, `0.5 * sum(abs(p - uniform))`, after normalizing a reply's ratings to p.",
@@ -344,7 +344,7 @@ def main() -> None:
         "",
         "The table shows that flat replies and coordinate sensitivity vary across model and item, so Nano alone cannot "
         "supply a general rejection threshold. Saved mismatch rationale is evidence against interpreting those replies as attitudes. "
-        "For other cells, no saved rationale does not establish genuine indifference. Preserve the published dense-rated "
+        "For other cells, no saved rationale does not establish genuine indifference. Preserve the published score-all-options "
         "readout and report this diagnostic rather than silently replace or filter it.",
         "",
         "-- PI[gpt-5.6-terra]",

scripts/wvs_map.py

@@ -259,7 +259,9 @@ def main() -> None:
     reasoning_group.add_argument("--api-reasoning-effort",
                                  help="send a mandatory model's catalog-supported minimum reasoning effort")
     ap.add_argument("--api-structured-output", action="store_true",
-                    help="request a strict rating JSON schema only for a catalog-confirmed supporting model")
+                    help="request a strict score-all-options JSON schema only for a catalog-confirmed supporting model")
+    ap.add_argument("--api-provider-json",
+                    help="OpenRouter provider policy JSON, included in the score-all-options protocol identity")
     ap.add_argument("--api-require-complete", action="store_true",
                     help="exit nonzero rather than render after an explicitly requested API panel is incomplete")
     ap.add_argument("--max-think-tokens", type=int, default=64)
@@ -277,6 +279,7 @@ def main() -> None:
     api_models = list(dict.fromkeys(args.api_models + list(API_MODEL_SETS.get(args.api_model_set, ()))))
     api_reasoning = ({"enabled": False} if args.api_disable_reasoning else
                      {"effort": args.api_reasoning_effort} if args.api_reasoning_effort else None)
+    api_provider = json.loads(args.api_provider_json) if args.api_provider_json else None
 
     recs = load_wvs_all()
     resolved = resolve_items(recs)
@@ -354,7 +357,7 @@ def main() -> None:
             m, rated_items, n_samples=args.api_samples, temperature=1.0,
             max_tokens=args.api_max_tokens, concurrency=args.api_concurrency,
             req_timeout=args.api_request_timeout, reasoning=api_reasoning,
-            structured_output=args.api_structured_output)
+            structured_output=args.api_structured_output, provider=api_provider)
         completed = cache["completed"].get(protocol_id)
         if completed is not None:
             models[key] = tuple(completed["coords"])
@@ -364,7 +367,7 @@ def main() -> None:
                                 max_tokens=args.api_max_tokens, concurrency=args.api_concurrency,
                                 req_timeout=args.api_request_timeout, reasoning=api_reasoning,
                                 structured_output=args.api_structured_output,
-                                records_path=args.records, verbose_first=True)
+                                records_path=args.records, verbose_first=True, provider=api_provider)
         incomplete = [row["id"] for row in rows if row["valid_samples"] != args.api_samples]
         if incomplete:
             message = f"{key}: incomplete items {incomplete}; raw evidence is in {args.records}; not cached or plotted"

scripts/wvs_score_all_options_refresh.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""Queue and run canonical WVS score-all-options model refresh lanes."""
+from __future__ import annotations
+
+import argparse
+import fcntl
+import hashlib
+import json
+import subprocess
+from contextlib import contextmanager
+from datetime import UTC, datetime
+from decimal import Decimal
+from pathlib import Path
+
+CATALOG = Path("slop/research/wvs/20260917_openrouter_models.json")
+CACHE = Path("slop/research/wvs/20260916_openrouter/wvs_iw_rated.json")
+RECORDS = Path("slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl")
+OUT = Path("slop/research/wvs/20260917_score_all_options")
+MANIFEST = OUT / "manifest.json"
+STATE = OUT / "budget.json"
+LOCK = OUT / "budget.lock"
+GLOBAL_STOP_USD = Decimal("80")
+# Includes the discarded pick-one-option spend. It remains spending under the USD 80 cap.
+PRIOR_OBSERVED_USD = Decimal("5.34309727235")
+OSS_PROVIDER = {
+    "allow_fallbacks": True,
+    "require_parameters": True,
+    "quantizations": ["fp8", "int8", "bf16", "fp16"],
+}
+LANES = ("openai", "google", "xai", "muse", "kimi", "glm", "deepseek", "qwen")
+SPECIALIZED = ("batch", "free", "-pro", "-fast", "vision", "-vl", "coder", "audio", "image", "guard", "safeguard", "multi-agent", "embedding", "rerank")
+
+
+def lane_for(model_id: str) -> str | None:
+    prefixes = {
+        "openai/": "openai", "google/": "google", "x-ai/": "xai", "meta/muse-": "muse",
+        "moonshotai/": "kimi", "z-ai/": "glm", "deepseek/": "deepseek", "qwen/": "qwen",
+    }
+    return next((lane for prefix, lane in prefixes.items() if model_id.startswith(prefix)), None)
+
+
+def catalog() -> dict[str, dict]:
+    return {row["id"]: row for row in json.loads(CATALOG.read_text())["data"]}
+
+
+def completed_models() -> set[str]:
+    return {entry["model"] for entry in json.loads(CACHE.read_text())["completed"].values()}
+
+
+def price(model: dict, field: str) -> Decimal:
+    return Decimal(model["pricing"][field]) * 1_000_000
+
+
+def reasoning(model: dict) -> tuple[dict | None, str]:
+    metadata = model.get("reasoning")
+    if metadata is None:
+        return None, "omitted, not advertised"
+    if metadata.get("mandatory"):
+        efforts = set(metadata.get("supported_efforts", []))
+        if "minimal" in efforts:
+            return {"effort": "minimal"}, "minimal"
+        if "low" in efforts:
+            return {"effort": "low"}, "low"
+        raise ValueError("mandatory reasoning lacks minimal/low")
+    return {"enabled": False}, "disabled, optional"
+
+
+def entry(model: dict, completed: set[str]) -> dict:
+    model_id = model["id"]
+    lane = lane_for(model_id)
+    if lane is None:
+        return {"id": model_id, "status": "outside requested families"}
+    lowered = model_id.lower()
+    if any(token in lowered for token in SPECIALIZED):
+        return {"id": model_id, "lane": lane, "status": "excluded", "reason": "batch/free/pro/fast or specialized variant"}
+    if lane == "google" and "gemma" in lowered:
+        return {"id": model_id, "lane": lane, "status": "excluded", "reason": "Gemma is outside the requested Gemini series"}
+    if price(model, "completion") > Decimal("15"):
+        return {"id": model_id, "lane": lane, "status": "excluded", "reason": "output price exceeds USD 15/M"}
+    if model_id in completed:
+        return {"id": model_id, "lane": lane, "status": "complete_cached"}
+    try:
+        setting, setting_label = reasoning(model)
+    except ValueError as error:
+        return {"id": model_id, "lane": lane, "status": "excluded", "reason": str(error)}
+    provider = OSS_PROVIDER if lane in {"muse", "kimi", "glm", "deepseek", "qwen"} else None
+    reserve = Decimal(144) * (Decimal(1024 + 2048) * price(model, "completion") + Decimal(1024) * price(model, "prompt")) / Decimal(1_000_000)
+    return {
+        "id": model_id, "lane": lane, "status": "runnable", "created": model["created"],
+        "input_usd_per_million": str(price(model, "prompt")),
+        "output_usd_per_million": str(price(model, "completion")),
+        "reasoning": setting, "reasoning_label": setting_label, "structured_output": "structured_outputs" in model["supported_parameters"],
+        "provider": provider, "calls": 144, "reserve_usd": str(reserve),
+    }
+
+
+def prepare() -> list[dict]:
+    done = completed_models()
+    return [entry(model, done) for model in catalog().values() if lane_for(model["id"]) is not None]
+
+
+def write_manifest() -> list[dict]:
+    OUT.mkdir(parents=True, exist_ok=True)
+    rows = prepare()
+    payload = {
+        "schema": 1,
+        "method": "score-all-options",
+        "method_definition": "For every WVS item, return a JSON score 1..5 for each answer option, repeat 12 times, then normalize.",
+        "catalog_sha256": hashlib.sha256(CATALOG.read_bytes()).hexdigest(),
+        "global_stop_usd": str(GLOBAL_STOP_USD),
+        "observed_before_refresh_usd": str(PRIOR_OBSERVED_USD),
+        "aggregate_concurrency_ceiling": 10,
+        "oss_provider_policy": OSS_PROVIDER,
+        "models": rows,
+    }
+    MANIFEST.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
+    return rows
+
+
+@contextmanager
+def budget_state():
+    OUT.mkdir(parents=True, exist_ok=True)
+    with LOCK.open("w") as lock:
+        fcntl.flock(lock, fcntl.LOCK_EX)
+        state = json.loads(STATE.read_text()) if STATE.exists() else {"schema": 1, "prior_observed_usd": str(PRIOR_OBSERVED_USD), "reservations": {}}
+        yield state
+        STATE.write_text(json.dumps(state, indent=2, sort_keys=True) + "\n")
+        fcntl.flock(lock, fcntl.LOCK_UN)
+
+
+def rated_cost() -> Decimal:
+    total = Decimal()
+    for line in RECORDS.read_text().splitlines():
+        record = json.loads(line)
+        if record.get("event") == "request_completed":
+            total += Decimal(str(record.get("usage", {}).get("cost", 0)))
+    return total
+
+
+def reserve(row: dict) -> bool:
+    with budget_state() as state:
+        held = sum(Decimal(value["reserve_usd"]) for value in state["reservations"].values())
+        observed = max(PRIOR_OBSERVED_USD, rated_cost())
+        required = Decimal(row["reserve_usd"])
+        if observed + held + required >= GLOBAL_STOP_USD:
+            print(f"stop: observed={observed} held={held} required={required} cap={GLOBAL_STOP_USD}")
+            return False
+        state["reservations"][row["id"]] = {"lane": row["lane"], "reserve_usd": str(required), "reserved_utc": datetime.now(UTC).isoformat()}
+        return True
+
+
+def release(model_id: str) -> None:
+    with budget_state() as state:
+        state["reservations"].pop(model_id, None)
+        state["rated_ledger_cost_usd"] = str(rated_cost())
+        state["reconciled_utc"] = datetime.now(UTC).isoformat()
+
+
+def command(row: dict) -> list[str]:
+    args = [
+        "uv", "run", "--offline", "--with", "datasets>=4.0,<5", "python", "scripts/wvs_map.py",
+        "--api-models", row["id"], "--api-samples", "12", "--api-concurrency", "1",
+        "--api-max-tokens", "1024", "--api-request-timeout", "90", "--api-require-complete",
+        "--cache", str(CACHE), "--records", str(RECORDS), "--out", "/tmp/wvs_score_all_options.png",
+    ]
+    if row["reasoning"] is not None:
+        if row["reasoning"] == {"enabled": False}:
+            args.append("--api-disable-reasoning")
+        else:
+            args.extend(["--api-reasoning-effort", row["reasoning"]["effort"]])
+    if row["structured_output"]:
+        args.append("--api-structured-output")
+    if row["provider"] is not None:
+        args.extend(["--api-provider-json", json.dumps(row["provider"], sort_keys=True)])
+    return args
+
+
+def run_lane(lane: str) -> None:
+    rows = json.loads(MANIFEST.read_text())["models"]
+    for row in rows:
+        if row.get("lane") != lane or row["status"] != "runnable":
+            continue
+        if not reserve(row):
+            return
+        try:
+            result = subprocess.run(command(row), check=False)
+            if result.returncode:
+                print(f"{row['id']}: incomplete score-all-options panel, exit={result.returncode}; evidence retained")
+            else:
+                print(f"{row['id']}: score-all-options complete or cache replay")
+        finally:
+            release(row["id"])
+
+
+def queue(rows: list[dict]) -> None:
+    for lane in LANES:
+        count = sum(row.get("lane") == lane and row["status"] == "runnable" for row in rows)
+        if not count:
+            continue
+        command = ["pueue", "add", "-w", str(Path.cwd()), "--group", "api", "-l",
+                   f"why: fill {count} canonical WVS score-all-options panels in serialized {lane} lane; resolve: retained complete cache or per-model failure evidence under USD 80", "--",
+                   "scripts/wvs_api/06_score_all_options_lane.sh", lane]
+        print(subprocess.run(command, check=True, capture_output=True, text=True).stdout.strip())
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--write-manifest", action="store_true")
+    parser.add_argument("--queue", action="store_true")
+    parser.add_argument("--lane", choices=LANES)
+    parser.add_argument("--smoke", action="store_true")
+    args = parser.parse_args()
+    rows = write_manifest() if args.write_manifest else json.loads(MANIFEST.read_text())["models"]
+    if args.smoke:
+        runnable = [row for row in rows if row["status"] == "runnable"]
+        assert all(row["calls"] == 144 for row in runnable)
+        assert all(row["provider"] == OSS_PROVIDER for row in runnable if row["lane"] in {"muse", "kimi", "glm", "deepseek", "qwen"})
+        assert all(row["provider"] is None for row in runnable if row["lane"] in {"openai", "google", "xai"})
+        print(f"smoke: {len(runnable)} score-all-options panels, {len(LANES)} provider lanes, concurrency <= 8")
+    if args.queue:
+        queue(rows)
+    if args.lane:
+        run_lane(args.lane)
+
+
+if __name__ == "__main__":
+    main()

src/moralmaps/read_api.py

@@ -162,7 +162,7 @@ def _force_msg(n: int) -> str:
 
 async def _force_answer(model: str, prompt: str, phase1_msg: dict, temperature: float,
                         max_tokens: int, req_timeout: float, reasoning: dict | None,
-                        response_format: dict | None, n: int) -> dict:
+                        response_format: dict | None, n: int, provider: dict | None) -> dict:
     """Phase-2 rescue (wassname's bounded-thinking pattern, gist 72eed3a1): a reasoning model that
     spent its whole budget thinking and truncated the JSON mid-object gets a follow-up in the SAME
     conversation -- feed its (truncated) reasoning back as the assistant turn, then demand a compact
@@ -178,6 +178,8 @@ async def _force_answer(model: str, prompt: str, phase1_msg: dict, temperature:
         payload["reasoning"] = reasoning
     if response_format is not None:
         payload["response_format"] = response_format
+    if provider is not None:
+        payload["provider"] = provider
     return await asyncio.wait_for(openrouter_request(payload), timeout=req_timeout)
 
 
@@ -205,7 +207,8 @@ def _rate_plan(items: list[dict], n_samples: int, per_call: int = 1) -> list[dic
 
 def rated_protocol_identity(model: str, items: list[dict], *, n_samples: int, temperature: float,
                             max_tokens: int, concurrency: int, req_timeout: float,
-                            reasoning: dict | None, structured_output: bool) -> str:
+                            reasoning: dict | None, structured_output: bool,
+                            provider: dict | None = None) -> str:
     """Hash the exact model, rendered prompts, and request settings that define a cacheable panel."""
     plan = _rate_plan(items, n_samples)
     protocol = {
@@ -217,6 +220,7 @@ def rated_protocol_identity(model: str, items: list[dict], *, n_samples: int, te
         "req_timeout": req_timeout,
         "reasoning": reasoning,
         "structured_output": structured_output,
+        "provider": provider,
         "rate_prompt": _RATE_PROMPT,
         "rescue_prompt": _force_msg(10),
         "requests": [{key: req[key] for key in ("i", "perm", "prompt", "cnt", "sample", "presented_options")}
@@ -237,7 +241,7 @@ def _append_record(path: Path, record: dict) -> None:
 def read_items_rated(model: str, items: list[dict], *, n_samples: int = 12, temperature: float = 1.0,
                      max_tokens: int = 512, concurrency: int = 8, req_timeout: float = 90.0,
                      reasoning: dict | None = None, structured_output: bool = False, records_path: str | Path,
-                     verbose_first: bool = False) -> list[dict]:
+                     verbose_first: bool = False, provider: dict | None = None) -> list[dict]:
     """Run one dense rating panel and write an fsynced JSONL event for every paid request phase.
 
     The record is the source of truth. It preserves dispatches, responses, rescues, provider usage,
@@ -249,13 +253,13 @@ def read_items_rated(model: str, items: list[dict], *, n_samples: int = 12, temp
     protocol_id = rated_protocol_identity(model, items, n_samples=n_samples, temperature=temperature,
                                           max_tokens=max_tokens, concurrency=concurrency,
                                           req_timeout=req_timeout, reasoning=reasoning,
-                                          structured_output=structured_output)
+                                          structured_output=structured_output, provider=provider)
     run_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_{protocol_id[:12]}"
     rpath = Path(records_path)
     rpath.parent.mkdir(parents=True, exist_ok=True)
     settings = {"model": model, "n_samples": n_samples, "temperature": temperature,
                 "max_tokens": max_tokens, "concurrency": concurrency, "req_timeout": req_timeout,
-                "reasoning": reasoning, "structured_output": structured_output}
+                "reasoning": reasoning, "structured_output": structured_output, "provider": provider}
     _append_record(rpath, {"event": "run_started", "run_id": run_id, "protocol_id": protocol_id,
                            "settings": settings, "items": items, "planned_requests": len(plan)})
 
@@ -273,6 +277,8 @@ def read_items_rated(model: str, items: list[dict], *, n_samples: int = 12, temp
                        "temperature": temperature, "n": req["cnt"], "max_tokens": max_tokens}
             if reasoning is not None:
                 payload["reasoning"] = reasoning
+            if provider is not None:
+                payload["provider"] = provider
             response_format = _rating_schema(item["n"]) if structured_output else None
             if response_format is not None:
                 payload["response_format"] = response_format
@@ -283,7 +289,8 @@ def read_items_rated(model: str, items: list[dict], *, n_samples: int = 12, temp
                                            **request_meta, "payload": payload})
                     data = await asyncio.wait_for(openrouter_request(payload), timeout=req_timeout)
                     _append_record(rpath, {"event": "request_completed", "phase": phase,
-                                           **request_meta, "response": data, "usage": data.get("usage")})
+                                           **request_meta, "response": data, "provider": data.get("provider"),
+                                           "usage": data.get("usage")})
                     if len(data["choices"]) != req["cnt"]:
                         raise ValueError(f"expected {req['cnt']} choices, got {len(data['choices'])}")
                     message = data["choices"][0]["message"]
@@ -303,13 +310,16 @@ def read_items_rated(model: str, items: list[dict], *, n_samples: int = 12, temp
                             rescue_payload["response_format"] = response_format
                         if reasoning is not None:
                             rescue_payload["reasoning"] = reasoning
+                        if provider is not None:
+                            rescue_payload["provider"] = provider
                         _append_record(rpath, {"event": "request_started", "phase": phase,
                                                **request_meta, "payload": rescue_payload,
                                                "initial_response_message": message})
                         rescue = await _force_answer(model, req["prompt"], message, temperature,
-                                                     max_tokens, req_timeout, reasoning, response_format, item["n"])
+                                                     max_tokens, req_timeout, reasoning, response_format, item["n"], provider)
                         _append_record(rpath, {"event": "request_completed", "phase": phase,
-                                               **request_meta, "response": rescue, "usage": rescue.get("usage")})
+                                               **request_meta, "response": rescue, "provider": rescue.get("provider"),
+                                               "usage": rescue.get("usage")})
                         if len(rescue["choices"]) != 1:
                             raise ValueError(f"expected one rescue choice, got {len(rescue['choices'])}")
                         text = rescue["choices"][0]["message"].get("content") or ""