Bootstrap WVS score-all-options response means

Co-Authored-By: PI[openai-codex] <288921227+claudypoo@users.noreply.github.com>

scripts/wvs_map.py

@@ -184,12 +184,32 @@ def read_model(rows: list[dict], meta: dict[str, dict]) -> dict[str, np.ndarray]
     return {r["id"]: np.asarray(r["p"], float)[: meta[r["id"]]["n"]] for r in rows}
 
 
+def _sample_only_coord_se(psamples: dict[str, np.ndarray], resolved: dict[str, list[dict]],
+                          rng: np.random.Generator, n_draws: int, B: int = 500) -> tuple[float, float]:
+    """Response-mean bootstrap SE with the WVS item set held fixed."""
+    samples = []
+    for _ in range(B):
+        xy = []
+        for axis in (X_AXIS, Y_AXIS):
+            vals = []
+            for it in resolved[axis]:
+                ps = psamples[it["suffix"]]
+                mean_p = ps[rng.integers(0, len(ps), n_draws)].mean(0)
+                vals.append(positiveness(mean_p, it["pole_idx"], it["n"]))
+            xy.append(float(np.mean(vals)))
+        samples.append(xy)
+    return tuple(np.std(samples, axis=0))
+
+
 def model_coord_ci(psamples: dict[str, np.ndarray], resolved: dict[str, list[dict]],
                    rng: np.random.Generator, B: int = 500) -> tuple[float, float, float, float]:
-    """(x, y, x_se, y_se). Point estimate = mean positiveness over each axis's items on the mean-over-
-    samples p. The SE is a bootstrap over BOTH noise sources: resample the axis's items with
-    replacement (item-set noise, only ~3-7 items/axis) and, per item, draw one of its N rating samples
-    (readout noise). std over B replicates -> the CI drawn as error bars on the map."""
+    """(x, y, x_se, y_se), combined item-and-response-mean bootstrap uncertainty.
+
+    The point uses each item's mean over its N ratings. Each replicate resamples items and, for every
+    selected item, resamples N ratings then averages them. This estimates uncertainty of the N-sample
+    mean rather than uncertainty of a single response. The separate `_sample_only_coord_se` keeps the
+    fixed-item response component available for diagnostics.
+    """
     def axis_coords(getp) -> list[float]:
         return [float(np.mean([positiveness(getp(it), it["pole_idx"], it["n"]) for it in resolved[axis]]))
                 for axis in (X_AXIS, Y_AXIS)]
@@ -199,17 +219,30 @@ def model_coord_ci(psamples: dict[str, np.ndarray], resolved: dict[str, list[dic
         xy = []
         for axis in (X_AXIS, Y_AXIS):
             items = resolved[axis]
-            idx = rng.integers(0, len(items), len(items))
             vals = []
-            for j in idx:
+            for j in rng.integers(0, len(items), len(items)):
                 it = items[j]
                 ps = psamples[it["suffix"]]
-                vals.append(positiveness(ps[int(rng.integers(0, len(ps)))], it["pole_idx"], it["n"]))
+                mean_p = ps[rng.integers(0, len(ps), len(ps))].mean(0)
+                vals.append(positiveness(mean_p, it["pole_idx"], it["n"]))
             xy.append(float(np.mean(vals)))
         bx.append(xy[0]); by.append(xy[1])
     return x, y, float(np.std(bx)), float(np.std(by))
 
 
+def ci_bootstrap_smoke() -> None:
+    """Sample-only SE must shrink by roughly sqrt(N) when ratings are averaged."""
+    resolved = {axis: [{"suffix": f"{axis}{i}", "pole_idx": 1, "n": 2} for i in range(3)]
+                for axis in (X_AXIS, Y_AXIS)}
+    psamples = {item["suffix"]: np.tile([[1.0, 0.0], [0.0, 1.0]], (128, 1))
+                for items in resolved.values() for item in items}
+    se_one = _sample_only_coord_se(psamples, resolved, np.random.default_rng(0), n_draws=1, B=10_000)
+    se_sixteen = _sample_only_coord_se(psamples, resolved, np.random.default_rng(1), n_draws=16, B=10_000)
+    ratio = float(np.mean(se_one) / np.mean(se_sixteen))
+    assert 3.5 < ratio < 4.5, f"sample-only SE ratio {ratio:.3f}, expected sqrt(16)=4"
+    print(f"ci smoke: sample-only SE ratio N=1/N=16 is {ratio:.3f}, expected 4.000")
+
+
 def cluster_outlier_sd(countries: list[str], P: np.ndarray, models: dict[str, tuple],
                        min_n: int = 8) -> list[tuple]:
     """How odd each model looks as a member of each human macro-zone, in cluster SDs.
@@ -257,6 +290,8 @@ def main() -> None:
     ap.add_argument("--api-request-timeout", type=float, default=90.0)
     ap.add_argument("--api-max-tokens", type=int, default=1024,
                     help="output budget per rating call; large enough that a reasoning model finishes the JSON")
+    ap.add_argument("--ci-smoke", action="store_true",
+                    help="check that response-mean bootstrap SE falls approximately as 1/sqrt(N)")
     reasoning_group = ap.add_mutually_exclusive_group()
     reasoning_group.add_argument("--api-disable-reasoning", action="store_true",
                                  help="send reasoning.enabled=false for models whose catalog metadata says optional")
@@ -282,6 +317,9 @@ def main() -> None:
     ap.add_argument("--include-all-cached", action="store_true",
                     help="render every complete durable cache entry without making an API request")
     args = ap.parse_args()
+    if args.ci_smoke:
+        ci_bootstrap_smoke()
+        return
     api_models = list(dict.fromkeys(args.api_models + list(API_MODEL_SETS.get(args.api_model_set, ()))))
     api_reasoning = ({"enabled": False} if args.api_disable_reasoning else
                      {"effort": args.api_reasoning_effort} if args.api_reasoning_effort else None)
@@ -392,6 +430,7 @@ def main() -> None:
             "protocol_id": protocol_id,
             "n_items": len(rows),
             "n_samples": args.api_samples,
+            "ci_method": "combined item and N-response-mean bootstrap",
         }
         save_cache()
         x, y, xs, ys = models[key]

scripts/wvs_score_all_options_recover_cache.py

@@ -13,7 +13,7 @@ from pathlib import Path
 import numpy as np
 
 from moralmaps.iw_axes import X_AXIS, Y_AXIS, resolve_items
-from moralmaps.rated_cache import merge_completed
+from moralmaps.rated_cache import merge_completed, update_coords
 from wvs_map import load_wvs_all, model_coord_ci
 
 CACHE = Path("slop/research/wvs/20260916_openrouter/wvs_iw_rated.json")
@@ -99,6 +99,8 @@ def recovery_nonoverwriting_smoke() -> None:
 def main() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument("--smoke", action="store_true")
+    parser.add_argument("--refresh-ci", action="store_true",
+                        help="recompute only existing complete panels' coordinate CI summaries from the ledger")
     args = parser.parse_args()
     if args.smoke:
         concurrency_smoke()
@@ -106,14 +108,21 @@ def main() -> None:
         print("smoke: concurrent writers merge, and recovery never overwrites an existing entry")
     records = read_records()
     existing = json.loads(CACHE.read_text())["completed"] if CACHE.exists() else {}
-    existing_hash = hashlib.sha256(json.dumps(existing, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+    def stable_entry(entry: dict) -> dict:
+        return {key: value for key, value in entry.items() if key not in {"coords", "ci_method"}}
+    existing_hash = hashlib.sha256(json.dumps({key: stable_entry(value) for key, value in existing.items()},
+                                              sort_keys=True, separators=(",", ":")).encode()).hexdigest()
     entries = recovered_entries(records)
     additions = {key: value for key, value in entries.items() if key not in existing}
     overlaps = {key: entry for key, entry in entries.items() if key in existing}
     point_coordinates_match = all(existing[key]["coords"][:2] == entry["coords"][:2] for key, entry in overlaps.items())
-    merged = merge_completed(CACHE, additions)
+    if args.refresh_ci:
+        merged = update_coords(CACHE, {key: entry["coords"] for key, entry in overlaps.items()})
+    else:
+        merged = merge_completed(CACHE, additions)
     preserved = {key: merged["completed"][key] for key in existing}
-    preserved_hash = hashlib.sha256(json.dumps(preserved, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+    preserved_hash = hashlib.sha256(json.dumps({key: stable_entry(value) for key, value in preserved.items()},
+                                               sort_keys=True, separators=(",", ":")).encode()).hexdigest()
     audit = {
         "ledger": str(RECORDS),
         "ledger_valid_lines": len(records),
@@ -121,6 +130,7 @@ def main() -> None:
         "existing_entries_sha256_before": existing_hash,
         "existing_entries_sha256_after": preserved_hash,
         "new_complete_runs_added": len(additions),
+        "ci_summaries_refreshed": len(overlaps) if args.refresh_ci else 0,
         "new_protocol_ids": sorted(additions),
         "new_models": sorted(entry["model"] for entry in additions.values()),
         "overlap_complete_runs": len(overlaps),

slop/audits/20260917_wvs_score_all_options_cache_recovery.json

@@ -1,23 +1,14 @@
 {
-  "cache_completed_entries_after_merge": 53,
-  "existing_entries_preserved_count": 49,
-  "existing_entries_sha256_after": "0866bae2697e7f7b7a14edc2454968319fbbc26ad8b134ea05186840d28f99c2",
-  "existing_entries_sha256_before": "0866bae2697e7f7b7a14edc2454968319fbbc26ad8b134ea05186840d28f99c2",
+  "cache_completed_entries_after_merge": 86,
+  "ci_summaries_refreshed": 86,
+  "existing_entries_preserved_count": 86,
+  "existing_entries_sha256_after": "2fa381c2744633f337988c3fc2a753227849a733aaa906eceb507fe60b5b187f",
+  "existing_entries_sha256_before": "2fa381c2744633f337988c3fc2a753227849a733aaa906eceb507fe60b5b187f",
   "ledger": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
-  "ledger_valid_lines": 32461,
-  "new_complete_runs_added": 4,
-  "new_models": [
-    "deepseek/deepseek-v4-flash",
-    "deepseek/deepseek-v4-flash-0731",
-    "meta/muse-glimmer-30b",
-    "moonshotai/kimi-k2.6"
-  ],
-  "new_protocol_ids": [
-    "333ece6448f97447c57f24f9e9fcf0a7fb72a1b0f96696aef3c38a9c1d2d2308",
-    "a2df2f1a32fde5dae2c1545120a4fd72cea4743bafe71a1768997dc3d125b4f4",
-    "c64ced76e82ec32cf6be896420d813403a95a421474eefcbbcbba421f8189192",
-    "d21d1e81010d87b7f79bd9ceb0d5bb226485fe6393e266e169474968e6bf26f8"
-  ],
-  "overlap_complete_runs": 49,
+  "ledger_valid_lines": 49466,
+  "new_complete_runs_added": 0,
+  "new_models": [],
+  "new_protocol_ids": [],
+  "overlap_complete_runs": 86,
   "overlap_point_coordinates_equal": true
 }

src/moralmaps/rated_cache.py

@@ -7,15 +7,14 @@ import os
 from pathlib import Path
 
 
-def merge_completed(path: Path, additions: dict[str, dict]) -> dict:
-    """Merge complete entries while holding the cache lock across reread and replacement."""
+def _write_locked(path: Path, update) -> dict:
     lock_path = path.with_suffix(path.suffix + ".lock")
     with lock_path.open("w") as lock:
         fcntl.flock(lock, fcntl.LOCK_EX)
         cache = json.loads(path.read_text()) if path.exists() else {"schema": 2, "completed": {}}
         if cache["schema"] != 2:
             raise ValueError(f"unsupported WVS cache schema {cache['schema']}")
-        cache["completed"].update(additions)
+        update(cache["completed"])
         temp = path.with_suffix(path.suffix + ".tmp")
         temp.write_text(json.dumps(cache, indent=2, sort_keys=True) + "\n")
         with temp.open("r+") as fh:
@@ -24,3 +23,17 @@ def merge_completed(path: Path, additions: dict[str, dict]) -> dict:
         temp.replace(path)
         fcntl.flock(lock, fcntl.LOCK_UN)
     return cache
+
+
+def merge_completed(path: Path, additions: dict[str, dict]) -> dict:
+    """Merge complete entries while holding the cache lock across reread and replacement."""
+    return _write_locked(path, lambda completed: completed.update(additions))
+
+
+def update_coords(path: Path, coords: dict[str, list[float]]) -> dict:
+    """Replace only the derived coordinate summaries for complete protocol records."""
+    def update(completed: dict[str, dict]) -> None:
+        for protocol_id, values in coords.items():
+            completed[protocol_id]["coords"] = values
+            completed[protocol_id]["ci_method"] = "combined item and N-response-mean bootstrap"
+    return _write_locked(path, update)