diff --git a/scripts/wvs_map.py b/scripts/wvs_map.py index c33f6c8..511223c 100644 --- a/scripts/wvs_map.py +++ b/scripts/wvs_map.py @@ -184,12 +184,32 @@ def read_model(rows: list[dict], meta: dict[str, dict]) -> dict[str, np.ndarray] return {r["id"]: np.asarray(r["p"], float)[: meta[r["id"]]["n"]] for r in rows} +def _sample_only_coord_se(psamples: dict[str, np.ndarray], resolved: dict[str, list[dict]], + rng: np.random.Generator, n_draws: int, B: int = 500) -> tuple[float, float]: + """Response-mean bootstrap SE with the WVS item set held fixed.""" + samples = [] + for _ in range(B): + xy = [] + for axis in (X_AXIS, Y_AXIS): + vals = [] + for it in resolved[axis]: + ps = psamples[it["suffix"]] + mean_p = ps[rng.integers(0, len(ps), n_draws)].mean(0) + vals.append(positiveness(mean_p, it["pole_idx"], it["n"])) + xy.append(float(np.mean(vals))) + samples.append(xy) + return tuple(np.std(samples, axis=0)) + + def model_coord_ci(psamples: dict[str, np.ndarray], resolved: dict[str, list[dict]], rng: np.random.Generator, B: int = 500) -> tuple[float, float, float, float]: - """(x, y, x_se, y_se). Point estimate = mean positiveness over each axis's items on the mean-over- - samples p. The SE is a bootstrap over BOTH noise sources: resample the axis's items with - replacement (item-set noise, only ~3-7 items/axis) and, per item, draw one of its N rating samples - (readout noise). std over B replicates -> the CI drawn as error bars on the map.""" + """(x, y, x_se, y_se), combined item-and-response-mean bootstrap uncertainty. + + The point uses each item's mean over its N ratings. Each replicate resamples items and, for every + selected item, resamples N ratings then averages them. This estimates uncertainty of the N-sample + mean rather than uncertainty of a single response. The separate `_sample_only_coord_se` keeps the + fixed-item response component available for diagnostics. + """ def axis_coords(getp) -> list[float]: return [float(np.mean([positiveness(getp(it), it["pole_idx"], it["n"]) for it in resolved[axis]])) for axis in (X_AXIS, Y_AXIS)] @@ -199,17 +219,30 @@ def model_coord_ci(psamples: dict[str, np.ndarray], resolved: dict[str, list[dic xy = [] for axis in (X_AXIS, Y_AXIS): items = resolved[axis] - idx = rng.integers(0, len(items), len(items)) vals = [] - for j in idx: + for j in rng.integers(0, len(items), len(items)): it = items[j] ps = psamples[it["suffix"]] - vals.append(positiveness(ps[int(rng.integers(0, len(ps)))], it["pole_idx"], it["n"])) + mean_p = ps[rng.integers(0, len(ps), len(ps))].mean(0) + vals.append(positiveness(mean_p, it["pole_idx"], it["n"])) xy.append(float(np.mean(vals))) bx.append(xy[0]); by.append(xy[1]) return x, y, float(np.std(bx)), float(np.std(by)) +def ci_bootstrap_smoke() -> None: + """Sample-only SE must shrink by roughly sqrt(N) when ratings are averaged.""" + resolved = {axis: [{"suffix": f"{axis}{i}", "pole_idx": 1, "n": 2} for i in range(3)] + for axis in (X_AXIS, Y_AXIS)} + psamples = {item["suffix"]: np.tile([[1.0, 0.0], [0.0, 1.0]], (128, 1)) + for items in resolved.values() for item in items} + se_one = _sample_only_coord_se(psamples, resolved, np.random.default_rng(0), n_draws=1, B=10_000) + se_sixteen = _sample_only_coord_se(psamples, resolved, np.random.default_rng(1), n_draws=16, B=10_000) + ratio = float(np.mean(se_one) / np.mean(se_sixteen)) + assert 3.5 < ratio < 4.5, f"sample-only SE ratio {ratio:.3f}, expected sqrt(16)=4" + print(f"ci smoke: sample-only SE ratio N=1/N=16 is {ratio:.3f}, expected 4.000") + + def cluster_outlier_sd(countries: list[str], P: np.ndarray, models: dict[str, tuple], min_n: int = 8) -> list[tuple]: """How odd each model looks as a member of each human macro-zone, in cluster SDs. @@ -257,6 +290,8 @@ def main() -> None: ap.add_argument("--api-request-timeout", type=float, default=90.0) ap.add_argument("--api-max-tokens", type=int, default=1024, help="output budget per rating call; large enough that a reasoning model finishes the JSON") + ap.add_argument("--ci-smoke", action="store_true", + help="check that response-mean bootstrap SE falls approximately as 1/sqrt(N)") reasoning_group = ap.add_mutually_exclusive_group() reasoning_group.add_argument("--api-disable-reasoning", action="store_true", help="send reasoning.enabled=false for models whose catalog metadata says optional") @@ -282,6 +317,9 @@ def main() -> None: ap.add_argument("--include-all-cached", action="store_true", help="render every complete durable cache entry without making an API request") args = ap.parse_args() + if args.ci_smoke: + ci_bootstrap_smoke() + return api_models = list(dict.fromkeys(args.api_models + list(API_MODEL_SETS.get(args.api_model_set, ())))) api_reasoning = ({"enabled": False} if args.api_disable_reasoning else {"effort": args.api_reasoning_effort} if args.api_reasoning_effort else None) @@ -392,6 +430,7 @@ def main() -> None: "protocol_id": protocol_id, "n_items": len(rows), "n_samples": args.api_samples, + "ci_method": "combined item and N-response-mean bootstrap", } save_cache() x, y, xs, ys = models[key] diff --git a/scripts/wvs_score_all_options_recover_cache.py b/scripts/wvs_score_all_options_recover_cache.py index 7abe3fa..5da7e35 100644 --- a/scripts/wvs_score_all_options_recover_cache.py +++ b/scripts/wvs_score_all_options_recover_cache.py @@ -13,7 +13,7 @@ from pathlib import Path import numpy as np from moralmaps.iw_axes import X_AXIS, Y_AXIS, resolve_items -from moralmaps.rated_cache import merge_completed +from moralmaps.rated_cache import merge_completed, update_coords from wvs_map import load_wvs_all, model_coord_ci CACHE = Path("slop/research/wvs/20260916_openrouter/wvs_iw_rated.json") @@ -99,6 +99,8 @@ def recovery_nonoverwriting_smoke() -> None: def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--smoke", action="store_true") + parser.add_argument("--refresh-ci", action="store_true", + help="recompute only existing complete panels' coordinate CI summaries from the ledger") args = parser.parse_args() if args.smoke: concurrency_smoke() @@ -106,14 +108,21 @@ def main() -> None: print("smoke: concurrent writers merge, and recovery never overwrites an existing entry") records = read_records() existing = json.loads(CACHE.read_text())["completed"] if CACHE.exists() else {} - existing_hash = hashlib.sha256(json.dumps(existing, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + def stable_entry(entry: dict) -> dict: + return {key: value for key, value in entry.items() if key not in {"coords", "ci_method"}} + existing_hash = hashlib.sha256(json.dumps({key: stable_entry(value) for key, value in existing.items()}, + sort_keys=True, separators=(",", ":")).encode()).hexdigest() entries = recovered_entries(records) additions = {key: value for key, value in entries.items() if key not in existing} overlaps = {key: entry for key, entry in entries.items() if key in existing} point_coordinates_match = all(existing[key]["coords"][:2] == entry["coords"][:2] for key, entry in overlaps.items()) - merged = merge_completed(CACHE, additions) + if args.refresh_ci: + merged = update_coords(CACHE, {key: entry["coords"] for key, entry in overlaps.items()}) + else: + merged = merge_completed(CACHE, additions) preserved = {key: merged["completed"][key] for key in existing} - preserved_hash = hashlib.sha256(json.dumps(preserved, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + preserved_hash = hashlib.sha256(json.dumps({key: stable_entry(value) for key, value in preserved.items()}, + sort_keys=True, separators=(",", ":")).encode()).hexdigest() audit = { "ledger": str(RECORDS), "ledger_valid_lines": len(records), @@ -121,6 +130,7 @@ def main() -> None: "existing_entries_sha256_before": existing_hash, "existing_entries_sha256_after": preserved_hash, "new_complete_runs_added": len(additions), + "ci_summaries_refreshed": len(overlaps) if args.refresh_ci else 0, "new_protocol_ids": sorted(additions), "new_models": sorted(entry["model"] for entry in additions.values()), "overlap_complete_runs": len(overlaps), diff --git a/slop/audits/20260917_wvs_score_all_options_cache_recovery.json b/slop/audits/20260917_wvs_score_all_options_cache_recovery.json index ec5784e..dbb4cb1 100644 --- a/slop/audits/20260917_wvs_score_all_options_cache_recovery.json +++ b/slop/audits/20260917_wvs_score_all_options_cache_recovery.json @@ -1,23 +1,14 @@ { - "cache_completed_entries_after_merge": 53, - "existing_entries_preserved_count": 49, - "existing_entries_sha256_after": "0866bae2697e7f7b7a14edc2454968319fbbc26ad8b134ea05186840d28f99c2", - "existing_entries_sha256_before": "0866bae2697e7f7b7a14edc2454968319fbbc26ad8b134ea05186840d28f99c2", + "cache_completed_entries_after_merge": 86, + "ci_summaries_refreshed": 86, + "existing_entries_preserved_count": 86, + "existing_entries_sha256_after": "2fa381c2744633f337988c3fc2a753227849a733aaa906eceb507fe60b5b187f", + "existing_entries_sha256_before": "2fa381c2744633f337988c3fc2a753227849a733aaa906eceb507fe60b5b187f", "ledger": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl", - "ledger_valid_lines": 32461, - "new_complete_runs_added": 4, - "new_models": [ - "deepseek/deepseek-v4-flash", - "deepseek/deepseek-v4-flash-0731", - "meta/muse-glimmer-30b", - "moonshotai/kimi-k2.6" - ], - "new_protocol_ids": [ - "333ece6448f97447c57f24f9e9fcf0a7fb72a1b0f96696aef3c38a9c1d2d2308", - "a2df2f1a32fde5dae2c1545120a4fd72cea4743bafe71a1768997dc3d125b4f4", - "c64ced76e82ec32cf6be896420d813403a95a421474eefcbbcbba421f8189192", - "d21d1e81010d87b7f79bd9ceb0d5bb226485fe6393e266e169474968e6bf26f8" - ], - "overlap_complete_runs": 49, + "ledger_valid_lines": 49466, + "new_complete_runs_added": 0, + "new_models": [], + "new_protocol_ids": [], + "overlap_complete_runs": 86, "overlap_point_coordinates_equal": true } diff --git a/slop/research/wvs/20260916_openrouter/wvs_iw_rated.json b/slop/research/wvs/20260916_openrouter/wvs_iw_rated.json index 737544a..f43f819 100644 --- a/slop/research/wvs/20260916_openrouter/wvs_iw_rated.json +++ b/slop/research/wvs/20260916_openrouter/wvs_iw_rated.json @@ -1,11 +1,12 @@ { "completed": { "00220612ec353054f53baca3eccb90ea2bea8632c2c2fdae99ef7a074bd75d05": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5467309986851505, 0.5724812695525422, - 0.06104656289606268, - 0.10460593729500693 + 0.04247123290804327, + 0.09480053501952798 ], "display_key": "grok-4.3 (rated)", "model": "x-ai/grok-4.3", @@ -16,11 +17,12 @@ "run_id": "20260917T122425Z_00220612ec35" }, "0043a43d1a2188f7c728ccda08eee7b062232366d8a4e912c72bdc938e37fe44": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6214472858835399, 0.6710244268262847, - 0.03786483811416081, - 0.06918409770409496 + 0.020535409516408785, + 0.060983115136406967 ], "display_key": "kimi-k3 (rated)", "model": "moonshotai/kimi-k3", @@ -31,11 +33,12 @@ "run_id": "20260916T152010Z_0043a43d1a21" }, "009e9bcb85966922ddafb0bb61fd0f750528edf0a13a21cf954b226476428210": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.3863756613756613, 0.6708921222810112, - 0.05550492956346581, - 0.07459772141856083 + 0.04714117942693544, + 0.06471194730894558 ], "display_key": "gemini-2.5-flash-lite (rated)", "model": "google/gemini-2.5-flash-lite", @@ -46,11 +49,12 @@ "run_id": "20260917T130434Z_009e9bcb8596" }, "02d386b067d55e0639d33db8b6f1e5c708c30acbada3125dcbabde47dc90edcf": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5849525883130219, 0.59521139256589, - 0.037643038378990704, - 0.07695278664473065 + 0.02321119834823242, + 0.0670252510886532 ], "display_key": "qwen2.5-vl-72b-instruct (rated)", "model": "qwen/qwen2.5-vl-72b-instruct", @@ -61,11 +65,12 @@ "run_id": "20260916T200346Z_02d386b067d5" }, "0a798666aeed20785b4661c8c2fe4a6fc6853796bc2f3c45133863469e5b3f82": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5335737179487179, 0.5337214372928658, - 0.029444543942379133, - 0.09738610052745941 + 0.022941344814671694, + 0.09410390751379963 ], "display_key": "qwen3-vl-30b-a3b-instruct (rated)", "model": "qwen/qwen3-vl-30b-a3b-instruct", @@ -76,11 +81,12 @@ "run_id": "20260916T192223Z_0a798666aeed" }, "11929820262d34605efabc00ce84137857e03ef6c53d827ea6c405555b905f26": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6151755041791274, 0.6364257870168759, - 0.06075682882091517, - 0.0850053325551206 + 0.05262633347758275, + 0.0809846685946923 ], "display_key": "gpt-5.4 (rated)", "model": "openai/gpt-5.4", @@ -91,11 +97,12 @@ "run_id": "20260917T124555Z_11929820262d" }, "134c21f0377c1a88f8ecfe9e9191341b0f53e0afbadb05d36ca0ba36c641f340": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5672038109538109, 0.6522156084656084, - 0.0657887759000581, - 0.0790074678539591 + 0.047966367122167806, + 0.06566980777538725 ], "display_key": "gemini-3.5-flash-lite (rated)", "model": "google/gemini-3.5-flash-lite", @@ -106,11 +113,12 @@ "run_id": "20260917T124448Z_134c21f0377c" }, "1486c2f54e34ad1075f7b519a1dfc2b43f9f7e052d563a083fbc139fc41bd6ee": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5941946410337214, 0.6441922202164339, - 0.06777104459972463, - 0.07883221721567056 + 0.03986801497595867, + 0.05382488437105076 ], "display_key": "glm-4.5-air (rated)", "model": "z-ai/glm-4.5-air", @@ -121,11 +129,12 @@ "run_id": "20260917T125444Z_1486c2f54e34" }, "182a7e88869698133f1175f18e53e7f6f2f056739559d325b211bd065235a71c": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6175297619047618, 0.5976147806792967, - 0.035727022605345674, - 0.07361977335041572 + 0.03468986624126052, + 0.0429044958309632 ], "display_key": "qwen3-coder-plus (rated)", "model": "qwen/qwen3-coder-plus", @@ -136,11 +145,12 @@ "run_id": "20260916T193258Z_182a7e888696" }, "1a14a0f088ae0c3cc56637a32bd4c2290e4dde93dfeebb0f888a5a68c784af0d": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5780312225900461, 0.608531746031746, - 0.06478224722015172, - 0.101427539255001 + 0.047255133267023915, + 0.09642087745368229 ], "display_key": "gemini-3.1-flash-lite-preview (rated)", "model": "google/gemini-3.1-flash-lite-preview", @@ -151,11 +161,12 @@ "run_id": "20260917T125623Z_1a14a0f088ae" }, "1a70f47789af20900f2799992b09e83af9490be6579098a6178d613d5ba7856c": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5274474357644292, 0.6650744376984001, - 0.044561828476471346, - 0.06617035838663828 + 0.039325438704147446, + 0.06431091953768533 ], "display_key": "gpt-5.6-sol (rated)", "model": "openai/gpt-5.6-sol", @@ -166,11 +177,12 @@ "run_id": "20260916T152843Z_1a70f47789af" }, "1c31668055e399de28805046bc08075035efbad0cdb1ff7d8926841a282e07a5": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5036638266417679, 0.6503283973522069, - 0.057017134313755345, - 0.06039652604026301 + 0.04391918485419595, + 0.041591327239427205 ], "display_key": "qwen3.7-max (rated)", "model": "qwen/qwen3.7-max", @@ -181,11 +193,12 @@ "run_id": "20260916T135329Z_1c31668055e3" }, "1d7f4015decbcbe955adb79817b970981ca1d3165ed121b458b213ba1060f675": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6061818582651916, 0.6004689754689754, - 0.029738189920548014, - 0.07794129978765893 + 0.029517269745380506, + 0.08002704064836072 ], "display_key": "qwen3-vl-8b-instruct (rated)", "model": "qwen/qwen3-vl-8b-instruct", @@ -196,11 +209,12 @@ "run_id": "20260916T192044Z_1d7f4015decb" }, "27cc1b32407e6ebca063ce550d1848751a64f4393f93b6af393a01b4f62fe162": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5442434494517827, 0.6391828271087302, - 0.04443386786526795, - 0.054686704767473275 + 0.025233816825009546, + 0.04299513986767339 ], "display_key": "glm-5.3-flash (rated)", "model": "z-ai/glm-5.3-flash", @@ -211,11 +225,12 @@ "run_id": "20260917T114916Z_27cc1b32407e" }, "29b17fd9e04c4b7f93e1763b460892f9ec2f4fc4698b6dd7571fcfebc76b8500": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5429175317746331, 0.694532627865961, - 0.06309666696112717, - 0.07571315515523741 + 0.05076181516930638, + 0.06853941563706838 ], "display_key": "glm-5.1 (rated)", "model": "z-ai/glm-5.1", @@ -226,11 +241,12 @@ "run_id": "20260917T120826Z_29b17fd9e04c" }, "3016d2c1ab8d8f88f01723d2d695be996c7f1fdcd442a51cfbb6c7b7110a25e9": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5017202097104058, 0.5916881018254176, - 0.04660057418362078, - 0.07020408831610975 + 0.014139200341870085, + 0.04877802971162936 ], "display_key": "qwen3.5-9b (rated)", "model": "qwen/qwen3.5-9b", @@ -241,11 +257,12 @@ "run_id": "20260916T142212Z_3016d2c1ab8d" }, "304db4b1aa6c80d734f802749c75143feddfcdb740c4131385e927148ba7d05b": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.598778659611993, 0.6204503168788883, - 0.04059052018905529, - 0.08949774510806 + 0.031879884568457334, + 0.08890048164637801 ], "display_key": "qwen3-coder (rated)", "model": "qwen/qwen3-coder", @@ -256,11 +273,12 @@ "run_id": "20260916T195554Z_304db4b1aa6c" }, "3194398c99baf1d5ebb273314207d4aa1bfe5d12c8b16cb192818aacd78d759f": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.4215300954189843, 0.7190489551600663, - 0.07543950734136469, - 0.06523372720374179 + 0.06886574238472291, + 0.058216440434159446 ], "display_key": "grok-4.6 (rated)", "model": "x-ai/grok-4.6", @@ -271,11 +289,12 @@ "run_id": "20260917T114916Z_3194398c99ba" }, "333ece6448f97447c57f24f9e9fcf0a7fb72a1b0f96696aef3c38a9c1d2d2308": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5482108382155133, 0.5760765801607652, - 0.06725327571313976, - 0.08275579176334262 + 0.05615879899255038, + 0.06900628612372126 ], "display_key": "deepseek-v4-flash-0731 (rated)", "model": "deepseek/deepseek-v4-flash-0731", @@ -286,11 +305,12 @@ "run_id": "20260917T112151Z_333ece6448f9" }, "3443c17ae66d3fb529a058128b662024c4bc0601394e614bb693b388ec4c988b": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.4153054353054353, 0.6346365065494385, - 0.07206749839259896, - 0.07710357656753936 + 0.06387070667136088, + 0.06142907585948666 ], "display_key": "qwen3.8-flash (rated)", "model": "qwen/qwen3.8-flash", @@ -301,11 +321,12 @@ "run_id": "20260916T133720Z_3443c17ae66d" }, "39659697ca3437f62be4fb8e129e6b3902a91b5acac3243ca914dfc917dd563b": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5030952380952382, 0.5846967846967848, - 0.046958286642217247, - 0.08746316369004264 + 0.02599338482636756, + 0.08346488455925544 ], "display_key": "qwen3-coder-30b-a3b-instruct (rated)", "model": "qwen/qwen3-coder-30b-a3b-instruct", @@ -316,11 +337,12 @@ "run_id": "20260916T194257Z_39659697ca34" }, "3c9d4fedebcab41719f4247ca94fa35430f57d0a34f3a539012e0632e35413de": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.4253104631546661, 0.731799663031547, - 0.0936048666617984, - 0.060012518938080954 + 0.08709838706511942, + 0.05122062529170766 ], "display_key": "muse-spark-1.3 (rated)", "model": "meta/muse-spark-1.3", @@ -331,11 +353,12 @@ "run_id": "20260916T155331Z_3c9d4fedebca" }, "4230ccaa1e160c793a906aef6b786d735d9fdbf71c3333691b0eba17dcf33928": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5790919024614678, 0.6085929478125246, - 0.038145029256266345, - 0.025980758516149186 + 0.03990434167170201, + 0.023380867881070636 ], "display_key": "claude-fable-5.1 (rated)", "model": "anthropic/claude-fable-5.1", @@ -346,11 +369,12 @@ "run_id": "20260916T153341Z_4230ccaa1e16" }, "43ddc43fb5a29c7fbf21c8f32b42a909c2118feb9255959ecaeab7dadb1f3dde": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5966931216931217, 0.5338064713064713, - 0.011307868884765215, - 0.10204509849192553 + 0.011836625928696285, + 0.10630349458372183 ], "display_key": "qwen3-vl-32b-instruct (rated)", "model": "qwen/qwen3-vl-32b-instruct", @@ -361,11 +385,12 @@ "run_id": "20260916T191809Z_43ddc43fb5a2" }, "49ae9f7a09090534a15fac734c2611c445a17139cad6c5684fa0581722f3fd70": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6137896825396826, 0.5819444444444445, - 0.015958730849444745, - 0.0834429418953706 + 0.009821057676311395, + 0.08482387932410497 ], "display_key": "qwen3-14b (rated)", "model": "qwen/qwen3-14b", @@ -376,11 +401,12 @@ "run_id": "20260916T150121Z_49ae9f7a0909" }, "4f0d6ed370f069e233ea6c4cb83000e79398c3a2aa10b355b10f99c8633c47f6": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5923129031462364, 0.6683453376721874, - 0.042111115261344804, - 0.08216742430172869 + 0.02592908625180882, + 0.07625311311101751 ], "display_key": "glm-5 (rated)", "model": "z-ai/glm-5", @@ -391,11 +417,12 @@ "run_id": "20260917T123002Z_4f0d6ed370f0" }, "5478eac564ec02f01e161ce017e1814954f4239b9fa1bedfe73172c5c630848f": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.4566709514410664, 0.670101976591992, - 0.05775584971748114, - 0.07962287120217941 + 0.0421869487162949, + 0.07271432406347927 ], "display_key": "gpt-5.4-mini (rated)", "model": "openai/gpt-5.4-mini", @@ -406,11 +433,12 @@ "run_id": "20260917T124216Z_5478eac564ec" }, "54e721734c251e2e13cf43beab962703409108bfc5203c19e7747770ce13a801": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.593257275132275, 0.6186534585344109, - 0.03571288070419747, - 0.06312818330951275 + 0.03287309998202738, + 0.06078538490729567 ], "display_key": "gemini-3.6-flash (rated)", "model": "google/gemini-3.6-flash", @@ -421,11 +449,12 @@ "run_id": "20260917T124016Z_54e721734c25" }, "568bb3fc2517dd40b1399c71e0cf52033dcce79dc7537c4943ace860fe3a1297": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6123449635867936, 0.7514717187100795, - 0.04454897242429451, - 0.049069808495596626 + 0.03722472860594401, + 0.04262080359788279 ], "display_key": "gpt-5.1 (rated)", "model": "openai/gpt-5.1", @@ -436,11 +465,12 @@ "run_id": "20260917T125409Z_568bb3fc2517" }, "57bed80fd5b94157effe68fe998c7c12612fea210c1371dbf230097a7e8e0717": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6363133579397021, 0.5792251640447779, - 0.027397680484314686, - 0.06607943334210371 + 0.015532918535503142, + 0.04894510480027706 ], "display_key": "qwen3-coder-next (rated)", "model": "qwen/qwen3-coder-next", @@ -451,11 +481,12 @@ "run_id": "20260916T191559Z_57bed80fd5b9" }, "580a5840948209ecb177dc6fdaf36e265a32ccfe223d791eb3235fea9269f27e": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6064902599972044, 0.5413085527420844, - 0.028266888268597475, - 0.07251640240693584 + 0.018354281932994144, + 0.043773731258587176 ], "display_key": "qwen3-32b (rated)", "model": "qwen/qwen3-32b", @@ -466,11 +497,12 @@ "run_id": "20260916T150406Z_580a58409482" }, "589e10fa7924ecf8f88650d6587ff37c440f47c6c4cc4325c7009ef8d4eeb4ec": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6217724867724868, 0.5391326046087952, - 0.015436127385306617, - 0.05984330714155522 + 0.013368066261339313, + 0.03510995792353358 ], "display_key": "qwen3-8b (rated)", "model": "qwen/qwen3-8b", @@ -481,11 +513,12 @@ "run_id": "20260916T181355Z_589e10fa7924" }, "5c920b68e50c5b4fd43542f379ac25b98e8d79107ea1722c1db12aa22fa34211": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5907043650793651, 0.5776102655592599, - 0.04137996061944377, - 0.07845514485495916 + 0.037783384172145155, + 0.07466018958883854 ], "display_key": "gemini-3.5-flash (rated)", "model": "google/gemini-3.5-flash", @@ -496,11 +529,12 @@ "run_id": "20260917T124812Z_5c920b68e50c" }, "6067c0208cc500e5837d123052d6b33b3568d616e5c157f601baa8dcb7ef159b": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5189766385048643, 0.6372836916088428, - 0.052897459249729474, - 0.08224247455566645 + 0.0337635437945517, + 0.059640726072916046 ], "display_key": "grok-4.20 (rated)", "model": "x-ai/grok-4.20", @@ -511,11 +545,12 @@ "run_id": "20260917T122709Z_6067c0208cc5" }, "61ab1adae3781ca3893328a845d4d517daa5fb47e07ea189da1df45388f62d42": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5993276014109347, 0.5462479278681369, - 0.024660010076102412, - 0.10518394004968958 + 0.01958469751960041, + 0.1037406905684859 ], "display_key": "qwen3-vl-235b-a22b-instruct (rated)", "model": "qwen/qwen3-vl-235b-a22b-instruct", @@ -526,11 +561,12 @@ "run_id": "20260916T192432Z_61ab1adae378" }, "66ad076e1a37493e4d3db982a0220048e520103459cefc709bc8f09f3fef6c9a": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5509667037841642, 0.5895236753978316, - 0.02658132977411291, - 0.08063463160776993 + 0.013511574117086913, + 0.07299126669851562 ], "display_key": "qwen-2.5-7b-instruct (rated)", "model": "qwen/qwen-2.5-7b-instruct", @@ -541,11 +577,12 @@ "run_id": "20260916T200857Z_66ad076e1a37" }, "67a70b1b03ba85be79dc122cc888a080fde239205624135a9147168e7e77293d": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5614798614630517, 0.6880988993033613, - 0.050774441812280406, - 0.057780026773918566 + 0.04645572992641227, + 0.046994343694421285 ], "display_key": "inkling (rated)", "model": "thinkingmachines/inkling", @@ -556,11 +593,12 @@ "run_id": "20260916T191343Z_67a70b1b03ba" }, "6af9f017856f4c733214bf8e26992a4af3c7cb94dac8a5a4b881e8669b3917a7": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5220108722106056, 0.6395004417804221, - 0.04704769940966264, - 0.09470717560391326 + 0.03076973529726908, + 0.084649994895056 ], "display_key": "gpt-5-mini (rated)", "model": "openai/gpt-5-mini", @@ -571,11 +609,12 @@ "run_id": "20260917T130236Z_6af9f017856f" }, "6d6ce30e9fbc5fdb07229b0a08e067d77511857069a4e572613d3e48b7c8b261": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5856201899951899, 0.6494018547589976, - 0.06114886163791156, - 0.08129405199775336 + 0.032344011142651626, + 0.08396736786485802 ], "display_key": "qwen3.5-plus-20260420 (rated)", "model": "qwen/qwen3.5-plus-20260420", @@ -586,11 +625,12 @@ "run_id": "20260916T135722Z_6d6ce30e9fbc" }, "6f7a7bbc4d70e7cfceff8e25b2da2008fbf6105538825f42bfbbc47ff3459387": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5605366161616162, 0.573015873015873, - 0.07008299148165956, - 0.09640175208725496 + 0.07041699717338515, + 0.0955132767170434 ], "display_key": "gemini-3.1-flash-lite (rated)", "model": "google/gemini-3.1-flash-lite", @@ -601,11 +641,12 @@ "run_id": "20260917T125248Z_6f7a7bbc4d70" }, "7058adf367d238c6c92e1b83ae71f6556362e9fc7a243297b6fc65e54e2c2886": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5974338624338624, 0.6515634387658198, - 0.023795241546514957, - 0.07922810144513995 + 0.0151114134522649, + 0.07890400348024484 ], "display_key": "qwen3.5-27b (rated)", "model": "qwen/qwen3.5-27b", @@ -616,11 +657,12 @@ "run_id": "20260916T143235Z_7058adf367d2" }, "74e3649a6961000bab284e801fe811d5c8fc4cf1db15af52bb10959f5a5a37c1": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5372045855379188, 0.6595114541543113, - 0.04180484013035134, - 0.08450011787948054 + 0.03701076833492129, + 0.08376087073198286 ], "display_key": "qwen3.5-plus-02-15 (rated)", "model": "qwen/qwen3.5-plus-02-15", @@ -631,11 +673,12 @@ "run_id": "20260916T144205Z_74e3649a6961" }, "7809cffa4ab1327c0a20b2d9ec23d0647b9382ddef65dd36c2ab5669da4f7d9f": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5595998356229528, 0.5926290303237558, - 0.04045393842948997, - 0.058480296424785895 + 0.02693093173377315, + 0.042768178388407095 ], "display_key": "glm-4.7-flash (rated)", "model": "z-ai/glm-4.7-flash", @@ -646,11 +689,12 @@ "run_id": "20260917T123743Z_7809cffa4ab1" }, "7ee20ca2b88f69dd06b9de3018deea4cb7a418e3119e17862cd0b32146daae1a": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.538440257742794, 0.5530118458934986, - 0.07277582530416211, - 0.0898393391241473 + 0.03744969864324434, + 0.06998170957293813 ], "display_key": "qwen3.5-35b-a3b (rated)", "model": "qwen/qwen3.5-35b-a3b", @@ -661,11 +705,12 @@ "run_id": "20260916T142959Z_7ee20ca2b88f" }, "7fe76f95937ed145b996116a31855dae4ab8c3b67e3e7ccfb84a000437e27f96": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.4526851851851852, 0.6334977783064814, - 0.057818785990183905, - 0.08298095716984875 + 0.036885948359860296, + 0.07744723420845886 ], "display_key": "gpt-5-nano (rated)", "model": "openai/gpt-5-nano", @@ -676,11 +721,12 @@ "run_id": "20260917T020418Z_7fe76f95937e" }, "82875b6ee164d0d980eba738bf05f69959234567b91f9cfd01fcabca0bd70dac": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6529354469060352, 0.5885508561103798, - 0.049299882651159914, - 0.08073268479006059 + 0.0314816537313644, + 0.04799521946452996 ], "display_key": "qwen3.7-flash (rated)", "model": "qwen/qwen3.7-flash", @@ -691,11 +737,12 @@ "run_id": "20260916T133001Z_82875b6ee164" }, "840da516c1c04e69e5d80dd104803f019a4d6c6640359ad0e4bbcb71e722aa6c": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5595320767195766, 0.6334656084656085, - 0.05158017344603731, - 0.06609164737112991 + 0.0479345416110304, + 0.06407749490095412 ], "display_key": "gemini-3-flash-preview (rated)", "model": "google/gemini-3-flash-preview", @@ -706,11 +753,12 @@ "run_id": "20260917T125926Z_840da516c1c0" }, "842298c634ec0f57848b605f67fca161631a852a68192cae1a5377b1d9f31566": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6427511094043352, 0.5554953612269281, - 0.03535208249381765, - 0.08497138486454137 + 0.03253928279038274, + 0.07779247297520335 ], "display_key": "qwen3-235b-a22b (rated)", "model": "qwen/qwen3-235b-a22b", @@ -721,11 +769,12 @@ "run_id": "20260916T150950Z_842298c634ec" }, "8700e79d3eccf358a0de1029f7b40e2db339bb6d11403b8e69cb7a66a7b0c668": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6707823388714307, 0.6060018656291949, - 0.03451993837104684, - 0.09876091230627733 + 0.022702792353656083, + 0.08121800045234231 ], "display_key": "qwen3-30b-a3b (rated)", "model": "qwen/qwen3-30b-a3b", @@ -736,11 +785,12 @@ "run_id": "20260916T145439Z_8700e79d3ecc" }, "88ba5af838eda5d298b312553d7c941e72a37cbf28890cdb9d8436b5f40e7cfe": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6025617283950616, 0.6287037037037038, - 0.03396367302553316, - 0.09406697361625513 + 0.030065943475536042, + 0.09420661909505727 ], "display_key": "qwen-plus (rated)", "model": "qwen/qwen-plus", @@ -751,11 +801,12 @@ "run_id": "20260916T200626Z_88ba5af838ed" }, "88ed922f271a76721ad18f0b7bb9693a48377e2507d02a4b78fa08b451972d83": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6450907949109246, 0.5437992771640354, - 0.023312776383531648, - 0.10024881542221994 + 0.014480282059694862, + 0.10414669190782924 ], "display_key": "qwen3-30b-a3b-instruct-2507 (rated)", "model": "qwen/qwen3-30b-a3b-instruct-2507", @@ -766,11 +817,12 @@ "run_id": "20260916T195128Z_88ed922f271a" }, "8c27d32cd851de5f217cb354b15dc270cc6f4ec2d02fb15dbb20fde0790c3571": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5419602854216832, 0.595896730624977, - 0.05193570378692634, - 0.08265065423220756 + 0.044286454927165, + 0.06410119405336592 ], "display_key": "deepseek-v4.1-flash (rated)", "model": "deepseek/deepseek-v4.1-flash", @@ -781,11 +833,12 @@ "run_id": "20260916T151414Z_8c27d32cd851" }, "921707cb0d3db143e948e9d78cd8191f4ee720eaf56b6934e2d13d9ce9e3382f": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5408168991502326, 0.6493278669204595, - 0.03917823180041766, - 0.08760189869507796 + 0.03453680502309858, + 0.08951355314543964 ], "display_key": "qwen3.5-397b-a17b (rated)", "model": "qwen/qwen3.5-397b-a17b", @@ -796,11 +849,12 @@ "run_id": "20260916T144546Z_921707cb0d3d" }, "94428060f3776ab79ab3d146382e869d248c259c86858f4e5dbc9c5a1f4633db": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.48311026165192833, 0.6007697469838861, - 0.05313376200674668, - 0.06690255611782707 + 0.028453155979336744, + 0.04236719989377499 ], "display_key": "gpt-5.4-nano (rated)", "model": "openai/gpt-5.4-nano", @@ -811,11 +865,12 @@ "run_id": "20260917T123850Z_94428060f377" }, "95bb4d3939e9937823357b5cd87adb1a6640cc5b04e0841dfec03095331a5e44": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.4569019748367575, 0.6751145899955423, - 0.07644348564506928, - 0.0567289888770763 + 0.07186088017315642, + 0.04558197261286701 ], "display_key": "gpt-6-astra (rated)", "model": "openai/gpt-6-astra", @@ -826,11 +881,12 @@ "run_id": "20260916T154406Z_95bb4d3939e9" }, "96d86b64bf7ceff8eed0daff1aa0396ee69609b5867942a7c2c9d33093f03c9d": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5196778711484593, 0.6698994127565557, - 0.042934824935707064, - 0.07801121120269483 + 0.03377693173708851, + 0.06927660087614809 ], "display_key": "qwen3.6-plus (rated)", "model": "qwen/qwen3.6-plus", @@ -841,11 +897,12 @@ "run_id": "20260916T141839Z_96d86b64bf7c" }, "9c1d40b30945565e488b212177e40c29f730c2e96d4b07d09cbc135ba75a81f0": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6026546601546601, 0.5957470912477916, - 0.03274570154412936, - 0.08673540897470654 + 0.024251712034204544, + 0.08684627683495146 ], "display_key": "qwen-2.5-72b-instruct (rated)", "model": "qwen/qwen-2.5-72b-instruct", @@ -856,11 +913,12 @@ "run_id": "20260916T201119Z_9c1d40b30945" }, "9d8a5189c078672eb884784938b183944fb161d0e716e2c44722540b17603e34": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6652380952380952, 0.7022990677752582, - 0.026433969418704885, - 0.06269095940430941 + 0.021284606679632966, + 0.04934275921145735 ], "display_key": "qwen3-next-80b-a3b-instruct (rated)", "model": "qwen/qwen3-next-80b-a3b-instruct", @@ -871,11 +929,12 @@ "run_id": "20260916T193846Z_9d8a5189c078" }, "a2df2f1a32fde5dae2c1545120a4fd72cea4743bafe71a1768997dc3d125b4f4": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.610243091658148, 0.6687657686659433, - 0.04414498817375987, - 0.08085439770187478 + 0.03675410818998801, + 0.06737179091474468 ], "display_key": "kimi-k2.6 (rated)", "model": "moonshotai/kimi-k2.6", @@ -886,11 +945,12 @@ "run_id": "20260917T112151Z_a2df2f1a32fd" }, "a544b24e1874f8d1930042f65a61ea53299057a6299b0eb2ded110691c0d4edb": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6298593073593075, 0.6495670432204361, - 0.03865121082002784, - 0.08563182969926275 + 0.040434814343758965, + 0.07471544119511193 ], "display_key": "gemini-2.5-flash (rated)", "model": "google/gemini-2.5-flash", @@ -901,11 +961,12 @@ "run_id": "20260917T130734Z_a544b24e1874" }, "a742f028f72ec35cc6586661ca261df77a51eecce778ff080062968942d822ae": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5253620889368438, 0.6943301071633757, - 0.06029287144663362, - 0.07390235325207269 + 0.03998834571383578, + 0.06608021212309273 ], "display_key": "glm-5.2 (rated)", "model": "z-ai/glm-5.2", @@ -916,11 +977,12 @@ "run_id": "20260917T115959Z_a742f028f72e" }, "a9a69a3ef43e7544f73290e6fd9d3c9f37013b15a9e50be755d4d227d6cc30eb": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.524187458188895, 0.708147541423977, - 0.05959262807078437, - 0.06778136701328388 + 0.04698755670397484, + 0.0578745248478481 ], "display_key": "grok-4.5 (rated)", "model": "x-ai/grok-4.5", @@ -931,11 +993,12 @@ "run_id": "20260917T121142Z_a9a69a3ef43e" }, "abf45a27f4bafad0eb264aac33b1e49e9d0404e141966954d0707930d56665e3": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.45740779531102105, 0.6795887184190553, - 0.09233310052166174, - 0.07166972793043255 + 0.07173891141983318, + 0.06455376828361989 ], "display_key": "qwen3.6-27b (rated)", "model": "qwen/qwen3.6-27b", @@ -946,11 +1009,12 @@ "run_id": "20260916T141139Z_abf45a27f4ba" }, "ac52652ff1f09e1f1972a8b751e1b1deca329cb555df54755aada79791bed940": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6417361111111111, 0.5127110103300581, - 0.03807268153101321, - 0.08042423628815949 + 0.03306211082399865, + 0.06459320760155234 ], "display_key": "qwen-plus-2025-07-28 (rated)", "model": "qwen/qwen-plus-2025-07-28", @@ -961,11 +1025,12 @@ "run_id": "20260916T194044Z_ac52652ff1f0" }, "b35b916c22649e91ded175aede34f3937a16f42a354f7abab01a17bf2b0dfc97": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6062280543530544, 0.5943180448406931, - 0.04853610852408334, - 0.0737314840555176 + 0.03134706527371011, + 0.04910757543789796 ], "display_key": "qwen3.6-35b-a3b (rated)", "model": "qwen/qwen3.6-35b-a3b", @@ -976,11 +1041,12 @@ "run_id": "20260916T140418Z_b35b916c2264" }, "b5fdfe76425ee50c9f475587863990f2f6fd088d7f64c5c5119e7062a6a1396b": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.552530712157149, 0.63810610515893, - 0.053278822950175, - 0.08499253722116974 + 0.0377567739081018, + 0.08012242145033936 ], "display_key": "gpt-5.2 (rated)", "model": "openai/gpt-5.2", @@ -991,11 +1057,12 @@ "run_id": "20260917T125001Z_b5fdfe76425e" }, "bca0745bdc1554718d57891098f366214f05e9e896007a916d14a06efa33471a": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5778139029180694, 0.6959770802908057, - 0.03666290866643542, - 0.07382269298741832 + 0.034466398666598765, + 0.07823187669222507 ], "display_key": "qwen3-max-thinking (rated)", "model": "qwen/qwen3-max-thinking", @@ -1006,11 +1073,12 @@ "run_id": "20260916T145109Z_bca0745bdc15" }, "be8b6978ea49ace083544a82e1837e17898ca264b959e08939da01a63424e2f9": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.56922050588427, 0.7223026320129816, - 0.036665367096187215, - 0.053812885345722886 + 0.02710061973263113, + 0.0504019572127584 ], "display_key": "gpt-5 (rated)", "model": "openai/gpt-5", @@ -1021,11 +1089,12 @@ "run_id": "20260917T125811Z_be8b6978ea49" }, "c2911249c065c9b0cd2f69104d1f931bf8783940bd503fd60aa99ec6c6c0a761": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5489586575370888, 0.6356739354917721, - 0.05257011593980756, - 0.0883451278287498 + 0.043048462200454045, + 0.08272361735803685 ], "display_key": "gpt-5.6-terra (rated)", "model": "openai/gpt-5.6-terra", @@ -1036,11 +1105,12 @@ "run_id": "20260917T123400Z_c2911249c065" }, "c4e310179ab86b9d786c221d8bab6b6891f42d18836a8affc64249769eaffdef": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5713445083420574, 0.5644094213660266, - 0.04068128275908349, - 0.08447489368900887 + 0.022694588339844418, + 0.07410136506739047 ], "display_key": "deepseek-v3.2 (rated)", "model": "deepseek/deepseek-v3.2", @@ -1051,11 +1121,12 @@ "run_id": "20260917T114916Z_c4e310179ab8" }, "c64ced76e82ec32cf6be896420d813403a95a421474eefcbbcbba421f8189192": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.48767589171684006, 0.6482142857142857, - 0.04852805951852839, - 0.06690143204728374 + 0.024847625255712682, + 0.0636935529346349 ], "display_key": "muse-glimmer-30b (rated)", "model": "meta/muse-glimmer-30b", @@ -1066,11 +1137,12 @@ "run_id": "20260917T111805Z_c64ced76e82e" }, "c7e906b267d13cb638b0a646e9643e15bd8917152cb4ef6a2e888ba309501565": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5803524402579809, 0.6040068811440846, - 0.04571011970861469, - 0.08797307424268631 + 0.03394817417921382, + 0.07258813270003692 ], "display_key": "deepseek-v3.2-exp (rated)", "model": "deepseek/deepseek-v3.2-exp", @@ -1081,11 +1153,12 @@ "run_id": "20260917T115739Z_c7e906b267d1" }, "cc5d6fdd3ce85c73322e963fe44186c44015427f93cdbcecec838da8b2dd4ab4": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5896480994850559, 0.5981282109666797, - 0.035112198239544395, - 0.08647066879117997 + 0.03137406859274463, + 0.08079858766631393 ], "display_key": "deepseek-chat-v3-0324 (rated)", "model": "deepseek/deepseek-chat-v3-0324", @@ -1096,11 +1169,12 @@ "run_id": "20260917T122110Z_cc5d6fdd3ce8" }, "cd5db529649a179032180cecafe4fd98aff124ee3654f7d60996de704e2b63ef": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.4988977072310405, 0.6262540369683227, - 0.01537866158910887, - 0.05342525613259921 + 0.006279760963488564, + 0.04730514024812738 ], "display_key": "gemini-3.7-flash (rated)", "model": "google/gemini-3.7-flash", @@ -1111,11 +1185,12 @@ "run_id": "20260916T172946Z_cd5db529649a" }, "d21d1e81010d87b7f79bd9ceb0d5bb226485fe6393e266e169474968e6bf26f8": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5886813794915646, 0.6339803982919925, - 0.04800287084122153, - 0.07143925775678789 + 0.04261590693561221, + 0.04660781411126606 ], "display_key": "deepseek-v4-flash (rated)", "model": "deepseek/deepseek-v4-flash", @@ -1126,11 +1201,12 @@ "run_id": "20260917T112521Z_d21d1e81010d" }, "d4580ea120a65129dc484875ee13aa8a721bb133a93a18b19f974cc876f00178": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.619008820472186, 0.6035800722347154, - 0.0369775280569868, - 0.08938873815444547 + 0.020634824655178182, + 0.07295306220868648 ], "display_key": "deepseek-chat-v3.1 (rated)", "model": "deepseek/deepseek-chat-v3.1", @@ -1141,11 +1217,12 @@ "run_id": "20260917T121651Z_d4580ea120a6" }, "d81f7e66b3c4c720f8dd80768d3dfc25de6edeaf6a352cfed40e961f21fcfa22": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5535321928702047, 0.6422322099171051, - 0.05892062733798582, - 0.06715990340387798 + 0.03217817886332268, + 0.05342139005960689 ], "display_key": "glm-5.3 (rated)", "model": "z-ai/glm-5.3", @@ -1156,11 +1233,12 @@ "run_id": "20260916T161456Z_d81f7e66b3c4" }, "da8a3ff2c0318113bd43c5d8b1813b9d0947fcd6330bfe600ec28a203e6507b8": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5974867724867725, 0.5792094488085178, - 0.04341830148688508, - 0.09828751212663685 + 0.03608202052858709, + 0.0970663887466463 ], "display_key": "gpt-5.6-luna (rated)", "model": "openai/gpt-5.6-luna", @@ -1171,11 +1249,12 @@ "run_id": "20260917T122950Z_da8a3ff2c031" }, "dc09c3fe433c95591b336b279603c546152daf6534d90699323b3bcde754f5bf": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6353621031746031, 0.5685931857310826, - 0.03688413980467388, - 0.08024381413619414 + 0.035301389540358157, + 0.055707793544840394 ], "display_key": "qwen3-235b-a22b-2507 (rated)", "model": "qwen/qwen3-235b-a22b-2507", @@ -1186,11 +1265,12 @@ "run_id": "20260916T195827Z_dc09c3fe433c" }, "dc89e0b249b6189f2f85b9c5eebd98efb1595ea416761f9c2803ca57324275f6": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.3810994858696008, 0.6397686012218032, - 0.08204078442843406, - 0.039637685403670346 + 0.07770722831755433, + 0.03612058315515798 ], "display_key": "gemini-3.8-flash (rated)", "model": "google/gemini-3.8-flash", @@ -1201,11 +1281,12 @@ "run_id": "20260917T123103Z_dc89e0b249b6" }, "e0a9daef251d975c5e805e159f3813dd0c55b3f0919876b828e572b0530bea88": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.48469169991228817, 0.6190010643383659, - 0.07699729945533007, - 0.07041604518532184 + 0.032452612491036585, + 0.06512265958871256 ], "display_key": "qwen3.5-122b-a10b (rated)", "model": "qwen/qwen3.5-122b-a10b", @@ -1216,11 +1297,12 @@ "run_id": "20260916T143453Z_e0a9daef251d" }, "e3abbf27f8d036c12dd728740d2c263bb9be98f160437d5af3cb05b33dbfa761": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5255845653943481, 0.6176120675327024, - 0.03757041296270222, - 0.06469679141919639 + 0.030723613091297796, + 0.05017065754814348 ], "display_key": "qwen3.8-2.4t-a95b (rated)", "model": "qwen/qwen3.8-2.4t-a95b", @@ -1231,11 +1313,12 @@ "run_id": "20260917T122025Z_e3abbf27f8d0" }, "ed8190c48b2a3778bba8afff7381bb9f1578211b5e8f750d21321b62617c82c3": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.431934218610816, 0.6881570439623662, - 0.08092059548185657, - 0.07158456159738981 + 0.06843286443972658, + 0.05955561989504253 ], "display_key": "qwen3.8-27b (rated)", "model": "qwen/qwen3.8-27b", @@ -1246,11 +1329,12 @@ "run_id": "20260916T134026Z_ed8190c48b2a" }, "f095e0d2dbafdb91b51be3d0b3bd5a597730c8e3c397e0461023c87e924d92f8": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.5078196649029982, 0.6664359119716262, - 0.04898292658545207, - 0.07513374670738206 + 0.040332760807123835, + 0.07218684667619854 ], "display_key": "qwen3.7-plus (rated)", "model": "qwen/qwen3.7-plus", @@ -1261,11 +1345,12 @@ "run_id": "20260916T134925Z_f095e0d2dbaf" }, "fa37fc90604e5032183ec460bcc2632507566c3be3e9085f61a6ca85582b7d47": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.6514346395123706, 0.583014517676874, - 0.044355827535678856, - 0.07772092035914374 + 0.04021271468677619, + 0.04160867116289502 ], "display_key": "qwen3.6-flash (rated)", "model": "qwen/qwen3.6-flash", @@ -1276,11 +1361,12 @@ "run_id": "20260916T140130Z_fa37fc90604e" }, "fa9ff591d389ad2d8a4b7e3fc1410ed588508fe286430c06a941180b0590422b": { + "ci_method": "combined item and N-response-mean bootstrap", "coords": [ 0.4805424128340796, 0.6905543787488233, - 0.08346396425450253, - 0.07197747047181104 + 0.03427460911867346, + 0.06690339494976041 ], "display_key": "qwen3.6-max-preview (rated)", "model": "qwen/qwen3.6-max-preview", diff --git a/src/moralmaps/rated_cache.py b/src/moralmaps/rated_cache.py index 8481f47..4d19177 100644 --- a/src/moralmaps/rated_cache.py +++ b/src/moralmaps/rated_cache.py @@ -7,15 +7,14 @@ import os from pathlib import Path -def merge_completed(path: Path, additions: dict[str, dict]) -> dict: - """Merge complete entries while holding the cache lock across reread and replacement.""" +def _write_locked(path: Path, update) -> dict: lock_path = path.with_suffix(path.suffix + ".lock") with lock_path.open("w") as lock: fcntl.flock(lock, fcntl.LOCK_EX) cache = json.loads(path.read_text()) if path.exists() else {"schema": 2, "completed": {}} if cache["schema"] != 2: raise ValueError(f"unsupported WVS cache schema {cache['schema']}") - cache["completed"].update(additions) + update(cache["completed"]) temp = path.with_suffix(path.suffix + ".tmp") temp.write_text(json.dumps(cache, indent=2, sort_keys=True) + "\n") with temp.open("r+") as fh: @@ -24,3 +23,17 @@ def merge_completed(path: Path, additions: dict[str, dict]) -> dict: temp.replace(path) fcntl.flock(lock, fcntl.LOCK_UN) return cache + + +def merge_completed(path: Path, additions: dict[str, dict]) -> dict: + """Merge complete entries while holding the cache lock across reread and replacement.""" + return _write_locked(path, lambda completed: completed.update(additions)) + + +def update_coords(path: Path, coords: dict[str, list[float]]) -> dict: + """Replace only the derived coordinate summaries for complete protocol records.""" + def update(completed: dict[str, dict]) -> None: + for protocol_id, values in coords.items(): + completed[protocol_id]["coords"] = values + completed[protocol_id]["ci_method"] = "combined item and N-response-mean bootstrap" + return _write_locked(path, update)