diff --git a/scripts/wvs_score_all_options_recover_cache.py b/scripts/wvs_score_all_options_recover_cache.py index 1b929f9..7abe3fa 100644 --- a/scripts/wvs_score_all_options_recover_cache.py +++ b/scripts/wvs_score_all_options_recover_cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import argparse +import hashlib import json import multiprocessing import tempfile @@ -42,7 +43,6 @@ def recovered_entries(records: list[dict]) -> dict[str, dict]: resolved = resolve_items(load_wvs_all()) expected_ids = [item["suffix"] for axis in (X_AXIS, Y_AXIS) for item in resolved[axis]] entries = {} - rng = np.random.default_rng(0) for finish in finished: run_id = finish["run_id"] start = starts[run_id] @@ -52,7 +52,7 @@ def recovered_entries(records: list[dict]) -> dict[str, dict]: if any(row["valid_samples"] != 12 for row in rows.values()): raise ValueError(f"complete run {run_id} has non-12 item samples") psamples = {item_id: np.asarray(rows[item_id]["p_samples"]) for item_id in expected_ids} - coords = model_coord_ci(psamples, resolved, rng) + coords = model_coord_ci(psamples, resolved, np.random.default_rng(0)) model = finish["model"] entries[finish["protocol_id"]] = { "model": model, @@ -84,22 +84,48 @@ def concurrency_smoke() -> None: assert set(json.loads(path.read_text())["completed"]) == {"a", "b"} +def recovery_nonoverwriting_smoke() -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "cache.json" + original = {"schema": 2, "completed": {"existing": {"coords": [1, 2, 3, 4]}}} + path.write_text(json.dumps(original, sort_keys=True)) + recovered = {"existing": {"coords": [9, 9, 9, 9]}, "missing": {"coords": [5, 6, 7, 8]}} + additions = {key: value for key, value in recovered.items() if key not in original["completed"]} + merged = merge_completed(path, additions) + assert merged["completed"]["existing"] == original["completed"]["existing"] + assert merged["completed"]["missing"] == recovered["missing"] + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--smoke", action="store_true") args = parser.parse_args() if args.smoke: concurrency_smoke() - print("smoke: two concurrent cache writers preserve both completed entries") + recovery_nonoverwriting_smoke() + print("smoke: concurrent writers merge, and recovery never overwrites an existing entry") records = read_records() + existing = json.loads(CACHE.read_text())["completed"] if CACHE.exists() else {} + existing_hash = hashlib.sha256(json.dumps(existing, sort_keys=True, separators=(",", ":")).encode()).hexdigest() entries = recovered_entries(records) - merged = merge_completed(CACHE, entries) + additions = {key: value for key, value in entries.items() if key not in existing} + overlaps = {key: entry for key, entry in entries.items() if key in existing} + point_coordinates_match = all(existing[key]["coords"][:2] == entry["coords"][:2] for key, entry in overlaps.items()) + merged = merge_completed(CACHE, additions) + preserved = {key: merged["completed"][key] for key in existing} + preserved_hash = hashlib.sha256(json.dumps(preserved, sort_keys=True, separators=(",", ":")).encode()).hexdigest() audit = { "ledger": str(RECORDS), "ledger_valid_lines": len(records), - "recovered_complete_runs": len(entries), + "existing_entries_preserved_count": len(existing), + "existing_entries_sha256_before": existing_hash, + "existing_entries_sha256_after": preserved_hash, + "new_complete_runs_added": len(additions), + "new_protocol_ids": sorted(additions), + "new_models": sorted(entry["model"] for entry in additions.values()), + "overlap_complete_runs": len(overlaps), + "overlap_point_coordinates_equal": point_coordinates_match, "cache_completed_entries_after_merge": len(merged["completed"]), - "recovered_models": sorted(entry["model"] for entry in entries.values()), } AUDIT.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n") print(json.dumps(audit, indent=2, sort_keys=True)) diff --git a/slop/audits/20260917_wvs_score_all_options_cache_recovery.json b/slop/audits/20260917_wvs_score_all_options_cache_recovery.json index 9ee2631..ec5784e 100644 --- a/slop/audits/20260917_wvs_score_all_options_cache_recovery.json +++ b/slop/audits/20260917_wvs_score_all_options_cache_recovery.json @@ -1,61 +1,23 @@ { "cache_completed_entries_after_merge": 53, + "existing_entries_preserved_count": 49, + "existing_entries_sha256_after": "0866bae2697e7f7b7a14edc2454968319fbbc26ad8b134ea05186840d28f99c2", + "existing_entries_sha256_before": "0866bae2697e7f7b7a14edc2454968319fbbc26ad8b134ea05186840d28f99c2", "ledger": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl", "ledger_valid_lines": 32461, - "recovered_complete_runs": 53, - "recovered_models": [ - "anthropic/claude-fable-5.1", + "new_complete_runs_added": 4, + "new_models": [ "deepseek/deepseek-v4-flash", "deepseek/deepseek-v4-flash-0731", - "deepseek/deepseek-v4.1-flash", - "google/gemini-3.7-flash", "meta/muse-glimmer-30b", - "meta/muse-spark-1.3", - "moonshotai/kimi-k2.6", - "moonshotai/kimi-k3", - "openai/gpt-5-nano", - "openai/gpt-5.6-sol", - "openai/gpt-6-astra", - "qwen/qwen-2.5-72b-instruct", - "qwen/qwen-2.5-7b-instruct", - "qwen/qwen-plus", - "qwen/qwen-plus-2025-07-28", - "qwen/qwen2.5-vl-72b-instruct", - "qwen/qwen3-14b", - "qwen/qwen3-235b-a22b", - "qwen/qwen3-235b-a22b-2507", - "qwen/qwen3-30b-a3b", - "qwen/qwen3-30b-a3b-instruct-2507", - "qwen/qwen3-32b", - "qwen/qwen3-8b", - "qwen/qwen3-coder", - "qwen/qwen3-coder-30b-a3b-instruct", - "qwen/qwen3-coder-next", - "qwen/qwen3-coder-plus", - "qwen/qwen3-max-thinking", - "qwen/qwen3-next-80b-a3b-instruct", - "qwen/qwen3-vl-235b-a22b-instruct", - "qwen/qwen3-vl-30b-a3b-instruct", - "qwen/qwen3-vl-32b-instruct", - "qwen/qwen3-vl-8b-instruct", - "qwen/qwen3.5-122b-a10b", - "qwen/qwen3.5-27b", - "qwen/qwen3.5-35b-a3b", - "qwen/qwen3.5-397b-a17b", - "qwen/qwen3.5-9b", - "qwen/qwen3.5-plus-02-15", - "qwen/qwen3.5-plus-20260420", - "qwen/qwen3.6-27b", - "qwen/qwen3.6-35b-a3b", - "qwen/qwen3.6-flash", - "qwen/qwen3.6-max-preview", - "qwen/qwen3.6-plus", - "qwen/qwen3.7-flash", - "qwen/qwen3.7-max", - "qwen/qwen3.7-plus", - "qwen/qwen3.8-27b", - "qwen/qwen3.8-flash", - "thinkingmachines/inkling", - "z-ai/glm-5.3" - ] + "moonshotai/kimi-k2.6" + ], + "new_protocol_ids": [ + "333ece6448f97447c57f24f9e9fcf0a7fb72a1b0f96696aef3c38a9c1d2d2308", + "a2df2f1a32fde5dae2c1545120a4fd72cea4743bafe71a1768997dc3d125b4f4", + "c64ced76e82ec32cf6be896420d813403a95a421474eefcbbcbba421f8189192", + "d21d1e81010d87b7f79bd9ceb0d5bb226485fe6393e266e169474968e6bf26f8" + ], + "overlap_complete_runs": 49, + "overlap_point_coordinates_equal": true } diff --git a/slop/research/wvs/20260916_openrouter/wvs_iw_rated.json b/slop/research/wvs/20260916_openrouter/wvs_iw_rated.json index e3a674b..db92b55 100644 --- a/slop/research/wvs/20260916_openrouter/wvs_iw_rated.json +++ b/slop/research/wvs/20260916_openrouter/wvs_iw_rated.json @@ -4,8 +4,8 @@ "coords": [ 0.6214472858835399, 0.6710244268262847, - 0.03498102851088573, - 0.06859664714201569 + 0.03786483811416081, + 0.06918409770409496 ], "display_key": "kimi-k3 (rated)", "model": "moonshotai/kimi-k3", @@ -19,8 +19,8 @@ "coords": [ 0.5849525883130219, 0.59521139256589, - 0.0388043837051704, - 0.07867897818636554 + 0.037643038378990704, + 0.07695278664473065 ], "display_key": "qwen2.5-vl-72b-instruct (rated)", "model": "qwen/qwen2.5-vl-72b-instruct", @@ -34,8 +34,8 @@ "coords": [ 0.5335737179487179, 0.5337214372928658, - 0.02927867567843363, - 0.09371226473048877 + 0.029444543942379133, + 0.09738610052745941 ], "display_key": "qwen3-vl-30b-a3b-instruct (rated)", "model": "qwen/qwen3-vl-30b-a3b-instruct", @@ -49,8 +49,8 @@ "coords": [ 0.6175297619047618, 0.5976147806792967, - 0.03541797753120733, - 0.06871303341417663 + 0.035727022605345674, + 0.07361977335041572 ], "display_key": "qwen3-coder-plus (rated)", "model": "qwen/qwen3-coder-plus", @@ -64,8 +64,8 @@ "coords": [ 0.5274474357644292, 0.6650744376984001, - 0.04284951309217784, - 0.06880740953633888 + 0.044561828476471346, + 0.06617035838663828 ], "display_key": "gpt-5.6-sol (rated)", "model": "openai/gpt-5.6-sol", @@ -79,8 +79,8 @@ "coords": [ 0.5036638266417679, 0.6503283973522069, - 0.056572571946403354, - 0.06023989073439502 + 0.057017134313755345, + 0.06039652604026301 ], "display_key": "qwen3.7-max (rated)", "model": "qwen/qwen3.7-max", @@ -94,8 +94,8 @@ "coords": [ 0.6061818582651916, 0.6004689754689754, - 0.02896221739482271, - 0.08351751936863892 + 0.029738189920548014, + 0.07794129978765893 ], "display_key": "qwen3-vl-8b-instruct (rated)", "model": "qwen/qwen3-vl-8b-instruct", @@ -109,8 +109,8 @@ "coords": [ 0.5017202097104058, 0.5916881018254176, - 0.04329249642491525, - 0.0665062125461305 + 0.04660057418362078, + 0.07020408831610975 ], "display_key": "qwen3.5-9b (rated)", "model": "qwen/qwen3.5-9b", @@ -124,8 +124,8 @@ "coords": [ 0.598778659611993, 0.6204503168788883, - 0.03672846655615712, - 0.08900654804546242 + 0.04059052018905529, + 0.08949774510806 ], "display_key": "qwen3-coder (rated)", "model": "qwen/qwen3-coder", @@ -139,8 +139,8 @@ "coords": [ 0.5482108382155133, 0.5760765801607652, - 0.06993277323172409, - 0.08010838046301313 + 0.06725327571313976, + 0.08275579176334262 ], "display_key": "deepseek-v4-flash-0731 (rated)", "model": "deepseek/deepseek-v4-flash-0731", @@ -154,8 +154,8 @@ "coords": [ 0.4153054353054353, 0.6346365065494385, - 0.07767226786811594, - 0.07694753838347716 + 0.07206749839259896, + 0.07710357656753936 ], "display_key": "qwen3.8-flash (rated)", "model": "qwen/qwen3.8-flash", @@ -169,8 +169,8 @@ "coords": [ 0.5030952380952382, 0.5846967846967848, - 0.04532986462569823, - 0.09037513909634669 + 0.046958286642217247, + 0.08746316369004264 ], "display_key": "qwen3-coder-30b-a3b-instruct (rated)", "model": "qwen/qwen3-coder-30b-a3b-instruct", @@ -184,8 +184,8 @@ "coords": [ 0.4253104631546661, 0.731799663031547, - 0.09130591901488724, - 0.06265630083510712 + 0.0936048666617984, + 0.060012518938080954 ], "display_key": "muse-spark-1.3 (rated)", "model": "meta/muse-spark-1.3", @@ -199,8 +199,8 @@ "coords": [ 0.5790919024614678, 0.6085929478125246, - 0.042481181470475825, - 0.026900473313497777 + 0.038145029256266345, + 0.025980758516149186 ], "display_key": "claude-fable-5.1 (rated)", "model": "anthropic/claude-fable-5.1", @@ -214,8 +214,8 @@ "coords": [ 0.5966931216931217, 0.5338064713064713, - 0.011999091335169519, - 0.10525911258820818 + 0.011307868884765215, + 0.10204509849192553 ], "display_key": "qwen3-vl-32b-instruct (rated)", "model": "qwen/qwen3-vl-32b-instruct", @@ -229,8 +229,8 @@ "coords": [ 0.6137896825396826, 0.5819444444444445, - 0.016073689650860815, - 0.08458895983430899 + 0.015958730849444745, + 0.0834429418953706 ], "display_key": "qwen3-14b (rated)", "model": "qwen/qwen3-14b", @@ -244,8 +244,8 @@ "coords": [ 0.6363133579397021, 0.5792251640447779, - 0.026429560820398015, - 0.06316211550201252 + 0.027397680484314686, + 0.06607943334210371 ], "display_key": "qwen3-coder-next (rated)", "model": "qwen/qwen3-coder-next", @@ -259,8 +259,8 @@ "coords": [ 0.6064902599972044, 0.5413085527420844, - 0.027826894148137167, - 0.06753477500466712 + 0.028266888268597475, + 0.07251640240693584 ], "display_key": "qwen3-32b (rated)", "model": "qwen/qwen3-32b", @@ -274,8 +274,8 @@ "coords": [ 0.6217724867724868, 0.5391326046087952, - 0.01514321596739316, - 0.05660900381630799 + 0.015436127385306617, + 0.05984330714155522 ], "display_key": "qwen3-8b (rated)", "model": "qwen/qwen3-8b", @@ -289,8 +289,8 @@ "coords": [ 0.5993276014109347, 0.5462479278681369, - 0.025633101038546928, - 0.10120032015015096 + 0.024660010076102412, + 0.10518394004968958 ], "display_key": "qwen3-vl-235b-a22b-instruct (rated)", "model": "qwen/qwen3-vl-235b-a22b-instruct", @@ -304,8 +304,8 @@ "coords": [ 0.5509667037841642, 0.5895236753978316, - 0.02679137212415509, - 0.08184125856535546 + 0.02658132977411291, + 0.08063463160776993 ], "display_key": "qwen-2.5-7b-instruct (rated)", "model": "qwen/qwen-2.5-7b-instruct", @@ -319,8 +319,8 @@ "coords": [ 0.5614798614630517, 0.6880988993033613, - 0.05786677240929432, - 0.05700827650004879 + 0.050774441812280406, + 0.057780026773918566 ], "display_key": "inkling (rated)", "model": "thinkingmachines/inkling", @@ -334,8 +334,8 @@ "coords": [ 0.5856201899951899, 0.6494018547589976, - 0.060949781665560955, - 0.08885439746145042 + 0.06114886163791156, + 0.08129405199775336 ], "display_key": "qwen3.5-plus-20260420 (rated)", "model": "qwen/qwen3.5-plus-20260420", @@ -349,8 +349,8 @@ "coords": [ 0.5974338624338624, 0.6515634387658198, - 0.02326665279699495, - 0.08397986216363947 + 0.023795241546514957, + 0.07922810144513995 ], "display_key": "qwen3.5-27b (rated)", "model": "qwen/qwen3.5-27b", @@ -364,8 +364,8 @@ "coords": [ 0.5372045855379188, 0.6595114541543113, - 0.043163200110489575, - 0.08370748405181158 + 0.04180484013035134, + 0.08450011787948054 ], "display_key": "qwen3.5-plus-02-15 (rated)", "model": "qwen/qwen3.5-plus-02-15", @@ -379,8 +379,8 @@ "coords": [ 0.538440257742794, 0.5530118458934986, - 0.07158681387163214, - 0.09053896523651496 + 0.07277582530416211, + 0.0898393391241473 ], "display_key": "qwen3.5-35b-a3b (rated)", "model": "qwen/qwen3.5-35b-a3b", @@ -394,8 +394,8 @@ "coords": [ 0.4526851851851852, 0.6334977783064814, - 0.05996398964814259, - 0.08320455765016928 + 0.057818785990183905, + 0.08298095716984875 ], "display_key": "gpt-5-nano (rated)", "model": "openai/gpt-5-nano", @@ -424,8 +424,8 @@ "coords": [ 0.6427511094043352, 0.5554953612269281, - 0.03400287356919112, - 0.09086207311764848 + 0.03535208249381765, + 0.08497138486454137 ], "display_key": "qwen3-235b-a22b (rated)", "model": "qwen/qwen3-235b-a22b", @@ -439,8 +439,8 @@ "coords": [ 0.6707823388714307, 0.6060018656291949, - 0.03345632993101348, - 0.09299505374530567 + 0.03451993837104684, + 0.09876091230627733 ], "display_key": "qwen3-30b-a3b (rated)", "model": "qwen/qwen3-30b-a3b", @@ -454,8 +454,8 @@ "coords": [ 0.6025617283950616, 0.6287037037037038, - 0.03435593378616836, - 0.09044283339419919 + 0.03396367302553316, + 0.09406697361625513 ], "display_key": "qwen-plus (rated)", "model": "qwen/qwen-plus", @@ -469,8 +469,8 @@ "coords": [ 0.6450907949109246, 0.5437992771640354, - 0.022113968893118275, - 0.10180231078172153 + 0.023312776383531648, + 0.10024881542221994 ], "display_key": "qwen3-30b-a3b-instruct-2507 (rated)", "model": "qwen/qwen3-30b-a3b-instruct-2507", @@ -484,8 +484,8 @@ "coords": [ 0.5419602854216832, 0.595896730624977, - 0.053731321649797846, - 0.08321090383068265 + 0.05193570378692634, + 0.08265065423220756 ], "display_key": "deepseek-v4.1-flash (rated)", "model": "deepseek/deepseek-v4.1-flash", @@ -499,8 +499,8 @@ "coords": [ 0.5408168991502326, 0.6493278669204595, - 0.03596105265682425, - 0.08640277131980666 + 0.03917823180041766, + 0.08760189869507796 ], "display_key": "qwen3.5-397b-a17b (rated)", "model": "qwen/qwen3.5-397b-a17b", @@ -514,8 +514,8 @@ "coords": [ 0.4569019748367575, 0.6751145899955423, - 0.07783687662394496, - 0.05577857978656997 + 0.07644348564506928, + 0.0567289888770763 ], "display_key": "gpt-6-astra (rated)", "model": "openai/gpt-6-astra", @@ -529,8 +529,8 @@ "coords": [ 0.5196778711484593, 0.6698994127565557, - 0.04339901270073358, - 0.07906452366543723 + 0.042934824935707064, + 0.07801121120269483 ], "display_key": "qwen3.6-plus (rated)", "model": "qwen/qwen3.6-plus", @@ -544,8 +544,8 @@ "coords": [ 0.6026546601546601, 0.5957470912477916, - 0.033015451958604745, - 0.09106122338341402 + 0.03274570154412936, + 0.08673540897470654 ], "display_key": "qwen-2.5-72b-instruct (rated)", "model": "qwen/qwen-2.5-72b-instruct", @@ -559,8 +559,8 @@ "coords": [ 0.6652380952380952, 0.7022990677752582, - 0.026124454690573333, - 0.05656973783290617 + 0.026433969418704885, + 0.06269095940430941 ], "display_key": "qwen3-next-80b-a3b-instruct (rated)", "model": "qwen/qwen3-next-80b-a3b-instruct", @@ -574,8 +574,8 @@ "coords": [ 0.610243091658148, 0.6687657686659433, - 0.04596153462279471, - 0.07682588731641059 + 0.04414498817375987, + 0.08085439770187478 ], "display_key": "kimi-k2.6 (rated)", "model": "moonshotai/kimi-k2.6", @@ -589,8 +589,8 @@ "coords": [ 0.45740779531102105, 0.6795887184190553, - 0.0930227110336478, - 0.07587935840780517 + 0.09233310052166174, + 0.07166972793043255 ], "display_key": "qwen3.6-27b (rated)", "model": "qwen/qwen3.6-27b", @@ -604,8 +604,8 @@ "coords": [ 0.6417361111111111, 0.5127110103300581, - 0.035533783509182494, - 0.08247734174209306 + 0.03807268153101321, + 0.08042423628815949 ], "display_key": "qwen-plus-2025-07-28 (rated)", "model": "qwen/qwen-plus-2025-07-28", @@ -619,8 +619,8 @@ "coords": [ 0.6062280543530544, 0.5943180448406931, - 0.04816007701260514, - 0.072116166212997 + 0.04853610852408334, + 0.0737314840555176 ], "display_key": "qwen3.6-35b-a3b (rated)", "model": "qwen/qwen3.6-35b-a3b", @@ -634,8 +634,8 @@ "coords": [ 0.5778139029180694, 0.6959770802908057, - 0.03693419149935657, - 0.08351595033327236 + 0.03666290866643542, + 0.07382269298741832 ], "display_key": "qwen3-max-thinking (rated)", "model": "qwen/qwen3-max-thinking", @@ -649,8 +649,8 @@ "coords": [ 0.48767589171684006, 0.6482142857142857, - 0.0426101913142831, - 0.06478149056076417 + 0.04852805951852839, + 0.06690143204728374 ], "display_key": "muse-glimmer-30b (rated)", "model": "meta/muse-glimmer-30b", @@ -664,8 +664,8 @@ "coords": [ 0.4988977072310405, 0.6262540369683227, - 0.013608591135547174, - 0.04901151771282093 + 0.01537866158910887, + 0.05342525613259921 ], "display_key": "gemini-3.7-flash (rated)", "model": "google/gemini-3.7-flash", @@ -679,8 +679,8 @@ "coords": [ 0.5886813794915646, 0.6339803982919925, - 0.05257271891027728, - 0.07140876500169609 + 0.04800287084122153, + 0.07143925775678789 ], "display_key": "deepseek-v4-flash (rated)", "model": "deepseek/deepseek-v4-flash", @@ -694,8 +694,8 @@ "coords": [ 0.5535321928702047, 0.6422322099171051, - 0.05621697080592131, - 0.061403096129926435 + 0.05892062733798582, + 0.06715990340387798 ], "display_key": "glm-5.3 (rated)", "model": "z-ai/glm-5.3", @@ -709,8 +709,8 @@ "coords": [ 0.6353621031746031, 0.5685931857310826, - 0.03785687360618813, - 0.08248362706680903 + 0.03688413980467388, + 0.08024381413619414 ], "display_key": "qwen3-235b-a22b-2507 (rated)", "model": "qwen/qwen3-235b-a22b-2507", @@ -724,8 +724,8 @@ "coords": [ 0.48469169991228817, 0.6190010643383659, - 0.07549082925386548, - 0.07517115878830224 + 0.07699729945533007, + 0.07041604518532184 ], "display_key": "qwen3.5-122b-a10b (rated)", "model": "qwen/qwen3.5-122b-a10b", @@ -739,8 +739,8 @@ "coords": [ 0.431934218610816, 0.6881570439623662, - 0.08341797207800665, - 0.06980159408625697 + 0.08092059548185657, + 0.07158456159738981 ], "display_key": "qwen3.8-27b (rated)", "model": "qwen/qwen3.8-27b", @@ -754,8 +754,8 @@ "coords": [ 0.5078196649029982, 0.6664359119716262, - 0.049856523015613566, - 0.0792534671436552 + 0.04898292658545207, + 0.07513374670738206 ], "display_key": "qwen3.7-plus (rated)", "model": "qwen/qwen3.7-plus", @@ -769,8 +769,8 @@ "coords": [ 0.6514346395123706, 0.583014517676874, - 0.04639237632705332, - 0.077088244066197 + 0.044355827535678856, + 0.07772092035914374 ], "display_key": "qwen3.6-flash (rated)", "model": "qwen/qwen3.6-flash", @@ -784,8 +784,8 @@ "coords": [ 0.4805424128340796, 0.6905543787488233, - 0.08437549450275793, - 0.07478325635344732 + 0.08346396425450253, + 0.07197747047181104 ], "display_key": "qwen3.6-max-preview (rated)", "model": "qwen/qwen3.6-max-preview",