wassname/moral-maps
1
0
Fork 0
mirror of https://github.com/wassname/moral-maps.git synced 2026-09-26 14:00:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

Preserve canonical cache entries during recovery

Browse Source 
Co-Authored-By: PI[openai-codex] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-09-17 19:45:55 +08:00
co-authored by PI[openai-codex]
parent 82fa098b28
commit b444e1fcf7
3 changed files with 151 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/wvs_score_all_options_recover_cache.py
+32 -6
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import argparse
import hashlib
import json
import multiprocessing
import tempfile
@@ -42,7 +43,6 @@ def recovered_entries(records: list[dict]) -> dict[str, dict]:
resolved = resolve_items(load_wvs_all())
expected_ids = [item["suffix"] for axis in (X_AXIS, Y_AXIS) for item in resolved[axis]]
entries = {}
rng = np.random.default_rng(0)
for finish in finished:
run_id = finish["run_id"]
start = starts[run_id]
@@ -52,7 +52,7 @@ def recovered_entries(records: list[dict]) -> dict[str, dict]:
if any(row["valid_samples"] != 12 for row in rows.values()):
raise ValueError(f"complete run {run_id} has non-12 item samples")
psamples = {item_id: np.asarray(rows[item_id]["p_samples"]) for item_id in expected_ids}
coords = model_coord_ci(psamples, resolved, rng)
coords = model_coord_ci(psamples, resolved, np.random.default_rng(0))
model = finish["model"]
entries[finish["protocol_id"]] = {
"model": model,
@@ -84,22 +84,48 @@ def concurrency_smoke() -> None:
assert set(json.loads(path.read_text())["completed"]) == {"a", "b"}
def recovery_nonoverwriting_smoke() -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "cache.json"
original = {"schema": 2, "completed": {"existing": {"coords": [1, 2, 3, 4]}}}
path.write_text(json.dumps(original, sort_keys=True))
recovered = {"existing": {"coords": [9, 9, 9, 9]}, "missing": {"coords": [5, 6, 7, 8]}}
additions = {key: value for key, value in recovered.items() if key not in original["completed"]}
merged = merge_completed(path, additions)
assert merged["completed"]["existing"] == original["completed"]["existing"]
assert merged["completed"]["missing"] == recovered["missing"]
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--smoke", action="store_true")
args = parser.parse_args()
if args.smoke:
concurrency_smoke()
print("smoke: two concurrent cache writers preserve both completed entries")
recovery_nonoverwriting_smoke()
print("smoke: concurrent writers merge, and recovery never overwrites an existing entry")
records = read_records()
existing = json.loads(CACHE.read_text())["completed"] if CACHE.exists() else {}
existing_hash = hashlib.sha256(json.dumps(existing, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
entries = recovered_entries(records)
merged = merge_completed(CACHE, entries)
additions = {key: value for key, value in entries.items() if key not in existing}
overlaps = {key: entry for key, entry in entries.items() if key in existing}
point_coordinates_match = all(existing[key]["coords"][:2] == entry["coords"][:2] for key, entry in overlaps.items())
merged = merge_completed(CACHE, additions)
preserved = {key: merged["completed"][key] for key in existing}
preserved_hash = hashlib.sha256(json.dumps(preserved, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
audit = {
"ledger": str(RECORDS),
"ledger_valid_lines": len(records),
"recovered_complete_runs": len(entries),
"existing_entries_preserved_count": len(existing),
"existing_entries_sha256_before": existing_hash,
"existing_entries_sha256_after": preserved_hash,
"new_complete_runs_added": len(additions),
"new_protocol_ids": sorted(additions),
"new_models": sorted(entry["model"] for entry in additions.values()),
"overlap_complete_runs": len(overlaps),
"overlap_point_coordinates_equal": point_coordinates_match,
"cache_completed_entries_after_merge": len(merged["completed"]),
"recovered_models": sorted(entry["model"] for entry in entries.values()),
}
AUDIT.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n")
print(json.dumps(audit, indent=2, sort_keys=True))
slop/audits/20260917_wvs_score_all_options_cache_recovery.json
+15 -53
View File
@@ -1,61 +1,23 @@
{
"cache_completed_entries_after_merge": 53,
"existing_entries_preserved_count": 49,
"existing_entries_sha256_after": "0866bae2697e7f7b7a14edc2454968319fbbc26ad8b134ea05186840d28f99c2",
"existing_entries_sha256_before": "0866bae2697e7f7b7a14edc2454968319fbbc26ad8b134ea05186840d28f99c2",
"ledger": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"ledger_valid_lines": 32461,
"recovered_complete_runs": 53,
"recovered_models": [
"anthropic/claude-fable-5.1",
"new_complete_runs_added": 4,
"new_models": [
"deepseek/deepseek-v4-flash",
"deepseek/deepseek-v4-flash-0731",
"deepseek/deepseek-v4.1-flash",
"google/gemini-3.7-flash",
"meta/muse-glimmer-30b",
"meta/muse-spark-1.3",
"moonshotai/kimi-k2.6",
"moonshotai/kimi-k3",
"openai/gpt-5-nano",
"openai/gpt-5.6-sol",
"openai/gpt-6-astra",
"qwen/qwen-2.5-72b-instruct",
"qwen/qwen-2.5-7b-instruct",
"qwen/qwen-plus",
"qwen/qwen-plus-2025-07-28",
"qwen/qwen2.5-vl-72b-instruct",
"qwen/qwen3-14b",
"qwen/qwen3-235b-a22b",
"qwen/qwen3-235b-a22b-2507",
"qwen/qwen3-30b-a3b",
"qwen/qwen3-30b-a3b-instruct-2507",
"qwen/qwen3-32b",
"qwen/qwen3-8b",
"qwen/qwen3-coder",
"qwen/qwen3-coder-30b-a3b-instruct",
"qwen/qwen3-coder-next",
"qwen/qwen3-coder-plus",
"qwen/qwen3-max-thinking",
"qwen/qwen3-next-80b-a3b-instruct",
"qwen/qwen3-vl-235b-a22b-instruct",
"qwen/qwen3-vl-30b-a3b-instruct",
"qwen/qwen3-vl-32b-instruct",
"qwen/qwen3-vl-8b-instruct",
"qwen/qwen3.5-122b-a10b",
"qwen/qwen3.5-27b",
"qwen/qwen3.5-35b-a3b",
"qwen/qwen3.5-397b-a17b",
"qwen/qwen3.5-9b",
"qwen/qwen3.5-plus-02-15",
"qwen/qwen3.5-plus-20260420",
"qwen/qwen3.6-27b",
"qwen/qwen3.6-35b-a3b",
"qwen/qwen3.6-flash",
"qwen/qwen3.6-max-preview",
"qwen/qwen3.6-plus",
"qwen/qwen3.7-flash",
"qwen/qwen3.7-max",
"qwen/qwen3.7-plus",
"qwen/qwen3.8-27b",
"qwen/qwen3.8-flash",
"thinkingmachines/inkling",
"z-ai/glm-5.3"
]
"moonshotai/kimi-k2.6"
],
"new_protocol_ids": [
"333ece6448f97447c57f24f9e9fcf0a7fb72a1b0f96696aef3c38a9c1d2d2308",
"a2df2f1a32fde5dae2c1545120a4fd72cea4743bafe71a1768997dc3d125b4f4",
"c64ced76e82ec32cf6be896420d813403a95a421474eefcbbcbba421f8189192",
"d21d1e81010d87b7f79bd9ceb0d5bb226485fe6393e266e169474968e6bf26f8"
],
"overlap_complete_runs": 49,
"overlap_point_coordinates_equal": true
}
slop/research/wvs/20260916_openrouter/wvs_iw_rated.json
+104 -104
View File
@@ -4,8 +4,8 @@
"coords": [
0.6214472858835399,
0.6710244268262847,
0.03498102851088573,
0.06859664714201569
0.03786483811416081,
0.06918409770409496
],
"display_key": "kimi-k3 (rated)",
"model": "moonshotai/kimi-k3",
@@ -19,8 +19,8 @@
"coords": [
0.5849525883130219,
0.59521139256589,
0.0388043837051704,
0.07867897818636554
0.037643038378990704,
0.07695278664473065
],
"display_key": "qwen2.5-vl-72b-instruct (rated)",
"model": "qwen/qwen2.5-vl-72b-instruct",
@@ -34,8 +34,8 @@
"coords": [
0.5335737179487179,
0.5337214372928658,
0.02927867567843363,
0.09371226473048877
0.029444543942379133,
0.09738610052745941
],
"display_key": "qwen3-vl-30b-a3b-instruct (rated)",
"model": "qwen/qwen3-vl-30b-a3b-instruct",
@@ -49,8 +49,8 @@
"coords": [
0.6175297619047618,
0.5976147806792967,
0.03541797753120733,
0.06871303341417663
0.035727022605345674,
0.07361977335041572
],
"display_key": "qwen3-coder-plus (rated)",
"model": "qwen/qwen3-coder-plus",
@@ -64,8 +64,8 @@
"coords": [
0.5274474357644292,
0.6650744376984001,
0.04284951309217784,
0.06880740953633888
0.044561828476471346,
0.06617035838663828
],
"display_key": "gpt-5.6-sol (rated)",
"model": "openai/gpt-5.6-sol",
@@ -79,8 +79,8 @@
"coords": [
0.5036638266417679,
0.6503283973522069,
0.056572571946403354,
0.06023989073439502
0.057017134313755345,
0.06039652604026301
],
"display_key": "qwen3.7-max (rated)",
"model": "qwen/qwen3.7-max",
@@ -94,8 +94,8 @@
"coords": [
0.6061818582651916,
0.6004689754689754,
0.02896221739482271,
0.08351751936863892
0.029738189920548014,
0.07794129978765893
],
"display_key": "qwen3-vl-8b-instruct (rated)",
"model": "qwen/qwen3-vl-8b-instruct",
@@ -109,8 +109,8 @@
"coords": [
0.5017202097104058,
0.5916881018254176,
0.04329249642491525,
0.0665062125461305
0.04660057418362078,
0.07020408831610975
],
"display_key": "qwen3.5-9b (rated)",
"model": "qwen/qwen3.5-9b",
@@ -124,8 +124,8 @@
"coords": [
0.598778659611993,
0.6204503168788883,
0.03672846655615712,
0.08900654804546242
0.04059052018905529,
0.08949774510806
],
"display_key": "qwen3-coder (rated)",
"model": "qwen/qwen3-coder",
@@ -139,8 +139,8 @@
"coords": [
0.5482108382155133,
0.5760765801607652,
0.06993277323172409,
0.08010838046301313
0.06725327571313976,
0.08275579176334262
],
"display_key": "deepseek-v4-flash-0731 (rated)",
"model": "deepseek/deepseek-v4-flash-0731",
@@ -154,8 +154,8 @@
"coords": [
0.4153054353054353,
0.6346365065494385,
0.07767226786811594,
0.07694753838347716
0.07206749839259896,
0.07710357656753936
],
"display_key": "qwen3.8-flash (rated)",
"model": "qwen/qwen3.8-flash",
@@ -169,8 +169,8 @@
"coords": [
0.5030952380952382,
0.5846967846967848,
0.04532986462569823,
0.09037513909634669
0.046958286642217247,
0.08746316369004264
],
"display_key": "qwen3-coder-30b-a3b-instruct (rated)",
"model": "qwen/qwen3-coder-30b-a3b-instruct",
@@ -184,8 +184,8 @@
"coords": [
0.4253104631546661,
0.731799663031547,
0.09130591901488724,
0.06265630083510712
0.0936048666617984,
0.060012518938080954
],
"display_key": "muse-spark-1.3 (rated)",
"model": "meta/muse-spark-1.3",
@@ -199,8 +199,8 @@
"coords": [
0.5790919024614678,
0.6085929478125246,
0.042481181470475825,
0.026900473313497777
0.038145029256266345,
0.025980758516149186
],
"display_key": "claude-fable-5.1 (rated)",
"model": "anthropic/claude-fable-5.1",
@@ -214,8 +214,8 @@
"coords": [
0.5966931216931217,
0.5338064713064713,
0.011999091335169519,
0.10525911258820818
0.011307868884765215,
0.10204509849192553
],
"display_key": "qwen3-vl-32b-instruct (rated)",
"model": "qwen/qwen3-vl-32b-instruct",
@@ -229,8 +229,8 @@
"coords": [
0.6137896825396826,
0.5819444444444445,
0.016073689650860815,
0.08458895983430899
0.015958730849444745,
0.0834429418953706
],
"display_key": "qwen3-14b (rated)",
"model": "qwen/qwen3-14b",
@@ -244,8 +244,8 @@
"coords": [
0.6363133579397021,
0.5792251640447779,
0.026429560820398015,
0.06316211550201252
0.027397680484314686,
0.06607943334210371
],
"display_key": "qwen3-coder-next (rated)",
"model": "qwen/qwen3-coder-next",
@@ -259,8 +259,8 @@
"coords": [
0.6064902599972044,
0.5413085527420844,
0.027826894148137167,
0.06753477500466712
0.028266888268597475,
0.07251640240693584
],
"display_key": "qwen3-32b (rated)",
"model": "qwen/qwen3-32b",
@@ -274,8 +274,8 @@
"coords": [
0.6217724867724868,
0.5391326046087952,
0.01514321596739316,
0.05660900381630799
0.015436127385306617,
0.05984330714155522
],
"display_key": "qwen3-8b (rated)",
"model": "qwen/qwen3-8b",
@@ -289,8 +289,8 @@
"coords": [
0.5993276014109347,
0.5462479278681369,
0.025633101038546928,
0.10120032015015096
0.024660010076102412,
0.10518394004968958
],
"display_key": "qwen3-vl-235b-a22b-instruct (rated)",
"model": "qwen/qwen3-vl-235b-a22b-instruct",
@@ -304,8 +304,8 @@
"coords": [
0.5509667037841642,
0.5895236753978316,
0.02679137212415509,
0.08184125856535546
0.02658132977411291,
0.08063463160776993
],
"display_key": "qwen-2.5-7b-instruct (rated)",
"model": "qwen/qwen-2.5-7b-instruct",
@@ -319,8 +319,8 @@
"coords": [
0.5614798614630517,
0.6880988993033613,
0.05786677240929432,
0.05700827650004879
0.050774441812280406,
0.057780026773918566
],
"display_key": "inkling (rated)",
"model": "thinkingmachines/inkling",
@@ -334,8 +334,8 @@
"coords": [
0.5856201899951899,
0.6494018547589976,
0.060949781665560955,
0.08885439746145042
0.06114886163791156,
0.08129405199775336
],
"display_key": "qwen3.5-plus-20260420 (rated)",
"model": "qwen/qwen3.5-plus-20260420",
@@ -349,8 +349,8 @@
"coords": [
0.5974338624338624,
0.6515634387658198,
0.02326665279699495,
0.08397986216363947
0.023795241546514957,
0.07922810144513995
],
"display_key": "qwen3.5-27b (rated)",
"model": "qwen/qwen3.5-27b",
@@ -364,8 +364,8 @@
"coords": [
0.5372045855379188,
0.6595114541543113,
0.043163200110489575,
0.08370748405181158
0.04180484013035134,
0.08450011787948054
],
"display_key": "qwen3.5-plus-02-15 (rated)",
"model": "qwen/qwen3.5-plus-02-15",
@@ -379,8 +379,8 @@
"coords": [
0.538440257742794,
0.5530118458934986,
0.07158681387163214,
0.09053896523651496
0.07277582530416211,
0.0898393391241473
],
"display_key": "qwen3.5-35b-a3b (rated)",
"model": "qwen/qwen3.5-35b-a3b",
@@ -394,8 +394,8 @@
"coords": [
0.4526851851851852,
0.6334977783064814,
0.05996398964814259,
0.08320455765016928
0.057818785990183905,
0.08298095716984875
],
"display_key": "gpt-5-nano (rated)",
"model": "openai/gpt-5-nano",
@@ -424,8 +424,8 @@
"coords": [
0.6427511094043352,
0.5554953612269281,
0.03400287356919112,
0.09086207311764848
0.03535208249381765,
0.08497138486454137
],
"display_key": "qwen3-235b-a22b (rated)",
"model": "qwen/qwen3-235b-a22b",
@@ -439,8 +439,8 @@
"coords": [
0.6707823388714307,
0.6060018656291949,
0.03345632993101348,
0.09299505374530567
0.03451993837104684,
0.09876091230627733
],
"display_key": "qwen3-30b-a3b (rated)",
"model": "qwen/qwen3-30b-a3b",
@@ -454,8 +454,8 @@
"coords": [
0.6025617283950616,
0.6287037037037038,
0.03435593378616836,
0.09044283339419919
0.03396367302553316,
0.09406697361625513
],
"display_key": "qwen-plus (rated)",
"model": "qwen/qwen-plus",
@@ -469,8 +469,8 @@
"coords": [
0.6450907949109246,
0.5437992771640354,
0.022113968893118275,
0.10180231078172153
0.023312776383531648,
0.10024881542221994
],
"display_key": "qwen3-30b-a3b-instruct-2507 (rated)",
"model": "qwen/qwen3-30b-a3b-instruct-2507",
@@ -484,8 +484,8 @@
"coords": [
0.5419602854216832,
0.595896730624977,
0.053731321649797846,
0.08321090383068265
0.05193570378692634,
0.08265065423220756
],
"display_key": "deepseek-v4.1-flash (rated)",
"model": "deepseek/deepseek-v4.1-flash",
@@ -499,8 +499,8 @@
"coords": [
0.5408168991502326,
0.6493278669204595,
0.03596105265682425,
0.08640277131980666
0.03917823180041766,
0.08760189869507796
],
"display_key": "qwen3.5-397b-a17b (rated)",
"model": "qwen/qwen3.5-397b-a17b",
@@ -514,8 +514,8 @@
"coords": [
0.4569019748367575,
0.6751145899955423,
0.07783687662394496,
0.05577857978656997
0.07644348564506928,
0.0567289888770763
],
"display_key": "gpt-6-astra (rated)",
"model": "openai/gpt-6-astra",
@@ -529,8 +529,8 @@
"coords": [
0.5196778711484593,
0.6698994127565557,
0.04339901270073358,
0.07906452366543723
0.042934824935707064,
0.07801121120269483
],
"display_key": "qwen3.6-plus (rated)",
"model": "qwen/qwen3.6-plus",
@@ -544,8 +544,8 @@
"coords": [
0.6026546601546601,
0.5957470912477916,
0.033015451958604745,
0.09106122338341402
0.03274570154412936,
0.08673540897470654
],
"display_key": "qwen-2.5-72b-instruct (rated)",
"model": "qwen/qwen-2.5-72b-instruct",
@@ -559,8 +559,8 @@
"coords": [
0.6652380952380952,
0.7022990677752582,
0.026124454690573333,
0.05656973783290617
0.026433969418704885,
0.06269095940430941
],
"display_key": "qwen3-next-80b-a3b-instruct (rated)",
"model": "qwen/qwen3-next-80b-a3b-instruct",
@@ -574,8 +574,8 @@
"coords": [
0.610243091658148,
0.6687657686659433,
0.04596153462279471,
0.07682588731641059
0.04414498817375987,
0.08085439770187478
],
"display_key": "kimi-k2.6 (rated)",
"model": "moonshotai/kimi-k2.6",
@@ -589,8 +589,8 @@
"coords": [
0.45740779531102105,
0.6795887184190553,
0.0930227110336478,
0.07587935840780517
0.09233310052166174,
0.07166972793043255
],
"display_key": "qwen3.6-27b (rated)",
"model": "qwen/qwen3.6-27b",
@@ -604,8 +604,8 @@
"coords": [
0.6417361111111111,
0.5127110103300581,
0.035533783509182494,
0.08247734174209306
0.03807268153101321,
0.08042423628815949
],
"display_key": "qwen-plus-2025-07-28 (rated)",
"model": "qwen/qwen-plus-2025-07-28",
@@ -619,8 +619,8 @@
"coords": [
0.6062280543530544,
0.5943180448406931,
0.04816007701260514,
0.072116166212997
0.04853610852408334,
0.0737314840555176
],
"display_key": "qwen3.6-35b-a3b (rated)",
"model": "qwen/qwen3.6-35b-a3b",
@@ -634,8 +634,8 @@
"coords": [
0.5778139029180694,
0.6959770802908057,
0.03693419149935657,
0.08351595033327236
0.03666290866643542,
0.07382269298741832
],
"display_key": "qwen3-max-thinking (rated)",
"model": "qwen/qwen3-max-thinking",
@@ -649,8 +649,8 @@
"coords": [
0.48767589171684006,
0.6482142857142857,
0.0426101913142831,
0.06478149056076417
0.04852805951852839,
0.06690143204728374
],
"display_key": "muse-glimmer-30b (rated)",
"model": "meta/muse-glimmer-30b",
@@ -664,8 +664,8 @@
"coords": [
0.4988977072310405,
0.6262540369683227,
0.013608591135547174,
0.04901151771282093
0.01537866158910887,
0.05342525613259921
],
"display_key": "gemini-3.7-flash (rated)",
"model": "google/gemini-3.7-flash",
@@ -679,8 +679,8 @@
"coords": [
0.5886813794915646,
0.6339803982919925,
0.05257271891027728,
0.07140876500169609
0.04800287084122153,
0.07143925775678789
],
"display_key": "deepseek-v4-flash (rated)",
"model": "deepseek/deepseek-v4-flash",
@@ -694,8 +694,8 @@
"coords": [
0.5535321928702047,
0.6422322099171051,
0.05621697080592131,
0.061403096129926435
0.05892062733798582,
0.06715990340387798
],
"display_key": "glm-5.3 (rated)",
"model": "z-ai/glm-5.3",
@@ -709,8 +709,8 @@
"coords": [
0.6353621031746031,
0.5685931857310826,
0.03785687360618813,
0.08248362706680903
0.03688413980467388,
0.08024381413619414
],
"display_key": "qwen3-235b-a22b-2507 (rated)",
"model": "qwen/qwen3-235b-a22b-2507",
@@ -724,8 +724,8 @@
"coords": [
0.48469169991228817,
0.6190010643383659,
0.07549082925386548,
0.07517115878830224
0.07699729945533007,
0.07041604518532184
],
"display_key": "qwen3.5-122b-a10b (rated)",
"model": "qwen/qwen3.5-122b-a10b",
@@ -739,8 +739,8 @@
"coords": [
0.431934218610816,
0.6881570439623662,
0.08341797207800665,
0.06980159408625697
0.08092059548185657,
0.07158456159738981
],
"display_key": "qwen3.8-27b (rated)",
"model": "qwen/qwen3.8-27b",
@@ -754,8 +754,8 @@
"coords": [
0.5078196649029982,
0.6664359119716262,
0.049856523015613566,
0.0792534671436552
0.04898292658545207,
0.07513374670738206
],
"display_key": "qwen3.7-plus (rated)",
"model": "qwen/qwen3.7-plus",
@@ -769,8 +769,8 @@
"coords": [
0.6514346395123706,
0.583014517676874,
0.04639237632705332,
0.077088244066197
0.044355827535678856,
0.07772092035914374
],
"display_key": "qwen3.6-flash (rated)",
"model": "qwen/qwen3.6-flash",
@@ -784,8 +784,8 @@
"coords": [
0.4805424128340796,
0.6905543787488233,
0.08437549450275793,
0.07478325635344732
0.08346396425450253,
0.07197747047181104
],
"display_key": "qwen3.6-max-preview (rated)",
"model": "qwen/qwen3.6-max-preview",