wassname/moral-maps
1
0
Fork 0
mirror of https://github.com/wassname/moral-maps.git synced 2026-09-21 13:10:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Preserve score-all-options cache across lanes

Browse Source 
Co-Authored-By: PI[openai-codex] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-09-17 19:42:06 +08:00
co-authored by PI[openai-codex]
parent 83741adb9c
commit 82fa098b28
8 changed files with 4939 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/wvs_map.py
+5 -6
View File
@@ -46,6 +46,7 @@ from moralmaps.zones import zones_for, zone_of, IW_MACRO
from moralmaps.instrument import Instrument, InstrItem
from moralmaps.read import read_items, resolve_answer_ids
from moralmaps.read_api import rated_protocol_identity, read_items_rated
from moralmaps.rated_cache import merge_completed
from moralmaps.iw_axes import AXIS_ITEMS, X_AXIS, Y_AXIS, SKIP, resolve_items, positiveness
# option labels are single digits 0..n-1 -- single-token (unlike '10' on the justifiable scale) and
@@ -326,10 +327,9 @@ def main() -> None:
models[entry["display_key"]] = tuple(entry["coords"])
def save_cache() -> None:
"""Atomic cache replacement after a complete model panel, so interruption cannot fabricate a hit."""
temp = cpath.with_suffix(cpath.suffix + ".tmp")
temp.write_text(json.dumps(cache, indent=2, sort_keys=True) + "\n")
temp.replace(cpath)
"""Merge completed panels under a lock so parallel lanes cannot erase one another."""
nonlocal cache
cache = merge_completed(cpath, cache["completed"])
rng = np.random.default_rng(0) # deterministic bootstrap
@@ -359,8 +359,7 @@ def main() -> None:
m, rated_items, n_samples=args.api_samples, temperature=1.0,
max_tokens=args.api_max_tokens, concurrency=args.api_concurrency,
req_timeout=args.api_request_timeout, reasoning=api_reasoning,
structured_output=args.api_structured_output, provider=api_provider,
probe_first=args.api_probe_first)
structured_output=args.api_structured_output, provider=api_provider)
completed = cache["completed"].get(protocol_id)
if completed is not None:
models[key] = tuple(completed["coords"])
scripts/wvs_score_all_options_recover_cache.py
+109
View File
@@ -0,0 +1,109 @@
#!/usr/bin/env python3
"""Recover complete score-all-options panels from the durable request ledger."""
from __future__ import annotations
import argparse
import json
import multiprocessing
import tempfile
from collections import defaultdict
from pathlib import Path
import numpy as np
from moralmaps.iw_axes import X_AXIS, Y_AXIS, resolve_items
from moralmaps.rated_cache import merge_completed
from wvs_map import load_wvs_all, model_coord_ci
CACHE = Path("slop/research/wvs/20260916_openrouter/wvs_iw_rated.json")
RECORDS = Path("slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl")
AUDIT = Path("slop/audits/20260917_wvs_score_all_options_cache_recovery.json")
def read_records() -> list[dict]:
records = []
for line_number, line in enumerate(RECORDS.read_text().splitlines(), start=1):
try:
records.append(json.loads(line))
except json.JSONDecodeError as error:
raise ValueError(f"invalid JSONL record at {RECORDS}:{line_number}") from error
return records
def recovered_entries(records: list[dict]) -> dict[str, dict]:
starts = {record["run_id"]: record for record in records if record.get("event") == "run_started"}
finished = [record for record in records if record.get("event") == "run_finished"
and record["planned_requests"] == 144 and record["valid_samples"] == 144
and record["failed_samples"] == 0]
item_results: dict[str, dict[str, dict]] = defaultdict(dict)
for record in records:
if record.get("event") == "item_result":
item_results[record["run_id"]][record["id"]] = record
resolved = resolve_items(load_wvs_all())
expected_ids = [item["suffix"] for axis in (X_AXIS, Y_AXIS) for item in resolved[axis]]
entries = {}
rng = np.random.default_rng(0)
for finish in finished:
run_id = finish["run_id"]
start = starts[run_id]
rows = item_results[run_id]
if set(rows) != set(expected_ids):
raise ValueError(f"complete run {run_id} has item results {sorted(rows)}, expected {expected_ids}")
if any(row["valid_samples"] != 12 for row in rows.values()):
raise ValueError(f"complete run {run_id} has non-12 item samples")
psamples = {item_id: np.asarray(rows[item_id]["p_samples"]) for item_id in expected_ids}
coords = model_coord_ci(psamples, resolved, rng)
model = finish["model"]
entries[finish["protocol_id"]] = {
"model": model,
"display_key": model.split("/")[-1] + " (rated)",
"coords": list(coords),
"records_path": str(RECORDS),
"run_id": run_id,
"protocol_id": finish["protocol_id"],
"n_items": len(rows),
"n_samples": 12,
}
return entries
def _writer(path: str, key: str) -> None:
merge_completed(Path(path), {key: {"model": key}})
def concurrency_smoke() -> None:
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / "cache.json"
processes = [multiprocessing.Process(target=_writer, args=(str(path), key)) for key in ("a", "b")]
for process in processes:
process.start()
for process in processes:
process.join()
if process.exitcode != 0:
raise RuntimeError(f"cache writer exited {process.exitcode}")
assert set(json.loads(path.read_text())["completed"]) == {"a", "b"}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--smoke", action="store_true")
args = parser.parse_args()
if args.smoke:
concurrency_smoke()
print("smoke: two concurrent cache writers preserve both completed entries")
records = read_records()
entries = recovered_entries(records)
merged = merge_completed(CACHE, entries)
audit = {
"ledger": str(RECORDS),
"ledger_valid_lines": len(records),
"recovered_complete_runs": len(entries),
"cache_completed_entries_after_merge": len(merged["completed"]),
"recovered_models": sorted(entry["model"] for entry in entries.values()),
}
AUDIT.write_text(json.dumps(audit, indent=2, sort_keys=True) + "\n")
print(json.dumps(audit, indent=2, sort_keys=True))
if __name__ == "__main__":
main()
slop/audits/20260917_wvs_score_all_options_cache_recovery.json
+61
View File
@@ -0,0 +1,61 @@
{
"cache_completed_entries_after_merge": 53,
"ledger": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"ledger_valid_lines": 32461,
"recovered_complete_runs": 53,
"recovered_models": [
"anthropic/claude-fable-5.1",
"deepseek/deepseek-v4-flash",
"deepseek/deepseek-v4-flash-0731",
"deepseek/deepseek-v4.1-flash",
"google/gemini-3.7-flash",
"meta/muse-glimmer-30b",
"meta/muse-spark-1.3",
"moonshotai/kimi-k2.6",
"moonshotai/kimi-k3",
"openai/gpt-5-nano",
"openai/gpt-5.6-sol",
"openai/gpt-6-astra",
"qwen/qwen-2.5-72b-instruct",
"qwen/qwen-2.5-7b-instruct",
"qwen/qwen-plus",
"qwen/qwen-plus-2025-07-28",
"qwen/qwen2.5-vl-72b-instruct",
"qwen/qwen3-14b",
"qwen/qwen3-235b-a22b",
"qwen/qwen3-235b-a22b-2507",
"qwen/qwen3-30b-a3b",
"qwen/qwen3-30b-a3b-instruct-2507",
"qwen/qwen3-32b",
"qwen/qwen3-8b",
"qwen/qwen3-coder",
"qwen/qwen3-coder-30b-a3b-instruct",
"qwen/qwen3-coder-next",
"qwen/qwen3-coder-plus",
"qwen/qwen3-max-thinking",
"qwen/qwen3-next-80b-a3b-instruct",
"qwen/qwen3-vl-235b-a22b-instruct",
"qwen/qwen3-vl-30b-a3b-instruct",
"qwen/qwen3-vl-32b-instruct",
"qwen/qwen3-vl-8b-instruct",
"qwen/qwen3.5-122b-a10b",
"qwen/qwen3.5-27b",
"qwen/qwen3.5-35b-a3b",
"qwen/qwen3.5-397b-a17b",
"qwen/qwen3.5-9b",
"qwen/qwen3.5-plus-02-15",
"qwen/qwen3.5-plus-20260420",
"qwen/qwen3.6-27b",
"qwen/qwen3.6-35b-a3b",
"qwen/qwen3.6-flash",
"qwen/qwen3.6-max-preview",
"qwen/qwen3.6-plus",
"qwen/qwen3.7-flash",
"qwen/qwen3.7-max",
"qwen/qwen3.7-plus",
"qwen/qwen3.8-27b",
"qwen/qwen3.8-flash",
"thinkingmachines/inkling",
"z-ai/glm-5.3"
]
}
slop/research/wvs/20260916_openrouter/wvs_iw_rated.json
+156 -96
View File
@@ -4,8 +4,8 @@
"coords": [
0.6214472858835399,
0.6710244268262847,
0.03786483811416081,
0.06918409770409496
0.03498102851088573,
0.06859664714201569
],
"display_key": "kimi-k3 (rated)",
"model": "moonshotai/kimi-k3",
@@ -19,8 +19,8 @@
"coords": [
0.5849525883130219,
0.59521139256589,
0.037643038378990704,
0.07695278664473065
0.0388043837051704,
0.07867897818636554
],
"display_key": "qwen2.5-vl-72b-instruct (rated)",
"model": "qwen/qwen2.5-vl-72b-instruct",
@@ -34,8 +34,8 @@
"coords": [
0.5335737179487179,
0.5337214372928658,
0.029444543942379133,
0.09738610052745941
0.02927867567843363,
0.09371226473048877
],
"display_key": "qwen3-vl-30b-a3b-instruct (rated)",
"model": "qwen/qwen3-vl-30b-a3b-instruct",
@@ -49,8 +49,8 @@
"coords": [
0.6175297619047618,
0.5976147806792967,
0.035727022605345674,
0.07361977335041572
0.03541797753120733,
0.06871303341417663
],
"display_key": "qwen3-coder-plus (rated)",
"model": "qwen/qwen3-coder-plus",
@@ -64,8 +64,8 @@
"coords": [
0.5274474357644292,
0.6650744376984001,
0.044561828476471346,
0.06617035838663828
0.04284951309217784,
0.06880740953633888
],
"display_key": "gpt-5.6-sol (rated)",
"model": "openai/gpt-5.6-sol",
@@ -79,8 +79,8 @@
"coords": [
0.5036638266417679,
0.6503283973522069,
0.057017134313755345,
0.06039652604026301
0.056572571946403354,
0.06023989073439502
],
"display_key": "qwen3.7-max (rated)",
"model": "qwen/qwen3.7-max",
@@ -94,8 +94,8 @@
"coords": [
0.6061818582651916,
0.6004689754689754,
0.029738189920548014,
0.07794129978765893
0.02896221739482271,
0.08351751936863892
],
"display_key": "qwen3-vl-8b-instruct (rated)",
"model": "qwen/qwen3-vl-8b-instruct",
@@ -109,8 +109,8 @@
"coords": [
0.5017202097104058,
0.5916881018254176,
0.04660057418362078,
0.07020408831610975
0.04329249642491525,
0.0665062125461305
],
"display_key": "qwen3.5-9b (rated)",
"model": "qwen/qwen3.5-9b",
@@ -124,8 +124,8 @@
"coords": [
0.598778659611993,
0.6204503168788883,
0.04059052018905529,
0.08949774510806
0.03672846655615712,
0.08900654804546242
],
"display_key": "qwen3-coder (rated)",
"model": "qwen/qwen3-coder",
@@ -135,12 +135,27 @@
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"run_id": "20260916T195554Z_304db4b1aa6c"
},
"333ece6448f97447c57f24f9e9fcf0a7fb72a1b0f96696aef3c38a9c1d2d2308": {
"coords": [
0.5482108382155133,
0.5760765801607652,
0.06993277323172409,
0.08010838046301313
],
"display_key": "deepseek-v4-flash-0731 (rated)",
"model": "deepseek/deepseek-v4-flash-0731",
"n_items": 12,
"n_samples": 12,
"protocol_id": "333ece6448f97447c57f24f9e9fcf0a7fb72a1b0f96696aef3c38a9c1d2d2308",
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"run_id": "20260917T112151Z_333ece6448f9"
},
"3443c17ae66d3fb529a058128b662024c4bc0601394e614bb693b388ec4c988b": {
"coords": [
0.4153054353054353,
0.6346365065494385,
0.07206749839259896,
0.07710357656753936
0.07767226786811594,
0.07694753838347716
],
"display_key": "qwen3.8-flash (rated)",
"model": "qwen/qwen3.8-flash",
@@ -154,8 +169,8 @@
"coords": [
0.5030952380952382,
0.5846967846967848,
0.046958286642217247,
0.08746316369004264
0.04532986462569823,
0.09037513909634669
],
"display_key": "qwen3-coder-30b-a3b-instruct (rated)",
"model": "qwen/qwen3-coder-30b-a3b-instruct",
@@ -169,8 +184,8 @@
"coords": [
0.4253104631546661,
0.731799663031547,
0.0936048666617984,
0.060012518938080954
0.09130591901488724,
0.06265630083510712
],
"display_key": "muse-spark-1.3 (rated)",
"model": "meta/muse-spark-1.3",
@@ -184,8 +199,8 @@
"coords": [
0.5790919024614678,
0.6085929478125246,
0.038145029256266345,
0.025980758516149186
0.042481181470475825,
0.026900473313497777
],
"display_key": "claude-fable-5.1 (rated)",
"model": "anthropic/claude-fable-5.1",
@@ -199,8 +214,8 @@
"coords": [
0.5966931216931217,
0.5338064713064713,
0.011307868884765215,
0.10204509849192553
0.011999091335169519,
0.10525911258820818
],
"display_key": "qwen3-vl-32b-instruct (rated)",
"model": "qwen/qwen3-vl-32b-instruct",
@@ -214,8 +229,8 @@
"coords": [
0.6137896825396826,
0.5819444444444445,
0.015958730849444745,
0.0834429418953706
0.016073689650860815,
0.08458895983430899
],
"display_key": "qwen3-14b (rated)",
"model": "qwen/qwen3-14b",
@@ -229,8 +244,8 @@
"coords": [
0.6363133579397021,
0.5792251640447779,
0.027397680484314686,
0.06607943334210371
0.026429560820398015,
0.06316211550201252
],
"display_key": "qwen3-coder-next (rated)",
"model": "qwen/qwen3-coder-next",
@@ -244,8 +259,8 @@
"coords": [
0.6064902599972044,
0.5413085527420844,
0.028266888268597475,
0.07251640240693584
0.027826894148137167,
0.06753477500466712
],
"display_key": "qwen3-32b (rated)",
"model": "qwen/qwen3-32b",
@@ -259,8 +274,8 @@
"coords": [
0.6217724867724868,
0.5391326046087952,
0.015436127385306617,
0.05984330714155522
0.01514321596739316,
0.05660900381630799
],
"display_key": "qwen3-8b (rated)",
"model": "qwen/qwen3-8b",
@@ -274,8 +289,8 @@
"coords": [
0.5993276014109347,
0.5462479278681369,
0.024660010076102412,
0.10518394004968958
0.025633101038546928,
0.10120032015015096
],
"display_key": "qwen3-vl-235b-a22b-instruct (rated)",
"model": "qwen/qwen3-vl-235b-a22b-instruct",
@@ -289,8 +304,8 @@
"coords": [
0.5509667037841642,
0.5895236753978316,
0.02658132977411291,
0.08063463160776993
0.02679137212415509,
0.08184125856535546
],
"display_key": "qwen-2.5-7b-instruct (rated)",
"model": "qwen/qwen-2.5-7b-instruct",
@@ -304,8 +319,8 @@
"coords": [
0.5614798614630517,
0.6880988993033613,
0.050774441812280406,
0.057780026773918566
0.05786677240929432,
0.05700827650004879
],
"display_key": "inkling (rated)",
"model": "thinkingmachines/inkling",
@@ -319,8 +334,8 @@
"coords": [
0.5856201899951899,
0.6494018547589976,
0.06114886163791156,
0.08129405199775336
0.060949781665560955,
0.08885439746145042
],
"display_key": "qwen3.5-plus-20260420 (rated)",
"model": "qwen/qwen3.5-plus-20260420",
@@ -334,8 +349,8 @@
"coords": [
0.5974338624338624,
0.6515634387658198,
0.023795241546514957,
0.07922810144513995
0.02326665279699495,
0.08397986216363947
],
"display_key": "qwen3.5-27b (rated)",
"model": "qwen/qwen3.5-27b",
@@ -349,8 +364,8 @@
"coords": [
0.5372045855379188,
0.6595114541543113,
0.04180484013035134,
0.08450011787948054
0.043163200110489575,
0.08370748405181158
],
"display_key": "qwen3.5-plus-02-15 (rated)",
"model": "qwen/qwen3.5-plus-02-15",
@@ -364,8 +379,8 @@
"coords": [
0.538440257742794,
0.5530118458934986,
0.07277582530416211,
0.0898393391241473
0.07158681387163214,
0.09053896523651496
],
"display_key": "qwen3.5-35b-a3b (rated)",
"model": "qwen/qwen3.5-35b-a3b",
@@ -379,8 +394,8 @@
"coords": [
0.4526851851851852,
0.6334977783064814,
0.057818785990183905,
0.08298095716984875
0.05996398964814259,
0.08320455765016928
],
"display_key": "gpt-5-nano (rated)",
"model": "openai/gpt-5-nano",
@@ -409,8 +424,8 @@
"coords": [
0.6427511094043352,
0.5554953612269281,
0.03535208249381765,
0.08497138486454137
0.03400287356919112,
0.09086207311764848
],
"display_key": "qwen3-235b-a22b (rated)",
"model": "qwen/qwen3-235b-a22b",
@@ -424,8 +439,8 @@
"coords": [
0.6707823388714307,
0.6060018656291949,
0.03451993837104684,
0.09876091230627733
0.03345632993101348,
0.09299505374530567
],
"display_key": "qwen3-30b-a3b (rated)",
"model": "qwen/qwen3-30b-a3b",
@@ -439,8 +454,8 @@
"coords": [
0.6025617283950616,
0.6287037037037038,
0.03396367302553316,
0.09406697361625513
0.03435593378616836,
0.09044283339419919
],
"display_key": "qwen-plus (rated)",
"model": "qwen/qwen-plus",
@@ -454,8 +469,8 @@
"coords": [
0.6450907949109246,
0.5437992771640354,
0.023312776383531648,
0.10024881542221994
0.022113968893118275,
0.10180231078172153
],
"display_key": "qwen3-30b-a3b-instruct-2507 (rated)",
"model": "qwen/qwen3-30b-a3b-instruct-2507",
@@ -469,8 +484,8 @@
"coords": [
0.5419602854216832,
0.595896730624977,
0.05193570378692634,
0.08265065423220756
0.053731321649797846,
0.08321090383068265
],
"display_key": "deepseek-v4.1-flash (rated)",
"model": "deepseek/deepseek-v4.1-flash",
@@ -484,8 +499,8 @@
"coords": [
0.5408168991502326,
0.6493278669204595,
0.03917823180041766,
0.08760189869507796
0.03596105265682425,
0.08640277131980666
],
"display_key": "qwen3.5-397b-a17b (rated)",
"model": "qwen/qwen3.5-397b-a17b",
@@ -499,8 +514,8 @@
"coords": [
0.4569019748367575,
0.6751145899955423,
0.07644348564506928,
0.0567289888770763
0.07783687662394496,
0.05577857978656997
],
"display_key": "gpt-6-astra (rated)",
"model": "openai/gpt-6-astra",
@@ -514,8 +529,8 @@
"coords": [
0.5196778711484593,
0.6698994127565557,
0.042934824935707064,
0.07801121120269483
0.04339901270073358,
0.07906452366543723
],
"display_key": "qwen3.6-plus (rated)",
"model": "qwen/qwen3.6-plus",
@@ -529,8 +544,8 @@
"coords": [
0.6026546601546601,
0.5957470912477916,
0.03274570154412936,
0.08673540897470654
0.033015451958604745,
0.09106122338341402
],
"display_key": "qwen-2.5-72b-instruct (rated)",
"model": "qwen/qwen-2.5-72b-instruct",
@@ -544,8 +559,8 @@
"coords": [
0.6652380952380952,
0.7022990677752582,
0.026433969418704885,
0.06269095940430941
0.026124454690573333,
0.05656973783290617
],
"display_key": "qwen3-next-80b-a3b-instruct (rated)",
"model": "qwen/qwen3-next-80b-a3b-instruct",
@@ -555,12 +570,27 @@
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"run_id": "20260916T193846Z_9d8a5189c078"
},
"a2df2f1a32fde5dae2c1545120a4fd72cea4743bafe71a1768997dc3d125b4f4": {
"coords": [
0.610243091658148,
0.6687657686659433,
0.04596153462279471,
0.07682588731641059
],
"display_key": "kimi-k2.6 (rated)",
"model": "moonshotai/kimi-k2.6",
"n_items": 12,
"n_samples": 12,
"protocol_id": "a2df2f1a32fde5dae2c1545120a4fd72cea4743bafe71a1768997dc3d125b4f4",
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"run_id": "20260917T112151Z_a2df2f1a32fd"
},
"abf45a27f4bafad0eb264aac33b1e49e9d0404e141966954d0707930d56665e3": {
"coords": [
0.45740779531102105,
0.6795887184190553,
0.09233310052166174,
0.07166972793043255
0.0930227110336478,
0.07587935840780517
],
"display_key": "qwen3.6-27b (rated)",
"model": "qwen/qwen3.6-27b",
@@ -574,8 +604,8 @@
"coords": [
0.6417361111111111,
0.5127110103300581,
0.03807268153101321,
0.08042423628815949
0.035533783509182494,
0.08247734174209306
],
"display_key": "qwen-plus-2025-07-28 (rated)",
"model": "qwen/qwen-plus-2025-07-28",
@@ -589,8 +619,8 @@
"coords": [
0.6062280543530544,
0.5943180448406931,
0.04853610852408334,
0.0737314840555176
0.04816007701260514,
0.072116166212997
],
"display_key": "qwen3.6-35b-a3b (rated)",
"model": "qwen/qwen3.6-35b-a3b",
@@ -604,8 +634,8 @@
"coords": [
0.5778139029180694,
0.6959770802908057,
0.03666290866643542,
0.07382269298741832
0.03693419149935657,
0.08351595033327236
],
"display_key": "qwen3-max-thinking (rated)",
"model": "qwen/qwen3-max-thinking",
@@ -615,12 +645,27 @@
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"run_id": "20260916T145109Z_bca0745bdc15"
},
"c64ced76e82ec32cf6be896420d813403a95a421474eefcbbcbba421f8189192": {
"coords": [
0.48767589171684006,
0.6482142857142857,
0.0426101913142831,
0.06478149056076417
],
"display_key": "muse-glimmer-30b (rated)",
"model": "meta/muse-glimmer-30b",
"n_items": 12,
"n_samples": 12,
"protocol_id": "c64ced76e82ec32cf6be896420d813403a95a421474eefcbbcbba421f8189192",
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"run_id": "20260917T111805Z_c64ced76e82e"
},
"cd5db529649a179032180cecafe4fd98aff124ee3654f7d60996de704e2b63ef": {
"coords": [
0.4988977072310405,
0.6262540369683227,
0.01537866158910887,
0.05342525613259921
0.013608591135547174,
0.04901151771282093
],
"display_key": "gemini-3.7-flash (rated)",
"model": "google/gemini-3.7-flash",
@@ -630,12 +675,27 @@
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"run_id": "20260916T172946Z_cd5db529649a"
},
"d21d1e81010d87b7f79bd9ceb0d5bb226485fe6393e266e169474968e6bf26f8": {
"coords": [
0.5886813794915646,
0.6339803982919925,
0.05257271891027728,
0.07140876500169609
],
"display_key": "deepseek-v4-flash (rated)",
"model": "deepseek/deepseek-v4-flash",
"n_items": 12,
"n_samples": 12,
"protocol_id": "d21d1e81010d87b7f79bd9ceb0d5bb226485fe6393e266e169474968e6bf26f8",
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
"run_id": "20260917T112521Z_d21d1e81010d"
},
"d81f7e66b3c4c720f8dd80768d3dfc25de6edeaf6a352cfed40e961f21fcfa22": {
"coords": [
0.5535321928702047,
0.6422322099171051,
0.05892062733798582,
0.06715990340387798
0.05621697080592131,
0.061403096129926435
],
"display_key": "glm-5.3 (rated)",
"model": "z-ai/glm-5.3",
@@ -649,8 +709,8 @@
"coords": [
0.6353621031746031,
0.5685931857310826,
0.03688413980467388,
0.08024381413619414
0.03785687360618813,
0.08248362706680903
],
"display_key": "qwen3-235b-a22b-2507 (rated)",
"model": "qwen/qwen3-235b-a22b-2507",
@@ -664,8 +724,8 @@
"coords": [
0.48469169991228817,
0.6190010643383659,
0.07699729945533007,
0.07041604518532184
0.07549082925386548,
0.07517115878830224
],
"display_key": "qwen3.5-122b-a10b (rated)",
"model": "qwen/qwen3.5-122b-a10b",
@@ -679,8 +739,8 @@
"coords": [
0.431934218610816,
0.6881570439623662,
0.08092059548185657,
0.07158456159738981
0.08341797207800665,
0.06980159408625697
],
"display_key": "qwen3.8-27b (rated)",
"model": "qwen/qwen3.8-27b",
@@ -694,8 +754,8 @@
"coords": [
0.5078196649029982,
0.6664359119716262,
0.04898292658545207,
0.07513374670738206
0.049856523015613566,
0.0792534671436552
],
"display_key": "qwen3.7-plus (rated)",
"model": "qwen/qwen3.7-plus",
@@ -709,8 +769,8 @@
"coords": [
0.6514346395123706,
0.583014517676874,
0.044355827535678856,
0.07772092035914374
0.04639237632705332,
0.077088244066197
],
"display_key": "qwen3.6-flash (rated)",
"model": "qwen/qwen3.6-flash",
@@ -724,8 +784,8 @@
"coords": [
0.4805424128340796,
0.6905543787488233,
0.08346396425450253,
0.07197747047181104
0.08437549450275793,
0.07478325635344732
],
"display_key": "qwen3.6-max-preview (rated)",
"model": "qwen/qwen3.6-max-preview",
slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl
+4578
View File
File diff suppressed because one or more lines are too long
slop/research/wvs/20260917_score_all_options/manifest.json
+2 -42
View File
@@ -272,29 +272,9 @@
"structured_output": true
},
{
"calls": 144,
"created": 1786302394,
"id": "meta/muse-glimmer-30b",
"input_usd_per_million": "0.35000000",
"lane": "muse",
"output_usd_per_million": "1.5000000",
"provider": {
"allow_fallbacks": true,
"quantizations": [
"fp8",
"int8",
"bf16",
"fp16"
],
"require_parameters": true
},
"reasoning": {
"effort": "low"
},
"reasoning_label": "low",
"reserve_usd": "0.71516160",
"status": "runnable",
"structured_output": true
"status": "complete_cached"
},
{
"id": "meta/muse-glimmer-30b:batch",
@@ -746,29 +726,9 @@
"status": "excluded"
},
{
"calls": 144,
"created": 1776699402,
"id": "moonshotai/kimi-k2.6",
"input_usd_per_million": "0.95000000",
"lane": "kimi",
"output_usd_per_million": "4.000000",
"provider": {
"allow_fallbacks": true,
"quantizations": [
"fp8",
"int8",
"bf16",
"fp16"
],
"require_parameters": true
},
"reasoning": {
"enabled": false
},
"reasoning_label": "disabled, optional",
"reserve_usd": "1.90955520",
"status": "runnable",
"structured_output": true
"status": "complete_cached"
},
{
"calls": 144,
src/moralmaps/rated_cache.py
+26
View File
@@ -0,0 +1,26 @@
"""Atomic merge persistence for completed score-all-options panels."""
from __future__ import annotations
import fcntl
import json
import os
from pathlib import Path
def merge_completed(path: Path, additions: dict[str, dict]) -> dict:
"""Merge complete entries while holding the cache lock across reread and replacement."""
lock_path = path.with_suffix(path.suffix + ".lock")
with lock_path.open("w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX)
cache = json.loads(path.read_text()) if path.exists() else {"schema": 2, "completed": {}}
if cache["schema"] != 2:
raise ValueError(f"unsupported WVS cache schema {cache['schema']}")
cache["completed"].update(additions)
temp = path.with_suffix(path.suffix + ".tmp")
temp.write_text(json.dumps(cache, indent=2, sort_keys=True) + "\n")
with temp.open("r+") as fh:
fh.flush()
os.fsync(fh.fileno())
temp.replace(path)
fcntl.flock(lock, fcntl.LOCK_UN)
return cache
src/moralmaps/read_api.py
+2 -4
View File
@@ -208,7 +208,7 @@ def _rate_plan(items: list[dict], n_samples: int, per_call: int = 1) -> list[dic
def rated_protocol_identity(model: str, items: list[dict], *, n_samples: int, temperature: float,
max_tokens: int, concurrency: int, req_timeout: float,
reasoning: dict | None, structured_output: bool,
provider: dict | None = None, probe_first: bool = False) -> str:
provider: dict | None = None) -> str:
"""Hash the exact model, rendered prompts, and request settings that define a cacheable panel."""
plan = _rate_plan(items, n_samples)
protocol = {
@@ -221,7 +221,6 @@ def rated_protocol_identity(model: str, items: list[dict], *, n_samples: int, te
"reasoning": reasoning,
"structured_output": structured_output,
"provider": provider,
"probe_first": probe_first,
"rate_prompt": _RATE_PROMPT,
"rescue_prompt": _force_msg(10),
"requests": [{key: req[key] for key in ("i", "perm", "prompt", "cnt", "sample", "presented_options")}
@@ -255,8 +254,7 @@ def read_items_rated(model: str, items: list[dict], *, n_samples: int = 12, temp
protocol_id = rated_protocol_identity(model, items, n_samples=n_samples, temperature=temperature,
max_tokens=max_tokens, concurrency=concurrency,
req_timeout=req_timeout, reasoning=reasoning,
structured_output=structured_output, provider=provider,
probe_first=probe_first)
structured_output=structured_output, provider=provider)
run_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_{protocol_id[:12]}"
rpath = Path(records_path)
rpath.parent.mkdir(parents=True, exist_ok=True)