From 1b6f2d7ef130e7f7cc8fa38b77a45c9aba1a732a Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Thu, 17 Sep 2026 21:35:20 +0800 Subject: [PATCH] Add versioned DeepSeek WVS reliability pilot Co-Authored-By: PI[openai-codex] <288921227+claudypoo@users.noreply.github.com> --- scripts/wvs_deepseek_reliability_pilot.py | 261 + .../manifest.json | 6469 +++++++++++++++++ src/moralmaps/read_api.py | 24 +- 3 files changed, 6749 insertions(+), 5 deletions(-) create mode 100644 scripts/wvs_deepseek_reliability_pilot.py create mode 100644 slop/research/wvs/20260917_deepseek_reliability/manifest.json diff --git a/scripts/wvs_deepseek_reliability_pilot.py b/scripts/wvs_deepseek_reliability_pilot.py new file mode 100644 index 0000000..5c79db9 --- /dev/null +++ b/scripts/wvs_deepseek_reliability_pilot.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Replicate seven complete DeepSeek WVS v1 panels without touching canonical cache or Pages data.""" +from __future__ import annotations + +import argparse +import fcntl +import hashlib +import json +from collections import Counter +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path + +import numpy as np + +from moralmaps.iw_axes import X_AXIS, Y_AXIS, resolve_items +from moralmaps.read_api import rated_protocol_identity, read_items_rated +from wvs_map import _sample_only_coord_se, load_wvs_all, model_coord_ci +from wvs_score_all_options_refresh import OSS_PROVIDER, release, reserve + +EVAL_VERSION = "wvs-score-all-options-v1" +MODELS = ( + "deepseek/deepseek-chat-v3-0324", + "deepseek/deepseek-chat-v3.1", + "deepseek/deepseek-v3.2", + "deepseek/deepseek-v3.2-exp", + "deepseek/deepseek-v4-flash", + "deepseek/deepseek-v4-flash-0731", + "deepseek/deepseek-v4.1-flash", +) +REPLICATES = 3 +N_SAMPLES = 24 +OUT = Path("slop/research/wvs/20260917_deepseek_reliability") +MANIFEST = OUT / "manifest.json" +RESULTS = OUT / "results.json" +STATE = OUT / "budget.json" +LOCK = OUT / "budget.lock" +CACHE = Path("slop/research/wvs/20260916_openrouter/wvs_iw_rated.json") +LEDGER = Path("slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl") +PILOT_RESERVATION_ID = "pilot/deepseek-wvs-v1-reliability" +PILOT_CAP_USD = Decimal("1") + + +def atomic_json(path: Path, value: object) -> None: + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + temp.replace(path) + + +def pilot_state(update) -> dict: + OUT.mkdir(parents=True, exist_ok=True) + with LOCK.open("w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + state = json.loads(STATE.read_text()) if STATE.exists() else { + "schema": 1, "pilot_cap_usd": str(PILOT_CAP_USD), "spent_usd": "0", "reserved_usd": "0", + } + update(state) + atomic_json(STATE, state) + fcntl.flock(lock, fcntl.LOCK_UN) + return state + + +def deterministic_seeds(model: str, replicate: int, count: int) -> list[int]: + seeds = [] + for sequence in range(count): + digest = hashlib.sha256(f"{EVAL_VERSION}|{model}|{replicate}|{sequence}".encode()).digest() + seeds.append(int.from_bytes(digest[:4], "big") & 0x7FFF_FFFF) + if len(set(seeds)) != len(seeds): + raise RuntimeError(f"seed collision for {model} replicate {replicate}") + return seeds + + +def completed_v1_settings() -> dict[str, dict]: + starts = {} + finished = set() + for line in LEDGER.read_text().splitlines(): + record = json.loads(line) + if record.get("model") not in MODELS: + continue + if record.get("event") in {"run_started", "request_started"}: + starts[record["run_id"]] = record + if record.get("event") == "run_finished" and record["valid_samples"] == 144 and record["failed_samples"] == 0: + finished.add(record["run_id"]) + settings = {} + for run_id in sorted(finished): + start = starts[run_id] + settings[start["model"]] = start["settings"] + if set(settings) != set(MODELS): + raise RuntimeError(f"missing complete v1 DeepSeek settings: {sorted(set(MODELS) - set(settings))}") + return settings + + +def rated_items() -> tuple[list[dict], dict]: + resolved = resolve_items(load_wvs_all()) + items, seen = [], set() + for axis in (X_AXIS, Y_AXIS): + for item in resolved[axis]: + if item["suffix"] in seen: + continue + seen.add(item["suffix"]) + items.append({"id": item["suffix"], "question": item["rec"]["q"], "options": item["rec"]["opts"], "n": item["n"]}) + return items, resolved + + +def catalog_seed_support() -> dict[str, bool]: + catalog = json.loads(Path("slop/research/wvs/20260917_openrouter_models.json").read_text())["data"] + by_id = {row["id"]: row for row in catalog} + return {model: "seed" in by_id[model].get("supported_parameters", []) for model in MODELS} + + +def protocol_settings(model: str, v1: dict, seeds: list[int]) -> dict: + return { + "model": model, "n_samples": N_SAMPLES, "temperature": v1["temperature"], + "max_tokens": v1["max_tokens"], "concurrency": 1, "req_timeout": v1["req_timeout"], + "reasoning": v1["reasoning"], "structured_output": v1["structured_output"], + "provider": v1.get("provider", OSS_PROVIDER), "eval_version": EVAL_VERSION, "seed_schedule": seeds, + } + + +def manifest() -> dict: + items, _ = rated_items() + settings = completed_v1_settings() + seed_support = catalog_seed_support() + rows = [] + for model in MODELS: + replicates = [] + for replicate in range(REPLICATES): + seeds = deterministic_seeds(model, replicate, len(items) * N_SAMPLES) + cfg = protocol_settings(model, settings[model], seeds) + protocol_id = rated_protocol_identity(model, items, **{key: cfg[key] for key in ( + "n_samples", "temperature", "max_tokens", "concurrency", "req_timeout", "reasoning", + "structured_output", "provider", "eval_version", "seed_schedule")}) + replicates.append({"replicate": replicate, "protocol_id": protocol_id, "seed_schedule": seeds, + "records": str(OUT / "records" / model.replace("/", "__") / f"replicate_{replicate}.jsonl")}) + rows.append({"id": model, "v1_settings": settings[model], "provider_policy": OSS_PROVIDER, + "endpoint_advertises_seed": seed_support[model], "replicates": replicates}) + return { + "schema": 1, "eval_version": EVAL_VERSION, "purpose": "reliability replication, not a new evaluator", + "models": rows, "items": len(items), "replicates": REPLICATES, "samples_per_item": N_SAMPLES, + "expected_calls": len(MODELS) * REPLICATES * len(items) * N_SAMPLES, + "pilot_cap_usd": str(PILOT_CAP_USD), "expected_cost_usd": "0.318363267912", + "global_cap_reservation_id": PILOT_RESERVATION_ID, + "hidden_reasoning": "unavailable when a provider does not return it; returned response fields are preserved verbatim", + "created_utc": datetime.now(UTC).isoformat(), + } + + +def usage(records: Path) -> tuple[Decimal, Counter, int, int]: + cost, providers, rescues, failures = Decimal(), Counter(), 0, 0 + for line in records.read_text().splitlines(): + record = json.loads(line) + if record["event"] == "request_completed": + cost += Decimal(str(record.get("usage", {}).get("cost", 0))) + providers[record.get("provider")] += 1 + if record["event"] == "item_result": + rescues += record["rescued_samples"] + failures += record["failed_samples"] + return cost, providers, rescues, failures + + +def v1_coords(model: str) -> list[float]: + cache = json.loads(CACHE.read_text())["completed"] + entries = [entry for entry in cache.values() if entry["model"] == model and entry.get("n_samples") == 12] + if not entries: + raise RuntimeError(f"missing canonical v1 cache entry for {model}") + return entries[-1]["coords"] + + +def run_replicate(model: str, replicate: dict, v1: dict, items: list[dict], resolved: dict) -> dict: + records = Path(replicate["records"]) + records.parent.mkdir(parents=True, exist_ok=True) + seeds = replicate["seed_schedule"] + rows = read_items_rated(model, items, n_samples=N_SAMPLES, temperature=v1["temperature"], + max_tokens=v1["max_tokens"], concurrency=1, req_timeout=v1["req_timeout"], + reasoning=v1["reasoning"], structured_output=v1["structured_output"], provider=v1.get("provider", OSS_PROVIDER), + records_path=records, verbose_first=True, probe_first=True, eval_version=EVAL_VERSION, + identity_eval_version=EVAL_VERSION, seed_schedule=seeds) + if any(row["valid_samples"] != N_SAMPLES for row in rows): + raise RuntimeError(f"incomplete replicate {model} {replicate['replicate']}") + psamples = {row["id"]: np.asarray(row["p_samples"]) for row in rows} + coords = model_coord_ci(psamples, resolved, np.random.default_rng(0)) + response_se = _sample_only_coord_se(psamples, resolved, np.random.default_rng(1), n_draws=N_SAMPLES) + cost, providers, rescues, failures = usage(records) + return {"replicate": replicate["replicate"], "protocol_id": replicate["protocol_id"], "records": str(records), + "coords": list(coords), "response_mean_se": list(response_se), "provider_requests": dict(providers), + "rescues": rescues, "failures": failures, "usage_cost_usd": str(cost)} + + +def run() -> None: + data = json.loads(MANIFEST.read_text()) + items, resolved = rated_items() + if not reserve({"id": PILOT_RESERVATION_ID, "lane": "deepseek", "reserve_usd": str(PILOT_CAP_USD)}): + raise RuntimeError("global USD 80 cap would be exceeded by the USD 1 pilot reservation") + pilot_state(lambda state: state.update({"reserved_usd": str(PILOT_CAP_USD), "started_utc": datetime.now(UTC).isoformat()})) + results = {"eval_version": EVAL_VERSION, "models": [], "not_published": True} + try: + for row in data["models"]: + reps = [] + for replicate in row["replicates"]: + result = run_replicate(row["id"], replicate, row["v1_settings"], items, resolved) + spent = pilot_state(lambda state: state.update({"spent_usd": str(Decimal(state["spent_usd"]) + Decimal(result["usage_cost_usd"]))})) + if Decimal(spent["spent_usd"]) >= PILOT_CAP_USD: + raise RuntimeError(f"pilot cap reached: {spent['spent_usd']} >= {PILOT_CAP_USD}") + reps.append(result) + atomic_json(RESULTS, results | {"models": results["models"] + [{"id": row["id"], "replicates": reps}]}) + values = np.asarray([rep["coords"][:2] for rep in reps]) + aggregate_samples = {} + for rep in reps: + rows = [json.loads(line) for line in Path(rep["records"]).read_text().splitlines()] + for record in rows: + if record["event"] == "item_result": + aggregate_samples.setdefault(record["id"], []).extend(record["p_samples"]) + aggregate = model_coord_ci({key: np.asarray(value) for key, value in aggregate_samples.items()}, resolved, np.random.default_rng(2)) + v1 = v1_coords(row["id"]) + results["models"].append({"id": row["id"], "v1_coords": v1, "replicates": reps, + "between_replicate_coordinate_sd": np.std(values, axis=0, ddof=1).tolist(), + "aggregate_n72_coords": list(aggregate), + "delta_aggregate_minus_v1": (np.asarray(aggregate[:2]) - np.asarray(v1[:2])).tolist()}) + atomic_json(RESULTS, results) + finally: + release(PILOT_RESERVATION_ID) + pilot_state(lambda state: state.update({"reserved_usd": "0", "finished_utc": datetime.now(UTC).isoformat()})) + + +def smoke() -> None: + data = json.loads(MANIFEST.read_text()) + row, replicate = data["models"][0], data["models"][0]["replicates"][0] + items, _ = rated_items() + full_identity = replicate["protocol_id"] + record = OUT / "smoke.jsonl" + rows = read_items_rated(row["id"], [items[0]], n_samples=1, temperature=row["v1_settings"]["temperature"], + max_tokens=row["v1_settings"]["max_tokens"], concurrency=1, req_timeout=row["v1_settings"]["req_timeout"], + reasoning=row["v1_settings"]["reasoning"], structured_output=row["v1_settings"]["structured_output"], + provider=row["v1_settings"].get("provider", OSS_PROVIDER), records_path=record, probe_first=True, + eval_version=EVAL_VERSION, identity_eval_version=EVAL_VERSION, + seed_schedule=[replicate["seed_schedule"][0]]) + if rows[0]["valid_samples"] != 1: + raise RuntimeError("one-request smoke was not parse-valid") + atomic_json(OUT / "smoke.json", {"full_panel_protocol_id": full_identity, "smoke_model": row["id"], + "smoke_item": items[0]["id"], "seed": replicate["seed_schedule"][0], + "settings": row["v1_settings"], "endpoint_advertises_seed": row["endpoint_advertises_seed"], + "result": rows[0]}) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--write-manifest", action="store_true") + parser.add_argument("--smoke", action="store_true") + parser.add_argument("--run", action="store_true") + args = parser.parse_args() + if args.write_manifest: + OUT.mkdir(parents=True, exist_ok=True) + atomic_json(MANIFEST, manifest()) + if args.smoke: + smoke() + if args.run: + run() + + +if __name__ == "__main__": + main() diff --git a/slop/research/wvs/20260917_deepseek_reliability/manifest.json b/slop/research/wvs/20260917_deepseek_reliability/manifest.json new file mode 100644 index 0000000..9be51bc --- /dev/null +++ b/slop/research/wvs/20260917_deepseek_reliability/manifest.json @@ -0,0 +1,6469 @@ +{ + "created_utc": "2026-09-17T13:34:57.191567+00:00", + "eval_version": "wvs-score-all-options-v1", + "expected_calls": 6048, + "expected_cost_usd": "0.318363267912", + "global_cap_reservation_id": "pilot/deepseek-wvs-v1-reliability", + "hidden_reasoning": "unavailable when a provider does not return it; returned response fields are preserved verbatim", + "items": 12, + "models": [ + { + "endpoint_advertises_seed": true, + "id": "deepseek/deepseek-chat-v3-0324", + "provider_policy": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "replicates": [ + { + "protocol_id": "e0b3257086235cad916ee8246bce41a93eb33be27eafd9900fbb831b70549598", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-chat-v3-0324/replicate_0.jsonl", + "replicate": 0, + "seed_schedule": [ + 591495052, + 1023133551, + 1318840843, + 81955722, + 2100656482, + 2083768764, + 84263090, + 1148339119, + 166070994, + 629546349, + 1276137432, + 852713831, + 2102635560, + 653691923, + 1136106561, + 1012128310, + 1545140081, + 2052659510, + 1304488691, + 1011921132, + 1791305495, + 1820382507, + 149184932, + 2078354482, + 1157844634, + 806524172, + 992900612, + 1763914444, + 29436939, + 262457228, + 1080410720, + 1608924838, + 1785369278, + 258765614, + 1211687825, + 1238292371, + 688355536, + 414367806, + 714772921, + 790972161, + 1446762879, + 1625211330, + 1946058210, + 1963450514, + 316343529, + 368668038, + 847329834, + 171370208, + 1099664489, + 610698031, + 74383830, + 1699433843, + 1856982080, + 416017845, + 1346618932, + 283831520, + 254214588, + 894414233, + 1402132072, + 2095308584, + 763496643, + 823285259, + 1971008217, + 1337562608, + 2023696645, + 589895719, + 707711603, + 579525912, + 1995555828, + 1037263293, + 532001789, + 917675134, + 1409783271, + 1482707183, + 22974669, + 511899875, + 529332620, + 1583330442, + 1140522720, + 110923412, + 1166095466, + 1489170918, + 1771052390, + 1265750084, + 1085616447, + 2023901872, + 1768179484, + 2125755835, + 1590500353, + 1611414584, + 2042819868, + 1903298846, + 1743923127, + 1982731537, + 426185563, + 96320020, + 1879522102, + 2612587, + 801931187, + 1045475965, + 340270843, + 813148306, + 566171182, + 1436392494, + 110973932, + 1881201442, + 1731591618, + 1121731004, + 225311362, + 2119104645, + 957342555, + 1085038527, + 1154337514, + 324809006, + 994509685, + 1185876651, + 610953064, + 1609355991, + 984829568, + 1650900428, + 1280870703, + 1368271652, + 1259947613, + 1680111596, + 1117430213, + 66510756, + 1082950186, + 1088501985, + 1621351588, + 1415974456, + 376902244, + 1197986892, + 752538730, + 1327391819, + 1706289842, + 646707603, + 782284921, + 1195157244, + 964751532, + 1387261925, + 958854272, + 320931666, + 415843269, + 1773270923, + 1389227316, + 2102216544, + 1129907615, + 2081421959, + 1861913588, + 901295534, + 1230099674, + 1082644540, + 755684266, + 103801912, + 1661927181, + 941464943, + 2132307343, + 2066172865, + 1429925227, + 159972731, + 365916299, + 1483130032, + 1145780562, + 1858242822, + 229865179, + 325617378, + 194987992, + 1404361060, + 586744251, + 1122901766, + 1876656803, + 152347077, + 1153632332, + 166962496, + 160641370, + 1183916578, + 1018884553, + 587643541, + 1290916236, + 435544169, + 1403898802, + 1287168983, + 1026822631, + 610419570, + 1786870961, + 308788517, + 1234062503, + 1770004231, + 1767383471, + 1806782575, + 1754862995, + 475381201, + 945398188, + 686346450, + 1235171841, + 1521604136, + 754114956, + 1609832272, + 1147329853, + 1433193811, + 546061648, + 685613577, + 1498331716, + 1309743939, + 1304323606, + 2037833572, + 1878469420, + 433495064, + 795520802, + 272504939, + 101124050, + 269646658, + 1004680559, + 604812551, + 1755878521, + 1980006817, + 1888113797, + 321278457, + 1274938409, + 433593192, + 1100715074, + 1508357376, + 2089393208, + 131301342, + 1629771942, + 1948691451, + 250691375, + 725985446, + 471862229, + 940778368, + 1605685231, + 1033747028, + 107856222, + 943795254, + 1173818987, + 381246489, + 1687567232, + 802241412, + 905525532, + 881775263, + 495394900, + 2050721598, + 800925240, + 55127620, + 1940517881, + 210377355, + 281079043, + 1539263120, + 1153781899, + 1076980531, + 1859182747, + 1151717600, + 558107834, + 1742252297, + 1640771044, + 966804460, + 175699482, + 1535846203, + 89826351, + 934385911, + 588555518, + 1978487367, + 1448776211, + 356521821, + 1383242711, + 1847824643, + 240244743, + 1721459398, + 1416642618, + 1646400065, + 1192285029, + 1601661128, + 2132291876, + 878742097, + 1960928779, + 937973667, + 1819466008, + 155749787, + 1681784482, + 1239848533, + 839816797, + 1605113950, + 249716251, + 688890299, + 1518945411, + 924536793, + 363118698, + 422640931 + ] + }, + { + "protocol_id": "6a52a998b7b86cf0f74d5982404e1e6e8f1d710fcf8bcd578f423aff6dde2e71", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-chat-v3-0324/replicate_1.jsonl", + "replicate": 1, + "seed_schedule": [ + 1297279086, + 819727637, + 1005339435, + 633969377, + 1925994499, + 1210386847, + 223293691, + 147779355, + 762159461, + 585361544, + 1247029905, + 1185218077, + 460646683, + 1572679224, + 1335413544, + 1893344991, + 2054772948, + 517333148, + 1638931237, + 241242677, + 63651445, + 2051076201, + 404097525, + 1825793625, + 1910383637, + 478631479, + 1133266200, + 1035115388, + 1495186022, + 1223609482, + 1927514940, + 1720041797, + 32360481, + 403130636, + 621310821, + 35045215, + 1753554868, + 423897874, + 497564511, + 1296166718, + 683580661, + 1022910836, + 1790873459, + 2054536827, + 1759974638, + 418670777, + 1781997746, + 1494083222, + 1905891649, + 35267987, + 759131439, + 508299208, + 1096890529, + 1369341411, + 2061369623, + 507423425, + 656207707, + 98446958, + 118213160, + 1531318894, + 224021555, + 994676543, + 1455102901, + 534201053, + 1863676958, + 268762518, + 1842933390, + 1909598503, + 2093914024, + 151033995, + 927666281, + 1729037632, + 1400245274, + 1818057633, + 52283089, + 473867634, + 1532085216, + 871334521, + 158110062, + 1260468154, + 1116194069, + 229698339, + 1820327451, + 2036133801, + 1297531401, + 1357892372, + 1025184945, + 859501852, + 1284956375, + 447501487, + 787358548, + 1198318778, + 2014046262, + 1592545149, + 137744009, + 1730400710, + 1636691791, + 1525404528, + 2146188037, + 493774356, + 322911698, + 1150525, + 1712556238, + 1248647042, + 1903189786, + 318415385, + 429827610, + 1905028312, + 678445413, + 653859206, + 236410389, + 1794105097, + 2091327122, + 73051753, + 260914628, + 1203747575, + 1387284152, + 256276324, + 2127930413, + 816284511, + 1460285181, + 546662141, + 1531441049, + 636076088, + 688871140, + 1654185050, + 1476744993, + 1415678452, + 2017071110, + 1647544589, + 1037102607, + 183368359, + 1611313862, + 1912791756, + 1318387279, + 906690211, + 802974657, + 481854771, + 174284840, + 521167064, + 1108920131, + 162470425, + 810865302, + 256034629, + 1087425961, + 1178173090, + 51091218, + 254867033, + 1843872149, + 78738148, + 1019152977, + 952490835, + 382807269, + 28696948, + 1502791396, + 1663139814, + 184510059, + 163248618, + 1187962619, + 1455379262, + 1310726783, + 1443225481, + 1593870515, + 920799895, + 1174359894, + 952971081, + 785992589, + 1678626068, + 310323773, + 1591456992, + 200452811, + 1971470979, + 980141097, + 673826046, + 594227411, + 1345731589, + 918338245, + 448206828, + 1489884401, + 1616310382, + 1965639157, + 455409582, + 281022593, + 1719102770, + 1187042796, + 961219995, + 1176354484, + 1381186877, + 1677708536, + 1766578326, + 439407415, + 280258857, + 1034199351, + 2062652629, + 649244651, + 1595905151, + 1633890287, + 317355646, + 626832251, + 1989362885, + 365553575, + 1490774655, + 1413849481, + 1676410413, + 285527452, + 383966229, + 1085696439, + 1365730717, + 2024857845, + 2129628378, + 1486530679, + 1046554703, + 589072756, + 1687340215, + 1371246934, + 564128092, + 1505229959, + 198178870, + 1333053305, + 1916774627, + 126228311, + 684405445, + 39627053, + 1914497289, + 1979588100, + 1865889119, + 1456470069, + 879855259, + 168571277, + 361884512, + 747392838, + 2111446978, + 1588317318, + 569912815, + 2061606790, + 1346540077, + 1687045267, + 1844574391, + 1653853617, + 1835917752, + 1813764674, + 1632500526, + 1542502427, + 995531881, + 1129517562, + 2053916245, + 1623742222, + 594813783, + 237867458, + 1441804461, + 1972368567, + 1568821659, + 1648820159, + 794146798, + 889787159, + 113548430, + 1655498137, + 523494286, + 426306642, + 1090752249, + 1922609681, + 1452901745, + 2037810405, + 1245011078, + 998980130, + 1739194927, + 531061414, + 810318694, + 952837246, + 500176026, + 121305252, + 1051320770, + 587743804, + 373760548, + 349943413, + 642835361, + 2117858613, + 476865761, + 2085387898, + 178466605, + 246558381, + 1261650338, + 1852768168, + 986061809, + 576836329, + 2050871361, + 1956340149, + 1496068014 + ] + }, + { + "protocol_id": "ca47e3a931bce20c5c094a8f211fb6114978138f61238974906af627b1a406fd", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-chat-v3-0324/replicate_2.jsonl", + "replicate": 2, + "seed_schedule": [ + 430359729, + 1132653041, + 1204726469, + 282038724, + 132070016, + 55793613, + 702017148, + 986693657, + 1897668524, + 1758657245, + 1217237685, + 1944049646, + 372349328, + 982755872, + 2119741300, + 1724098840, + 306923264, + 546815693, + 1770657288, + 994787699, + 1881228491, + 1673178428, + 325656510, + 1414885608, + 588478979, + 147761332, + 603869497, + 1476602226, + 1100191821, + 74873168, + 2038085546, + 26261883, + 411888940, + 1761784854, + 1620849843, + 643402620, + 813204310, + 1286036579, + 1610212548, + 853760325, + 667623599, + 16035663, + 1521014860, + 1101684049, + 958621062, + 148001138, + 2036359669, + 1768658767, + 1228307935, + 701374746, + 880882518, + 929733677, + 1977422986, + 843979526, + 179936534, + 837802492, + 1213467286, + 1892795890, + 1555622525, + 918348399, + 1766656974, + 1408471302, + 770670234, + 504621247, + 1651044319, + 939753123, + 383586297, + 1648324951, + 81028578, + 1777743968, + 1510411882, + 1295654153, + 278188700, + 998004431, + 1947833924, + 1815105924, + 1910463121, + 956116525, + 872011520, + 1425549549, + 373871142, + 1936325104, + 1363610931, + 1325484236, + 1933206051, + 1895325084, + 1229690524, + 1348177807, + 153975632, + 847065615, + 705611049, + 1600292714, + 1720372130, + 737679372, + 1782651991, + 1006981702, + 127204236, + 1566168728, + 1012839211, + 1098770782, + 540228534, + 2109250613, + 676242730, + 171465002, + 1310404968, + 624438502, + 645752286, + 1252619753, + 802903523, + 1253926626, + 972596564, + 1247953973, + 1023210585, + 1054021912, + 943210114, + 294136442, + 1950322318, + 1023218209, + 1457925, + 1893281866, + 653945626, + 450446745, + 2080813573, + 1750162078, + 1351108875, + 842506127, + 80500461, + 944559918, + 289650809, + 1784827683, + 654457116, + 349553054, + 1240097018, + 1288198125, + 1925083972, + 599936660, + 1442551638, + 1540965583, + 964567148, + 993688573, + 1881473394, + 720164894, + 265120402, + 266231257, + 798979150, + 355362550, + 1151507070, + 1859152016, + 2061902090, + 1845388014, + 200488724, + 437043511, + 401626493, + 967254668, + 1044042650, + 235227659, + 1757252671, + 1159072413, + 2129612285, + 1016944896, + 970563704, + 305943806, + 336493657, + 1983525383, + 866041229, + 1597557971, + 14358365, + 608313728, + 1714225408, + 1609576169, + 1879400719, + 777272140, + 1020662307, + 1879403843, + 1231717786, + 76567635, + 1839138473, + 1139234786, + 1877867431, + 387952997, + 959086169, + 1434271906, + 1148368120, + 1610988713, + 1183421659, + 1760969030, + 1374900242, + 1058414651, + 491757020, + 24370224, + 240467956, + 1586453859, + 1866600792, + 1792338983, + 1904175711, + 1683380542, + 597538966, + 1298569086, + 345001924, + 1343033529, + 775331673, + 1922267094, + 1449916777, + 1400126001, + 224002559, + 1076340431, + 118354648, + 1534412989, + 376286638, + 924075670, + 830163838, + 2061850364, + 1946978141, + 73929690, + 642643835, + 1114088311, + 1914155157, + 417016242, + 507352577, + 1364282855, + 1621542753, + 923910244, + 1409927426, + 1238079456, + 582194544, + 566089858, + 2138203548, + 103311954, + 344708982, + 34756989, + 391435691, + 476786877, + 217892663, + 197573766, + 993654907, + 1815375365, + 239310309, + 1528267252, + 1366154654, + 767455267, + 1758255070, + 1653080716, + 280196181, + 1256814137, + 1283464476, + 1835174537, + 1928896068, + 1013621720, + 1687019042, + 652813719, + 943973025, + 1905529538, + 1225893348, + 1248974438, + 1913562576, + 1883748917, + 1694726331, + 206960118, + 967181380, + 1037407459, + 1683637296, + 433505406, + 482006127, + 766466, + 1104086073, + 959246071, + 927637890, + 1126295591, + 496285848, + 1485050600, + 667010875, + 1935962631, + 1639326678, + 400151181, + 234475862, + 1042414288, + 249737602, + 1462374136, + 1743155624, + 411247046, + 2008200350, + 1357248682, + 1196235777, + 97233985, + 781871851, + 746994582, + 795026971, + 305027430 + ] + } + ], + "v1_settings": { + "concurrency": 1, + "max_tokens": 1024, + "model": "deepseek/deepseek-chat-v3-0324", + "n_samples": 12, + "probe_first": true, + "provider": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "reasoning": null, + "req_timeout": 90.0, + "structured_output": true, + "temperature": 1.0 + } + }, + { + "endpoint_advertises_seed": true, + "id": "deepseek/deepseek-chat-v3.1", + "provider_policy": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "replicates": [ + { + "protocol_id": "f811e2ecacecd2fe39b12677e4190f5a2f5878b8c44482f94007d554f0a1bc5b", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-chat-v3.1/replicate_0.jsonl", + "replicate": 0, + "seed_schedule": [ + 1654459904, + 111581769, + 1200243979, + 1447280353, + 1463419994, + 601213585, + 76843542, + 1296589004, + 1065140942, + 1933988356, + 105500704, + 1429026948, + 707351848, + 1055204419, + 541642958, + 650366034, + 1131662072, + 1613451543, + 1135977205, + 1929242993, + 761654081, + 770367637, + 1631889372, + 1100543766, + 1510993940, + 1675735322, + 808750282, + 1101829052, + 1288959742, + 1909527901, + 1795467537, + 1957816145, + 1657323989, + 875975085, + 1880234215, + 1332664338, + 802599740, + 15253875, + 2132819691, + 987940446, + 1678815203, + 617719488, + 1668933220, + 1110717681, + 1371075908, + 432286323, + 1411913823, + 242258962, + 1433460151, + 914480838, + 728543202, + 403668616, + 911424251, + 2094873297, + 856583448, + 1694032129, + 621460488, + 128488853, + 1872407117, + 579067440, + 245524695, + 334099055, + 866270821, + 1170160434, + 1170706953, + 5057668, + 1337488355, + 318240309, + 1257660998, + 1821874817, + 588983694, + 1466107035, + 746421577, + 1249254527, + 678800909, + 1922210996, + 387197145, + 1006157676, + 223135924, + 1181701272, + 1155751169, + 999805363, + 509616512, + 1478564561, + 1517049052, + 217056578, + 2076365836, + 363800580, + 784762242, + 605433059, + 1131324544, + 1392959663, + 1937333296, + 1116015742, + 51217876, + 716482792, + 1456247178, + 2120425404, + 1291162626, + 538096709, + 788869374, + 1009476727, + 1984825715, + 443409677, + 88903406, + 689638988, + 1784172636, + 867596114, + 570330513, + 1550436690, + 305441477, + 216887187, + 1758762590, + 205507722, + 1736720696, + 1403119936, + 1021420146, + 138282772, + 1446278820, + 1510773481, + 380223279, + 1399979581, + 564403882, + 635742512, + 926618121, + 395312122, + 521245349, + 278410425, + 683462495, + 627178267, + 1071991815, + 1708505189, + 1999800830, + 1208490039, + 1377190485, + 767884323, + 2052737913, + 283325272, + 1877307097, + 1059216235, + 1804818020, + 732301734, + 460197026, + 953673408, + 1641437478, + 678451926, + 488398205, + 2022052391, + 1070804995, + 1388895215, + 221250131, + 1478792380, + 540711617, + 240599420, + 755164414, + 1964744672, + 670293105, + 1997421978, + 1697273190, + 787897126, + 999386193, + 1544498906, + 77670540, + 1289095630, + 1489431038, + 412581339, + 1313146605, + 1177769594, + 259681055, + 713664103, + 1402221172, + 1304030997, + 1951351928, + 1573405183, + 69332691, + 228723794, + 724682532, + 1160774348, + 593652515, + 85841684, + 553899023, + 1532121708, + 997320693, + 1119358281, + 1202900678, + 572535848, + 260349891, + 1140471539, + 1835518769, + 2094784695, + 1139167461, + 109346298, + 700330301, + 180693201, + 1198176420, + 505259416, + 293099745, + 1355771015, + 19048180, + 123365468, + 4118053, + 1050292353, + 1962829434, + 104328478, + 1397947957, + 1049152416, + 335329575, + 1636474194, + 1102901623, + 463099940, + 1946804038, + 1459288266, + 1920022059, + 20598982, + 1944159874, + 34528785, + 2071888475, + 1122714757, + 1169871907, + 926239847, + 742573178, + 2050198467, + 1399029639, + 845903811, + 1069537242, + 1889493975, + 516516224, + 997672830, + 1479729685, + 1100512764, + 1901762652, + 815300118, + 2067585534, + 1947122015, + 538279300, + 2086637870, + 657773686, + 1897357614, + 851024569, + 746238055, + 1374705781, + 248138465, + 1819416851, + 235127544, + 1862216904, + 1432271806, + 1597847885, + 2123683677, + 946202566, + 186421493, + 1884043312, + 66366730, + 1512324329, + 1224287100, + 685883026, + 1770152151, + 548750917, + 1166302919, + 455057022, + 1948493079, + 1081714210, + 1794966165, + 1008178363, + 781793039, + 2050002895, + 878308272, + 1394179665, + 874596661, + 1713513772, + 539891674, + 36967109, + 789258804, + 1964152654, + 1730713947, + 468823446, + 1074049089, + 585165567, + 917875541, + 973257683, + 24813884, + 688921870, + 56747523, + 1831470456, + 1466944666, + 1141402944, + 977346773, + 718810184, + 973590221 + ] + }, + { + "protocol_id": "790e4f86bbafc6d07a24dba9d9d69f6a711bd0690a2f0003450eebf2239867a9", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-chat-v3.1/replicate_1.jsonl", + "replicate": 1, + "seed_schedule": [ + 1609241699, + 377384165, + 2055787344, + 538178309, + 799899020, + 2051023012, + 1247374550, + 703433693, + 1170030111, + 1780406178, + 2061481472, + 1128822999, + 426166320, + 1360768575, + 475619068, + 1278843413, + 1278701458, + 2089755354, + 919197404, + 1518055853, + 669045008, + 2003480649, + 477875231, + 2083467200, + 481165064, + 1756414535, + 1308801182, + 93407422, + 1196908223, + 321241915, + 1664702484, + 1689812409, + 1634296137, + 2090056332, + 1844607347, + 1845079670, + 1039427159, + 219910108, + 642126910, + 1570280212, + 1732783595, + 561852401, + 2021612742, + 51370342, + 361235866, + 990458048, + 1781358602, + 33171330, + 2042963329, + 996851723, + 1420229654, + 1322421423, + 273873811, + 317800501, + 429057543, + 1455589646, + 308246272, + 1360480486, + 970909972, + 970578526, + 1448880621, + 824996391, + 1162755279, + 187170481, + 1411738848, + 262698100, + 775728236, + 1498783845, + 868265410, + 190637423, + 810746054, + 163218162, + 1933494382, + 883486084, + 204923106, + 1786395388, + 2119563265, + 783707390, + 2057332092, + 2092258851, + 1047284960, + 355717544, + 2089980203, + 2049820030, + 200775103, + 1972509706, + 574440212, + 191854017, + 315565810, + 2130227659, + 353129663, + 800003271, + 1241524637, + 56498805, + 1342660504, + 593254438, + 1723004581, + 843526590, + 709565698, + 1853257939, + 1357650332, + 2107294873, + 969709098, + 1228615135, + 472693907, + 274856104, + 1647422813, + 1074574939, + 1457560187, + 1212874024, + 1741758009, + 1849769495, + 1903570093, + 1975296599, + 803456981, + 1175854375, + 1750818733, + 1918373971, + 1339241997, + 1116591772, + 1959313345, + 749733042, + 60480330, + 1455160606, + 454882211, + 1510043818, + 469670785, + 1374943952, + 1241925650, + 337080794, + 491984964, + 686694502, + 1486836245, + 1560581366, + 688509760, + 1850188240, + 1072563399, + 1688890439, + 1114646607, + 425552043, + 813273351, + 2030789466, + 676660731, + 453955787, + 1587258810, + 1954266166, + 1601400839, + 1113434309, + 225613611, + 735013923, + 1776055274, + 541977024, + 17541655, + 414700176, + 1049959042, + 1220257997, + 1722276411, + 929838679, + 594818949, + 1949099587, + 160692505, + 385829555, + 1445632406, + 1853489449, + 2074901626, + 1632493619, + 308563427, + 1350537359, + 569825927, + 1792354743, + 104576721, + 2048980659, + 1251826419, + 1715372281, + 798637895, + 393638192, + 825173461, + 873581892, + 7468068, + 1998642932, + 1382323808, + 1531188044, + 2118630792, + 1863123583, + 242383063, + 694401161, + 20687458, + 1453988082, + 69825101, + 1520041800, + 228722873, + 57164596, + 1687915015, + 1725706850, + 1113163409, + 186113213, + 405736278, + 735087460, + 1111950504, + 1495396057, + 1849063717, + 2140304507, + 1563584695, + 978159024, + 1456135018, + 1897350227, + 1665075544, + 1395568906, + 623287274, + 1256883014, + 277970523, + 1771503464, + 756511740, + 855124504, + 127745319, + 2033944371, + 1220234432, + 1339134293, + 867742822, + 939710035, + 1488733614, + 1661135928, + 599664796, + 1378290425, + 449301604, + 396432349, + 2083992298, + 426298296, + 418871164, + 1348616589, + 745720836, + 2088442221, + 1786361681, + 2122369505, + 517129849, + 1196258029, + 1451698914, + 1811417128, + 455343007, + 1010210510, + 787745044, + 1458996357, + 1815830869, + 2099040826, + 2105968843, + 1695504963, + 1918669638, + 1085116058, + 664941313, + 846128991, + 1693405165, + 1868000864, + 1311246067, + 790671847, + 101068573, + 678734846, + 1551895404, + 1965319005, + 550864414, + 1141872604, + 1547582508, + 1885399728, + 116739592, + 1003789286, + 1428291617, + 279449694, + 1525213129, + 685826403, + 1980191553, + 2128329481, + 54264141, + 1654762051, + 2123414404, + 1353421453, + 83501933, + 1332707682, + 1291455837, + 645948158, + 759421489, + 2027647552, + 269929372, + 1354150943, + 406645579, + 513081764, + 1041150348, + 2019680022, + 887110300, + 1348018705 + ] + }, + { + "protocol_id": "737337e2cf6ca66abb03523df4d41971d4a568d72d1ba038c6d1ed2003d96e86", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-chat-v3.1/replicate_2.jsonl", + "replicate": 2, + "seed_schedule": [ + 453921235, + 1756168470, + 2029691878, + 1907646388, + 1353529778, + 2095068575, + 312981498, + 1382190023, + 1626839002, + 2032666536, + 1371909907, + 1458914180, + 1116772694, + 1427449659, + 510193363, + 1683013705, + 438714357, + 699791147, + 1790235356, + 423254584, + 395581472, + 488769661, + 1756542331, + 336626274, + 203188429, + 777725468, + 252968464, + 1413423252, + 1034493188, + 1458893458, + 2097036218, + 1394934332, + 26160704, + 784487829, + 1093496428, + 2001914318, + 2051035477, + 1646782275, + 924685464, + 241702379, + 1182861024, + 1182315191, + 244399454, + 18214077, + 678109089, + 2001007133, + 1492465819, + 1311193921, + 256624775, + 1194761284, + 1103776741, + 2063040263, + 821965588, + 1946071886, + 787466829, + 1417058679, + 570550112, + 796574127, + 358793526, + 1421744610, + 1112750885, + 882027111, + 349219909, + 774907564, + 335793010, + 1069167051, + 170921340, + 656732742, + 1955979428, + 2106448205, + 1538721841, + 1491390095, + 1578862862, + 1935045132, + 218062822, + 1611830812, + 1607972067, + 394665552, + 162285332, + 816817461, + 847914384, + 2037687310, + 1288348920, + 349691885, + 1033271132, + 537796879, + 256294325, + 1972240306, + 2020632805, + 16047959, + 1198962197, + 944850354, + 1721103689, + 149379810, + 837044422, + 1959424725, + 748423786, + 1810963050, + 1263343896, + 2145946428, + 997182827, + 2038822216, + 1501886036, + 1813623734, + 1441512326, + 76775181, + 266169767, + 594816505, + 444731379, + 1560065428, + 571763966, + 1512465797, + 1642829257, + 126948043, + 1876091803, + 996172197, + 1755169829, + 1972021213, + 1015298100, + 1178723428, + 832010604, + 372181299, + 1549173273, + 1007072108, + 2016071010, + 1908714409, + 431970009, + 1217465362, + 654312606, + 1862462237, + 1421887867, + 356736037, + 1245389158, + 358540381, + 495493727, + 1046628649, + 835058988, + 135024315, + 816716056, + 1730311769, + 833785166, + 931096813, + 1810711225, + 464926295, + 719333122, + 1265045567, + 1967208587, + 673104588, + 1696056451, + 1759595121, + 1736450050, + 1801306545, + 860796372, + 1287576944, + 318437649, + 1829148282, + 2129739505, + 202496732, + 869727341, + 1500289516, + 80273258, + 1917565817, + 439876385, + 2090496355, + 199101950, + 830155462, + 1356680267, + 1899444838, + 1440601296, + 538070585, + 1015710805, + 456789484, + 1734688122, + 41449563, + 1683748159, + 185135523, + 1447489703, + 1181875938, + 1570667256, + 733164364, + 1235468616, + 32167099, + 1976011226, + 967042804, + 419138887, + 1395259061, + 1163030998, + 281960204, + 481476217, + 1143193951, + 1636774220, + 911434280, + 1854045613, + 1376186602, + 985769069, + 136325380, + 620220739, + 930212572, + 346946093, + 236379621, + 2063252143, + 177094469, + 1282115015, + 1266950682, + 763545942, + 133753398, + 753823554, + 1736156723, + 896931063, + 528388999, + 1269294301, + 1352364243, + 1213881329, + 1249314687, + 862716785, + 1214041456, + 1175667579, + 256477896, + 1929137430, + 379852038, + 1885610771, + 843193978, + 1083767949, + 72332041, + 1897666763, + 867710306, + 1090799071, + 908657282, + 1337792934, + 933075772, + 1735831611, + 859536395, + 1142006807, + 191274093, + 1518964166, + 308599381, + 344168480, + 2022716521, + 1195078189, + 993541257, + 1835514781, + 1340144289, + 1632164264, + 1541338743, + 1146920962, + 136760479, + 864350329, + 354190190, + 1678123883, + 1896810271, + 697774901, + 1623520439, + 515707047, + 787483950, + 836102633, + 1135730740, + 23586989, + 197562749, + 1294621243, + 2016970218, + 476707479, + 960168584, + 345474956, + 670528206, + 749129366, + 1241103587, + 484801132, + 527828686, + 311562095, + 668325641, + 1232514975, + 523139109, + 490066909, + 1473628240, + 2124203465, + 1974574258, + 1226890446, + 1723590245, + 253173527, + 571534159, + 1062385633, + 430956052, + 278987048, + 1863151845, + 1413929868, + 1533362030, + 1508648937, + 728607748 + ] + } + ], + "v1_settings": { + "concurrency": 1, + "max_tokens": 1024, + "model": "deepseek/deepseek-chat-v3.1", + "n_samples": 12, + "probe_first": true, + "provider": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "reasoning": { + "enabled": false + }, + "req_timeout": 90.0, + "structured_output": true, + "temperature": 1.0 + } + }, + { + "endpoint_advertises_seed": true, + "id": "deepseek/deepseek-v3.2", + "provider_policy": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "replicates": [ + { + "protocol_id": "25ac8be54f2b9a688124eb43f76244c1fcc8783e55ef00c1f81b4d4f583c7ada", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v3.2/replicate_0.jsonl", + "replicate": 0, + "seed_schedule": [ + 201407516, + 37232452, + 1105711169, + 72173605, + 1495303160, + 704825276, + 530019450, + 148719461, + 1308988384, + 1804072586, + 990539766, + 1060002548, + 1607120933, + 408774515, + 509098126, + 1500922761, + 1567888650, + 422773534, + 1032784940, + 1655629410, + 219530860, + 533588778, + 115304291, + 779724006, + 164829772, + 828479823, + 1339955702, + 1397548238, + 540062412, + 772501125, + 1470251538, + 817038323, + 1913677170, + 60268527, + 934478177, + 939144658, + 1380779532, + 1405444312, + 712906484, + 1141508240, + 139062175, + 179924767, + 51204606, + 1412223275, + 569121975, + 1187623708, + 1220711017, + 1170233569, + 363792621, + 512979446, + 1337326319, + 714080403, + 639276799, + 996720827, + 1115630567, + 1828834810, + 3538397, + 1564584027, + 1798434424, + 104326592, + 1048146444, + 473418871, + 1290507596, + 1941925956, + 679564188, + 1726127634, + 1656997239, + 858808341, + 1541444390, + 158360774, + 1791612718, + 1205114569, + 1833993161, + 1504613501, + 1504326892, + 249224350, + 1530874950, + 99084857, + 288045256, + 68102683, + 1343442178, + 2111061593, + 103015369, + 1635909534, + 1481477177, + 1205391946, + 753899526, + 1674477277, + 462514018, + 1593214368, + 1577038467, + 304904814, + 1111837420, + 421869093, + 335558150, + 745614784, + 547341242, + 936230004, + 128931043, + 974177069, + 849206049, + 430591228, + 1181776994, + 1775758279, + 1825178428, + 1091053614, + 204079567, + 1898716215, + 323330294, + 1377750596, + 1863379453, + 1773352107, + 390696461, + 1698234905, + 68078157, + 1106317530, + 1342409061, + 1503651229, + 1495617487, + 144708217, + 1218661859, + 65480742, + 1821945071, + 1853990388, + 914352472, + 1423861598, + 1332935579, + 2070710248, + 1562560335, + 1939733898, + 2106115527, + 448374593, + 886287040, + 1901099538, + 85157502, + 923248115, + 1595833401, + 1150002037, + 732188219, + 638177675, + 1221622810, + 1943134460, + 806649810, + 2124956234, + 256741056, + 1008231714, + 483450516, + 446876285, + 334897621, + 284227618, + 1999687064, + 1997955922, + 26533092, + 368879038, + 1781816016, + 1230344108, + 1505412514, + 674108013, + 600450967, + 545757262, + 31084294, + 1178389851, + 987335851, + 586966925, + 2092175527, + 732231320, + 869819588, + 1090715236, + 1014111730, + 618067460, + 147337347, + 16943203, + 1198102352, + 1779965276, + 761074854, + 1920404288, + 2082298168, + 1397208811, + 94330407, + 884705042, + 291245578, + 81527499, + 273404045, + 2090736758, + 1909704062, + 1198790202, + 169360653, + 1218091144, + 1790958202, + 272268810, + 690059147, + 468513729, + 1118109828, + 1032932699, + 547660904, + 1401777531, + 794264859, + 922209376, + 158076879, + 935023382, + 1345925480, + 1786357578, + 1418924930, + 80993241, + 860305776, + 181341493, + 1680399651, + 1282568511, + 320619685, + 2081058763, + 2055818773, + 1521646172, + 892845138, + 1417537954, + 810566448, + 2097857463, + 761230303, + 143111545, + 1472402985, + 1719799205, + 2045143360, + 1584971901, + 2001900067, + 340848694, + 1512100252, + 42699574, + 502552093, + 2040973669, + 1198858745, + 1063494091, + 1927698646, + 979472689, + 1162914899, + 1997054265, + 2085397049, + 123709299, + 1223107205, + 288529086, + 486731740, + 1430127305, + 1608975900, + 1528420885, + 2037427714, + 777182795, + 1056063270, + 1325729037, + 2063606040, + 1845625702, + 397956189, + 1417469915, + 882193006, + 1163104486, + 1042069801, + 1117395041, + 832195749, + 2036957859, + 1589911195, + 704160280, + 596104676, + 786011527, + 1479143264, + 655092024, + 1439136864, + 940765688, + 930009591, + 1053885730, + 2121686225, + 1505539347, + 992789080, + 651476867, + 661145812, + 1113444679, + 440271057, + 971263027, + 1815898708, + 2145937789, + 485198558, + 534841319, + 1382994034, + 1105721723, + 1664297512, + 1171705225, + 589426726, + 1770684754, + 1760759571, + 1041261902, + 232329939, + 967842431 + ] + }, + { + "protocol_id": "98436c4ea1d3906f28aa5cdb1d45d2f505b2b0879713045979ab75ef5a55e5cf", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v3.2/replicate_1.jsonl", + "replicate": 1, + "seed_schedule": [ + 892284495, + 681266164, + 1809691221, + 893063750, + 1988077160, + 2104062635, + 1335092496, + 376735767, + 256689756, + 1306157114, + 2084836150, + 591562959, + 1390027355, + 2043647833, + 390663231, + 337500999, + 706542164, + 1902661513, + 1358301243, + 690466272, + 1367797609, + 144987382, + 116058069, + 1577279995, + 1641964083, + 728139397, + 1385044011, + 294876561, + 1433360378, + 1415905635, + 1173005324, + 1106429667, + 1940627258, + 853988729, + 221531624, + 1668176290, + 344305362, + 1786228630, + 1811132962, + 857677798, + 1258019691, + 1033070757, + 782246936, + 514482745, + 312924519, + 1968282415, + 427817224, + 501621998, + 314800131, + 660572665, + 1544064953, + 553516019, + 1611986427, + 1349606496, + 1396307117, + 832730734, + 631316559, + 541854161, + 1368895426, + 306667815, + 1586292821, + 917040278, + 771367210, + 1793435981, + 1870816363, + 517943579, + 765029439, + 1185545655, + 888753058, + 1464951563, + 285717427, + 492958272, + 858357618, + 901721827, + 901529044, + 1662163872, + 112202029, + 996373120, + 46866080, + 1527787365, + 1809826821, + 915235860, + 988037742, + 1353272934, + 1171117777, + 567796735, + 2074789335, + 651056508, + 401002872, + 51136563, + 1194503892, + 1646253366, + 266680864, + 1662619651, + 169029786, + 238674655, + 2140724067, + 999550910, + 875183280, + 106490581, + 992725255, + 1665448672, + 628104510, + 222968014, + 760803116, + 178240931, + 1850290864, + 133112822, + 1608055590, + 1147226838, + 1836833610, + 1300590509, + 1823594709, + 838898809, + 416388336, + 777199058, + 717515859, + 1696510483, + 812819979, + 942353537, + 1436667686, + 16742420, + 21317373, + 2018590109, + 636467122, + 1238123691, + 44643472, + 1909719801, + 1428298960, + 1504200624, + 871888068, + 618855292, + 1528822383, + 1960755306, + 919301470, + 2078955560, + 274865741, + 1056265846, + 1840554102, + 1197380552, + 706903225, + 1088901603, + 1982002197, + 831231497, + 282215946, + 1830878890, + 1817230942, + 1043682801, + 1421361717, + 607694247, + 254990165, + 333305093, + 563075095, + 505264847, + 54755283, + 1821801886, + 1497214217, + 2116630902, + 1274044489, + 835832969, + 1526045904, + 1405907956, + 938936413, + 1460856336, + 909518651, + 834557435, + 942616909, + 1846104057, + 493901764, + 1530847269, + 1480786786, + 843496774, + 367801242, + 2004299688, + 64858836, + 30876251, + 369839887, + 307377745, + 1790447191, + 1284005254, + 1590353377, + 1652894551, + 1689550315, + 779368073, + 1508275132, + 1152055681, + 1404688804, + 1297119886, + 579496903, + 712346971, + 1774408257, + 1712805755, + 2076071669, + 1367430447, + 983514059, + 1745976287, + 1241831132, + 1574356590, + 1858460362, + 2123958514, + 1624469322, + 675285502, + 1932838795, + 33774282, + 1053378750, + 2143315853, + 275173811, + 867283741, + 1889850146, + 275722944, + 1512355988, + 352687364, + 1152421665, + 1694598140, + 1573240001, + 1154794254, + 1497157305, + 1274349205, + 634044971, + 315336890, + 214603180, + 1719128683, + 1942330293, + 1496998503, + 1566720372, + 1955791159, + 499295291, + 232970292, + 1230669575, + 1961009899, + 2044361883, + 663043846, + 1202490385, + 875169342, + 937181652, + 654816924, + 532391004, + 524355427, + 974368549, + 1554650503, + 1645769895, + 1478302971, + 775266012, + 346547424, + 1862078533, + 1253559727, + 1408886425, + 2031664082, + 46743250, + 1780009704, + 293259475, + 1722241552, + 1388435827, + 1442382120, + 241132329, + 758820919, + 1164106177, + 2015168550, + 985477586, + 1456324023, + 471220053, + 644598990, + 516225167, + 1331093610, + 938485030, + 1781300570, + 600096651, + 475122238, + 1598226078, + 1616761869, + 368755445, + 2031244799, + 1839475401, + 802051438, + 1849766415, + 1462172699, + 900654291, + 1602341034, + 942981653, + 578283043, + 1213609399, + 794970273, + 1383973295, + 447141821, + 285250022, + 1479396217, + 1461995794, + 38268840 + ] + }, + { + "protocol_id": "913cdf8aeb538dfb4073f2e60ad66c816474ddea0f54448b2760f3565ccb470e", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v3.2/replicate_2.jsonl", + "replicate": 2, + "seed_schedule": [ + 1048566503, + 723887437, + 271033544, + 2118192532, + 1960605585, + 2038149695, + 1844314849, + 1265767084, + 63687247, + 265883107, + 264749118, + 433137444, + 1975803429, + 1578627154, + 794587210, + 1486545172, + 1727137717, + 1161486939, + 68895987, + 1939988061, + 1791562504, + 1068837149, + 172039155, + 483777766, + 2065610872, + 1520252316, + 2126646099, + 177191528, + 262200754, + 244044672, + 1046094666, + 1855505420, + 1483016740, + 646565654, + 338258220, + 1844612215, + 748956026, + 82045590, + 437719003, + 1280784085, + 1859169659, + 308010730, + 192873527, + 384220477, + 46562500, + 2092871292, + 1922963162, + 795618295, + 1496781178, + 1750255671, + 858924391, + 436919945, + 1532145820, + 1919229328, + 373742192, + 1405961207, + 273476940, + 134367860, + 1003349813, + 1687071776, + 1525279112, + 1852162600, + 1746411066, + 2119651984, + 784430286, + 1983728555, + 1838364342, + 226526067, + 1320507059, + 1364207176, + 852649802, + 1622038757, + 1997486291, + 860049156, + 2038864423, + 383781592, + 625941712, + 1425016307, + 680277666, + 683525431, + 1064062134, + 427006338, + 319671867, + 1197849405, + 923399809, + 2015568644, + 430092427, + 1856514781, + 1573897507, + 1951479599, + 1432658098, + 946624412, + 1754107913, + 1241244307, + 928711376, + 1554139231, + 1300104592, + 1206792610, + 117147609, + 1933447648, + 448730473, + 1106863501, + 69567100, + 473472085, + 157859504, + 1603290619, + 110504795, + 319541740, + 236129542, + 252073023, + 1593265250, + 1720553313, + 1514587634, + 780225365, + 1961094753, + 1809901586, + 11070859, + 1293848811, + 300706259, + 442370129, + 592581564, + 1200235483, + 720480821, + 1548231632, + 1292091546, + 1777245972, + 1450760865, + 1407931373, + 38226101, + 1501340577, + 733100374, + 6818042, + 2095264616, + 278210832, + 258010138, + 1220546950, + 341617803, + 1448567696, + 534024225, + 1318342923, + 1965042235, + 1707649866, + 816617682, + 2140668932, + 1926028259, + 2073497104, + 2121055303, + 1867177623, + 1114558745, + 1549143343, + 1836418608, + 569877879, + 598147339, + 278992362, + 45039861, + 484705157, + 1513985254, + 1241923815, + 1836982764, + 2136684605, + 1703258064, + 1669196628, + 1440637676, + 430039348, + 1191923578, + 439790064, + 1558933494, + 224433886, + 1758675991, + 1511860310, + 1732987386, + 1590283956, + 2146263622, + 912373380, + 169327742, + 2003746907, + 1506631588, + 1129391641, + 2062302805, + 775257978, + 697515271, + 1657432229, + 837065831, + 745865022, + 617639181, + 583580288, + 1407720459, + 1993506774, + 1289423407, + 1547370568, + 682433188, + 657399633, + 1441577368, + 1010342675, + 290218047, + 2085218755, + 203393366, + 1841925880, + 1509692044, + 795592411, + 1390249195, + 966898961, + 1760552898, + 877656666, + 2000156203, + 822582105, + 1751233498, + 1308345359, + 2022421219, + 445695310, + 503801551, + 730403812, + 288392568, + 208885992, + 1903059490, + 483006303, + 1945785430, + 1693946901, + 1061079497, + 1141063984, + 49910351, + 952732384, + 1659534601, + 546044243, + 273829522, + 729002920, + 616809731, + 939614829, + 1489642177, + 380486719, + 1127962951, + 1749803152, + 2060986751, + 65492951, + 2030797427, + 163886861, + 353703722, + 308174754, + 1479431365, + 143158411, + 1854620005, + 923649711, + 1812660628, + 896750281, + 183983448, + 1221630051, + 487179512, + 772219234, + 1817881431, + 1052626720, + 1645641438, + 1753477894, + 1993065494, + 603819855, + 894159905, + 2072760418, + 1726618120, + 559742497, + 263900760, + 1688035861, + 1605079016, + 144758285, + 721534012, + 581363397, + 843164214, + 651550521, + 19674903, + 2079464461, + 1148219546, + 2123206425, + 920092279, + 207062351, + 1506064090, + 1761315404, + 1948502493, + 1376776605, + 2017862675, + 1519828052, + 38632444, + 480556415, + 1824473994, + 2004663147, + 1154666160, + 1552912427, + 491297986, + 1009115555, + 1576159016, + 1830817851 + ] + } + ], + "v1_settings": { + "concurrency": 1, + "max_tokens": 1024, + "model": "deepseek/deepseek-v3.2", + "n_samples": 12, + "probe_first": true, + "provider": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "reasoning": { + "enabled": false + }, + "req_timeout": 90.0, + "structured_output": true, + "temperature": 1.0 + } + }, + { + "endpoint_advertises_seed": true, + "id": "deepseek/deepseek-v3.2-exp", + "provider_policy": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "replicates": [ + { + "protocol_id": "943c510824b10bdee1c21d3ac91a713bf008261b6fdac851bf739043c7f32717", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v3.2-exp/replicate_0.jsonl", + "replicate": 0, + "seed_schedule": [ + 2071645268, + 686971990, + 1975065283, + 1574608339, + 2087739351, + 1961611253, + 430969348, + 2138640716, + 476351859, + 1410978956, + 1518290544, + 1657657523, + 1997239531, + 226989618, + 612377826, + 1881010088, + 1653682772, + 928233713, + 108102970, + 929311013, + 681491286, + 760254289, + 1297867638, + 1534648664, + 1958243867, + 802012275, + 82316131, + 1432766023, + 1321448232, + 1624379351, + 549766536, + 740645618, + 1759065411, + 1444016777, + 1840189809, + 1430191460, + 1113276990, + 382386447, + 1275432065, + 1209171585, + 1682024548, + 1561059457, + 1126409801, + 2064445581, + 487058637, + 1476923682, + 1318542477, + 495916686, + 116996972, + 1064232780, + 1463574320, + 1671097869, + 979968058, + 1580739476, + 7294677, + 1034057516, + 1681072404, + 1562209758, + 576090767, + 84783898, + 329979934, + 1079070831, + 1157535179, + 2062723917, + 1461389999, + 1386125791, + 1725639277, + 469876086, + 581389816, + 554087146, + 959478827, + 1686162340, + 1167721159, + 421255064, + 457132919, + 827640543, + 294277641, + 2147268071, + 1954625593, + 1705217276, + 1662085595, + 1461113967, + 1275481685, + 1173272486, + 1581671331, + 1203934835, + 1428876784, + 1775039002, + 240318093, + 1279628574, + 1972546801, + 1158679275, + 1364407677, + 239497272, + 954857824, + 429224116, + 441935158, + 833177432, + 1783579088, + 1382282255, + 1111393814, + 1160844373, + 1980904732, + 1330766598, + 204822502, + 778075844, + 1644582111, + 363250977, + 1421927032, + 107976813, + 900695014, + 81338319, + 2086860567, + 2107636942, + 388789787, + 1529085033, + 1439749389, + 775066119, + 252597867, + 1178726999, + 823870061, + 1604447300, + 524018149, + 211110895, + 691377636, + 813716557, + 468298214, + 2071068959, + 1419698423, + 1838663557, + 871317234, + 1628267493, + 1138941663, + 1691095148, + 1482838612, + 77114131, + 916561492, + 1386102295, + 220308910, + 1015193247, + 1790389849, + 107247450, + 581596188, + 314432002, + 2102922766, + 2039263043, + 79535647, + 1128355891, + 1823813606, + 1924859025, + 456082890, + 1435454159, + 261497514, + 1883655581, + 1100393699, + 1711694723, + 2056149328, + 162522301, + 50919785, + 699269693, + 1913644903, + 1006653966, + 589480968, + 350409294, + 1844041925, + 4690864, + 659589458, + 779195745, + 598143829, + 768535415, + 453856658, + 171963061, + 1619845641, + 313628909, + 927918199, + 2024511381, + 666155850, + 269346949, + 1949488097, + 2019950996, + 1824686851, + 311814052, + 1038649896, + 510542508, + 1339708148, + 100982912, + 1251938871, + 2056721753, + 240852587, + 1827681283, + 110508198, + 537734775, + 1953845554, + 1043837078, + 1036473589, + 172553585, + 692768146, + 1767482970, + 826493325, + 1286305213, + 1990114137, + 891468159, + 1252560195, + 1282094554, + 1822224994, + 455139837, + 1481103860, + 2077626193, + 1804746132, + 1411214077, + 1722601109, + 443330170, + 246483928, + 909420135, + 1435925149, + 2141204751, + 204196993, + 180693788, + 1031073770, + 768394362, + 971673784, + 596757892, + 427491056, + 1304796019, + 1774042818, + 191243709, + 1855179789, + 2064979185, + 2128349610, + 1479117626, + 1288016997, + 685356416, + 2003718717, + 946049654, + 626901652, + 562453957, + 242240961, + 588602522, + 472437823, + 1478179119, + 506241848, + 1207205894, + 612548031, + 889687139, + 564361103, + 1806004270, + 561910676, + 1289699757, + 1067785681, + 1056721742, + 666882640, + 1391383784, + 2048768401, + 617648930, + 510344375, + 1034250674, + 249840128, + 1482860519, + 1017303398, + 2055603729, + 607554557, + 285713792, + 1179588766, + 492728948, + 1845345697, + 115043083, + 2139150443, + 430435886, + 1854648950, + 1919756462, + 871771386, + 1105061857, + 1621589757, + 1352276785, + 310914077, + 1613239776, + 695681573, + 1155525958, + 1331528103, + 269258951, + 204095940, + 387144665, + 1550779926, + 1917237026, + 760132645, + 1068430875, + 342879356, + 720821986 + ] + }, + { + "protocol_id": "2d0fb258f66fc934373d86c7839678724234884baf177c5a5d01f58df422034b", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v3.2-exp/replicate_1.jsonl", + "replicate": 1, + "seed_schedule": [ + 2049237432, + 2115530073, + 522062942, + 683255633, + 1572412655, + 719761357, + 1046223253, + 884716379, + 2116072898, + 607174284, + 665318703, + 1562074238, + 1562452964, + 543942433, + 1212443343, + 1680856355, + 95271559, + 766573000, + 1305727171, + 62860398, + 1107680659, + 1633613206, + 553340601, + 878643612, + 264104050, + 2042295716, + 426344384, + 925090538, + 1040489135, + 1889659889, + 862844892, + 345936801, + 2070798220, + 115632391, + 1713640312, + 931863991, + 545021874, + 2021675481, + 1588240577, + 482283774, + 75199680, + 882007640, + 2076210451, + 859359308, + 1410620243, + 65466342, + 1840510873, + 1583686699, + 1787348030, + 1832215926, + 210614179, + 1080238196, + 2007549853, + 1981251699, + 36696503, + 1371209501, + 1095040315, + 276855172, + 1154742522, + 750818470, + 1623236189, + 711410377, + 213060200, + 1599882093, + 436381980, + 1715900191, + 1776037878, + 549281932, + 1605757846, + 1027346403, + 162299095, + 1498030964, + 1663860362, + 1179984750, + 1402658279, + 536860446, + 613503799, + 208964306, + 337608827, + 843213240, + 139658321, + 1049636428, + 139911364, + 255107517, + 1160537257, + 2068810743, + 1586297789, + 715605281, + 971839985, + 1902575791, + 1832970854, + 1369352425, + 333625338, + 1800568358, + 1235638212, + 1230599863, + 1478763085, + 1101336693, + 1073039722, + 1211858343, + 511687709, + 1701637065, + 506742087, + 1132476428, + 691132179, + 1473988238, + 178540410, + 2022861438, + 1076844952, + 177392278, + 1561042892, + 912632686, + 1499797878, + 85387912, + 1151154808, + 389557761, + 448366284, + 1336254374, + 179786592, + 448469406, + 145580764, + 1680035952, + 1324590780, + 1643558825, + 486863706, + 828666128, + 1556957993, + 560292570, + 1372675742, + 1440924521, + 1066144944, + 117802684, + 187862890, + 1370228982, + 184737464, + 885212254, + 713833054, + 1636800820, + 532255008, + 1525497973, + 1225984650, + 2051291640, + 194034887, + 340261752, + 1587332724, + 1832068320, + 405311478, + 1333899953, + 1107612923, + 534285038, + 273093415, + 650390718, + 1766716141, + 1897671239, + 2092078248, + 1036154624, + 945033331, + 678067491, + 1550969461, + 1113414839, + 1307985373, + 60264524, + 18178799, + 2145207633, + 1325055913, + 1392356794, + 742002022, + 1423638935, + 614178753, + 188851634, + 419247608, + 1820675172, + 1250555398, + 730349030, + 839165677, + 1949861529, + 1005148120, + 764801622, + 982564754, + 2141343155, + 103106419, + 68910558, + 1508720530, + 1448874185, + 728456162, + 2138559056, + 1483817929, + 750738095, + 669797554, + 1640297851, + 1941119529, + 1674484436, + 238765035, + 41216916, + 695269662, + 19900710, + 1056928327, + 1575673479, + 477795777, + 1412434408, + 100623355, + 1790913342, + 892037658, + 892941404, + 2073953158, + 636853904, + 1386551500, + 1514879972, + 1587320864, + 1179200966, + 1555268190, + 660032938, + 1810005444, + 1871956351, + 115915670, + 26440812, + 347033256, + 483882631, + 941828706, + 1375759538, + 634855920, + 418794244, + 319355377, + 1485308517, + 901063416, + 1640933264, + 1755407925, + 691793144, + 329973592, + 1018687788, + 952719377, + 503973250, + 975307369, + 812583749, + 414280544, + 1758861562, + 1783424974, + 2098617295, + 763663878, + 458866947, + 570350652, + 97024060, + 748413014, + 121150518, + 1750371769, + 1439260426, + 910397245, + 181988418, + 128029574, + 580586723, + 1870410544, + 1709685456, + 136442572, + 1138794906, + 703401911, + 1834143457, + 1323837202, + 1628071528, + 1712589109, + 752492676, + 2117422495, + 873589200, + 977935625, + 277777505, + 1453539057, + 1062607130, + 866825419, + 1745092, + 1078670576, + 362743424, + 79906800, + 1782759833, + 647414525, + 1785550374, + 265677646, + 1459331267, + 144323294, + 162752383, + 1795138167, + 2068223352, + 1495860047, + 1431634036, + 1928497193, + 24177962, + 1349292641, + 1921653569, + 264774996, + 1592187228 + ] + }, + { + "protocol_id": "ab8f0a9a643c82e91f3094dc19d631b81b8f6c79104a94019ee34035e0e91deb", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v3.2-exp/replicate_2.jsonl", + "replicate": 2, + "seed_schedule": [ + 30809642, + 664644053, + 1400961824, + 819114988, + 1070891080, + 647829720, + 1519389822, + 1871905650, + 670194022, + 1433021424, + 148624797, + 533058283, + 714669740, + 487361645, + 22604821, + 1191650205, + 3321179, + 1718477223, + 230937865, + 1589089942, + 874666036, + 1286924625, + 1932082082, + 385865193, + 341268829, + 1295371287, + 421141249, + 519069012, + 1485946181, + 667587298, + 1190153177, + 1589234217, + 1621354605, + 1885784374, + 1949395387, + 1825277553, + 303892347, + 337244675, + 162479247, + 1320346458, + 1865254530, + 753491108, + 1552776280, + 475388206, + 1485237653, + 609393856, + 910438952, + 1225113665, + 754872856, + 1581930204, + 1604404129, + 1407271364, + 1685898342, + 1199725594, + 266786093, + 1407984975, + 473395429, + 929656263, + 2094048608, + 60481955, + 1674872197, + 293332760, + 1566258627, + 2020517257, + 653972939, + 926506689, + 1830842326, + 1229203167, + 1758382332, + 1791063016, + 1608043481, + 1117156984, + 1628353130, + 2092296743, + 1013465817, + 329955523, + 139588210, + 895609993, + 382645429, + 1621488019, + 199290709, + 1319010223, + 701888278, + 1705538000, + 803787613, + 1889199668, + 788350913, + 193585963, + 1468924255, + 2139873131, + 1108208279, + 1517649843, + 1808778601, + 372001799, + 452807366, + 1361281800, + 1434384120, + 1137726167, + 1139514997, + 1854717938, + 1695480662, + 1308191323, + 118066918, + 892257376, + 1455024617, + 512717061, + 728130192, + 638393439, + 754022264, + 930424820, + 1065619089, + 1025643144, + 22669545, + 1313163821, + 2075428427, + 564338620, + 2101954061, + 1842016619, + 1549324999, + 2025570795, + 1770429058, + 885405681, + 1599146480, + 667603868, + 1654672363, + 1166151252, + 506804958, + 1092408504, + 461508095, + 154935463, + 1748027535, + 697665614, + 903039246, + 319983680, + 1124188088, + 1798135432, + 2092322357, + 715611407, + 935338879, + 2045005427, + 538358885, + 2056620727, + 1280140497, + 743449277, + 1382540096, + 1989099388, + 760938802, + 361764188, + 1346277616, + 1867745972, + 1354762961, + 1180580518, + 472743101, + 247780009, + 1539002069, + 600342259, + 708093634, + 1321387568, + 1962594444, + 1636310556, + 1197330864, + 1252920826, + 1686531648, + 1333927282, + 797060684, + 1691368542, + 1927679176, + 432275892, + 1737106271, + 799422059, + 1659091566, + 251096500, + 1245579917, + 1702748262, + 1886167777, + 154277696, + 1770915664, + 886660643, + 104087247, + 1811805842, + 547007066, + 446610196, + 1779009096, + 30396779, + 492374984, + 1054280683, + 1157247404, + 291335165, + 367483340, + 505098471, + 1014953813, + 2114988877, + 445044547, + 1087084183, + 2046783687, + 1531426590, + 1303410095, + 200617737, + 204480662, + 1354364967, + 442705182, + 2071463778, + 629594438, + 1159385347, + 932634793, + 996373803, + 509978912, + 1867461833, + 1232580531, + 558475962, + 412271764, + 350249705, + 22264320, + 412552185, + 1559624698, + 291463773, + 1168421972, + 761277503, + 2072734331, + 227470518, + 2044152501, + 2057913374, + 1442654614, + 1007389668, + 2121459891, + 971662030, + 2003892313, + 873819256, + 860343371, + 413433792, + 1841922228, + 943846298, + 613148378, + 1569645815, + 1302706077, + 930721528, + 1267323103, + 1798338939, + 1984952230, + 1214671865, + 1400791327, + 1062228002, + 1218919724, + 952843039, + 702973316, + 1818031253, + 1002341374, + 64288720, + 2135332750, + 25113940, + 68382279, + 1803559391, + 1993110155, + 521980986, + 1216885738, + 1469929567, + 558187374, + 1722554297, + 1408749509, + 725845601, + 1236485678, + 938935602, + 783464398, + 1276062211, + 539385113, + 1338612530, + 478957396, + 371738322, + 494433451, + 1689416830, + 1620652042, + 1605960299, + 1261807716, + 1411294245, + 838656374, + 1161260636, + 587373855, + 1974602953, + 657763269, + 1337634184, + 1260715918, + 1951303226, + 617230994, + 2144845916, + 1906230176, + 387179499, + 485897550, + 1813858548 + ] + } + ], + "v1_settings": { + "concurrency": 1, + "max_tokens": 1024, + "model": "deepseek/deepseek-v3.2-exp", + "n_samples": 12, + "probe_first": true, + "provider": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "reasoning": { + "enabled": false + }, + "req_timeout": 90.0, + "structured_output": true, + "temperature": 1.0 + } + }, + { + "endpoint_advertises_seed": true, + "id": "deepseek/deepseek-v4-flash", + "provider_policy": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "replicates": [ + { + "protocol_id": "402babb39511a0791f12f81f504c55d104ea15cd04e152c1d825afb7b2fb6f9a", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v4-flash/replicate_0.jsonl", + "replicate": 0, + "seed_schedule": [ + 1024517601, + 677879273, + 1066070070, + 208247269, + 2121431023, + 1940262941, + 458533123, + 1643912183, + 894137697, + 192894535, + 1752840096, + 696226657, + 711720165, + 580026148, + 1106048153, + 1433354617, + 1015850412, + 1397510396, + 1005236591, + 1545200633, + 867703679, + 1932893288, + 1344832615, + 1284620189, + 1973689738, + 687947295, + 20831004, + 79238762, + 2065244944, + 1230062133, + 997595462, + 1114658068, + 1603032949, + 668428786, + 1978934144, + 830436687, + 1663142580, + 1781442561, + 82809015, + 402407675, + 1531933669, + 1162124254, + 382235179, + 1839458864, + 31773695, + 1464171814, + 936525133, + 740058083, + 1934499123, + 995411859, + 1230484342, + 674090682, + 1083847976, + 1688895076, + 1223201302, + 913327328, + 1533102756, + 1127930583, + 286475620, + 1015074349, + 1606559395, + 1398460263, + 1281503218, + 2079290716, + 1036936203, + 1079383267, + 1248582179, + 2005462557, + 1310323910, + 1696187511, + 480585991, + 2040341245, + 1225097329, + 342944609, + 1546354720, + 678634948, + 1999152058, + 1807954947, + 165506824, + 533208389, + 2099798796, + 254566670, + 1162503549, + 565591796, + 660765613, + 1546219208, + 1142055644, + 1642503883, + 928311058, + 463432540, + 1175911736, + 1188892228, + 611411974, + 103536888, + 415165602, + 509533475, + 491085486, + 776385803, + 1310334196, + 807837033, + 1040809914, + 853512293, + 1036897614, + 2034853053, + 658632352, + 1066014861, + 1133934757, + 347667628, + 1225003552, + 1088347575, + 274615032, + 695832017, + 1609770676, + 9416890, + 1818941115, + 809400809, + 155955129, + 320323576, + 1144098130, + 1932402457, + 1271470217, + 1349076616, + 2094514641, + 436019120, + 211581736, + 755290557, + 1067090815, + 1961473816, + 1409251886, + 1782393546, + 248295496, + 1871627436, + 1008703573, + 1172867764, + 2139284537, + 214856471, + 1122349955, + 1924672865, + 708530908, + 264422319, + 1882684872, + 846714049, + 934427333, + 440466327, + 1763464472, + 1732075594, + 455259228, + 532420446, + 378056845, + 492075080, + 1179707118, + 442365912, + 2001432555, + 1732997780, + 1939248073, + 553312255, + 886148889, + 1656406434, + 1378245207, + 686547093, + 1926455948, + 88610404, + 1722602431, + 697219202, + 616928752, + 1745972459, + 2104054680, + 1195084867, + 951757306, + 1750402439, + 1060394565, + 1310942906, + 123627856, + 1229757019, + 1926993239, + 1499112387, + 75725733, + 8649486, + 964438313, + 1620180307, + 1341418296, + 482986574, + 2123433964, + 1042086216, + 1169704675, + 1085351909, + 1980708616, + 1379950092, + 1402904044, + 1465161597, + 1892130151, + 1142523155, + 1199082149, + 690654504, + 892483324, + 1170872541, + 612401079, + 743013383, + 1297446423, + 689417236, + 955648488, + 2073283908, + 198697749, + 709974333, + 379605113, + 1139602928, + 750092714, + 556074153, + 750888153, + 1903417375, + 524052108, + 14271050, + 1859228853, + 496157164, + 412203674, + 18821261, + 335614048, + 255512169, + 350764942, + 1426842279, + 738878427, + 549128098, + 1061181740, + 420520119, + 2140773062, + 1274817166, + 35313410, + 1163528786, + 635333417, + 230087478, + 1488634355, + 339780315, + 689894114, + 1577797030, + 1895397141, + 256279652, + 1946518867, + 1701606126, + 561261064, + 2082962284, + 1881892644, + 453398914, + 670570677, + 763854708, + 1447453843, + 544670204, + 438651567, + 215672898, + 1164211046, + 456356961, + 447048526, + 8991957, + 1091601395, + 1380278989, + 2070324273, + 1604906965, + 368246147, + 1421408277, + 530335419, + 715758038, + 2034194019, + 40949319, + 2136315926, + 143769719, + 754299425, + 724664728, + 1884491821, + 1427039241, + 1813011208, + 1234246515, + 1223768747, + 166047902, + 959352330, + 1726429251, + 48164752, + 1184791635, + 911115988, + 770794164, + 47634651, + 1381629319, + 1130129914, + 286756758, + 1350715527, + 1962570667, + 1060207655, + 230065359, + 1182361066, + 707705216 + ] + }, + { + "protocol_id": "d93d76684d43086e85e4c65ab44d3b875bf3a079497fae926a5c3b6c855d58e0", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v4-flash/replicate_1.jsonl", + "replicate": 1, + "seed_schedule": [ + 2056807050, + 2144155666, + 134974253, + 1032065599, + 1644457294, + 1488611757, + 2077609779, + 178276353, + 585300931, + 1654350572, + 1370050381, + 472218245, + 691102127, + 78130686, + 1721973759, + 51986636, + 491226729, + 1704216690, + 2091991394, + 1093771361, + 289404313, + 106855386, + 351002891, + 1833597914, + 295532607, + 929908396, + 121572642, + 134838139, + 798409598, + 497163763, + 1367202954, + 2079596297, + 111736830, + 1559841712, + 696813451, + 1973945438, + 1391667630, + 1794530006, + 1735666650, + 718053534, + 497865041, + 37538348, + 1132530321, + 380045075, + 758778677, + 671744581, + 1441459235, + 1327118124, + 1754859710, + 1384270743, + 1260478613, + 562275266, + 822328823, + 962108319, + 960476234, + 300733553, + 799498156, + 1724874034, + 1153402133, + 697181544, + 1548784301, + 1835962274, + 333383135, + 117174691, + 817529249, + 310507285, + 468427844, + 2063706991, + 2128352741, + 2091562519, + 712148367, + 1109484899, + 944868952, + 385291539, + 1660385557, + 1331132810, + 532627550, + 2086313616, + 90676352, + 1368215202, + 286863279, + 1908801701, + 1621991255, + 266773512, + 404408430, + 1362624770, + 1288917104, + 1492980770, + 1029780280, + 740091393, + 1819008901, + 29557912, + 1455159467, + 2115615629, + 1814810107, + 501454008, + 523504989, + 514276718, + 997572217, + 402244433, + 447884900, + 1170355460, + 744769252, + 1213423376, + 1692992273, + 1670212080, + 1168895459, + 1861860847, + 820708407, + 963521519, + 855137912, + 233711822, + 829201067, + 711476173, + 359092371, + 1004256834, + 960744846, + 28950833, + 1705670589, + 1914120721, + 1161767655, + 624730154, + 1083813481, + 1110938619, + 1486090753, + 1372400854, + 370362493, + 1251267832, + 949853700, + 28659412, + 16515136, + 1307018814, + 1518064139, + 1348274976, + 28413344, + 2078302546, + 199258741, + 220616456, + 1672109326, + 1238611895, + 1998904830, + 1219633814, + 317580180, + 1157109570, + 59391557, + 1971936785, + 2086521247, + 2121565545, + 1354153849, + 297606295, + 200104819, + 84635126, + 2137904522, + 1192940637, + 242806936, + 1945785983, + 2046674580, + 168735524, + 1370762114, + 496821282, + 282605603, + 1047308284, + 626545845, + 1193335092, + 1689235216, + 960793297, + 1974101673, + 868480366, + 784419183, + 728427410, + 52004307, + 1724699763, + 1737084425, + 990122705, + 1937821793, + 1741082045, + 893393303, + 518731811, + 1871909965, + 1188628360, + 1709435766, + 137551041, + 964438142, + 466473284, + 1953286078, + 1159909109, + 1703602812, + 862314602, + 812905282, + 745732851, + 423992050, + 280055826, + 1793648184, + 480848908, + 602419918, + 1970927646, + 262152317, + 1962767147, + 631139148, + 941191585, + 1475087512, + 62653068, + 1213710671, + 484025378, + 512271127, + 1795230416, + 796664515, + 1957896430, + 39817182, + 996931026, + 526400483, + 585368678, + 1936071226, + 1057509510, + 1327711747, + 763614526, + 1178106033, + 181541731, + 640364166, + 1987790864, + 1843998792, + 366117811, + 323209145, + 818527768, + 562795463, + 103014325, + 1598321212, + 1709133159, + 1224415064, + 1329025636, + 1587376693, + 1636659824, + 1547623891, + 1565784540, + 1510061850, + 1519310073, + 1180501670, + 2080674382, + 800103182, + 1713065465, + 94881222, + 1790566147, + 1480486641, + 707949346, + 2004306681, + 169287996, + 1838406039, + 2015272702, + 2115713768, + 472331069, + 933736819, + 2088690564, + 1171022712, + 1271841855, + 1301973335, + 660178500, + 533080556, + 479429562, + 1464034147, + 1653753534, + 850840534, + 743148211, + 1606364636, + 1948851646, + 1607444618, + 165416553, + 995106525, + 562257069, + 1541852789, + 697016525, + 1144101850, + 1084604706, + 480593298, + 1258042918, + 187005322, + 1280469341, + 122661699, + 809476013, + 2113087788, + 825405494, + 879566736, + 1240243442, + 1817026278, + 1079440650, + 1519535703, + 1553408363, + 1949235493, + 1358049148 + ] + }, + { + "protocol_id": "6219872a14841bd64813e3b576484ffdb9e5c6cc7655a6f201781386d28d472c", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v4-flash/replicate_2.jsonl", + "replicate": 2, + "seed_schedule": [ + 836407980, + 1683795460, + 1641406870, + 785862032, + 1894815468, + 807955517, + 1928104002, + 1082155141, + 1551678796, + 440071329, + 4795396, + 2117785446, + 18198423, + 1297236760, + 1282589309, + 415385885, + 398852315, + 15209862, + 169259424, + 1456336659, + 2065452747, + 579698122, + 58209748, + 1816536820, + 982442317, + 1534205760, + 1016923887, + 95685606, + 1037068547, + 414400891, + 739336519, + 1821601873, + 814903125, + 854799889, + 509284728, + 2038826856, + 1457247348, + 2107031799, + 651442319, + 1043619041, + 1879514378, + 1508358545, + 1340090825, + 861256582, + 655081056, + 807361339, + 60559936, + 2058652660, + 1931604865, + 812506144, + 763918202, + 959311276, + 1950411796, + 444816112, + 1999097604, + 1295655583, + 1569031412, + 609074705, + 691907735, + 563749167, + 1374022973, + 215416837, + 170106183, + 1482169247, + 1916569161, + 392179311, + 836425187, + 482753299, + 1579558243, + 867631722, + 967657948, + 1243906468, + 1641327249, + 1508812535, + 903384125, + 1712883679, + 385594103, + 208683063, + 1541574798, + 894656258, + 200769636, + 1622310687, + 634635809, + 820245977, + 111441929, + 747053281, + 1904609761, + 390967110, + 502268999, + 127933714, + 1019146175, + 575906991, + 1561709856, + 814074898, + 139092148, + 514318791, + 1102580910, + 1853097671, + 640880782, + 1495408834, + 401849609, + 496722649, + 518720256, + 1233497445, + 1450574405, + 944378003, + 2881778, + 1681862463, + 1277642305, + 1619875345, + 626153120, + 2091978097, + 1657059881, + 179296789, + 1514255038, + 894880537, + 375714693, + 1090409516, + 725833783, + 763616341, + 88145004, + 1499619036, + 1550441263, + 1850855921, + 1089066937, + 632795108, + 135862681, + 482171253, + 1491159500, + 233571044, + 747471283, + 699009629, + 162248433, + 1437517567, + 1476284512, + 1586194220, + 185234438, + 1748690380, + 1435992975, + 28232877, + 1876842674, + 1556363822, + 657007928, + 522122459, + 1378222065, + 344832220, + 2116841183, + 956080561, + 1785032370, + 157858704, + 1359324772, + 1327643793, + 409553990, + 63197307, + 1704176976, + 1267358455, + 1740893868, + 947364155, + 745544928, + 718364509, + 1718274928, + 1249010531, + 852497433, + 542516876, + 542382359, + 536162266, + 1654537391, + 1736737455, + 1775161732, + 87518044, + 620494846, + 1274606131, + 574979378, + 1509288761, + 472862664, + 382793617, + 1181087495, + 1862504367, + 1691413761, + 1335700348, + 574009718, + 587162616, + 31354338, + 1860136348, + 2097033684, + 2035188906, + 1771365502, + 1808413212, + 641939474, + 1509698431, + 1882163723, + 760411370, + 9274280, + 1454488690, + 1370705071, + 627522111, + 1496275139, + 871057718, + 83615811, + 2040249973, + 1021713871, + 1758760205, + 353441015, + 1411738900, + 752818078, + 260575356, + 1404784189, + 1873665131, + 154784628, + 2005449525, + 2106966774, + 1081527825, + 663819675, + 1704552535, + 1183991347, + 258277503, + 1806316985, + 1397719938, + 1364254222, + 583181277, + 1411291332, + 1155637182, + 619133849, + 966894381, + 482921324, + 9905455, + 1446475099, + 2107002122, + 336965089, + 4159379, + 166093013, + 1718634983, + 1115091886, + 1331913724, + 1237591347, + 406304963, + 331782113, + 1784334677, + 1194519885, + 1214918551, + 1662359579, + 1890081853, + 1277705118, + 828311438, + 1624091186, + 338198875, + 218692932, + 861860339, + 998297318, + 1178942412, + 1788953947, + 985285604, + 151360735, + 510350154, + 1791642442, + 1172776925, + 1775406273, + 1595295353, + 2048868075, + 1659508032, + 2051219026, + 1175694952, + 1135812015, + 511778750, + 2022006164, + 1525820737, + 422053027, + 795662428, + 988312576, + 1072990358, + 2145769954, + 1180246804, + 468222362, + 305723217, + 1090354499, + 1870937864, + 2096027767, + 1349882939, + 161398783, + 575921871, + 1017322220, + 1677572358, + 1840266628, + 46115696, + 474630248, + 569460353, + 1951332233, + 1394636497 + ] + } + ], + "v1_settings": { + "concurrency": 1, + "max_tokens": 1024, + "model": "deepseek/deepseek-v4-flash", + "n_samples": 12, + "provider": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "reasoning": { + "enabled": false + }, + "req_timeout": 90.0, + "structured_output": true, + "temperature": 1.0 + } + }, + { + "endpoint_advertises_seed": true, + "id": "deepseek/deepseek-v4-flash-0731", + "provider_policy": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "replicates": [ + { + "protocol_id": "5cef4da013dfbd74357ac942577d3738ac0efb4b370b1ea20be3a940c5fc74c9", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v4-flash-0731/replicate_0.jsonl", + "replicate": 0, + "seed_schedule": [ + 1393071768, + 2054045436, + 1383280616, + 858460827, + 719231152, + 1750635576, + 1804544165, + 1554409254, + 1527215838, + 1128305898, + 1609338060, + 1009361677, + 445054192, + 511460107, + 805120121, + 1191442581, + 1679034458, + 1859560520, + 1178890725, + 5675562, + 1997224811, + 1270930720, + 1121828812, + 856355783, + 1405965003, + 1018143523, + 663775012, + 207865320, + 1699129470, + 357244490, + 1845180489, + 85622908, + 168618937, + 227262802, + 1235311870, + 63616073, + 1415454176, + 763532764, + 355430860, + 1688691037, + 1534577556, + 368842606, + 1395022344, + 1542863347, + 1970908403, + 290119351, + 21933326, + 1133800881, + 32670900, + 1284942790, + 1968656752, + 681557091, + 1391117177, + 655193027, + 770692094, + 46623918, + 1207852184, + 497439133, + 1223846787, + 201304207, + 931440939, + 880204476, + 214872148, + 1818496521, + 601047260, + 2143560363, + 2043822553, + 1075597269, + 1123755935, + 1625877088, + 519420233, + 137028034, + 1347153002, + 1442406703, + 61304351, + 539836484, + 1342219699, + 644922559, + 1647856873, + 1582498818, + 868241762, + 1867328446, + 1804181636, + 1185208452, + 491863944, + 117061568, + 1703261477, + 975303003, + 1538248625, + 321900712, + 1429951549, + 1384698861, + 1989652482, + 1218145867, + 1560356819, + 626395856, + 1125402600, + 405830792, + 1214181706, + 261100009, + 1703782336, + 1803072237, + 218618154, + 563988027, + 123264429, + 631015191, + 539758498, + 401794736, + 718258373, + 1574404745, + 252425551, + 1574433547, + 330961671, + 337996956, + 1221960853, + 1650340584, + 1295964979, + 1635829074, + 100517244, + 427512257, + 342657557, + 44816703, + 2048135509, + 1701505980, + 1051123226, + 1263781931, + 1915684172, + 2040865559, + 357174446, + 1498490694, + 953723961, + 327202848, + 2071065399, + 1194603319, + 6078841, + 2010464013, + 1194842720, + 388376669, + 2028296348, + 2123119703, + 564963843, + 505887207, + 1113485245, + 385361018, + 88063792, + 65399450, + 437378470, + 1800981964, + 1338096325, + 1078810696, + 2065468684, + 111256262, + 370326895, + 7689216, + 1662797523, + 126560663, + 2049465054, + 1327627169, + 1945850356, + 1711016941, + 408065344, + 939002298, + 1325021916, + 1732128412, + 404474527, + 1431030888, + 1987245485, + 441056694, + 2062464904, + 2027322770, + 1281672982, + 837382793, + 1504207835, + 1749026423, + 871223020, + 2050646920, + 896771656, + 1419987504, + 1835593452, + 853404305, + 1683907956, + 2133379543, + 53810917, + 491579495, + 1223078419, + 2050874105, + 1143222193, + 41157620, + 1470670432, + 956788185, + 321559664, + 945486974, + 669674244, + 540771155, + 1453228688, + 685531981, + 908486254, + 975144328, + 1665101365, + 1646467023, + 576913, + 780783722, + 1289782562, + 2116299606, + 213413886, + 264009368, + 1937142777, + 37647321, + 1885712962, + 1968075805, + 1193426741, + 807355583, + 1140370854, + 753621043, + 573175566, + 1106997343, + 1181145420, + 1574883268, + 1189130912, + 1205405278, + 365025829, + 1138606452, + 1514713320, + 75184821, + 133900133, + 397392372, + 1204870912, + 879492715, + 1660289527, + 2113031976, + 318669791, + 337818548, + 1344974489, + 433013078, + 595665875, + 1890485123, + 544802272, + 1850827008, + 636039923, + 1571229000, + 1703758929, + 990252797, + 1501999884, + 1400335855, + 1173936189, + 234713229, + 205425046, + 543116709, + 1331908479, + 844886895, + 2133258086, + 204081594, + 1956262173, + 177483736, + 284886750, + 2110618467, + 1406166183, + 392161390, + 42725186, + 970062794, + 2023734515, + 914698729, + 1278538730, + 1665766059, + 1012416314, + 747224082, + 1422514482, + 1813270958, + 1111267046, + 941846592, + 1008864915, + 165699374, + 880926313, + 570211717, + 2099026184, + 1524531513, + 1440370631, + 1701861524, + 732275651, + 705491180, + 1322921355, + 229989023, + 1184987446, + 1584334060, + 649533783, + 667629235, + 1555373224, + 909009967 + ] + }, + { + "protocol_id": "6866dbcabfd31ea99dc739adc82fa3d640be303df8bdb391a85352a66e5f4695", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v4-flash-0731/replicate_1.jsonl", + "replicate": 1, + "seed_schedule": [ + 1823229884, + 1153670404, + 2032319880, + 96157663, + 1330461266, + 1661075017, + 1582413727, + 511824507, + 1204660313, + 1366050984, + 791981791, + 491847674, + 1087248685, + 1418349207, + 463717950, + 1936723258, + 1058500992, + 276657940, + 1086353030, + 2072821152, + 30151631, + 1037676577, + 1562159380, + 1675832722, + 1219500505, + 1195540882, + 1933652645, + 619645356, + 119919184, + 639675574, + 573819585, + 1309640749, + 1601811287, + 1328400674, + 748485888, + 261311524, + 1279704741, + 2139452160, + 1269787774, + 1667438235, + 971748900, + 2110463141, + 483266777, + 66881577, + 820677669, + 1632099402, + 152143491, + 1748741995, + 1599891365, + 1844902325, + 783684760, + 473824897, + 263340165, + 994925118, + 1762437607, + 562586800, + 1698991744, + 1675965077, + 1361807448, + 832800446, + 1805541538, + 2030575748, + 557406597, + 65059033, + 719250015, + 922286295, + 2033687819, + 1771682917, + 2085168298, + 491242460, + 1161469230, + 1501386682, + 637732982, + 1063666389, + 348085026, + 1033872924, + 877045136, + 341043296, + 474904734, + 1941160243, + 1198911416, + 225818600, + 1932422031, + 1943143632, + 393387128, + 1230352598, + 857015293, + 396547918, + 508557163, + 1672544328, + 1047817555, + 1739291794, + 2081460737, + 88835786, + 331267264, + 1588775294, + 459200336, + 2143569743, + 120378108, + 1276530659, + 2065704046, + 339752507, + 1923149515, + 70887982, + 1138233906, + 1331470227, + 729984941, + 1724852009, + 584859943, + 1083986813, + 173217137, + 4335051, + 1836668875, + 1205722592, + 1258783968, + 2070287995, + 726265341, + 1986190404, + 576072091, + 472789713, + 558863312, + 661253539, + 198835376, + 22117322, + 2079164007, + 21886994, + 1992013740, + 312327383, + 1469479796, + 1390539250, + 1923572776, + 173712146, + 203910359, + 724832325, + 980089981, + 2020017569, + 627280133, + 1147870132, + 281131767, + 501774804, + 667150919, + 410347898, + 1821132167, + 1250632568, + 1713921227, + 1411599268, + 1467201448, + 956183706, + 984727584, + 1107263014, + 84892468, + 1671448189, + 342728367, + 1590235314, + 333424622, + 971162541, + 353662776, + 769294048, + 1640166637, + 442410475, + 1100113532, + 1746646779, + 1652529766, + 1591723478, + 1113633389, + 1094780996, + 794604105, + 334517555, + 1319798537, + 1468484880, + 1767230439, + 825858892, + 916206135, + 332136, + 1920407010, + 1131221146, + 424887026, + 1389825272, + 1580503384, + 1756973217, + 351835026, + 1940850303, + 521627700, + 2069660353, + 1752619447, + 813218616, + 1766185058, + 407196826, + 86668749, + 1220646501, + 766635701, + 1778145788, + 1534879763, + 280434077, + 819588558, + 70366299, + 550362969, + 1158841365, + 716184932, + 1739601140, + 1618549181, + 1574693749, + 1428280487, + 944524551, + 8942628, + 391228302, + 1133687368, + 1202117315, + 1052457687, + 2090311733, + 1796830247, + 1541465205, + 1776739571, + 62417073, + 1502394610, + 46295762, + 1700812476, + 768242053, + 1567691708, + 1530717800, + 1251285166, + 761023220, + 259754162, + 281453722, + 1500167428, + 486238362, + 843421773, + 587616280, + 1042711478, + 593400659, + 1414964740, + 370042730, + 871834096, + 440767009, + 77076626, + 1501052862, + 1347981934, + 605285951, + 227378401, + 560855559, + 1898666951, + 1585302179, + 1650840919, + 2092107516, + 286888198, + 644291095, + 1285730256, + 75129599, + 1805446735, + 2049513096, + 1888605213, + 135783662, + 892185169, + 999114061, + 1834347137, + 1850414921, + 804649711, + 696318157, + 1019480911, + 352800460, + 971568473, + 1593798146, + 9758963, + 1888880912, + 2077575558, + 1072639889, + 767633672, + 532760363, + 1557894619, + 1802204348, + 406799925, + 1711595375, + 1999142760, + 661613903, + 919252261, + 798244705, + 546480546, + 1571868987, + 991062510, + 645942590, + 2056864285, + 1728128995, + 1322609768, + 1291959418, + 533438990, + 1413099234, + 680448014, + 41588175 + ] + }, + { + "protocol_id": "7dc717bddbde5d39ddddcefbb3ba051f571c62e07d12f0a07a9591f741395674", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v4-flash-0731/replicate_2.jsonl", + "replicate": 2, + "seed_schedule": [ + 273459365, + 148712935, + 1195189517, + 1834009222, + 75121255, + 1411382461, + 781555684, + 1136590494, + 338183172, + 209329714, + 208624080, + 1468333268, + 1234962958, + 661284671, + 1064189309, + 2121350187, + 118946830, + 996939900, + 197905706, + 1239177661, + 1417882433, + 1119837473, + 178108996, + 1702615569, + 615221626, + 2058761656, + 64564418, + 338826042, + 890976132, + 625801891, + 1275464391, + 1270680597, + 2035430754, + 1126257749, + 413938874, + 1006783449, + 563677126, + 2109520675, + 1768319719, + 2036947024, + 1198100393, + 505996188, + 1603181624, + 1441365962, + 578231044, + 1202013790, + 1698129364, + 2090291407, + 2131389909, + 1951094507, + 444902682, + 628210959, + 1610490598, + 1538193083, + 1459297355, + 485849428, + 994524745, + 1642166150, + 1564223500, + 423221673, + 948570165, + 451322290, + 1802462475, + 69693135, + 1970108205, + 865235087, + 133678820, + 971686612, + 1113855403, + 160992142, + 486665419, + 1753362635, + 419032724, + 215826209, + 358657270, + 2049309102, + 1818299560, + 825557346, + 595837355, + 1577507762, + 334829884, + 1217840501, + 1976295090, + 630227936, + 111107720, + 820137324, + 1742980594, + 2061102247, + 213730498, + 1191906465, + 2099055555, + 850679821, + 1076749822, + 343809066, + 1599134781, + 1865027156, + 910707972, + 2037872984, + 1751032785, + 680713276, + 769598936, + 1801130295, + 1163065696, + 1666245773, + 758566124, + 1421554003, + 2040375804, + 1907728934, + 2048225706, + 1865112589, + 805435330, + 223224877, + 609451381, + 483497478, + 1971782912, + 978200316, + 644881409, + 182837097, + 1704962426, + 1367103735, + 351199221, + 1184470404, + 993483460, + 39657312, + 421248713, + 1063067964, + 1554274865, + 1145392501, + 935406617, + 1412129267, + 1048707737, + 381129451, + 1453507055, + 1963013563, + 866676903, + 697304782, + 801336624, + 102614770, + 542459159, + 1756318398, + 232842139, + 87849097, + 98345079, + 1682735489, + 1466153472, + 30961689, + 155828061, + 1031333221, + 561300149, + 322867178, + 156013526, + 888050091, + 1033967115, + 74420698, + 560994311, + 375441451, + 2089756139, + 1384265938, + 309765520, + 1740750531, + 1422885399, + 1914544625, + 881644720, + 1174546700, + 1012033301, + 929293212, + 804302253, + 1676071691, + 1542702019, + 1988198809, + 532516096, + 166949451, + 908180517, + 2046110808, + 1084891766, + 252007208, + 381607264, + 1379565676, + 1291802576, + 2087390302, + 41472576, + 1596144686, + 819028506, + 1991969588, + 1871191136, + 1111520399, + 1638283281, + 546726171, + 1235818417, + 1224140625, + 1018289858, + 1131588715, + 234485546, + 1362796261, + 1978192883, + 526312109, + 391665598, + 1858430401, + 1043278688, + 1902618309, + 1229700006, + 1234294856, + 826265060, + 1132350028, + 678144585, + 394708125, + 629271122, + 1189858039, + 52479799, + 618319369, + 78788467, + 804372757, + 1558637331, + 2122590640, + 1270013819, + 459371824, + 1650150574, + 1389599439, + 1583732768, + 1050050670, + 1052669269, + 394631847, + 253032086, + 1951802208, + 91887926, + 1231050008, + 411871386, + 96875366, + 1837010259, + 2004892210, + 391093332, + 605586164, + 767986872, + 2101617103, + 834075827, + 1726487527, + 1036357576, + 75052535, + 929760456, + 2139793857, + 357448937, + 118977473, + 337205105, + 1262577486, + 195659562, + 1416913155, + 1089737742, + 688069594, + 2126667987, + 1073536451, + 1677026306, + 1334996350, + 867125469, + 1458724438, + 160024927, + 1166394982, + 1245248522, + 1343785842, + 354022508, + 176731618, + 915742333, + 1889985770, + 1762223222, + 162134977, + 811507930, + 2125116189, + 1297161997, + 1323135026, + 204377648, + 616425313, + 534241815, + 1166292433, + 1017317074, + 260276225, + 923977268, + 175716822, + 186916344, + 1001168518, + 130313738, + 483328896, + 619109387, + 1394672067, + 108544887, + 787992463, + 2049676178, + 712604797, + 1076893667, + 1939554989 + ] + } + ], + "v1_settings": { + "concurrency": 1, + "max_tokens": 1024, + "model": "deepseek/deepseek-v4-flash-0731", + "n_samples": 12, + "provider": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "reasoning": { + "enabled": false + }, + "req_timeout": 90.0, + "structured_output": true, + "temperature": 1.0 + } + }, + { + "endpoint_advertises_seed": true, + "id": "deepseek/deepseek-v4.1-flash", + "provider_policy": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, + "replicates": [ + { + "protocol_id": "623588c4663ed794870ded595b8b42373614107db091fe4fe3103fa24857ea4d", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v4.1-flash/replicate_0.jsonl", + "replicate": 0, + "seed_schedule": [ + 2143621977, + 1485575154, + 1605928193, + 709302044, + 2055117899, + 756626930, + 222564401, + 720003666, + 1651252410, + 23365043, + 707782723, + 1794484223, + 1360126398, + 1638606507, + 1182341419, + 2125195314, + 71086990, + 1776005719, + 706285789, + 2118705487, + 1506119351, + 891697686, + 948905481, + 1081422528, + 1003544826, + 21438982, + 1673077599, + 540136335, + 849116669, + 369375295, + 856916467, + 1507352667, + 692807651, + 1198504782, + 98138614, + 991774644, + 1232094777, + 1923925623, + 2046194046, + 1410449019, + 1719446102, + 2101724130, + 207756516, + 93442417, + 949781577, + 2089194729, + 172513607, + 1688147793, + 1094239064, + 1499814034, + 1237189479, + 473553631, + 1782024471, + 2010483674, + 280041115, + 365807965, + 1700907839, + 828430283, + 388806795, + 1597040408, + 1917786135, + 845378774, + 2064239855, + 870500411, + 1087827014, + 2122700955, + 1593825244, + 1622825466, + 2031435536, + 1423184772, + 906758585, + 296591542, + 1265451765, + 1810788968, + 1833117284, + 2006111027, + 1657893010, + 138115720, + 1794955153, + 1064047749, + 455467069, + 387028849, + 1495614691, + 352233581, + 1407056498, + 736809945, + 811607814, + 2029578537, + 305481516, + 754531139, + 680248472, + 141617108, + 876124398, + 1003802963, + 1328246803, + 1170422879, + 132060144, + 1106249207, + 1226675764, + 1190499873, + 1675388114, + 2070353744, + 90197995, + 1326245336, + 15329698, + 1177399203, + 1187242119, + 1236700306, + 821635232, + 67748088, + 584917539, + 1715015872, + 197760006, + 1961059585, + 1266476868, + 43275118, + 998606836, + 1075587466, + 682236835, + 796586416, + 119662805, + 930686195, + 1921125075, + 151842093, + 1258964270, + 310667573, + 1576640280, + 814703792, + 2044601232, + 166144053, + 1477599296, + 1326112080, + 1637667273, + 1858111257, + 616380764, + 1927178014, + 237679003, + 1112427736, + 1716158023, + 77424922, + 726373266, + 1739083298, + 58421151, + 895653665, + 424671510, + 534897500, + 1829408267, + 417169568, + 871901693, + 1685745290, + 1820104528, + 1077414515, + 693708574, + 2050571008, + 1800223126, + 1563837248, + 966722331, + 97595508, + 1586053294, + 337469722, + 1219464677, + 1282878953, + 994021531, + 1999575544, + 2119225755, + 1341036181, + 806102686, + 751989103, + 294475022, + 144684845, + 1419658361, + 1888725785, + 1877361604, + 657335257, + 16017367, + 947297386, + 861584898, + 45566467, + 458587651, + 747178638, + 998851762, + 45315380, + 1513441501, + 1377103752, + 868914807, + 1762675686, + 763557559, + 61573318, + 595692626, + 1253099483, + 231225616, + 378988084, + 395925770, + 951950776, + 629721043, + 616982071, + 627838808, + 709127600, + 1399923472, + 1457032324, + 1127072390, + 958387791, + 366998990, + 43571152, + 2067709034, + 167246497, + 1456090923, + 1003557007, + 1702754573, + 1845898429, + 617697263, + 1340551659, + 387922128, + 454968152, + 1939355750, + 229993516, + 1431526436, + 1971300186, + 875235269, + 1114805739, + 1220705330, + 1305255116, + 654320840, + 629880135, + 861255230, + 668382061, + 352733296, + 716756708, + 1999534160, + 904104101, + 846417256, + 1191880949, + 184420054, + 1952793365, + 1929493542, + 287802920, + 1577083003, + 1673969035, + 972962642, + 1242009489, + 6319144, + 1180283594, + 1206289067, + 16043610, + 1766861820, + 241926111, + 2028669344, + 167495443, + 1115336873, + 492835318, + 112488567, + 1688461257, + 1258348191, + 1238858729, + 589691925, + 435194386, + 211150572, + 1438018903, + 16184062, + 519851856, + 1467096280, + 978333640, + 744108295, + 97231006, + 592767538, + 299293333, + 1068430338, + 167232476, + 1140378458, + 1494883680, + 261353695, + 291633088, + 650723152, + 1995714929, + 423636778, + 1646148661, + 1894324030, + 374948903, + 745229322, + 977554558, + 145888673, + 1302924067, + 1959095237, + 1053351403, + 252365511, + 1972463716, + 1076345686, + 1457665318 + ] + }, + { + "protocol_id": "39b16666015ca0415f921d6b83786abd67ecba8547e399c88827c8a9a9251940", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v4.1-flash/replicate_1.jsonl", + "replicate": 1, + "seed_schedule": [ + 1673039805, + 1666601083, + 111524500, + 1982543421, + 1536966605, + 1443924729, + 1595618390, + 921965017, + 928232309, + 840992085, + 117447652, + 226034377, + 882718429, + 633575972, + 2095718144, + 533327575, + 1297863538, + 1022841667, + 662891896, + 424213385, + 2089838051, + 1139419334, + 733186451, + 1881368825, + 417582675, + 1982004145, + 391225609, + 1623450716, + 1112392100, + 1257703280, + 1314505165, + 959592900, + 862355175, + 1016666318, + 854414158, + 85557112, + 1530502113, + 1471147639, + 807998258, + 2104335886, + 2081197105, + 1046909771, + 185431016, + 1133749017, + 1746319067, + 146089738, + 721316301, + 930361875, + 1275579154, + 496093860, + 1213128999, + 906662056, + 316906871, + 1022629987, + 1186111111, + 677733275, + 1249642779, + 873646726, + 315591296, + 1739835102, + 1526767824, + 1048559777, + 1183486026, + 2104495841, + 741586627, + 1594650156, + 2004706297, + 2064311007, + 639642146, + 1821940558, + 273522877, + 1945954476, + 400806921, + 1754155264, + 1667808787, + 1517343676, + 20698437, + 347064552, + 1914881406, + 2035427430, + 674822887, + 1979664185, + 2081313895, + 1206727910, + 1942111156, + 1311754335, + 247887099, + 141927612, + 1167618544, + 1091508999, + 319837014, + 1008632662, + 985989437, + 2139324128, + 1923056281, + 21159253, + 2111410996, + 1050841923, + 1387477705, + 1199443841, + 1039294409, + 512079347, + 2001020546, + 2018012750, + 169850027, + 1762109201, + 1475715936, + 980670053, + 576823923, + 2088989504, + 2065296810, + 1830848258, + 1560419662, + 1164277764, + 1467598332, + 310552696, + 1278470697, + 442526670, + 38213159, + 10030, + 1849304407, + 806563684, + 932586736, + 416763894, + 1230124472, + 1783894735, + 1135567144, + 952258516, + 1692145050, + 205845544, + 1402203970, + 520591402, + 1034794040, + 1626361580, + 1479907813, + 1969603330, + 2049548537, + 680725564, + 1881996757, + 996368593, + 1828942756, + 1067647147, + 1664147104, + 1761800665, + 1455163000, + 204205196, + 344768350, + 299043768, + 39623480, + 637680915, + 1162963078, + 59742280, + 1939292909, + 499785661, + 46297378, + 1788859742, + 1726813832, + 1354511803, + 413285113, + 832113470, + 479804027, + 1368757738, + 1563917158, + 1997439446, + 712081911, + 822159340, + 822262302, + 1036188452, + 751436203, + 1094471700, + 604602575, + 737315974, + 1245447956, + 841058461, + 42265993, + 916512459, + 1002766844, + 2044846262, + 586181593, + 417864800, + 898055853, + 64637799, + 2116999513, + 443526095, + 951829928, + 1119264488, + 1693715235, + 1805082585, + 1710585798, + 190057134, + 1518746891, + 623169041, + 639215324, + 1483489165, + 2112560815, + 1743520264, + 1345481838, + 1408075290, + 1855337170, + 2041361455, + 1926547694, + 1716347448, + 560865277, + 1783138214, + 1126535999, + 1793849877, + 353144497, + 1328657359, + 1063631942, + 1858911560, + 649387382, + 1033384873, + 1363654905, + 809103935, + 2096375678, + 183004238, + 957346915, + 301773526, + 440687073, + 822006143, + 677062034, + 804287886, + 1508190365, + 778146276, + 115718876, + 515187036, + 1095764213, + 1848713164, + 1712399307, + 741245792, + 2054395531, + 840677098, + 1527852002, + 1002416975, + 295638745, + 1599375416, + 2071999396, + 33126544, + 1516511868, + 374138494, + 1216288126, + 226624831, + 296563251, + 604480836, + 968771210, + 1786301263, + 1295707802, + 64784495, + 123353783, + 1978315182, + 1836114443, + 1119676584, + 303255043, + 813053692, + 795068045, + 1017907011, + 1782257367, + 741747122, + 597629419, + 905646805, + 1616517131, + 994808023, + 2004781773, + 561543894, + 758640462, + 1226646902, + 2020137442, + 1630830152, + 244928245, + 87902376, + 153632909, + 2023360128, + 551128728, + 1587579032, + 34399752, + 559425808, + 1564295688, + 1981661887, + 1167747271, + 1374414326, + 1479277313, + 667921654, + 1259762158, + 1293866276, + 163181245, + 1852876677, + 25389410, + 1989806992 + ] + }, + { + "protocol_id": "f2733d2f962bd9584fe56c01509fc935e0f50be6100594e056cc49ec717a7e8c", + "records": "slop/research/wvs/20260917_deepseek_reliability/records/deepseek__deepseek-v4.1-flash/replicate_2.jsonl", + "replicate": 2, + "seed_schedule": [ + 1258699505, + 498570777, + 41605010, + 2133444376, + 1074734748, + 1452055695, + 247560858, + 875612473, + 31359041, + 1059933683, + 1660387218, + 125581245, + 982100126, + 1022623328, + 1741822819, + 292720713, + 48744630, + 1135675813, + 280259922, + 167387902, + 1451024000, + 1389314377, + 1338628577, + 293617777, + 1089222248, + 10064521, + 2066852114, + 778430686, + 1063225520, + 1353601313, + 410394758, + 1764206645, + 2043436926, + 1692165263, + 1599687047, + 343855720, + 1168392830, + 182437729, + 228243638, + 1459662408, + 824396695, + 1076821944, + 1085701919, + 1056436312, + 1418511453, + 318076065, + 1012707119, + 1992182984, + 1426075908, + 306303607, + 399234897, + 936599230, + 505120105, + 1501526059, + 290916062, + 1117155895, + 1774096980, + 1345184525, + 1597892304, + 20940036, + 1876011254, + 55279794, + 1903051108, + 993563354, + 1544554857, + 33706302, + 1097447637, + 613875390, + 2002925982, + 108991131, + 705003524, + 875195559, + 405259388, + 364099778, + 95182602, + 901326062, + 1232113920, + 257353954, + 225061017, + 261679643, + 605867070, + 197180831, + 650230758, + 1427880595, + 1724758035, + 1803429576, + 1223609660, + 1920021960, + 1353694589, + 430784726, + 906304996, + 1935490066, + 304439810, + 2143540193, + 926696548, + 302933483, + 58239068, + 54971762, + 484756321, + 546262693, + 772829277, + 1836072808, + 580419964, + 1073162351, + 348514775, + 791272764, + 288574975, + 1624882328, + 419872675, + 1379475708, + 1254356952, + 432002971, + 1406467141, + 982773250, + 2044926072, + 1060618488, + 519785953, + 2027881614, + 1923485803, + 909159380, + 87079018, + 421175232, + 1845670795, + 1797975273, + 404692059, + 1993127878, + 1394116423, + 907396780, + 274798064, + 1583361833, + 488243090, + 2057158269, + 1576699431, + 1657291247, + 1276123039, + 1303990469, + 1194375617, + 2000780742, + 1802346294, + 1942611602, + 1054807983, + 248757036, + 800931747, + 1102454696, + 1877250603, + 842183070, + 1956426802, + 536679716, + 95696656, + 2027792948, + 1004855628, + 511662706, + 1163560075, + 2146246864, + 392298910, + 827842889, + 58807775, + 925075629, + 1205225295, + 1014752921, + 2046362419, + 1977514115, + 837198411, + 1056553220, + 1223589393, + 912442430, + 898599863, + 1449904944, + 1678573845, + 913697084, + 1918499241, + 2143529917, + 641783247, + 1374633719, + 700725128, + 2140329583, + 1773354275, + 1162765924, + 1206153942, + 1520709758, + 701287821, + 1252816207, + 1614192872, + 320893528, + 1678451004, + 863466704, + 2144833297, + 1126289700, + 1564219369, + 560730492, + 1420932370, + 1517600624, + 861177785, + 1616819641, + 634344924, + 1096735351, + 901728444, + 1686528096, + 1764448723, + 447024475, + 418175337, + 2130246976, + 1847417438, + 1564390136, + 430884704, + 265870370, + 1001218049, + 1989105843, + 1283184549, + 2133186494, + 1390558055, + 1254160523, + 1470478121, + 1975733983, + 349612973, + 1446894679, + 1697228065, + 463376440, + 1912298256, + 468292794, + 1621222891, + 1821965756, + 166088349, + 1432151716, + 1051668742, + 1931954724, + 261927032, + 1956490409, + 202452501, + 1715247240, + 1555291508, + 223127211, + 1243771206, + 2630198, + 709561945, + 181736448, + 1773010681, + 1253911904, + 1292723224, + 170292398, + 788994290, + 1093461193, + 1729558037, + 597342212, + 1287440039, + 477875367, + 1256719808, + 146206289, + 2085901059, + 1262683974, + 574725592, + 603824248, + 689045901, + 1082916695, + 1422275628, + 2096901512, + 1456994488, + 1985842143, + 1082993206, + 526421961, + 2035434426, + 1812570520, + 308883968, + 1610790237, + 1247353266, + 433410079, + 2067000642, + 297338195, + 610501649, + 560802881, + 922850987, + 638577601, + 785567647, + 782655704, + 948473443, + 1302086854, + 239735213, + 1442284063, + 1206453756, + 7585188, + 693750227, + 960980458, + 123145524, + 1875518236, + 1389372761, + 691134005, + 120844727, + 921447537 + ] + } + ], + "v1_settings": { + "concurrency": 1, + "max_tokens": 1024, + "model": "deepseek/deepseek-v4.1-flash", + "n_samples": 12, + "reasoning": { + "enabled": false + }, + "req_timeout": 90.0, + "structured_output": true, + "temperature": 1.0 + } + } + ], + "pilot_cap_usd": "1", + "purpose": "reliability replication, not a new evaluator", + "replicates": 3, + "samples_per_item": 24, + "schema": 1 +} diff --git a/src/moralmaps/read_api.py b/src/moralmaps/read_api.py index e0679c7..5fc7d8c 100644 --- a/src/moralmaps/read_api.py +++ b/src/moralmaps/read_api.py @@ -208,7 +208,8 @@ def _rate_plan(items: list[dict], n_samples: int, per_call: int = 1) -> list[dic def rated_protocol_identity(model: str, items: list[dict], *, n_samples: int, temperature: float, max_tokens: int, concurrency: int, req_timeout: float, reasoning: dict | None, structured_output: bool, - provider: dict | None = None) -> str: + provider: dict | None = None, eval_version: str | None = None, + seed_schedule: list[int] | None = None) -> str: """Hash the exact model, rendered prompts, and request settings that define a cacheable panel.""" plan = _rate_plan(items, n_samples) protocol = { @@ -226,6 +227,10 @@ def rated_protocol_identity(model: str, items: list[dict], *, n_samples: int, te "requests": [{key: req[key] for key in ("i", "perm", "prompt", "cnt", "sample", "presented_options")} for req in plan], } + if eval_version is not None: + protocol["eval_version"] = eval_version + if seed_schedule is not None: + protocol["seed_schedule"] = seed_schedule encoded = json.dumps(protocol, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() return hashlib.sha256(encoded).hexdigest() @@ -242,7 +247,8 @@ def read_items_rated(model: str, items: list[dict], *, n_samples: int = 12, temp max_tokens: int = 512, concurrency: int = 8, req_timeout: float = 90.0, reasoning: dict | None = None, structured_output: bool = False, records_path: str | Path, verbose_first: bool = False, provider: dict | None = None, - probe_first: bool = False) -> list[dict]: + probe_first: bool = False, eval_version: str = "wvs-score-all-options-v1", + identity_eval_version: str | None = None, seed_schedule: list[int] | None = None) -> list[dict]: """Run one score-all-options panel and write an fsynced JSONL event for every paid request phase. The record is the source of truth. It preserves dispatches, responses, rescues, provider usage, @@ -251,17 +257,21 @@ def read_items_rated(model: str, items: list[dict], *, n_samples: int = 12, temp """ assert temperature > 0, "sampling readout needs temperature > 0" plan = _rate_plan(items, n_samples) + if seed_schedule is not None and len(seed_schedule) != len(plan): + raise ValueError(f"seed schedule has {len(seed_schedule)} entries, expected {len(plan)}") protocol_id = rated_protocol_identity(model, items, n_samples=n_samples, temperature=temperature, max_tokens=max_tokens, concurrency=concurrency, req_timeout=req_timeout, reasoning=reasoning, - structured_output=structured_output, provider=provider) + structured_output=structured_output, provider=provider, + eval_version=identity_eval_version, seed_schedule=seed_schedule) run_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_{protocol_id[:12]}" rpath = Path(records_path) rpath.parent.mkdir(parents=True, exist_ok=True) settings = {"model": model, "n_samples": n_samples, "temperature": temperature, "max_tokens": max_tokens, "concurrency": concurrency, "req_timeout": req_timeout, "reasoning": reasoning, "structured_output": structured_output, "provider": provider, - "probe_first": probe_first} + "probe_first": probe_first, "eval_version": eval_version, + "identity_eval_version": identity_eval_version, "seed_schedule": seed_schedule} _append_record(rpath, {"event": "run_started", "run_id": run_id, "protocol_id": protocol_id, "settings": settings, "items": items, "planned_requests": len(plan)}) @@ -271,12 +281,16 @@ def read_items_rated(model: str, items: list[dict], *, n_samples: int = 12, temp async def call(seq: int, req: dict) -> dict: item = items[req["i"]] request_id = f"{run_id}_{seq:03d}" + seed = seed_schedule[seq] if seed_schedule is not None else None request_meta = {"request_id": request_id, "run_id": run_id, "protocol_id": protocol_id, "model": model, "item_id": item["id"], "canonical_options": item["options"], "presented_options": req["presented_options"], "presented_order": req["perm"], - "sample": req["sample"], "prompt": req["prompt"], "settings": settings} + "sample": req["sample"], "prompt": req["prompt"], "settings": settings, + "eval_version": eval_version, "seed": seed} payload = {"model": model, "messages": [{"role": "user", "content": req["prompt"]}], "temperature": temperature, "n": req["cnt"], "max_tokens": max_tokens} + if seed is not None: + payload["seed"] = seed if reasoning is not None: payload["reasoning"] = reasoning if provider is not None: