diff --git a/scripts/wvs_deepseek_reliability_pilot.py b/scripts/wvs_deepseek_reliability_pilot.py index 66d530c..93da986 100644 --- a/scripts/wvs_deepseek_reliability_pilot.py +++ b/scripts/wvs_deepseek_reliability_pilot.py @@ -145,17 +145,47 @@ def manifest() -> dict: } -def usage(records: Path) -> tuple[Decimal, Counter, int, int]: - cost, providers, rescues, failures = Decimal(), Counter(), 0, 0 - for line in records.read_text().splitlines(): - record = json.loads(line) - if record["event"] == "request_completed": +def replicate_result(records: Path, replicate: dict, items: list[dict], resolved: dict) -> dict | None: + events = [json.loads(line) for line in records.read_text().splitlines()] if records.exists() else [] + finished = [record for record in events if record["event"] == "run_finished" + and record["protocol_id"] == replicate["protocol_id"]] + if not finished: + return None + final = finished[-1] + run_id = final["run_id"] + item_rows = {record["id"]: record for record in events + if record["event"] == "item_result" and record["run_id"] == run_id} + cost, providers = Decimal(), Counter() + for record in events: + if record.get("run_id") == run_id and record["event"] == "request_completed": cost += Decimal(str(record.get("usage", {}).get("cost", 0))) providers[record.get("provider")] += 1 - if record["event"] == "item_result": - rescues += record["rescued_samples"] - failures += record["failed_samples"] - return cost, providers, rescues, failures + result = { + "replicate": replicate["replicate"], "protocol_id": replicate["protocol_id"], + "records": str(records), "run_id": run_id, "provider_requests": dict(providers), + "rescues": final["rescued_samples"], "failures": final["failed_samples"], + "usage_cost_usd": str(cost), "valid_samples": final["valid_samples"], + } + complete = (final["valid_samples"] == len(items) * N_SAMPLES + and final["failed_samples"] == 0 + and len(item_rows) == len(items) + and all(row["valid_samples"] == N_SAMPLES for row in item_rows.values())) + if not complete: + return result | {"status": "incomplete"} + psamples = {item["id"]: np.asarray(item_rows[item["id"]]["p_samples"]) for item in items} + coords = model_coord_ci(psamples, resolved, np.random.default_rng(0)) + response_se = _sample_only_coord_se(psamples, resolved, np.random.default_rng(1), n_draws=N_SAMPLES) + return result | {"status": "complete", "coords": list(coords), "response_mean_se": list(response_se)} + + +def pilot_spend() -> Decimal: + cost = Decimal() + for records in (OUT / "records").glob("**/*.jsonl"): + for line in records.read_text().splitlines(): + record = json.loads(line) + if record["event"] == "request_completed": + cost += Decimal(str(record.get("usage", {}).get("cost", 0))) + return cost def v1_coords(model: str) -> list[float]: @@ -168,22 +198,19 @@ def v1_coords(model: str) -> list[float]: def run_replicate(model: str, replicate: dict, v1: dict, items: list[dict], resolved: dict) -> dict: records = Path(replicate["records"]) + prior = replicate_result(records, replicate, items, resolved) + if prior is not None: + return prior records.parent.mkdir(parents=True, exist_ok=True) - seeds = replicate["seed_schedule"] - rows = read_items_rated(model, items, n_samples=N_SAMPLES, temperature=v1["temperature"], - max_tokens=v1["max_tokens"], concurrency=1, req_timeout=v1["req_timeout"], - reasoning=v1["reasoning"], structured_output=v1["structured_output"], provider=v1.get("provider", OSS_PROVIDER), - records_path=records, verbose_first=True, probe_first=True, eval_version=EVAL_VERSION, - identity_eval_version=EVAL_VERSION, seed_schedule=seeds) - if any(row["valid_samples"] != N_SAMPLES for row in rows): - raise RuntimeError(f"incomplete replicate {model} {replicate['replicate']}") - psamples = {row["id"]: np.asarray(row["p_samples"]) for row in rows} - coords = model_coord_ci(psamples, resolved, np.random.default_rng(0)) - response_se = _sample_only_coord_se(psamples, resolved, np.random.default_rng(1), n_draws=N_SAMPLES) - cost, providers, rescues, failures = usage(records) - return {"replicate": replicate["replicate"], "protocol_id": replicate["protocol_id"], "records": str(records), - "coords": list(coords), "response_mean_se": list(response_se), "provider_requests": dict(providers), - "rescues": rescues, "failures": failures, "usage_cost_usd": str(cost)} + read_items_rated(model, items, n_samples=N_SAMPLES, temperature=v1["temperature"], + max_tokens=v1["max_tokens"], concurrency=1, req_timeout=v1["req_timeout"], + reasoning=v1["reasoning"], structured_output=v1["structured_output"], provider=v1.get("provider", OSS_PROVIDER), + records_path=records, verbose_first=True, probe_first=True, eval_version=EVAL_VERSION, + identity_eval_version=EVAL_VERSION, seed_schedule=replicate["seed_schedule"]) + result = replicate_result(records, replicate, items, resolved) + if result is None: + raise RuntimeError(f"replicate {model} {replicate['replicate']} did not write run_finished") + return result def run() -> None: @@ -191,31 +218,35 @@ def run() -> None: items, resolved = rated_items() if not reserve({"id": PILOT_RESERVATION_ID, "lane": "deepseek", "reserve_usd": str(PILOT_CAP_USD)}): raise RuntimeError("global USD 80 cap would be exceeded by the USD 1 pilot reservation") - pilot_state(lambda state: state.update({"reserved_usd": str(PILOT_CAP_USD), "started_utc": datetime.now(UTC).isoformat()})) + pilot_state(lambda state: state.update({"reserved_usd": str(PILOT_CAP_USD), "started_utc": datetime.now(UTC).isoformat(), + "spent_usd": str(pilot_spend())})) results = {"eval_version": EVAL_VERSION, "models": [], "not_published": True} try: for row in data["models"]: reps = [] for replicate in row["replicates"]: + if Decimal(pilot_state(lambda state: state)["spent_usd"]) >= PILOT_CAP_USD: + raise RuntimeError(f"pilot cap reached before {row['id']}: {pilot_spend()}") result = run_replicate(row["id"], replicate, row["v1_settings"], items, resolved) - spent = pilot_state(lambda state: state.update({"spent_usd": str(Decimal(state["spent_usd"]) + Decimal(result["usage_cost_usd"]))})) - if Decimal(spent["spent_usd"]) >= PILOT_CAP_USD: - raise RuntimeError(f"pilot cap reached: {spent['spent_usd']} >= {PILOT_CAP_USD}") + pilot_state(lambda state: state.update({"spent_usd": str(pilot_spend())})) reps.append(result) atomic_json(RESULTS, results | {"models": results["models"] + [{"id": row["id"], "replicates": reps}]}) - values = np.asarray([rep["coords"][:2] for rep in reps]) - aggregate_samples = {} - for rep in reps: - rows = [json.loads(line) for line in Path(rep["records"]).read_text().splitlines()] - for record in rows: - if record["event"] == "item_result": - aggregate_samples.setdefault(record["id"], []).extend(record["p_samples"]) - aggregate = model_coord_ci({key: np.asarray(value) for key, value in aggregate_samples.items()}, resolved, np.random.default_rng(2)) - v1 = v1_coords(row["id"]) - results["models"].append({"id": row["id"], "v1_coords": v1, "replicates": reps, - "between_replicate_coordinate_sd": np.std(values, axis=0, ddof=1).tolist(), - "aggregate_n72_coords": list(aggregate), - "delta_aggregate_minus_v1": (np.asarray(aggregate[:2]) - np.asarray(v1[:2])).tolist()}) + model_result = {"id": row["id"], "v1_coords": v1_coords(row["id"]), "replicates": reps} + if all(rep["status"] == "complete" for rep in reps): + values = np.asarray([rep["coords"][:2] for rep in reps]) + aggregate_samples = {} + for rep in reps: + records = [json.loads(line) for line in Path(rep["records"]).read_text().splitlines()] + for record in records: + if record["event"] == "item_result" and record["run_id"] == rep["run_id"]: + aggregate_samples.setdefault(record["id"], []).extend(record["p_samples"]) + aggregate = model_coord_ci({key: np.asarray(value) for key, value in aggregate_samples.items()}, resolved, np.random.default_rng(2)) + model_result |= {"status": "complete", "between_replicate_coordinate_sd": np.std(values, axis=0, ddof=1).tolist(), + "aggregate_n72_coords": list(aggregate), + "delta_aggregate_minus_v1": (np.asarray(aggregate[:2]) - np.asarray(model_result["v1_coords"][:2])).tolist()} + else: + model_result |= {"status": "incomplete", "aggregate_excluded_reason": "at least one replicate has fewer than 288 valid samples"} + results["models"].append(model_result) atomic_json(RESULTS, results) finally: release(PILOT_RESERVATION_ID) diff --git a/slop/audits/20260917_wvs_deepseek_reliability_task_1674.md b/slop/audits/20260917_wvs_deepseek_reliability_task_1674.md new file mode 100644 index 0000000..055d3d6 --- /dev/null +++ b/slop/audits/20260917_wvs_deepseek_reliability_task_1674.md @@ -0,0 +1,78 @@ +# Audit: DeepSeek reliability pilot task 1674 + +- Pueue task 1674 ran `sh scripts/wvs_api/07_deepseek_reliability_pilot.sh` from 21:38:41 to 22:31:21 +0800 and exited 1. +- full normalized Pueue log: 80/80 lines; raw log: 92 lines, both at `/tmp/wvs-audit-1674/`. +- scope: separate `wvs-score-all-options-v1` reliability records only. No canonical cache or Pages data changed. + +| stage | expected | observed | expected? | consequence | +|---|---|---|---|---| +| pilot reservation | USD 1 pilot and global reservation released on exit | pilot `reserved_usd: 0`; global `reservations: {}` | yes | no stale budget block | +| first model replicate 0 | 288 valid ratings | 288/288, GMICloud, USD 0.01570740 | yes | retained separately | +| replicate 1 | 288 valid ratings | 288/288, GMICloud, USD 0.01578264 | yes | retained separately | +| replicate 2 | 288 valid ratings | 287/288, one `ReadTimeout` at `Attending peaceful demonstrations`, sample 10 | no | no N=72 aggregate for this model | +| remaining six models | continue after a model/replicate failure | not attempted because the runner raised | no | the task's continuation condition was not met | + +## Evidence + +The final replicate record says: + +> `"event": "request_failed", "item_id": "Attending peaceful demonstrations", "sample": 10, "phase": "initial", "error_type": "ReadTimeout", "error": ""` +> +> `"event": "run_finished", "failed_samples": 1, "planned_requests": 288, "rescued_samples": 0, "valid_samples": 287` + +The complete first two replicates have matching `run_finished` records with `valid_samples: 288`, `failed_samples: 0`, and `rescued_samples: 0`. The terminal traceback is: + +> `RuntimeError: incomplete replicate deepseek/deepseek-chat-v3-0324 2` +> +> `File ".../scripts/wvs_deepseek_reliability_pilot.py", line 179, in run_replicate` + +The measured raw response cost is USD 0.04716160: USD 0.01570740 + USD 0.01578264 + USD 0.01567156. The old pilot state recorded only the two completed replicates, USD 0.03149004, because the exception occurred before state reconciliation. It did release reservations in `finally`. + +## ML-debug form, adapted to this API pilot + +| row | answer | +|---|---| +| config in log | DeepSeek chat v3-0324; three separate 12-item x 24-sample replicate records; deterministic seed schedules; GMICloud responses | +| SHOULD line | `valid rate near 1.0 -> coherent`; observed 288/288, 288/288, then 287/288 | +| null/baseline | canonical v1 coordinate exists but no completed pilot aggregate can yet compare to it | +| full samples | each replicate's first item emitted bare parse-valid JSON; raw records retain all request payloads and responses | +| surprising event | one empty-message `ReadTimeout`, following 287 successful responses in replicate 2 | +| missing metric | whether OpenRouter executed the timed-out request; no receipt or usage record exists | +| second cause | a network timeout and a provider/model response failure both produce one missing sample; the raw record identifies only `ReadTimeout` | + +## Hypotheses + +### H1 [harness | Almost Certain | 95%] + +- **Mechanism:** the runner treated one incomplete replicate as a process-level exception instead of retaining it and proceeding to later models. +- **Evidence:** the quoted traceback occurs immediately after the 287/288 final record; no files exist for the remaining six model directories. +- **Contrary evidence:** the original code did preserve the first two completed replicate records before failing. +- **Discriminating test:** re-read the three existing records without API calls. Expected result: statuses complete, complete, incomplete; no new request event. +- **Fix/action:** resume from durable `run_finished` records, label incomplete replicates, reconcile recorded cost, and continue later model/replicate work. +- **Interpretability:** yes, for the operational diagnosis. + +### H2 [harness | Likely | 65%] + +- **Mechanism:** the pilot state's USD 0.03149004 understated actual recorded spend because it increments only after a complete replicate returns. +- **Evidence:** the three raw record costs sum to USD 0.04716160 while `budget.json` contains USD 0.03149004. +- **Contrary evidence:** both the pilot and global reservations were correctly released, so the stale value did not leave the repository cap reserved. +- **Discriminating test:** reconcile state from every `request_completed` record before dispatch. Expected result: USD 0.04716160 before any new request. +- **Fix/action:** recompute pilot spend from durable records on resume. +- **Interpretability:** partial until reconciliation; it affects pilot accounting, not the raw response data. + +### H3 [data | Chances a little better than even | 50%] + +- **Mechanism:** the missing response is transient transport failure rather than a DeepSeek scoring incompatibility. +- **Evidence:** 863 other requests in this model's three replicates completed, including the same item in replicates 0 and 1. +- **Contrary evidence:** the timeout has no provider response, so the endpoint behavior is unobserved for that request. +- **Discriminating test:** not a silent retry. Keep this replicate incomplete and compare later completed model replicates; an owner-approved replacement protocol would be needed to estimate this model's N=72 aggregate. +- **Fix/action:** exclude the incomplete model aggregate for now. +- **Interpretability:** no aggregate claim for chat-v3-0324; the two complete replicate panels remain reusable reliability evidence. + +## Decision + +- **Resolve-condition verdict:** not met. The process did not produce seven model x three replicate results, and one failure stopped the remaining models. +- **Validity:** P(incomplete-replicate diagnosis is wrong) is Remote, about 5%; the raw record, run summary, and traceback agree. No numerical reliability conclusion is valid yet. +- **Recommended sequence:** resume using only durable completed records, mark chat-v3-0324 incomplete rather than replacing its schedule, and continue the other six models under the same USD 1 cap. Do not add any pilot result to the canonical cache or Pages. + +-- PI[k3]