mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-21 13:10:52 +08:00
Resume DeepSeek pilot after incomplete replicate
Co-Authored-By: PI[k3] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
@@ -145,17 +145,47 @@ def manifest() -> dict:
|
||||
}
|
||||
|
||||
|
||||
def usage(records: Path) -> tuple[Decimal, Counter, int, int]:
|
||||
cost, providers, rescues, failures = Decimal(), Counter(), 0, 0
|
||||
for line in records.read_text().splitlines():
|
||||
record = json.loads(line)
|
||||
if record["event"] == "request_completed":
|
||||
def replicate_result(records: Path, replicate: dict, items: list[dict], resolved: dict) -> dict | None:
|
||||
events = [json.loads(line) for line in records.read_text().splitlines()] if records.exists() else []
|
||||
finished = [record for record in events if record["event"] == "run_finished"
|
||||
and record["protocol_id"] == replicate["protocol_id"]]
|
||||
if not finished:
|
||||
return None
|
||||
final = finished[-1]
|
||||
run_id = final["run_id"]
|
||||
item_rows = {record["id"]: record for record in events
|
||||
if record["event"] == "item_result" and record["run_id"] == run_id}
|
||||
cost, providers = Decimal(), Counter()
|
||||
for record in events:
|
||||
if record.get("run_id") == run_id and record["event"] == "request_completed":
|
||||
cost += Decimal(str(record.get("usage", {}).get("cost", 0)))
|
||||
providers[record.get("provider")] += 1
|
||||
if record["event"] == "item_result":
|
||||
rescues += record["rescued_samples"]
|
||||
failures += record["failed_samples"]
|
||||
return cost, providers, rescues, failures
|
||||
result = {
|
||||
"replicate": replicate["replicate"], "protocol_id": replicate["protocol_id"],
|
||||
"records": str(records), "run_id": run_id, "provider_requests": dict(providers),
|
||||
"rescues": final["rescued_samples"], "failures": final["failed_samples"],
|
||||
"usage_cost_usd": str(cost), "valid_samples": final["valid_samples"],
|
||||
}
|
||||
complete = (final["valid_samples"] == len(items) * N_SAMPLES
|
||||
and final["failed_samples"] == 0
|
||||
and len(item_rows) == len(items)
|
||||
and all(row["valid_samples"] == N_SAMPLES for row in item_rows.values()))
|
||||
if not complete:
|
||||
return result | {"status": "incomplete"}
|
||||
psamples = {item["id"]: np.asarray(item_rows[item["id"]]["p_samples"]) for item in items}
|
||||
coords = model_coord_ci(psamples, resolved, np.random.default_rng(0))
|
||||
response_se = _sample_only_coord_se(psamples, resolved, np.random.default_rng(1), n_draws=N_SAMPLES)
|
||||
return result | {"status": "complete", "coords": list(coords), "response_mean_se": list(response_se)}
|
||||
|
||||
|
||||
def pilot_spend() -> Decimal:
|
||||
cost = Decimal()
|
||||
for records in (OUT / "records").glob("**/*.jsonl"):
|
||||
for line in records.read_text().splitlines():
|
||||
record = json.loads(line)
|
||||
if record["event"] == "request_completed":
|
||||
cost += Decimal(str(record.get("usage", {}).get("cost", 0)))
|
||||
return cost
|
||||
|
||||
|
||||
def v1_coords(model: str) -> list[float]:
|
||||
@@ -168,22 +198,19 @@ def v1_coords(model: str) -> list[float]:
|
||||
|
||||
def run_replicate(model: str, replicate: dict, v1: dict, items: list[dict], resolved: dict) -> dict:
|
||||
records = Path(replicate["records"])
|
||||
prior = replicate_result(records, replicate, items, resolved)
|
||||
if prior is not None:
|
||||
return prior
|
||||
records.parent.mkdir(parents=True, exist_ok=True)
|
||||
seeds = replicate["seed_schedule"]
|
||||
rows = read_items_rated(model, items, n_samples=N_SAMPLES, temperature=v1["temperature"],
|
||||
max_tokens=v1["max_tokens"], concurrency=1, req_timeout=v1["req_timeout"],
|
||||
reasoning=v1["reasoning"], structured_output=v1["structured_output"], provider=v1.get("provider", OSS_PROVIDER),
|
||||
records_path=records, verbose_first=True, probe_first=True, eval_version=EVAL_VERSION,
|
||||
identity_eval_version=EVAL_VERSION, seed_schedule=seeds)
|
||||
if any(row["valid_samples"] != N_SAMPLES for row in rows):
|
||||
raise RuntimeError(f"incomplete replicate {model} {replicate['replicate']}")
|
||||
psamples = {row["id"]: np.asarray(row["p_samples"]) for row in rows}
|
||||
coords = model_coord_ci(psamples, resolved, np.random.default_rng(0))
|
||||
response_se = _sample_only_coord_se(psamples, resolved, np.random.default_rng(1), n_draws=N_SAMPLES)
|
||||
cost, providers, rescues, failures = usage(records)
|
||||
return {"replicate": replicate["replicate"], "protocol_id": replicate["protocol_id"], "records": str(records),
|
||||
"coords": list(coords), "response_mean_se": list(response_se), "provider_requests": dict(providers),
|
||||
"rescues": rescues, "failures": failures, "usage_cost_usd": str(cost)}
|
||||
read_items_rated(model, items, n_samples=N_SAMPLES, temperature=v1["temperature"],
|
||||
max_tokens=v1["max_tokens"], concurrency=1, req_timeout=v1["req_timeout"],
|
||||
reasoning=v1["reasoning"], structured_output=v1["structured_output"], provider=v1.get("provider", OSS_PROVIDER),
|
||||
records_path=records, verbose_first=True, probe_first=True, eval_version=EVAL_VERSION,
|
||||
identity_eval_version=EVAL_VERSION, seed_schedule=replicate["seed_schedule"])
|
||||
result = replicate_result(records, replicate, items, resolved)
|
||||
if result is None:
|
||||
raise RuntimeError(f"replicate {model} {replicate['replicate']} did not write run_finished")
|
||||
return result
|
||||
|
||||
|
||||
def run() -> None:
|
||||
@@ -191,31 +218,35 @@ def run() -> None:
|
||||
items, resolved = rated_items()
|
||||
if not reserve({"id": PILOT_RESERVATION_ID, "lane": "deepseek", "reserve_usd": str(PILOT_CAP_USD)}):
|
||||
raise RuntimeError("global USD 80 cap would be exceeded by the USD 1 pilot reservation")
|
||||
pilot_state(lambda state: state.update({"reserved_usd": str(PILOT_CAP_USD), "started_utc": datetime.now(UTC).isoformat()}))
|
||||
pilot_state(lambda state: state.update({"reserved_usd": str(PILOT_CAP_USD), "started_utc": datetime.now(UTC).isoformat(),
|
||||
"spent_usd": str(pilot_spend())}))
|
||||
results = {"eval_version": EVAL_VERSION, "models": [], "not_published": True}
|
||||
try:
|
||||
for row in data["models"]:
|
||||
reps = []
|
||||
for replicate in row["replicates"]:
|
||||
if Decimal(pilot_state(lambda state: state)["spent_usd"]) >= PILOT_CAP_USD:
|
||||
raise RuntimeError(f"pilot cap reached before {row['id']}: {pilot_spend()}")
|
||||
result = run_replicate(row["id"], replicate, row["v1_settings"], items, resolved)
|
||||
spent = pilot_state(lambda state: state.update({"spent_usd": str(Decimal(state["spent_usd"]) + Decimal(result["usage_cost_usd"]))}))
|
||||
if Decimal(spent["spent_usd"]) >= PILOT_CAP_USD:
|
||||
raise RuntimeError(f"pilot cap reached: {spent['spent_usd']} >= {PILOT_CAP_USD}")
|
||||
pilot_state(lambda state: state.update({"spent_usd": str(pilot_spend())}))
|
||||
reps.append(result)
|
||||
atomic_json(RESULTS, results | {"models": results["models"] + [{"id": row["id"], "replicates": reps}]})
|
||||
values = np.asarray([rep["coords"][:2] for rep in reps])
|
||||
aggregate_samples = {}
|
||||
for rep in reps:
|
||||
rows = [json.loads(line) for line in Path(rep["records"]).read_text().splitlines()]
|
||||
for record in rows:
|
||||
if record["event"] == "item_result":
|
||||
aggregate_samples.setdefault(record["id"], []).extend(record["p_samples"])
|
||||
aggregate = model_coord_ci({key: np.asarray(value) for key, value in aggregate_samples.items()}, resolved, np.random.default_rng(2))
|
||||
v1 = v1_coords(row["id"])
|
||||
results["models"].append({"id": row["id"], "v1_coords": v1, "replicates": reps,
|
||||
"between_replicate_coordinate_sd": np.std(values, axis=0, ddof=1).tolist(),
|
||||
"aggregate_n72_coords": list(aggregate),
|
||||
"delta_aggregate_minus_v1": (np.asarray(aggregate[:2]) - np.asarray(v1[:2])).tolist()})
|
||||
model_result = {"id": row["id"], "v1_coords": v1_coords(row["id"]), "replicates": reps}
|
||||
if all(rep["status"] == "complete" for rep in reps):
|
||||
values = np.asarray([rep["coords"][:2] for rep in reps])
|
||||
aggregate_samples = {}
|
||||
for rep in reps:
|
||||
records = [json.loads(line) for line in Path(rep["records"]).read_text().splitlines()]
|
||||
for record in records:
|
||||
if record["event"] == "item_result" and record["run_id"] == rep["run_id"]:
|
||||
aggregate_samples.setdefault(record["id"], []).extend(record["p_samples"])
|
||||
aggregate = model_coord_ci({key: np.asarray(value) for key, value in aggregate_samples.items()}, resolved, np.random.default_rng(2))
|
||||
model_result |= {"status": "complete", "between_replicate_coordinate_sd": np.std(values, axis=0, ddof=1).tolist(),
|
||||
"aggregate_n72_coords": list(aggregate),
|
||||
"delta_aggregate_minus_v1": (np.asarray(aggregate[:2]) - np.asarray(model_result["v1_coords"][:2])).tolist()}
|
||||
else:
|
||||
model_result |= {"status": "incomplete", "aggregate_excluded_reason": "at least one replicate has fewer than 288 valid samples"}
|
||||
results["models"].append(model_result)
|
||||
atomic_json(RESULTS, results)
|
||||
finally:
|
||||
release(PILOT_RESERVATION_ID)
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
# Audit: DeepSeek reliability pilot task 1674
|
||||
|
||||
- Pueue task 1674 ran `sh scripts/wvs_api/07_deepseek_reliability_pilot.sh` from 21:38:41 to 22:31:21 +0800 and exited 1.
|
||||
- full normalized Pueue log: 80/80 lines; raw log: 92 lines, both at `/tmp/wvs-audit-1674/`.
|
||||
- scope: separate `wvs-score-all-options-v1` reliability records only. No canonical cache or Pages data changed.
|
||||
|
||||
| stage | expected | observed | expected? | consequence |
|
||||
|---|---|---|---|---|
|
||||
| pilot reservation | USD 1 pilot and global reservation released on exit | pilot `reserved_usd: 0`; global `reservations: {}` | yes | no stale budget block |
|
||||
| first model replicate 0 | 288 valid ratings | 288/288, GMICloud, USD 0.01570740 | yes | retained separately |
|
||||
| replicate 1 | 288 valid ratings | 288/288, GMICloud, USD 0.01578264 | yes | retained separately |
|
||||
| replicate 2 | 288 valid ratings | 287/288, one `ReadTimeout` at `Attending peaceful demonstrations`, sample 10 | no | no N=72 aggregate for this model |
|
||||
| remaining six models | continue after a model/replicate failure | not attempted because the runner raised | no | the task's continuation condition was not met |
|
||||
|
||||
## Evidence
|
||||
|
||||
The final replicate record says:
|
||||
|
||||
> `"event": "request_failed", "item_id": "Attending peaceful demonstrations", "sample": 10, "phase": "initial", "error_type": "ReadTimeout", "error": ""`
|
||||
>
|
||||
> `"event": "run_finished", "failed_samples": 1, "planned_requests": 288, "rescued_samples": 0, "valid_samples": 287`
|
||||
|
||||
The complete first two replicates have matching `run_finished` records with `valid_samples: 288`, `failed_samples: 0`, and `rescued_samples: 0`. The terminal traceback is:
|
||||
|
||||
> `RuntimeError: incomplete replicate deepseek/deepseek-chat-v3-0324 2`
|
||||
>
|
||||
> `File ".../scripts/wvs_deepseek_reliability_pilot.py", line 179, in run_replicate`
|
||||
|
||||
The measured raw response cost is USD 0.04716160: USD 0.01570740 + USD 0.01578264 + USD 0.01567156. The old pilot state recorded only the two completed replicates, USD 0.03149004, because the exception occurred before state reconciliation. It did release reservations in `finally`.
|
||||
|
||||
## ML-debug form, adapted to this API pilot
|
||||
|
||||
| row | answer |
|
||||
|---|---|
|
||||
| config in log | DeepSeek chat v3-0324; three separate 12-item x 24-sample replicate records; deterministic seed schedules; GMICloud responses |
|
||||
| SHOULD line | `valid rate near 1.0 -> coherent`; observed 288/288, 288/288, then 287/288 |
|
||||
| null/baseline | canonical v1 coordinate exists but no completed pilot aggregate can yet compare to it |
|
||||
| full samples | each replicate's first item emitted bare parse-valid JSON; raw records retain all request payloads and responses |
|
||||
| surprising event | one empty-message `ReadTimeout`, following 287 successful responses in replicate 2 |
|
||||
| missing metric | whether OpenRouter executed the timed-out request; no receipt or usage record exists |
|
||||
| second cause | a network timeout and a provider/model response failure both produce one missing sample; the raw record identifies only `ReadTimeout` |
|
||||
|
||||
## Hypotheses
|
||||
|
||||
### H1 [harness | Almost Certain | 95%]
|
||||
|
||||
- **Mechanism:** the runner treated one incomplete replicate as a process-level exception instead of retaining it and proceeding to later models.
|
||||
- **Evidence:** the quoted traceback occurs immediately after the 287/288 final record; no files exist for the remaining six model directories.
|
||||
- **Contrary evidence:** the original code did preserve the first two completed replicate records before failing.
|
||||
- **Discriminating test:** re-read the three existing records without API calls. Expected result: statuses complete, complete, incomplete; no new request event.
|
||||
- **Fix/action:** resume from durable `run_finished` records, label incomplete replicates, reconcile recorded cost, and continue later model/replicate work.
|
||||
- **Interpretability:** yes, for the operational diagnosis.
|
||||
|
||||
### H2 [harness | Likely | 65%]
|
||||
|
||||
- **Mechanism:** the pilot state's USD 0.03149004 understated actual recorded spend because it increments only after a complete replicate returns.
|
||||
- **Evidence:** the three raw record costs sum to USD 0.04716160 while `budget.json` contains USD 0.03149004.
|
||||
- **Contrary evidence:** both the pilot and global reservations were correctly released, so the stale value did not leave the repository cap reserved.
|
||||
- **Discriminating test:** reconcile state from every `request_completed` record before dispatch. Expected result: USD 0.04716160 before any new request.
|
||||
- **Fix/action:** recompute pilot spend from durable records on resume.
|
||||
- **Interpretability:** partial until reconciliation; it affects pilot accounting, not the raw response data.
|
||||
|
||||
### H3 [data | Chances a little better than even | 50%]
|
||||
|
||||
- **Mechanism:** the missing response is transient transport failure rather than a DeepSeek scoring incompatibility.
|
||||
- **Evidence:** 863 other requests in this model's three replicates completed, including the same item in replicates 0 and 1.
|
||||
- **Contrary evidence:** the timeout has no provider response, so the endpoint behavior is unobserved for that request.
|
||||
- **Discriminating test:** not a silent retry. Keep this replicate incomplete and compare later completed model replicates; an owner-approved replacement protocol would be needed to estimate this model's N=72 aggregate.
|
||||
- **Fix/action:** exclude the incomplete model aggregate for now.
|
||||
- **Interpretability:** no aggregate claim for chat-v3-0324; the two complete replicate panels remain reusable reliability evidence.
|
||||
|
||||
## Decision
|
||||
|
||||
- **Resolve-condition verdict:** not met. The process did not produce seven model x three replicate results, and one failure stopped the remaining models.
|
||||
- **Validity:** P(incomplete-replicate diagnosis is wrong) is Remote, about 5%; the raw record, run summary, and traceback agree. No numerical reliability conclusion is valid yet.
|
||||
- **Recommended sequence:** resume using only durable completed records, mark chat-v3-0324 incomplete rather than replacing its schedule, and continue the other six models under the same USD 1 cap. Do not add any pilot result to the canonical cache or Pages.
|
||||
|
||||
-- PI[k3]
|
||||
Reference in New Issue
Block a user