"""Summarize durable WVS request records without discarding provider billing fields.""" from __future__ import annotations import argparse import hashlib import json from collections import Counter from pathlib import Path USAGE_FIELDS = ( "prompt_tokens", "completion_tokens", "reasoning_tokens", "cache_read_input_tokens", "cache_write_input_tokens", "total_tokens", "cost", ) def sum_usage(records: list[dict]) -> dict[str, float | None]: totals: dict[str, float | None] = {} for field in USAGE_FIELDS: values = [record["usage"][field] for record in records if record["usage"] is not None and field in record["usage"]] totals[field] = sum(values) if values else None return totals def format_value(value: float | None) -> str: return "unknown" if value is None else f"{value:g}" def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--records", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--model") parser.add_argument("--run-id") args = parser.parse_args() rows = [json.loads(line) for line in args.records.read_text().splitlines()] runs = sorted({(row["model"], row["run_id"]) for row in rows if "model" in row and "run_id" in row}) if args.model is not None: runs = [run for run in runs if run[0] == args.model] if args.run_id is not None: runs = [run for run in runs if run[1] == args.run_id] lines = ["# WVS request-ledger audit", "", f"Source: `{args.records}`.", ""] for model, run_id in runs: records = [row for row in rows if row.get("model") == model and row.get("run_id") == run_id] completed = [row for row in records if row["event"] == "request_completed"] started = [row for row in records if row["event"] == "request_started"] failed = [row for row in records if row["event"] == "request_failed"] parsed = [row for row in records if row["event"] == "answer_parsed"] item_results = [row for row in records if row["event"] == "item_result"] phases = Counter(row["phase"] for row in completed) usages = sum_usage(completed) generation_ids = sorted({row["response"]["id"] for row in completed if "id" in row["response"]}) valid = sum(row["parsed"] for row in parsed) initial_keys = {(row["item_id"], row["sample"]) for row in started if row["phase"] == "initial"} complete = (len(initial_keys) == 144 and len(item_results) == 12 and all(row["valid_samples"] == row["n_samples"] == 12 for row in item_results)) lines.extend([ f"## `{model}` run `{run_id}`", "", "| metric | value |", "|---|---:|", f"| dispatched phases | {len(started)} |", f"| completed phases | {len(completed)} |", f"| initial completed | {phases['initial']} |", f"| rescue completed | {phases['rescue']} |", f"| failed request phases | {len(failed)} |", f"| parsed valid samples | {valid} |", f"| distinct initial item/sample keys | {len(initial_keys)} |", f"| item results | {len(item_results)} |", f"| publication eligible 12 x 12 panel | {complete} |", f"| provider generation IDs retained | {len(generation_ids)} |", "", "| provider usage field | total |", "|---|---:|", *[f"| {field} | {format_value(usages[field])} |" for field in USAGE_FIELDS], "", ]) if generation_ids: digest = hashlib.sha256("\n".join(generation_ids).encode()).hexdigest() lines.extend([ "Generation IDs are retained verbatim in the source ledger.", "", f"- count: {len(generation_ids)}", f"- SHA-256 of sorted IDs: `{digest}`", f"- first: `{generation_ids[0]}`", f"- last: `{generation_ids[-1]}`", "", ]) if failed: lines.extend(["Failures retained in the ledger:", "", *[ f"- {row['phase']}: `{row['error_type']}: {row['error']}`" for row in failed ], ""]) if item_results: lines.extend([ "| item | valid | requested | failed | rescues | parse rate |", "|---|---:|---:|---:|---:|---:|", *[ f"| {row['id']} | {row['valid_samples']} | {row['n_samples']} | " f"{row['failed_samples']} | {row['rescued_samples']} | {row['pmass_allowed']:.3f} |" for row in item_results ], "", ]) lines.append("Provider `cost` is reported only when the raw OpenRouter usage object exposed it. Missing usage fields are unknown, not zero. -- PI[gpt-5.6-terra]") args.out.parent.mkdir(parents=True, exist_ok=True) args.out.write_text("\n".join(lines) + "\n") if __name__ == "__main__": main()