Validate WVS catalog inventory fields

Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-09-17 08:27:32 +08:00
co-authored by PI[gpt-5.6-terra]
parent 1502ebfe21
commit f801fa356f
3 changed files with 47 additions and 2 deletions
+44
View File
@@ -0,0 +1,44 @@
"""Check dated/price/structured catalog fields quoted in the WVS family inventory."""
from __future__ import annotations
import json
import re
from datetime import UTC, datetime
from decimal import Decimal
from pathlib import Path
INVENTORY = Path("slop/research/wvs/20260917_openrouter_family_inventory.md")
CATALOG = Path("slop/research/wvs/20260917_openrouter_models.json")
ROW = re.compile(
r"^\| `(?P<id>[^`]+)` \| (?P<date>\d{4}-\d{2}-\d{2}) \| "
r"(?P<input>[0-9.]+) \| (?P<output>[0-9.]+) \| (?P<structured>yes|no) \|"
)
def main() -> None:
catalog = {model["id"]: model for model in json.loads(CATALOG.read_text())["data"]}
rows = [match.groupdict() for line in INVENTORY.read_text().splitlines() if (match := ROW.match(line))]
mismatches = []
for row in rows:
model = catalog[row["id"]]
actual = {
"date": datetime.fromtimestamp(model["created"], UTC).date().isoformat(),
"input": Decimal(model["pricing"]["prompt"]) * 1_000_000,
"output": Decimal(model["pricing"]["completion"]) * 1_000_000,
"structured": "yes" if "structured_outputs" in model["supported_parameters"] else "no",
}
expected = {
"date": row["date"],
"input": Decimal(row["input"]),
"output": Decimal(row["output"]),
"structured": row["structured"],
}
if actual != expected:
mismatches.append({"id": row["id"], "expected": expected, "actual": actual})
if mismatches:
raise ValueError(json.dumps(mismatches, default=str, indent=2))
print(f"validated {len(rows)} inventory date/price/structured rows against {len(catalog)} saved catalog records: 0 mismatches")
if __name__ == "__main__":
main()
@@ -0,0 +1 @@
validated 135 inventory date/price/structured rows against 444 saved catalog records: 0 mismatches
@@ -57,8 +57,8 @@ Live catalog observation:
|---|---:|---:|---:|---|---|
| `x-ai/grok-4.6` | 2026-08-12 | 2 | 6 | yes | exists, missing comparable candidate |
| `x-ai/grok-4.5` | 2026-07-08 | 2 | 6 | yes | exists, missing comparable candidate; saved WVS attempts incomplete |
| `x-ai/grok-4.3` | 2026-04-30 | 2 | 6 | yes | exists and plotted |
| `x-ai/grok-4.20` | 2026-03-30 | 2 | 6 | yes | exists and plotted |
| `x-ai/grok-4.3` | 2026-04-30 | 1.25 | 2.5 | yes | exists and plotted |
| `x-ai/grok-4.20` | 2026-03-31 | 1.25 | 2.5 | yes | exists and plotted |
| `x-ai/grok-4.4` | -- | -- | -- | -- | absent from this 2026-09-17 catalog snapshot |
This is a direct absence check in the saved catalog, not proof that no provider will ever host 4.4.