From db0ef75c09e58f37cfb44d49d8dff7969397d03a Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Thu, 17 Sep 2026 09:54:41 +0800 Subject: [PATCH] Add bounded WVS priority API manifest Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com> --- scripts/wvs_priority_manifest.py | 185 ++++++++++++++++++ .../wvs/20260917_priority_phase_manifest.md | 96 +++++++++ 2 files changed, 281 insertions(+) create mode 100644 scripts/wvs_priority_manifest.py create mode 100644 slop/research/wvs/20260917_priority_phase_manifest.md diff --git a/scripts/wvs_priority_manifest.py b/scripts/wvs_priority_manifest.py new file mode 100644 index 0000000..3b90d50 --- /dev/null +++ b/scripts/wvs_priority_manifest.py @@ -0,0 +1,185 @@ +"""Generate the saved-catalog manifest for the next bounded WVS API phase.""" +from __future__ import annotations + +import json +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path + +CATALOG = Path("slop/research/wvs/20260917_openrouter_models.json") +MAP = Path("docs/wvs/wvs_map_data.json") +LEDGER = Path("slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl") +OUT = Path("slop/research/wvs/20260917_priority_phase_manifest.md") +CALLS_PER_MODEL = 12 * 12 +PHASE_STOP_USD = Decimal("35") +GLOBAL_STOP_USD = Decimal("80") + +FAMILY_PREFIX = { + "claude": "anthropic/", "deepseek": "deepseek/", "gemini": "google/", + "gemma": "google/", "glm": "z-ai/", "gpt": "openai/", "grok": "x-ai/", + "inkling": "thinkingmachines/", "kimi": "moonshotai/", "llama": "meta-llama/", + "mistral": "mistralai/", "muse": "meta/", "qwen": "qwen/", +} + + +def displayed_id(model: dict[str, object]) -> str: + return FAMILY_PREFIX[model["family"]] + model["name"] + + +def date(model: dict[str, object]) -> str: + return datetime.fromtimestamp(model["created"], UTC).date().isoformat() + + +def price_per_million(model: dict[str, object], key: str) -> Decimal: + return Decimal(model["pricing"][key]) * 1_000_000 + + +def supports_structured(model: dict[str, object]) -> bool: + return "structured_outputs" in model["supported_parameters"] + + +def row(model: dict[str, object], rationale: str) -> str: + completion_bound = price_per_million(model, "completion") * Decimal(CALLS_PER_MODEL * 1024) / 1_000_000 + return ( + f"| `{model['id']}` | {date(model)} | {price_per_million(model, 'prompt'):g} | " + f"{price_per_million(model, 'completion'):g} | {'yes' if supports_structured(model) else 'no'} | " + f"USD {completion_bound:.4f} | {rationale} |" + ) + + +def ledger_cost() -> Decimal: + cost = Decimal() + for line in LEDGER.read_text().splitlines(): + event = json.loads(line) + if event["event"] == "request_completed": + cost += Decimal(str(event["usage"]["cost"])) + return cost + + +def main() -> None: + catalog = {model["id"]: model for model in json.loads(CATALOG.read_text())["data"]} + plotted = {displayed_id(model) for model in json.loads(MAP.read_text())["models"]} + + def eligible(model: dict[str, object]) -> bool: + return ( + not model["id"].endswith((":batch", ":free")) + and supports_structured(model) + and price_per_million(model, "completion") <= Decimal("15") + ) + + openai = [ + model for model in catalog.values() + if model["id"].startswith("openai/") + and eligible(model) + and model["id"] not in plotted + and "-pro" not in model["id"] + and "fast" not in model["id"] + and not any(word in model["id"] for word in ("codex", "image", "audio", "safeguard")) + and model["id"] != "openai/gpt-chat-latest" + and model["id"] not in {"openai/gpt-4o-2024-05-13", "openai/gpt-4o-mini-2024-07-18"} + ] + google = [ + model for model in catalog.values() + if model["id"].startswith("google/gemini") + and eligible(model) + and model["id"] not in plotted + and "-pro" not in model["id"] + and "fast" not in model["id"] + and not any(word in model["id"] for word in ("image", "customtools")) + ] + explicit = [ + "x-ai/grok-4.6", "x-ai/grok-4.5", "google/gemma-4-26b-a4b-it", + "meta/muse-spark-1.2", "meta/muse-spark-1.1", + ] + selected_ids = [model["id"] for model in openai + google] + explicit + selected = [catalog[model_id] for model_id in dict.fromkeys(selected_ids)] + if set(selected_ids) & plotted: + raise ValueError("priority manifest includes a plotted ID") + if not all(eligible(model) for model in selected): + raise ValueError("priority manifest includes an ineligible catalog entry") + + deferred_ids = [ + "qwen/qwen3.8-max-0902", "qwen/qwen3.8-2.4t-a95b", "z-ai/glm-5.3-flash", + "mistralai/mistral-medium-3-5", "mistralai/mistral-small-2603", + "mistralai/ministral-14b-2512", "mistralai/ministral-8b-2512", + "mistralai/ministral-3b-2512", + ] + deferred = [catalog[model_id] for model_id in deferred_ids] + total_completion_bound = sum( + price_per_million(model, "completion") * Decimal(CALLS_PER_MODEL * 1024) / 1_000_000 + for model in selected + ) + spend = ledger_cost() + + lines = [ + "# WVS priority API phase manifest, 2026-09-17", + "", + "This exact manifest is generated from the saved 444-record OpenRouter catalog snapshot, not a fresh paid request. It authorizes no calls by itself.", + "", + "## Guards", + "", + f"- A complete panel is {CALLS_PER_MODEL} initial calls, 12 items x 12 samples.", + f"- Existing ledger spend is USD {spend:.10f}; the global stop remains USD {GLOBAL_STOP_USD}.", + f"- This priority phase stops before USD {PHASE_STOP_USD} of new observed provider cost, even if the manifest has remaining models.", + f"- Sum of 1024-token completion-only ceilings for all listed priority panels is USD {total_completion_bound:.4f}. This excludes prompt tokens and rescues, so it is not a spend authorization or a cost prediction.", + "- Before every model, read cumulative `usage.cost` from the append-only ledger. Do not start a request whose conservative remaining cost could pass the phase or global stop.", + "- A panel is publishable only as one exact protocol/run with 144 distinct item/sample keys. Never merge the two old incomplete Grok 4.5 attempts.", + "", + "## Required first diagnostic", + "", + "Run only this panel before any other manifest model. Audit repeats, parser outcomes, rescues, refusals, cache replay and provider cost before dispatching the priority batch.", + "", + "| exact ID | created UTC | input USD/M | output USD/M | structured | completion-only 144x1024 ceiling | rationale |", + "|---|---:|---:|---:|---|---:|---|", + row(catalog["openai/gpt-5-nano"], "required cheapest new 144-call diagnostic"), + "", + "## Priority manifest after diagnostic pass", + "", + "The order is Grok, OpenAI, Google, then the requested Muse points. `Flash` entries are retained because the user excluded `Fast`, not `Flash`.", + "", + "| exact ID | created UTC | input USD/M | output USD/M | structured | completion-only 144x1024 ceiling | rationale |", + "|---|---:|---:|---:|---|---:|---|", + ] + diagnostic = catalog["openai/gpt-5-nano"] + if diagnostic not in openai: + raise ValueError("the required diagnostic is not eligible") + groups = [ + ("Grok", [catalog["x-ai/grok-4.6"], catalog["x-ai/grok-4.5"]]), + ("OpenAI", [model for model in sorted(openai, key=lambda model: (-model["created"], model["id"])) if model["id"] != diagnostic["id"]]), + ("Google", sorted(google + [catalog["google/gemma-4-26b-a4b-it"]], key=lambda model: (-model["created"], model["id"]))), + ("Muse", [catalog["meta/muse-spark-1.2"], catalog["meta/muse-spark-1.1"]]), + ] + for name, models in groups: + lines.append(f"| **{name}** | | | | | | |") + for model in models: + rationale = "clean new full attempt" if model["id"] == "x-ai/grok-4.5" else "new direct panel" + lines.append(row(model, rationale)) + + lines.extend([ + "", + "## Deferred approved shortlist", + "", + "These remain eligible only after the priority phase has a passing diagnostic and remaining observed budget. They are not queued by this manifest.", + "", + "| exact ID | created UTC | input USD/M | output USD/M | structured | completion-only 144x1024 ceiling | rationale |", + "|---|---:|---:|---:|---|---:|---|", + ]) + for model in deferred: + lines.append(row(model, "deferred Qwen/GLM/Mistral shortlist")) + lines.extend([ + "", + "## Exclusions checked", + "", + "- Already complete/plotted direct IDs, including Grok 4.3 and 4.20, are excluded.", + "- `:batch` and `:free` routes, `Pro` and `Fast` IDs, output prices above USD 15/M, and code/image/audio/safeguard/multi-agent variants are excluded.", + "- `x-ai/grok-4.4` remains absent from the saved catalog.", + "- `qwen/qwen3.5-flash-02-23` and the prior Grok 4.5 records are retained as incomplete evidence, not plotted or merged.", + "", + "-- PI[gpt-5.6-terra]", + ]) + OUT.write_text("\n".join(lines) + "\n") + print(f"wrote {OUT}: {len(selected)} priority models, {len(deferred)} deferred models, {CALLS_PER_MODEL * len(selected)} expected initial calls") + + +if __name__ == "__main__": + main() diff --git a/slop/research/wvs/20260917_priority_phase_manifest.md b/slop/research/wvs/20260917_priority_phase_manifest.md new file mode 100644 index 0000000..58cb072 --- /dev/null +++ b/slop/research/wvs/20260917_priority_phase_manifest.md @@ -0,0 +1,96 @@ +# WVS priority API phase manifest, 2026-09-17 + +This exact manifest is generated from the saved 444-record OpenRouter catalog snapshot, not a fresh paid request. It authorizes no calls by itself. + +## Guards + +- A complete panel is 144 initial calls, 12 items x 12 samples. +- Existing ledger spend is USD 3.5908606724; the global stop remains USD 80. +- This priority phase stops before USD 35 of new observed provider cost, even if the manifest has remaining models. +- Sum of 1024-token completion-only ceilings for all listed priority panels is USD 28.6138. This excludes prompt tokens and rescues, so it is not a spend authorization or a cost prediction. +- Before every model, read cumulative `usage.cost` from the append-only ledger. Do not start a request whose conservative remaining cost could pass the phase or global stop. +- A panel is publishable only as one exact protocol/run with 144 distinct item/sample keys. Never merge the two old incomplete Grok 4.5 attempts. + +## Required first diagnostic + +Run only this panel before any other manifest model. Audit repeats, parser outcomes, rescues, refusals, cache replay and provider cost before dispatching the priority batch. + +| exact ID | created UTC | input USD/M | output USD/M | structured | completion-only 144x1024 ceiling | rationale | +|---|---:|---:|---:|---|---:|---| +| `openai/gpt-5-nano` | 2025-08-07 | 0.05000000 | 0.4000000 | yes | USD 0.0590 | required cheapest new 144-call diagnostic | + +## Priority manifest after diagnostic pass + +The order is Grok, OpenAI, Google, then the requested Muse points. `Flash` entries are retained because the user excluded `Fast`, not `Flash`. + +| exact ID | created UTC | input USD/M | output USD/M | structured | completion-only 144x1024 ceiling | rationale | +|---|---:|---:|---:|---|---:|---| +| **Grok** | | | | | | | +| `x-ai/grok-4.6` | 2026-08-12 | 2.000000 | 6.000000 | yes | USD 0.8847 | new direct panel | +| `x-ai/grok-4.5` | 2026-07-08 | 2.000000 | 6.000000 | yes | USD 0.8847 | clean new full attempt | +| **OpenAI** | | | | | | | +| `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | yes | USD 0.1769 | new direct panel | +| `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | yes | USD 1.7695 | new direct panel | +| `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | yes | USD 0.1843 | new direct panel | +| `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | yes | USD 0.6636 | new direct panel | +| `openai/gpt-5.2-chat` | 2025-12-10 | 1.75000000 | 14.000000 | yes | USD 2.0644 | new direct panel | +| `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | yes | USD 2.0644 | new direct panel | +| `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | yes | USD 1.4746 | new direct panel | +| `openai/gpt-5` | 2025-08-07 | 1.25000000 | 10.00000 | yes | USD 1.4746 | new direct panel | +| `openai/gpt-5-mini` | 2025-08-07 | 0.25000000 | 2.000000 | yes | USD 0.2949 | new direct panel | +| `openai/gpt-oss-120b` | 2025-08-05 | 0.037000000 | 0.17000000 | yes | USD 0.0251 | new direct panel | +| `openai/gpt-oss-20b` | 2025-08-05 | 0.03000000 | 0.13000000 | yes | USD 0.0192 | new direct panel | +| `openai/o4-mini-high` | 2025-04-16 | 1.1000000 | 4.4000000 | yes | USD 0.6488 | new direct panel | +| `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | yes | USD 1.1796 | new direct panel | +| `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | yes | USD 0.6488 | new direct panel | +| `openai/gpt-4.1` | 2025-04-14 | 2.000000 | 8.000000 | yes | USD 1.1796 | new direct panel | +| `openai/gpt-4.1-mini` | 2025-04-14 | 0.4000000 | 1.6000000 | yes | USD 0.2359 | new direct panel | +| `openai/gpt-4.1-nano` | 2025-04-14 | 0.1000000 | 0.4000000 | yes | USD 0.0590 | new direct panel | +| `openai/o3-mini-high` | 2025-02-12 | 1.1000000 | 4.4000000 | yes | USD 0.6488 | new direct panel | +| `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | yes | USD 0.6488 | new direct panel | +| `openai/gpt-4o-2024-11-20` | 2024-11-20 | 2.5000000 | 10.00000 | yes | USD 1.4746 | new direct panel | +| `openai/gpt-4o-2024-08-06` | 2024-08-06 | 2.5000000 | 10.00000 | yes | USD 1.4746 | new direct panel | +| `openai/gpt-4o-mini` | 2024-07-18 | 0.15000000 | 0.6000000 | yes | USD 0.0885 | new direct panel | +| `openai/gpt-4o` | 2024-05-13 | 2.5000000 | 10.00000 | yes | USD 1.4746 | new direct panel | +| `openai/gpt-3.5-turbo-0613` | 2024-01-25 | 1.000000 | 2.000000 | yes | USD 0.2949 | new direct panel | +| `openai/gpt-3.5-turbo-instruct` | 2023-09-28 | 1.5000000 | 2.000000 | yes | USD 0.2949 | new direct panel | +| `openai/gpt-3.5-turbo-16k` | 2023-08-28 | 3.000000 | 4.000000 | yes | USD 0.5898 | new direct panel | +| `openai/gpt-3.5-turbo` | 2023-05-28 | 0.5000000 | 1.5000000 | yes | USD 0.2212 | new direct panel | +| **Google** | | | | | | | +| `google/gemini-3.8-flash` | 2026-09-02 | 0.75000000 | 3.75000000 | yes | USD 0.5530 | new direct panel | +| `google/gemini-3.6-flash` | 2026-07-21 | 0.75000000 | 3.75000000 | yes | USD 0.5530 | new direct panel | +| `google/gemini-3.5-flash-lite` | 2026-07-21 | 0.3000000 | 2.5000000 | yes | USD 0.3686 | new direct panel | +| `google/gemini-3.5-flash` | 2026-05-19 | 1.5000000 | 9.000000 | yes | USD 1.3271 | new direct panel | +| `google/gemini-3.1-flash-lite` | 2026-05-07 | 0.25000000 | 1.5000000 | yes | USD 0.2212 | new direct panel | +| `google/gemma-4-26b-a4b-it` | 2026-04-03 | 0.09000000 | 0.3000000 | yes | USD 0.0442 | new direct panel | +| `google/gemini-3.1-flash-lite-preview` | 2026-03-03 | 0.25000000 | 1.5000000 | yes | USD 0.2212 | new direct panel | +| `google/gemini-3-flash-preview` | 2025-12-17 | 0.5000000 | 3.000000 | yes | USD 0.4424 | new direct panel | +| `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | yes | USD 0.0590 | new direct panel | +| `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | yes | USD 0.3686 | new direct panel | +| **Muse** | | | | | | | +| `meta/muse-spark-1.2` | 2026-08-05 | 1.25000000 | 4.25000000 | yes | USD 0.6267 | new direct panel | +| `meta/muse-spark-1.1` | 2026-07-16 | 1.25000000 | 4.25000000 | yes | USD 0.6267 | new direct panel | + +## Deferred approved shortlist + +These remain eligible only after the priority phase has a passing diagnostic and remaining observed budget. They are not queued by this manifest. + +| exact ID | created UTC | input USD/M | output USD/M | structured | completion-only 144x1024 ceiling | rationale | +|---|---:|---:|---:|---|---:|---| +| `qwen/qwen3.8-max-0902` | 2026-09-03 | 2.000000 | 6.000000 | yes | USD 0.8847 | deferred Qwen/GLM/Mistral shortlist | +| `qwen/qwen3.8-2.4t-a95b` | 2026-08-12 | 2.000000 | 6.000000 | yes | USD 0.8847 | deferred Qwen/GLM/Mistral shortlist | +| `z-ai/glm-5.3-flash` | 2026-08-26 | 0.07000000 | 0.2333000000 | yes | USD 0.0344 | deferred Qwen/GLM/Mistral shortlist | +| `mistralai/mistral-medium-3-5` | 2026-04-30 | 1.5000000 | 7.5000000 | yes | USD 1.1059 | deferred Qwen/GLM/Mistral shortlist | +| `mistralai/mistral-small-2603` | 2026-03-16 | 0.15000000 | 0.6000000 | yes | USD 0.0885 | deferred Qwen/GLM/Mistral shortlist | +| `mistralai/ministral-14b-2512` | 2025-12-02 | 0.2000000 | 0.2000000 | yes | USD 0.0295 | deferred Qwen/GLM/Mistral shortlist | +| `mistralai/ministral-8b-2512` | 2025-12-02 | 0.15000000 | 0.15000000 | yes | USD 0.0221 | deferred Qwen/GLM/Mistral shortlist | +| `mistralai/ministral-3b-2512` | 2025-12-02 | 0.1000000 | 0.1000000 | yes | USD 0.0147 | deferred Qwen/GLM/Mistral shortlist | + +## Exclusions checked + +- Already complete/plotted direct IDs, including Grok 4.3 and 4.20, are excluded. +- `:batch` and `:free` routes, `Pro` and `Fast` IDs, output prices above USD 15/M, and code/image/audio/safeguard/multi-agent variants are excluded. +- `x-ai/grok-4.4` remains absent from the saved catalog. +- `qwen/qwen3.5-flash-02-23` and the prior Grok 4.5 records are retained as incomplete evidence, not plotted or merged. + +-- PI[gpt-5.6-terra]