From 02eb587fc250fa6e50e4579ac6ce60ab6a7a7b2f Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Thu, 17 Sep 2026 19:20:19 +0800 Subject: [PATCH] Filter specialized models from WVS lanes Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com> --- scripts/wvs_score_all_options_refresh.py | 6 +- .../20260917_score_all_options/manifest.json | 175 ++++-------------- 2 files changed, 42 insertions(+), 139 deletions(-) diff --git a/scripts/wvs_score_all_options_refresh.py b/scripts/wvs_score_all_options_refresh.py index c56b5e7..b8e0564 100644 --- a/scripts/wvs_score_all_options_refresh.py +++ b/scripts/wvs_score_all_options_refresh.py @@ -29,7 +29,7 @@ OSS_PROVIDER = { "quantizations": ["fp8", "int8", "bf16", "fp16"], } LANES = ("openai", "google", "xai", "muse", "kimi", "glm", "deepseek", "qwen") -SPECIALIZED = ("batch", "free", "-pro", "-fast", "vision", "-vl", "coder", "audio", "image", "guard", "safeguard", "multi-agent", "embedding", "rerank") +SPECIALIZED = ("batch", "free", "-pro", "-fast", "vision", "-vl", "-5v", "-4.6v", "-4.5v", "-code", "-codex", "coder", "audio", "clip", "image", "guard", "safeguard", "multi-agent", "embedding", "rerank") def lane_for(model_id: str) -> str | None: @@ -84,7 +84,7 @@ def entry(model: dict, completed: set[str]) -> dict: setting, setting_label = reasoning(model) except ValueError as error: return {"id": model_id, "lane": lane, "status": "excluded", "reason": str(error)} - provider = OSS_PROVIDER if lane in {"muse", "kimi", "glm", "deepseek", "qwen"} else None + provider = OSS_PROVIDER if lane in {"muse", "kimi", "glm", "deepseek", "qwen"} or model_id.startswith("openai/gpt-oss-") else None reserve = Decimal(144) * (Decimal(1024 + 2048) * price(model, "completion") + Decimal(1024) * price(model, "prompt")) / Decimal(1_000_000) return { "id": model_id, "lane": lane, "status": "runnable", "created": model["created"], @@ -215,7 +215,7 @@ def main() -> None: if args.smoke: runnable = [row for row in rows if row["status"] == "runnable"] assert all(row["calls"] == 144 for row in runnable) - assert all(row["provider"] == OSS_PROVIDER for row in runnable if row["lane"] in {"muse", "kimi", "glm", "deepseek", "qwen"}) + assert all(row["provider"] == OSS_PROVIDER for row in runnable if row["lane"] in {"muse", "kimi", "glm", "deepseek", "qwen"} or row["id"].startswith("openai/gpt-oss-")) assert all(row["provider"] is None for row in runnable if row["lane"] in {"openai", "google", "xai"}) print(f"smoke: {len(runnable)} score-all-options panels, {len(LANES)} provider lanes, concurrency <= 8") if args.queue: diff --git a/slop/research/wvs/20260917_score_all_options/manifest.json b/slop/research/wvs/20260917_score_all_options/manifest.json index d0ed8fc..bc152c7 100644 --- a/slop/research/wvs/20260917_score_all_options/manifest.json +++ b/slop/research/wvs/20260917_score_all_options/manifest.json @@ -607,7 +607,7 @@ { "id": "moonshotai/kimi-k2.7-code", "lane": "kimi", - "reason": "mandatory reasoning lacks minimal/low", + "reason": "batch/free/pro/fast or specialized variant", "status": "excluded" }, { @@ -864,29 +864,10 @@ "status": "complete_cached" }, { - "calls": 144, - "created": 1775061458, "id": "z-ai/glm-5v-turbo", - "input_usd_per_million": "1.2000000", "lane": "glm", - "output_usd_per_million": "4.000000", - "provider": { - "allow_fallbacks": true, - "quantizations": [ - "fp8", - "int8", - "bf16", - "fp16" - ], - "require_parameters": true - }, - "reasoning": { - "enabled": false - }, - "reasoning_label": "disabled, optional", - "reserve_usd": "1.9464192", - "status": "runnable", - "structured_output": false + "reason": "batch/free/pro/fast or specialized variant", + "status": "excluded" }, { "id": "x-ai/grok-4.20-multi-agent", @@ -917,18 +898,10 @@ "status": "excluded" }, { - "calls": 144, - "created": 1774907255, "id": "google/lyria-3-clip-preview", - "input_usd_per_million": "0", "lane": "google", - "output_usd_per_million": "0", - "provider": null, - "reasoning": null, - "reasoning_label": "omitted, not advertised", - "reserve_usd": "0", - "status": "runnable", - "structured_output": false + "reason": "batch/free/pro/fast or specialized variant", + "status": "excluded" }, { "calls": 144, @@ -1113,20 +1086,10 @@ "status": "excluded" }, { - "calls": 144, - "created": 1771959164, "id": "openai/gpt-5.3-codex", - "input_usd_per_million": "1.75000000", "lane": "openai", - "output_usd_per_million": "14.000000", - "provider": null, - "reasoning": { - "enabled": false - }, - "reasoning_label": "disabled, optional", - "reserve_usd": "6.45120000", - "status": "runnable", - "structured_output": true + "reason": "batch/free/pro/fast or specialized variant", + "status": "excluded" }, { "id": "google/gemini-3.1-pro-preview", @@ -1249,20 +1212,10 @@ "structured_output": true }, { - "calls": 144, - "created": 1768409315, "id": "openai/gpt-5.2-codex", - "input_usd_per_million": "1.75000000", "lane": "openai", - "output_usd_per_million": "14.000000", - "provider": null, - "reasoning": { - "effort": "low" - }, - "reasoning_label": "low", - "reserve_usd": "6.45120000", - "status": "runnable", - "structured_output": true + "reason": "batch/free/pro/fast or specialized variant", + "status": "excluded" }, { "calls": 144, @@ -1360,45 +1313,16 @@ "status": "excluded" }, { - "calls": 144, - "created": 1765207462, "id": "z-ai/glm-4.6v", - "input_usd_per_million": "0.3000000", "lane": "glm", - "output_usd_per_million": "0.9000000", - "provider": { - "allow_fallbacks": true, - "quantizations": [ - "fp8", - "int8", - "bf16", - "fp16" - ], - "require_parameters": true - }, - "reasoning": { - "enabled": false - }, - "reasoning_label": "disabled, optional", - "reserve_usd": "0.4423680", - "status": "runnable", - "structured_output": false + "reason": "batch/free/pro/fast or specialized variant", + "status": "excluded" }, { - "calls": 144, - "created": 1764878934, "id": "openai/gpt-5.1-codex-max", - "input_usd_per_million": "1.25000000", "lane": "openai", - "output_usd_per_million": "10.00000", - "provider": null, - "reasoning": { - "effort": "low" - }, - "reasoning_label": "low", - "reserve_usd": "4.60800000", - "status": "runnable", - "structured_output": true + "reason": "batch/free/pro/fast or specialized variant", + "status": "excluded" }, { "calls": 144, @@ -1454,36 +1378,16 @@ "status": "excluded" }, { - "calls": 144, - "created": 1763060298, "id": "openai/gpt-5.1-codex", - "input_usd_per_million": "1.25000000", "lane": "openai", - "output_usd_per_million": "10.00000", - "provider": null, - "reasoning": { - "effort": "low" - }, - "reasoning_label": "low", - "reserve_usd": "4.60800000", - "status": "runnable", - "structured_output": true + "reason": "batch/free/pro/fast or specialized variant", + "status": "excluded" }, { - "calls": 144, - "created": 1763057820, "id": "openai/gpt-5.1-codex-mini", - "input_usd_per_million": "0.25000000", "lane": "openai", - "output_usd_per_million": "2.000000", - "provider": null, - "reasoning": { - "enabled": false - }, - "reasoning_label": "disabled, optional", - "reserve_usd": "0.92160000", - "status": "runnable", - "structured_output": true + "reason": "batch/free/pro/fast or specialized variant", + "status": "excluded" }, { "id": "moonshotai/kimi-k2-thinking", @@ -1752,29 +1656,10 @@ "structured_output": true }, { - "calls": 144, - "created": 1754922288, "id": "z-ai/glm-4.5v", - "input_usd_per_million": "0.6000000", "lane": "glm", - "output_usd_per_million": "1.8000000", - "provider": { - "allow_fallbacks": true, - "quantizations": [ - "fp8", - "int8", - "bf16", - "fp16" - ], - "require_parameters": true - }, - "reasoning": { - "enabled": false - }, - "reasoning_label": "disabled, optional", - "reserve_usd": "0.8847360", - "status": "runnable", - "structured_output": false + "reason": "batch/free/pro/fast or specialized variant", + "status": "excluded" }, { "calls": 144, @@ -1838,7 +1723,16 @@ "input_usd_per_million": "0.037000000", "lane": "openai", "output_usd_per_million": "0.17000000", - "provider": null, + "provider": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, "reasoning": { "effort": "low" }, @@ -1860,7 +1754,16 @@ "input_usd_per_million": "0.03000000", "lane": "openai", "output_usd_per_million": "0.13000000", - "provider": null, + "provider": { + "allow_fallbacks": true, + "quantizations": [ + "fp8", + "int8", + "bf16", + "fp16" + ], + "require_parameters": true + }, "reasoning": { "effort": "low" },