Harden direct-choice priority preflight

Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com>

scripts/wvs_direct_choice_priority.py

@@ -47,7 +47,7 @@ GROUPS = {
     ),
     "Google": (
         "google/gemini-3.8-flash", "google/gemini-3.6-flash", "google/gemini-3.5-flash-lite",
-        "google/gemini-3.5-flash", "google/gemini-3.1-flash-lite", "google/gemma-4-26b-a4b-it",
+        "google/gemini-3.5-flash", "google/gemini-3.1-flash-lite",
         "google/gemini-3.1-flash-lite-preview", "google/gemini-3-flash-preview",
         "google/gemini-2.5-flash-lite", "google/gemini-2.5-flash",
     ),
@@ -68,13 +68,16 @@ def reasoning_setting(model: dict) -> tuple[dict | None, str]:
     if metadata is None:
         return None, "not advertised"
     efforts = set(metadata.get("supported_efforts", []))
+    optional = not metadata.get("mandatory")
+    if optional and "none" in efforts:
+        return {"enabled": False}, "disabled (optional, none advertised)"
+    if optional and not efforts and "reasoning" in model["supported_parameters"]:
+        return {"enabled": False}, "disabled (optional, parameter advertised without efforts)"
     if "minimal" in efforts:
         return {"effort": "minimal"}, "minimal"
     if "low" in efforts:
         return {"effort": "low"}, "low"
-    if not metadata.get("mandatory") and "none" in efforts:
-        return {"enabled": False}, "disabled (optional, none advertised)"
-    if not metadata.get("mandatory") and not efforts:
+    if optional and not efforts:
         return None, "not advertised (optional; omitted)"
     raise ValueError(f"no allowed minimal/low reasoning setting for {model['id']}: {metadata}")
 
@@ -100,7 +103,7 @@ def entry(model: dict, pilot_items: list[dict], request_plan: list[dict]) -> dic
         max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT,
         reasoning=reasoning, structured_output=True, prompt_instruction=PROMPT_INSTRUCTION,
         answer_instruction=ANSWER_INSTRUCTION, rescue_instruction=RESCUE_INSTRUCTION,
-        plan_override=request_plan,
+        plan_override=request_plan, fail_fast_first_request=True,
     )
     input_rate = rate_per_million(model, "prompt")
     output_rate = rate_per_million(model, "completion")
@@ -152,6 +155,7 @@ def write_manifest(priority: list[dict]) -> None:
             "initial_calls_per_model": 240, "schedule": "balanced_cyclic_rotations",
             "prompt_instruction": PROMPT_INSTRUCTION, "answer_instruction": ANSWER_INSTRUCTION,
             "rescue_instruction": RESCUE_INSTRUCTION, "strict_structured_output": True,
+            "fail_fast_first_request": True,
         },
         "stop_usd": {"priority_phase": str(PHASE_STOP_USD), "global": str(GLOBAL_STOP_USD)},
         "current_observed_cost_usd": str(current_cost),
@@ -174,6 +178,7 @@ def write_manifest(priority: list[dict]) -> None:
         f"- final response: `{ANSWER_INSTRUCTION}`",
         f"- rescue response: `{RESCUE_INSTRUCTION}`",
         "- strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID",
+        "- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests",
         "",
         "## Spend checks before any later dispatch",
         "",
@@ -185,7 +190,7 @@ def write_manifest(priority: list[dict]) -> None:
         "",
         "## Ordered panels",
         "",
-        "The order is Grok, OpenAI, Google, then Muse. `minimal` is used when catalog metadata advertises it; otherwise `low`; disabled is used only when the catalog says reasoning is optional and accepts `none`.",
+        "The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.",
         "",
         "| family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger |",
         "|---|---|---:|---:|---:|---|---|---|---:|---:|---:|---|",
@@ -204,6 +209,7 @@ def write_manifest(priority: list[dict]) -> None:
         "- Already plotted dense-rated IDs are not repeated in this prepared direct-choice list, including Grok 4.3/4.20, GPT-6 Astra, GPT-5.6 Sol, GPT-5.5, GPT-5.4, GPT-5.3 Chat, Gemini 3.7 Flash, Gemini 2.5 Pro, and Muse 1.3.",
         "- GPT-5 Nano is retained as a completed dense-rated protocol diagnostic, not silently relabelled as a direct-choice panel.",
         "- Pro/Fast, batch/free aliases, output price above USD 15/M, and code/image/audio/safeguard/multi-agent entries remain excluded. `Flash` is included where it is a general chat model.",
+        "- `google/gemma-4-26b-a4b-it` is excluded: it is Gemma, not an identified member of the requested Gemini release series.",
         "- `openai/o4-mini-high` and `openai/o3-mini-high` are excluded because their catalog entries advertise only `high` reasoning, not the registered minimal/low policy.",
         "- The deferred Qwen/GLM/Mistral shortlist remains outside this priority manifest until a direct-choice expansion decision is made.",
         "",
@@ -258,6 +264,7 @@ def main() -> None:
         structured_output=True, records_path=records_path, cache_path=cache_path,
         prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION,
         rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan,
+        fail_fast_first_request=True,
     )
     if result["cached"]:
         print(f"priority direct-choice cache hit: {args.model}, protocol={result['protocol_id'][:12]}")

scripts/wvs_direct_choice_priority_smoke.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Zero-network smoke checks for prepared direct-choice priority panels."""
+from __future__ import annotations
+
+import asyncio
+import json
+import tempfile
+from collections import Counter
+from pathlib import Path
+
+import moralmaps.read_direct_choice as reader
+from wvs_direct_choice_priority import entries
+from wvs_direct_choice_production_pilot import (
+    ANSWER_INSTRUCTION,
+    PROMPT_INSTRUCTION,
+    RESCUE_INSTRUCTION,
+    items,
+    schedule,
+)
+
+
+def main() -> None:
+    prepared = entries()
+    assert len(prepared) == 38
+    assert all(row["initial_calls"] == 240 and row["structured_output"] for row in prepared)
+    assert {str(row["reasoning"]) for row in prepared} == {"None", "{'enabled': False}", "{'effort': 'low'}", "{'effort': 'minimal'}"}
+    assert all(row["protocol_id"] for row in prepared)
+    print("smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings")
+
+    pilot_items = items()
+    request_plan = schedule(pilot_items)
+    for reasoning, expected in ((None, None), ({"enabled": False}, {"enabled": False})):
+        calls = []
+
+        async def fail(payload: dict) -> dict:
+            calls.append(payload)
+            raise RuntimeError("synthetic compatibility failure")
+
+        original = reader.openrouter_request
+        reader.openrouter_request = fail
+        try:
+            with tempfile.TemporaryDirectory() as directory:
+                records = Path(directory) / "records.jsonl"
+                cache = Path(directory) / "cache.json"
+                try:
+                    reader.read_items_direct_choice(
+                        "test/model", pilot_items, samples_per_order=10, temperature=1.0,
+                        max_tokens=1024, concurrency=1, request_timeout=1, reasoning=reasoning,
+                        structured_output=True, records_path=records, cache_path=cache,
+                        prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION,
+                        rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan,
+                        fail_fast_first_request=True,
+                    )
+                except RuntimeError as error:
+                    assert "first scheduled request failed before remaining 239 requests" in str(error)
+                else:
+                    raise AssertionError("synthetic first-request failure did not abort")
+                events = [json.loads(line) for line in records.read_text().splitlines()]
+                assert Counter(event["event"] for event in events) == Counter({
+                    "run_started": 1, "request_started": 1, "request_failed": 1, "run_finished": 1,
+                })
+                assert len(calls) == 1 and calls[0].get("reasoning") == expected
+                assert not cache.exists()
+        finally:
+            reader.openrouter_request = original
+    print("smoke: synthetic first request exits before remaining 239 and writes no cache")
+    print("smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false")
+
+
+if __name__ == "__main__":
+    main()