Harden direct-choice priority preflight

Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-09-17 12:16:23 +08:00
co-authored by PI[gpt-5.6-terra]
parent 82d992725b
commit f3603b4500
6 changed files with 239 additions and 137 deletions
+13 -6
View File
@@ -47,7 +47,7 @@ GROUPS = {
),
"Google": (
"google/gemini-3.8-flash", "google/gemini-3.6-flash", "google/gemini-3.5-flash-lite",
"google/gemini-3.5-flash", "google/gemini-3.1-flash-lite", "google/gemma-4-26b-a4b-it",
"google/gemini-3.5-flash", "google/gemini-3.1-flash-lite",
"google/gemini-3.1-flash-lite-preview", "google/gemini-3-flash-preview",
"google/gemini-2.5-flash-lite", "google/gemini-2.5-flash",
),
@@ -68,13 +68,16 @@ def reasoning_setting(model: dict) -> tuple[dict | None, str]:
if metadata is None:
return None, "not advertised"
efforts = set(metadata.get("supported_efforts", []))
optional = not metadata.get("mandatory")
if optional and "none" in efforts:
return {"enabled": False}, "disabled (optional, none advertised)"
if optional and not efforts and "reasoning" in model["supported_parameters"]:
return {"enabled": False}, "disabled (optional, parameter advertised without efforts)"
if "minimal" in efforts:
return {"effort": "minimal"}, "minimal"
if "low" in efforts:
return {"effort": "low"}, "low"
if not metadata.get("mandatory") and "none" in efforts:
return {"enabled": False}, "disabled (optional, none advertised)"
if not metadata.get("mandatory") and not efforts:
if optional and not efforts:
return None, "not advertised (optional; omitted)"
raise ValueError(f"no allowed minimal/low reasoning setting for {model['id']}: {metadata}")
@@ -100,7 +103,7 @@ def entry(model: dict, pilot_items: list[dict], request_plan: list[dict]) -> dic
max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT,
reasoning=reasoning, structured_output=True, prompt_instruction=PROMPT_INSTRUCTION,
answer_instruction=ANSWER_INSTRUCTION, rescue_instruction=RESCUE_INSTRUCTION,
plan_override=request_plan,
plan_override=request_plan, fail_fast_first_request=True,
)
input_rate = rate_per_million(model, "prompt")
output_rate = rate_per_million(model, "completion")
@@ -152,6 +155,7 @@ def write_manifest(priority: list[dict]) -> None:
"initial_calls_per_model": 240, "schedule": "balanced_cyclic_rotations",
"prompt_instruction": PROMPT_INSTRUCTION, "answer_instruction": ANSWER_INSTRUCTION,
"rescue_instruction": RESCUE_INSTRUCTION, "strict_structured_output": True,
"fail_fast_first_request": True,
},
"stop_usd": {"priority_phase": str(PHASE_STOP_USD), "global": str(GLOBAL_STOP_USD)},
"current_observed_cost_usd": str(current_cost),
@@ -174,6 +178,7 @@ def write_manifest(priority: list[dict]) -> None:
f"- final response: `{ANSWER_INSTRUCTION}`",
f"- rescue response: `{RESCUE_INSTRUCTION}`",
"- strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID",
"- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests",
"",
"## Spend checks before any later dispatch",
"",
@@ -185,7 +190,7 @@ def write_manifest(priority: list[dict]) -> None:
"",
"## Ordered panels",
"",
"The order is Grok, OpenAI, Google, then Muse. `minimal` is used when catalog metadata advertises it; otherwise `low`; disabled is used only when the catalog says reasoning is optional and accepts `none`.",
"The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.",
"",
"| family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger |",
"|---|---|---:|---:|---:|---|---|---|---:|---:|---:|---|",
@@ -204,6 +209,7 @@ def write_manifest(priority: list[dict]) -> None:
"- Already plotted dense-rated IDs are not repeated in this prepared direct-choice list, including Grok 4.3/4.20, GPT-6 Astra, GPT-5.6 Sol, GPT-5.5, GPT-5.4, GPT-5.3 Chat, Gemini 3.7 Flash, Gemini 2.5 Pro, and Muse 1.3.",
"- GPT-5 Nano is retained as a completed dense-rated protocol diagnostic, not silently relabelled as a direct-choice panel.",
"- Pro/Fast, batch/free aliases, output price above USD 15/M, and code/image/audio/safeguard/multi-agent entries remain excluded. `Flash` is included where it is a general chat model.",
"- `google/gemma-4-26b-a4b-it` is excluded: it is Gemma, not an identified member of the requested Gemini release series.",
"- `openai/o4-mini-high` and `openai/o3-mini-high` are excluded because their catalog entries advertise only `high` reasoning, not the registered minimal/low policy.",
"- The deferred Qwen/GLM/Mistral shortlist remains outside this priority manifest until a direct-choice expansion decision is made.",
"",
@@ -258,6 +264,7 @@ def main() -> None:
structured_output=True, records_path=records_path, cache_path=cache_path,
prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION,
rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan,
fail_fast_first_request=True,
)
if result["cached"]:
print(f"priority direct-choice cache hit: {args.model}, protocol={result['protocol_id'][:12]}")
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Zero-network smoke checks for prepared direct-choice priority panels."""
from __future__ import annotations
import asyncio
import json
import tempfile
from collections import Counter
from pathlib import Path
import moralmaps.read_direct_choice as reader
from wvs_direct_choice_priority import entries
from wvs_direct_choice_production_pilot import (
ANSWER_INSTRUCTION,
PROMPT_INSTRUCTION,
RESCUE_INSTRUCTION,
items,
schedule,
)
def main() -> None:
prepared = entries()
assert len(prepared) == 38
assert all(row["initial_calls"] == 240 and row["structured_output"] for row in prepared)
assert {str(row["reasoning"]) for row in prepared} == {"None", "{'enabled': False}", "{'effort': 'low'}", "{'effort': 'minimal'}"}
assert all(row["protocol_id"] for row in prepared)
print("smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings")
pilot_items = items()
request_plan = schedule(pilot_items)
for reasoning, expected in ((None, None), ({"enabled": False}, {"enabled": False})):
calls = []
async def fail(payload: dict) -> dict:
calls.append(payload)
raise RuntimeError("synthetic compatibility failure")
original = reader.openrouter_request
reader.openrouter_request = fail
try:
with tempfile.TemporaryDirectory() as directory:
records = Path(directory) / "records.jsonl"
cache = Path(directory) / "cache.json"
try:
reader.read_items_direct_choice(
"test/model", pilot_items, samples_per_order=10, temperature=1.0,
max_tokens=1024, concurrency=1, request_timeout=1, reasoning=reasoning,
structured_output=True, records_path=records, cache_path=cache,
prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION,
rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan,
fail_fast_first_request=True,
)
except RuntimeError as error:
assert "first scheduled request failed before remaining 239 requests" in str(error)
else:
raise AssertionError("synthetic first-request failure did not abort")
events = [json.loads(line) for line in records.read_text().splitlines()]
assert Counter(event["event"] for event in events) == Counter({
"run_started": 1, "request_started": 1, "request_failed": 1, "run_finished": 1,
})
assert len(calls) == 1 and calls[0].get("reasoning") == expected
assert not cache.exists()
finally:
reader.openrouter_request = original
print("smoke: synthetic first request exits before remaining 239 and writes no cache")
print("smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false")
if __name__ == "__main__":
main()
@@ -5,6 +5,7 @@
"design": {
"answer_instruction": "Respond with ONLY a JSON object with exactly one key named answer. Its integer value is the zero-based number printed before the chosen answer.",
"construct": "direct_choice",
"fail_fast_first_request": true,
"initial_calls_per_model": 240,
"items": 12,
"prompt_instruction": "Think briefly about which ONE answer best reflects the values expressed by your behavior as this AI assistant. Do not analyze the survey design or discuss lacking personal beliefs. Answer immediately.",
@@ -24,7 +25,7 @@
"initial_calls": 240,
"input_usd_per_million": "2.000000",
"output_usd_per_million": "6.000000",
"protocol_id": "f1dcc6e5e4651add7df5473e80ba5b5a5ec733d788a6a791625776501f8605d6",
"protocol_id": "34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6",
"reasoning": {
"effort": "low"
},
@@ -42,7 +43,7 @@
"initial_calls": 240,
"input_usd_per_million": "2.000000",
"output_usd_per_million": "6.000000",
"protocol_id": "fdb4ff16d7879cd840722b7f84654da2e67bc388cdf0b52ea64a1d12b487de18",
"protocol_id": "d1362e8b42a4e2ff54cbd66be8c6a65225f94c9679931299f0e5e359e4065c63",
"reasoning": {
"effort": "low"
},
@@ -60,11 +61,11 @@
"initial_calls": 240,
"input_usd_per_million": "0.2000000",
"output_usd_per_million": "1.2000000",
"protocol_id": "b9bd5e38934aeae851dcd6227403c474e41c52ebb0bf599b112203b30403e718",
"protocol_id": "8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95",
"reasoning": {
"effort": "low"
"enabled": false
},
"reasoning_label": "low",
"reasoning_label": "disabled (optional, none advertised)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl",
"structured_output": true
},
@@ -78,11 +79,11 @@
"initial_calls": 240,
"input_usd_per_million": "2.000000",
"output_usd_per_million": "12.000000",
"protocol_id": "76c8b34537d9ec301ec168c8566849a0a0d26fffa088b70b8da09acb760130b1",
"protocol_id": "08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305",
"reasoning": {
"effort": "low"
"enabled": false
},
"reasoning_label": "low",
"reasoning_label": "disabled (optional, none advertised)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl",
"structured_output": true
},
@@ -96,11 +97,11 @@
"initial_calls": 240,
"input_usd_per_million": "0.2000000",
"output_usd_per_million": "1.25000000",
"protocol_id": "a25533b578ada960251c7fa244a98b86e68a054346d641b6215888a09c92b09e",
"protocol_id": "35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5",
"reasoning": {
"effort": "low"
"enabled": false
},
"reasoning_label": "low",
"reasoning_label": "disabled (optional, none advertised)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl",
"structured_output": true
},
@@ -114,11 +115,11 @@
"initial_calls": 240,
"input_usd_per_million": "0.75000000",
"output_usd_per_million": "4.5000000",
"protocol_id": "42e8c091233072fd0925ab8f5554bc01e23c5cf3051e09ba7b45970af189fb10",
"protocol_id": "c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f",
"reasoning": {
"effort": "low"
"enabled": false
},
"reasoning_label": "low",
"reasoning_label": "disabled (optional, none advertised)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl",
"structured_output": true
},
@@ -132,7 +133,7 @@
"initial_calls": 240,
"input_usd_per_million": "1.75000000",
"output_usd_per_million": "14.000000",
"protocol_id": "6911c78f8c33d085e4d2e4a627c8e311059855cfb859b16a6a319185b6abb807",
"protocol_id": "4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl",
@@ -148,11 +149,11 @@
"initial_calls": 240,
"input_usd_per_million": "1.75000000",
"output_usd_per_million": "14.000000",
"protocol_id": "6d72c036f9f413fa500c0a2902750b0a2c29fc1145d77d3fa02ecf89b48054ff",
"protocol_id": "1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111",
"reasoning": {
"effort": "low"
"enabled": false
},
"reasoning_label": "low",
"reasoning_label": "disabled (optional, none advertised)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl",
"structured_output": true
},
@@ -166,11 +167,11 @@
"initial_calls": 240,
"input_usd_per_million": "1.25000000",
"output_usd_per_million": "10.00000",
"protocol_id": "063cbfba0812ff35df231154d69a498e77cd240961d61f18913e6465c1ac1938",
"protocol_id": "2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05",
"reasoning": {
"effort": "low"
"enabled": false
},
"reasoning_label": "low",
"reasoning_label": "disabled (optional, none advertised)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl",
"structured_output": true
},
@@ -184,7 +185,7 @@
"initial_calls": 240,
"input_usd_per_million": "1.25000000",
"output_usd_per_million": "10.00000",
"protocol_id": "48ffd6a149a5583355dd779b855e944ba26e4eab9b9cd4b5acd780cb3e3fb4e1",
"protocol_id": "4f7fe97763a5822831b2244e72b0d4cc01ff46da55090fed2485c79f97e4bdf0",
"reasoning": {
"effort": "minimal"
},
@@ -202,7 +203,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.25000000",
"output_usd_per_million": "2.000000",
"protocol_id": "00e8beddc47977c22afa2128d78af2c00acf8ddb87077f427bcb1360ed87a980",
"protocol_id": "5b115c7341384a37117745f735367dae10932cc9f16b3fdac5819b5adfe007f4",
"reasoning": {
"effort": "minimal"
},
@@ -220,7 +221,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.037000000",
"output_usd_per_million": "0.17000000",
"protocol_id": "25ac3627e4795573424e184ab883ed80e3abb3bfadb0adba94a8e5b1ce659725",
"protocol_id": "ae278f4f5e91f668448c4b921633e999ba90a6cd9cc3819c8ee4beac8484d38a",
"reasoning": {
"effort": "low"
},
@@ -238,7 +239,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.03000000",
"output_usd_per_million": "0.13000000",
"protocol_id": "06243dd30d6fb69a30c4b24a9b891b95a25975d7d41b7071a474e4365e856169",
"protocol_id": "2b45bec6a0271fb1390ea36e196eef7e75a53112c1dcd1e40b1b60b7631a9d7d",
"reasoning": {
"effort": "low"
},
@@ -256,9 +257,11 @@
"initial_calls": 240,
"input_usd_per_million": "2.000000",
"output_usd_per_million": "8.000000",
"protocol_id": "f4054a623913b60fa79beee9aacec15967aa058434835937e4e8af276adf9557",
"reasoning": null,
"reasoning_label": "not advertised (optional; omitted)",
"protocol_id": "0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298",
"reasoning": {
"enabled": false
},
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl",
"structured_output": true
},
@@ -272,9 +275,11 @@
"initial_calls": 240,
"input_usd_per_million": "1.1000000",
"output_usd_per_million": "4.4000000",
"protocol_id": "26873a2a9f3d6530a06368113b903fbc80f673849ed605f1f49963f1f7abb638",
"reasoning": null,
"reasoning_label": "not advertised (optional; omitted)",
"protocol_id": "eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367",
"reasoning": {
"enabled": false
},
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl",
"structured_output": true
},
@@ -288,7 +293,7 @@
"initial_calls": 240,
"input_usd_per_million": "2.000000",
"output_usd_per_million": "8.000000",
"protocol_id": "9a3bb10b7179275010d35313de4a9d8021ca42abb0901414450ef0c6a42f2aea",
"protocol_id": "1f6ad277fcb151d83dd6b0c9d2f9ed5d2cb7466a09b30c4368ec0c74a8d72854",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl",
@@ -304,7 +309,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.4000000",
"output_usd_per_million": "1.6000000",
"protocol_id": "59833d4bac77233732006cfada3136d655274a1fba463a994c4bd3107348b642",
"protocol_id": "ce0bb6d5ff9f57496b0f677bb5ac47fc196380e084b457b43f6203d2b7d1b6d1",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl",
@@ -320,7 +325,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.1000000",
"output_usd_per_million": "0.4000000",
"protocol_id": "91b5e653465ea8562e62d0a5c76895e4dfa9cc774dc777f04268be6643f04cb9",
"protocol_id": "0efb6591fbdbe87758af494f18fc0a5a1a4897451dc2bc05f45681e9b5d73758",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl",
@@ -336,9 +341,11 @@
"initial_calls": 240,
"input_usd_per_million": "1.1000000",
"output_usd_per_million": "4.4000000",
"protocol_id": "2a0f497f04f98ce1a6ff53412018721609b993a091a0da4e7c339b5a72b1859c",
"reasoning": null,
"reasoning_label": "not advertised (optional; omitted)",
"protocol_id": "3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6",
"reasoning": {
"enabled": false
},
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl",
"structured_output": true
},
@@ -352,7 +359,7 @@
"initial_calls": 240,
"input_usd_per_million": "2.5000000",
"output_usd_per_million": "10.00000",
"protocol_id": "e558a7bb4fdfbf9c4ddc6cd2f7505850a0779b7f8c02cde480dff902e709674b",
"protocol_id": "8c3bd515d1ac0e9ee598baa0ad42dc958817e6ad14cad7a73ecd710f71b003a6",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl",
@@ -368,7 +375,7 @@
"initial_calls": 240,
"input_usd_per_million": "2.5000000",
"output_usd_per_million": "10.00000",
"protocol_id": "ac49e31fe7abf39eadc966e042c72652b063d56d698cf437b76c53411e0a6979",
"protocol_id": "64377e05e282d8d9d5dc425635c6acdbf87ae8115aadfb39503c876ad5bbb9ac",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl",
@@ -384,7 +391,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.15000000",
"output_usd_per_million": "0.6000000",
"protocol_id": "1662dce1e899d6b1060b2dfaa19034935ff097e77114ae42936eb5b52ddc7296",
"protocol_id": "388af881427c7032c7a863540b75d7255292a83ee553dcb99eb2d07c7a1766df",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl",
@@ -400,7 +407,7 @@
"initial_calls": 240,
"input_usd_per_million": "2.5000000",
"output_usd_per_million": "10.00000",
"protocol_id": "11a70a549953db006a77522bc587bcf9464f9c601afc7b3d09b0119ade903d99",
"protocol_id": "eb4b8ad31eea6d046b4caff46dd0bfb56f113e65111be3871e02ac2e351086e6",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o_requests.jsonl",
@@ -416,7 +423,7 @@
"initial_calls": 240,
"input_usd_per_million": "1.000000",
"output_usd_per_million": "2.000000",
"protocol_id": "3b8af120356177e5159f76ee1736a89695006f54cd3055b2c2588ec9050374c6",
"protocol_id": "b0abbb72e5db3b335516292c218a5937a9bc4979fa19b616301b6b969f48c585",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-0613_requests.jsonl",
@@ -432,7 +439,7 @@
"initial_calls": 240,
"input_usd_per_million": "1.5000000",
"output_usd_per_million": "2.000000",
"protocol_id": "74b3ad3c9d06e892a0c08968419e330a7a8a91e5db7393e23c31f41c97139967",
"protocol_id": "3ccab07621e008211a4fed870619e72e1f6568a12aedf955dfd0d8cfc45016a2",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-instruct_requests.jsonl",
@@ -448,7 +455,7 @@
"initial_calls": 240,
"input_usd_per_million": "3.000000",
"output_usd_per_million": "4.000000",
"protocol_id": "07953ad5d1b968e61ff14f3ef8337e407d128295918855be42d7a0993acb0e32",
"protocol_id": "dfb7fd3c21a1cd90320c754b37d1d78e71d72eec2a1d3a1dfa3af320f7f5b189",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-16k_requests.jsonl",
@@ -464,7 +471,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.5000000",
"output_usd_per_million": "1.5000000",
"protocol_id": "446953742af525795631082baf5f3df86027c1710d1003ca6054f6e2a48dc18f",
"protocol_id": "b2f3aaa32e5e21e9e89a146ffe48e158fcae7b9b65937fb5bbf64df747bea23c",
"reasoning": null,
"reasoning_label": "not advertised",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo_requests.jsonl",
@@ -480,7 +487,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.75000000",
"output_usd_per_million": "3.75000000",
"protocol_id": "1e42dbfead6255ca8efab8f522b5c7684cd8eec6035059bfdf1fdac5d40dc3ae",
"protocol_id": "c1a151216652af24dca0a97c85b5d24f45fea8a28caa7075cdf09afb3fb646e7",
"reasoning": {
"effort": "low"
},
@@ -498,7 +505,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.75000000",
"output_usd_per_million": "3.75000000",
"protocol_id": "4524a90e3320fb15103d7468b5e8088ab99c73051539240d0c1ae14d353d7eac",
"protocol_id": "4e8a491693ca97ff0bf6ce704a893651a86b1756b0e56a5ebe5fac05c32c6e4f",
"reasoning": {
"effort": "minimal"
},
@@ -516,7 +523,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.3000000",
"output_usd_per_million": "2.5000000",
"protocol_id": "0f44f6e4ebb50ece2ada37707d79b988d36fc55a207f23235bb38740e7726b01",
"protocol_id": "24e7aee4cbb982a7b66cd14cbcf6e08a8f2ed80cb6cb132566b5ab1deed17245",
"reasoning": {
"effort": "minimal"
},
@@ -534,7 +541,7 @@
"initial_calls": 240,
"input_usd_per_million": "1.5000000",
"output_usd_per_million": "9.000000",
"protocol_id": "fab233d2ad1205b7fa4c2346875105509219846ceeeb7fd292d209615473c231",
"protocol_id": "4c5c5789977f56c13209d2babb47b3c79cd747fb2cd7cea568bd22301779d16f",
"reasoning": {
"effort": "minimal"
},
@@ -552,7 +559,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.25000000",
"output_usd_per_million": "1.5000000",
"protocol_id": "c325d9a280a083263f3705cf8d7fa8d4733877cb733526c2fe2290948d3ac4a4",
"protocol_id": "c9517bdfa0b53ccbf0aecb228cee50dc0f9c7abe4e5bf600132afa368b6eac8e",
"reasoning": {
"effort": "minimal"
},
@@ -560,22 +567,6 @@
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl",
"structured_output": true
},
{
"cache_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemma-4-26b-a4b-it_cache.json",
"completion_only_ceiling_usd": "0.0737280",
"conservative_reserve_usd": "0.24330240",
"created_utc": "2026-04-03",
"group": "Google",
"id": "google/gemma-4-26b-a4b-it",
"initial_calls": 240,
"input_usd_per_million": "0.09000000",
"output_usd_per_million": "0.3000000",
"protocol_id": "08d96b55bfee15010d2b9053f40c21d8e1d23d912ef84c645eb810eaa8803f1a",
"reasoning": null,
"reasoning_label": "not advertised (optional; omitted)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemma-4-26b-a4b-it_requests.jsonl",
"structured_output": true
},
{
"cache_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_cache.json",
"completion_only_ceiling_usd": "0.3686400",
@@ -586,7 +577,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.25000000",
"output_usd_per_million": "1.5000000",
"protocol_id": "4a85c0754b9bd2903c6cc978b2ded49ca381bc0e90199bff6eaf53e42d03e879",
"protocol_id": "851a22259abbe2da4fea19fd61b4fb7ce6e0406866e7bf2b54908f91d84cd860",
"reasoning": {
"effort": "minimal"
},
@@ -604,7 +595,7 @@
"initial_calls": 240,
"input_usd_per_million": "0.5000000",
"output_usd_per_million": "3.000000",
"protocol_id": "1c64e2725648b42570c0c0bd443655de81ee55c0477bdc835ff4c1087a76cded",
"protocol_id": "1529b1845000ea4830c5c7c88e898b6e1d75abaab1aa4a0d86951330cec17200",
"reasoning": {
"effort": "minimal"
},
@@ -622,9 +613,11 @@
"initial_calls": 240,
"input_usd_per_million": "0.1000000",
"output_usd_per_million": "0.4000000",
"protocol_id": "f5dbbdf216a4373df531ec7fef4a02fcec6e89609aa13553bb1f903dc8e0a43b",
"reasoning": null,
"reasoning_label": "not advertised (optional; omitted)",
"protocol_id": "028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7",
"reasoning": {
"enabled": false
},
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl",
"structured_output": true
},
@@ -638,9 +631,11 @@
"initial_calls": 240,
"input_usd_per_million": "0.3000000",
"output_usd_per_million": "2.5000000",
"protocol_id": "4d8587d1f7a72ee96a78e2693aec222cba6d313c5fcc978b82b27508d736a223",
"reasoning": null,
"reasoning_label": "not advertised (optional; omitted)",
"protocol_id": "0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428",
"reasoning": {
"enabled": false
},
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl",
"structured_output": true
},
@@ -654,7 +649,7 @@
"initial_calls": 240,
"input_usd_per_million": "1.25000000",
"output_usd_per_million": "4.25000000",
"protocol_id": "a716da6000adc3d9792e112c55d516e21c49fbe46acac6517d8b740e81b30e69",
"protocol_id": "5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2",
"reasoning": {
"effort": "minimal"
},
@@ -672,7 +667,7 @@
"initial_calls": 240,
"input_usd_per_million": "1.25000000",
"output_usd_per_million": "4.25000000",
"protocol_id": "c4cd8eedebc35b380c52d7c559f87c293623a04858b8acfeb091145340dde72d",
"protocol_id": "fcb191133bbd11dce82ac53f4d6b253a15c316d5abd71c54896f9dfdf644b964",
"reasoning": {
"effort": "minimal"
},
@@ -11,6 +11,7 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It
- final response: `Respond with ONLY a JSON object with exactly one key named answer. Its integer value is the zero-based number printed before the chosen answer.`
- rescue response: `Return only the one-key object required by the response schema. No explanation.`
- strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID
- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests
## Spend checks before any later dispatch
@@ -22,55 +23,55 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It
## Ordered panels
The order is Grok, OpenAI, Google, then Muse. `minimal` is used when catalog metadata advertises it; otherwise `low`; disabled is used only when the catalog says reasoning is optional and accepts `none`.
The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.
| family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger |
|---|---|---:|---:|---:|---|---|---|---:|---:|---:|---|
| Grok | `x-ai/grok-4.6` | 2026-08-12 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `f1dcc6e5e4651add7df5473e80ba5b5a5ec733d788a6a791625776501f8605d6` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.6_requests.jsonl` |
| Grok | `x-ai/grok-4.5` | 2026-07-08 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `fdb4ff16d7879cd840722b7f84654da2e67bc388cdf0b52ea64a1d12b487de18` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.5_requests.jsonl` |
| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"effort": "low"}` (low) | yes | `b9bd5e38934aeae851dcd6227403c474e41c52ebb0bf599b112203b30403e718` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` |
| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"effort": "low"}` (low) | yes | `76c8b34537d9ec301ec168c8566849a0a0d26fffa088b70b8da09acb760130b1` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` |
| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"effort": "low"}` (low) | yes | `a25533b578ada960251c7fa244a98b86e68a054346d641b6215888a09c92b09e` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` |
| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"effort": "low"}` (low) | yes | `42e8c091233072fd0925ab8f5554bc01e23c5cf3051e09ba7b45970af189fb10` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` |
| OpenAI | `openai/gpt-5.2-chat` | 2025-12-10 | 1.75000000 | 14.000000 | `null` (not advertised) | yes | `6911c78f8c33d085e4d2e4a627c8e311059855cfb859b16a6a319185b6abb807` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl` |
| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"effort": "low"}` (low) | yes | `6d72c036f9f413fa500c0a2902750b0a2c29fc1145d77d3fa02ecf89b48054ff` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` |
| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"effort": "low"}` (low) | yes | `063cbfba0812ff35df231154d69a498e77cd240961d61f18913e6465c1ac1938` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` |
| OpenAI | `openai/gpt-5` | 2025-08-07 | 1.25000000 | 10.00000 | `{"effort": "minimal"}` (minimal) | yes | `48ffd6a149a5583355dd779b855e944ba26e4eab9b9cd4b5acd780cb3e3fb4e1` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5_requests.jsonl` |
| OpenAI | `openai/gpt-5-mini` | 2025-08-07 | 0.25000000 | 2.000000 | `{"effort": "minimal"}` (minimal) | yes | `00e8beddc47977c22afa2128d78af2c00acf8ddb87077f427bcb1360ed87a980` | 240 | USD 0.4915 | USD 1.5360 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5-mini_requests.jsonl` |
| OpenAI | `openai/gpt-oss-120b` | 2025-08-05 | 0.037000000 | 0.17000000 | `{"effort": "low"}` (low) | yes | `25ac3627e4795573424e184ab883ed80e3abb3bfadb0adba94a8e5b1ce659725` | 240 | USD 0.0418 | USD 0.1344 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-120b_requests.jsonl` |
| OpenAI | `openai/gpt-oss-20b` | 2025-08-05 | 0.03000000 | 0.13000000 | `{"effort": "low"}` (low) | yes | `06243dd30d6fb69a30c4b24a9b891b95a25975d7d41b7071a474e4365e856169` | 240 | USD 0.0319 | USD 0.1032 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-20b_requests.jsonl` |
| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `null` (not advertised (optional; omitted)) | yes | `f4054a623913b60fa79beee9aacec15967aa058434835937e4e8af276adf9557` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` |
| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `null` (not advertised (optional; omitted)) | yes | `26873a2a9f3d6530a06368113b903fbc80f673849ed605f1f49963f1f7abb638` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` |
| OpenAI | `openai/gpt-4.1` | 2025-04-14 | 2.000000 | 8.000000 | `null` (not advertised) | yes | `9a3bb10b7179275010d35313de4a9d8021ca42abb0901414450ef0c6a42f2aea` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl` |
| OpenAI | `openai/gpt-4.1-mini` | 2025-04-14 | 0.4000000 | 1.6000000 | `null` (not advertised) | yes | `59833d4bac77233732006cfada3136d655274a1fba463a994c4bd3107348b642` | 240 | USD 0.3932 | USD 1.2780 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl` |
| OpenAI | `openai/gpt-4.1-nano` | 2025-04-14 | 0.1000000 | 0.4000000 | `null` (not advertised) | yes | `91b5e653465ea8562e62d0a5c76895e4dfa9cc774dc777f04268be6643f04cb9` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl` |
| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `null` (not advertised (optional; omitted)) | yes | `2a0f497f04f98ce1a6ff53412018721609b993a091a0da4e7c339b5a72b1859c` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` |
| OpenAI | `openai/gpt-4o-2024-11-20` | 2024-11-20 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `e558a7bb4fdfbf9c4ddc6cd2f7505850a0779b7f8c02cde480dff902e709674b` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl` |
| OpenAI | `openai/gpt-4o-2024-08-06` | 2024-08-06 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `ac49e31fe7abf39eadc966e042c72652b063d56d698cf437b76c53411e0a6979` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl` |
| OpenAI | `openai/gpt-4o-mini` | 2024-07-18 | 0.15000000 | 0.6000000 | `null` (not advertised) | yes | `1662dce1e899d6b1060b2dfaa19034935ff097e77114ae42936eb5b52ddc7296` | 240 | USD 0.1475 | USD 0.4792 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl` |
| OpenAI | `openai/gpt-4o` | 2024-05-13 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `11a70a549953db006a77522bc587bcf9464f9c601afc7b3d09b0119ade903d99` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o_requests.jsonl` |
| OpenAI | `openai/gpt-3.5-turbo-0613` | 2024-01-25 | 1.000000 | 2.000000 | `null` (not advertised) | yes | `3b8af120356177e5159f76ee1736a89695006f54cd3055b2c2588ec9050374c6` | 240 | USD 0.4915 | USD 1.7203 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-0613_requests.jsonl` |
| OpenAI | `openai/gpt-3.5-turbo-instruct` | 2023-09-28 | 1.5000000 | 2.000000 | `null` (not advertised) | yes | `74b3ad3c9d06e892a0c08968419e330a7a8a91e5db7393e23c31f41c97139967` | 240 | USD 0.4915 | USD 1.8432 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-instruct_requests.jsonl` |
| OpenAI | `openai/gpt-3.5-turbo-16k` | 2023-08-28 | 3.000000 | 4.000000 | `null` (not advertised) | yes | `07953ad5d1b968e61ff14f3ef8337e407d128295918855be42d7a0993acb0e32` | 240 | USD 0.9830 | USD 3.6864 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-16k_requests.jsonl` |
| OpenAI | `openai/gpt-3.5-turbo` | 2023-05-28 | 0.5000000 | 1.5000000 | `null` (not advertised) | yes | `446953742af525795631082baf5f3df86027c1710d1003ca6054f6e2a48dc18f` | 240 | USD 0.3686 | USD 1.2288 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo_requests.jsonl` |
| Google | `google/gemini-3.8-flash` | 2026-09-02 | 0.75000000 | 3.75000000 | `{"effort": "low"}` (low) | yes | `1e42dbfead6255ca8efab8f522b5c7684cd8eec6035059bfdf1fdac5d40dc3ae` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.8-flash_requests.jsonl` |
| Google | `google/gemini-3.6-flash` | 2026-07-21 | 0.75000000 | 3.75000000 | `{"effort": "minimal"}` (minimal) | yes | `4524a90e3320fb15103d7468b5e8088ab99c73051539240d0c1ae14d353d7eac` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.6-flash_requests.jsonl` |
| Google | `google/gemini-3.5-flash-lite` | 2026-07-21 | 0.3000000 | 2.5000000 | `{"effort": "minimal"}` (minimal) | yes | `0f44f6e4ebb50ece2ada37707d79b988d36fc55a207f23235bb38740e7726b01` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash-lite_requests.jsonl` |
| Google | `google/gemini-3.5-flash` | 2026-05-19 | 1.5000000 | 9.000000 | `{"effort": "minimal"}` (minimal) | yes | `fab233d2ad1205b7fa4c2346875105509219846ceeeb7fd292d209615473c231` | 240 | USD 2.2118 | USD 7.0042 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash_requests.jsonl` |
| Google | `google/gemini-3.1-flash-lite` | 2026-05-07 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `c325d9a280a083263f3705cf8d7fa8d4733877cb733526c2fe2290948d3ac4a4` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl` |
| Google | `google/gemma-4-26b-a4b-it` | 2026-04-03 | 0.09000000 | 0.3000000 | `null` (not advertised (optional; omitted)) | yes | `08d96b55bfee15010d2b9053f40c21d8e1d23d912ef84c645eb810eaa8803f1a` | 240 | USD 0.0737 | USD 0.2433 | `slop/research/wvs/20260917_direct_choice/priority/google__gemma-4-26b-a4b-it_requests.jsonl` |
| Google | `google/gemini-3.1-flash-lite-preview` | 2026-03-03 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `4a85c0754b9bd2903c6cc978b2ded49ca381bc0e90199bff6eaf53e42d03e879` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_requests.jsonl` |
| Google | `google/gemini-3-flash-preview` | 2025-12-17 | 0.5000000 | 3.000000 | `{"effort": "minimal"}` (minimal) | yes | `1c64e2725648b42570c0c0bd443655de81ee55c0477bdc835ff4c1087a76cded` | 240 | USD 0.7373 | USD 2.3347 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3-flash-preview_requests.jsonl` |
| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `null` (not advertised (optional; omitted)) | yes | `f5dbbdf216a4373df531ec7fef4a02fcec6e89609aa13553bb1f903dc8e0a43b` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` |
| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `null` (not advertised (optional; omitted)) | yes | `4d8587d1f7a72ee96a78e2693aec222cba6d313c5fcc978b82b27508d736a223` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` |
| Muse | `meta/muse-spark-1.2` | 2026-08-05 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `a716da6000adc3d9792e112c55d516e21c49fbe46acac6517d8b740e81b30e69` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.2_requests.jsonl` |
| Muse | `meta/muse-spark-1.1` | 2026-07-16 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `c4cd8eedebc35b380c52d7c559f87c293623a04858b8acfeb091145340dde72d` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.1_requests.jsonl` |
| Grok | `x-ai/grok-4.6` | 2026-08-12 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.6_requests.jsonl` |
| Grok | `x-ai/grok-4.5` | 2026-07-08 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `d1362e8b42a4e2ff54cbd66be8c6a65225f94c9679931299f0e5e359e4065c63` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.5_requests.jsonl` |
| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` |
| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` |
| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` |
| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` |
| OpenAI | `openai/gpt-5.2-chat` | 2025-12-10 | 1.75000000 | 14.000000 | `null` (not advertised) | yes | `4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl` |
| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` |
| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` |
| OpenAI | `openai/gpt-5` | 2025-08-07 | 1.25000000 | 10.00000 | `{"effort": "minimal"}` (minimal) | yes | `4f7fe97763a5822831b2244e72b0d4cc01ff46da55090fed2485c79f97e4bdf0` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5_requests.jsonl` |
| OpenAI | `openai/gpt-5-mini` | 2025-08-07 | 0.25000000 | 2.000000 | `{"effort": "minimal"}` (minimal) | yes | `5b115c7341384a37117745f735367dae10932cc9f16b3fdac5819b5adfe007f4` | 240 | USD 0.4915 | USD 1.5360 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5-mini_requests.jsonl` |
| OpenAI | `openai/gpt-oss-120b` | 2025-08-05 | 0.037000000 | 0.17000000 | `{"effort": "low"}` (low) | yes | `ae278f4f5e91f668448c4b921633e999ba90a6cd9cc3819c8ee4beac8484d38a` | 240 | USD 0.0418 | USD 0.1344 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-120b_requests.jsonl` |
| OpenAI | `openai/gpt-oss-20b` | 2025-08-05 | 0.03000000 | 0.13000000 | `{"effort": "low"}` (low) | yes | `2b45bec6a0271fb1390ea36e196eef7e75a53112c1dcd1e40b1b60b7631a9d7d` | 240 | USD 0.0319 | USD 0.1032 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-20b_requests.jsonl` |
| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` |
| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` |
| OpenAI | `openai/gpt-4.1` | 2025-04-14 | 2.000000 | 8.000000 | `null` (not advertised) | yes | `1f6ad277fcb151d83dd6b0c9d2f9ed5d2cb7466a09b30c4368ec0c74a8d72854` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl` |
| OpenAI | `openai/gpt-4.1-mini` | 2025-04-14 | 0.4000000 | 1.6000000 | `null` (not advertised) | yes | `ce0bb6d5ff9f57496b0f677bb5ac47fc196380e084b457b43f6203d2b7d1b6d1` | 240 | USD 0.3932 | USD 1.2780 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl` |
| OpenAI | `openai/gpt-4.1-nano` | 2025-04-14 | 0.1000000 | 0.4000000 | `null` (not advertised) | yes | `0efb6591fbdbe87758af494f18fc0a5a1a4897451dc2bc05f45681e9b5d73758` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl` |
| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` |
| OpenAI | `openai/gpt-4o-2024-11-20` | 2024-11-20 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `8c3bd515d1ac0e9ee598baa0ad42dc958817e6ad14cad7a73ecd710f71b003a6` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl` |
| OpenAI | `openai/gpt-4o-2024-08-06` | 2024-08-06 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `64377e05e282d8d9d5dc425635c6acdbf87ae8115aadfb39503c876ad5bbb9ac` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl` |
| OpenAI | `openai/gpt-4o-mini` | 2024-07-18 | 0.15000000 | 0.6000000 | `null` (not advertised) | yes | `388af881427c7032c7a863540b75d7255292a83ee553dcb99eb2d07c7a1766df` | 240 | USD 0.1475 | USD 0.4792 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl` |
| OpenAI | `openai/gpt-4o` | 2024-05-13 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `eb4b8ad31eea6d046b4caff46dd0bfb56f113e65111be3871e02ac2e351086e6` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o_requests.jsonl` |
| OpenAI | `openai/gpt-3.5-turbo-0613` | 2024-01-25 | 1.000000 | 2.000000 | `null` (not advertised) | yes | `b0abbb72e5db3b335516292c218a5937a9bc4979fa19b616301b6b969f48c585` | 240 | USD 0.4915 | USD 1.7203 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-0613_requests.jsonl` |
| OpenAI | `openai/gpt-3.5-turbo-instruct` | 2023-09-28 | 1.5000000 | 2.000000 | `null` (not advertised) | yes | `3ccab07621e008211a4fed870619e72e1f6568a12aedf955dfd0d8cfc45016a2` | 240 | USD 0.4915 | USD 1.8432 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-instruct_requests.jsonl` |
| OpenAI | `openai/gpt-3.5-turbo-16k` | 2023-08-28 | 3.000000 | 4.000000 | `null` (not advertised) | yes | `dfb7fd3c21a1cd90320c754b37d1d78e71d72eec2a1d3a1dfa3af320f7f5b189` | 240 | USD 0.9830 | USD 3.6864 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-16k_requests.jsonl` |
| OpenAI | `openai/gpt-3.5-turbo` | 2023-05-28 | 0.5000000 | 1.5000000 | `null` (not advertised) | yes | `b2f3aaa32e5e21e9e89a146ffe48e158fcae7b9b65937fb5bbf64df747bea23c` | 240 | USD 0.3686 | USD 1.2288 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo_requests.jsonl` |
| Google | `google/gemini-3.8-flash` | 2026-09-02 | 0.75000000 | 3.75000000 | `{"effort": "low"}` (low) | yes | `c1a151216652af24dca0a97c85b5d24f45fea8a28caa7075cdf09afb3fb646e7` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.8-flash_requests.jsonl` |
| Google | `google/gemini-3.6-flash` | 2026-07-21 | 0.75000000 | 3.75000000 | `{"effort": "minimal"}` (minimal) | yes | `4e8a491693ca97ff0bf6ce704a893651a86b1756b0e56a5ebe5fac05c32c6e4f` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.6-flash_requests.jsonl` |
| Google | `google/gemini-3.5-flash-lite` | 2026-07-21 | 0.3000000 | 2.5000000 | `{"effort": "minimal"}` (minimal) | yes | `24e7aee4cbb982a7b66cd14cbcf6e08a8f2ed80cb6cb132566b5ab1deed17245` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash-lite_requests.jsonl` |
| Google | `google/gemini-3.5-flash` | 2026-05-19 | 1.5000000 | 9.000000 | `{"effort": "minimal"}` (minimal) | yes | `4c5c5789977f56c13209d2babb47b3c79cd747fb2cd7cea568bd22301779d16f` | 240 | USD 2.2118 | USD 7.0042 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash_requests.jsonl` |
| Google | `google/gemini-3.1-flash-lite` | 2026-05-07 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `c9517bdfa0b53ccbf0aecb228cee50dc0f9c7abe4e5bf600132afa368b6eac8e` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl` |
| Google | `google/gemini-3.1-flash-lite-preview` | 2026-03-03 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `851a22259abbe2da4fea19fd61b4fb7ce6e0406866e7bf2b54908f91d84cd860` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_requests.jsonl` |
| Google | `google/gemini-3-flash-preview` | 2025-12-17 | 0.5000000 | 3.000000 | `{"effort": "minimal"}` (minimal) | yes | `1529b1845000ea4830c5c7c88e898b6e1d75abaab1aa4a0d86951330cec17200` | 240 | USD 0.7373 | USD 2.3347 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3-flash-preview_requests.jsonl` |
| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` |
| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` |
| Muse | `meta/muse-spark-1.2` | 2026-08-05 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.2_requests.jsonl` |
| Muse | `meta/muse-spark-1.1` | 2026-07-16 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `fcb191133bbd11dce82ac53f4d6b253a15c316d5abd71c54896f9dfdf644b964` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.1_requests.jsonl` |
## Exclusions
- Already plotted dense-rated IDs are not repeated in this prepared direct-choice list, including Grok 4.3/4.20, GPT-6 Astra, GPT-5.6 Sol, GPT-5.5, GPT-5.4, GPT-5.3 Chat, Gemini 3.7 Flash, Gemini 2.5 Pro, and Muse 1.3.
- GPT-5 Nano is retained as a completed dense-rated protocol diagnostic, not silently relabelled as a direct-choice panel.
- Pro/Fast, batch/free aliases, output price above USD 15/M, and code/image/audio/safeguard/multi-agent entries remain excluded. `Flash` is included where it is a general chat model.
- `google/gemma-4-26b-a4b-it` is excluded: it is Gemma, not an identified member of the requested Gemini release series.
- `openai/o4-mini-high` and `openai/o3-mini-high` are excluded because their catalog entries advertise only `high` reasoning, not the registered minimal/low policy.
- The deferred Qwen/GLM/Mistral shortlist remains outside this priority manifest until a direct-choice expansion decision is made.
@@ -1,16 +1,22 @@
$ python scripts/wvs_direct_choice_priority.py --write-manifest
wrote slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md and slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json
$ python scripts/wvs_direct_choice_priority.py --model x-ai/grok-4.6 --smoke
smoke: x-ai/grok-4.6, 240 direct-choice requests, protocol=f1dcc6e5e4651add7df5473e80ba5b5a5ec733d788a6a791625776501f8605d6
smoke: x-ai/grok-4.6, 240 direct-choice requests, protocol=34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6
smoke: reasoning={'effort': 'low'}, reserve=USD 4.915200
$ python scripts/wvs_direct_choice_priority.py --model openai/gpt-5.6-luna --smoke
smoke: openai/gpt-5.6-luna, 240 direct-choice requests, protocol=8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95
smoke: reasoning={'enabled': False}, reserve=USD 0.9338880
$ python scripts/wvs_direct_choice_priority.py --model openai/o3 --smoke
smoke: openai/o3, 240 direct-choice requests, protocol=0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298
smoke: reasoning={'enabled': False}, reserve=USD 6.389760
$ python scripts/wvs_direct_choice_priority.py --model openai/gpt-5.2-chat --smoke
smoke: openai/gpt-5.2-chat, 240 direct-choice requests, protocol=6911c78f8c33d085e4d2e4a627c8e311059855cfb859b16a6a319185b6abb807
smoke: openai/gpt-5.2-chat, 240 direct-choice requests, protocol=4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca
smoke: reasoning=None, reserve=USD 10.75200000
$ python scripts/wvs_direct_choice_priority.py --model google/gemini-3.6-flash --smoke
smoke: google/gemini-3.6-flash, 240 direct-choice requests, protocol=4524a90e3320fb15103d7468b5e8088ab99c73051539240d0c1ae14d353d7eac
smoke: reasoning={'effort': 'minimal'}, reserve=USD 2.94912000
$ python scripts/wvs_direct_choice_priority.py --model meta/muse-spark-1.2 --smoke
smoke: meta/muse-spark-1.2, 240 direct-choice requests, protocol=a716da6000adc3d9792e112c55d516e21c49fbe46acac6517d8b740e81b30e69
smoke: meta/muse-spark-1.2, 240 direct-choice requests, protocol=5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2
smoke: reasoning={'effort': 'minimal'}, reserve=USD 3.44064000
verified manifest: 39 unique structured direct-choice protocols, 240 calls each, output <= USD 15/M
wrapper syntax: valid; no --run execution invoked
$ python scripts/wvs_direct_choice_priority_smoke.py
smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings
smoke: synthetic first request exits before remaining 239 and writes no cache
smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false
verified manifest: 38 distinct strict-schema 240-call protocols; Gemma excluded; fail-fast probe encoded
+29 -7
View File
@@ -142,7 +142,8 @@ def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_pe
prompt_instruction: str = _PROMPT_INSTRUCTION_OWN_VIEW,
answer_instruction: str = _ANSWER_INSTRUCTION_WITH_EXAMPLE,
rescue_instruction: str | None = None,
plan_override: list[dict] | None = None) -> str:
plan_override: list[dict] | None = None,
fail_fast_first_request: bool = False) -> str:
plan = _plan(items, samples_per_order, answer_instruction) if plan_override is None else plan_override
protocol = {
"schema": 1,
@@ -163,6 +164,8 @@ def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_pe
if plan_override is not None:
protocol.pop("samples_per_order")
protocol["samples_per_item"] = {item["id"]: sum(request["item_id"] == item["id"] for request in plan) for item in items}
if fail_fast_first_request:
protocol["fail_fast_first_request"] = True
encoded = json.dumps(protocol, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()
return hashlib.sha256(encoded).hexdigest()
@@ -180,7 +183,8 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
prompt_instruction: str = _PROMPT_INSTRUCTION_OWN_VIEW,
answer_instruction: str = _ANSWER_INSTRUCTION_WITH_EXAMPLE,
rescue_instruction: str | None = None,
plan_override: list[dict] | None = None) -> dict:
plan_override: list[dict] | None = None,
fail_fast_first_request: bool = False) -> dict:
"""Sample exactly one selected option per prompt, including canonical and reversed option orders.
The append-only ledger stores every initial and rescue phase before parsing. A cache entry is written only
@@ -197,6 +201,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
reasoning=reasoning, structured_output=structured_output, prompt_instruction=prompt_instruction,
answer_instruction=answer_instruction,
rescue_instruction=rescue_instruction, plan_override=plan_override,
fail_fast_first_request=fail_fast_first_request,
)
cache_file = Path(cache_path)
cache = json.loads(cache_file.read_text()) if cache_file.exists() else {"schema": 1, "completed": {}}
@@ -211,7 +216,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
"model": model, "samples_per_order": samples_per_order, "temperature": temperature,
"max_tokens": max_tokens, "concurrency": concurrency, "request_timeout": request_timeout,
"reasoning": reasoning, "structured_output": structured_output,
"prompt_instruction": prompt_instruction,
"prompt_instruction": prompt_instruction, "fail_fast_first_request": fail_fast_first_request,
}
if plan_override is not None:
settings.pop("samples_per_order")
@@ -242,9 +247,10 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
response_format = _choice_schema(item["n"])
payload = {
"model": model, "messages": [{"role": "user", "content": request["prompt"]}],
"temperature": temperature, "max_tokens": max_tokens, "reasoning": reasoning,
"response_format": response_format,
"temperature": temperature, "max_tokens": max_tokens, "response_format": response_format,
}
if reasoning is not None:
payload["reasoning"] = reasoning
phase = "initial"
async with semaphore:
try:
@@ -266,8 +272,10 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
{"role": "assistant", "content": assistant_tail},
{"role": "user", "content": rescue_instruction or _force_choice(item["n"])},
], "temperature": temperature, "max_tokens": max(max_tokens, 2048),
"reasoning": reasoning, "response_format": response_format,
"response_format": response_format,
}
if reasoning is not None:
rescue_payload["reasoning"] = reasoning
_append_record(records, {"event": "request_started", "phase": phase, **request_meta,
"payload": rescue_payload, "initial_response_message": message})
response = await asyncio.wait_for(openrouter_request(rescue_payload), timeout=request_timeout)
@@ -283,9 +291,23 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
"error_type": type(exc).__name__, "error": str(exc)})
return {"text": None, "rescued": phase == "rescue", "error": f"{type(exc).__name__}: {exc}"}
return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan)))
if not fail_fast_first_request:
return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan)))
first = await call(0, plan[0])
if first["error"] is not None:
return [first]
remaining = await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan[1:], start=1)))
return [first, *remaining]
results = asyncio.run(run_all())
if fail_fast_first_request and results[0]["error"] is not None:
summary = {
"run_id": run_id, "protocol_id": protocol_id, "model": model, "settings": settings,
"planned_requests": len(plan), "failed_requests": 1, "rescued_requests": int(results[0]["rescued"]),
"complete": False, "items": [], "failure": results[0]["error"],
}
_append_record(records, {"event": "run_finished", "construct": "direct_choice", **summary})
raise RuntimeError(f"first scheduled request failed before remaining {len(plan) - 1} requests: {results[0]['error']}")
by_item = {item["id"]: [] for item in items}
failed = 0
rescues = 0