From f3603b4500a7c541fc692a54a3ea7e2b84385d02 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:16:23 +0800 Subject: [PATCH] Harden direct-choice priority preflight Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com> --- scripts/wvs_direct_choice_priority.py | 19 ++- scripts/wvs_direct_choice_priority_smoke.py | 71 +++++++++ .../priority_direct_choice_manifest.json | 147 +++++++++--------- .../priority_direct_choice_manifest.md | 81 +++++----- .../priority_preflight_smoke.log | 22 ++- src/moralmaps/read_direct_choice.py | 36 ++++- 6 files changed, 239 insertions(+), 137 deletions(-) create mode 100644 scripts/wvs_direct_choice_priority_smoke.py diff --git a/scripts/wvs_direct_choice_priority.py b/scripts/wvs_direct_choice_priority.py index f50dcc4..c8c0d40 100644 --- a/scripts/wvs_direct_choice_priority.py +++ b/scripts/wvs_direct_choice_priority.py @@ -47,7 +47,7 @@ GROUPS = { ), "Google": ( "google/gemini-3.8-flash", "google/gemini-3.6-flash", "google/gemini-3.5-flash-lite", - "google/gemini-3.5-flash", "google/gemini-3.1-flash-lite", "google/gemma-4-26b-a4b-it", + "google/gemini-3.5-flash", "google/gemini-3.1-flash-lite", "google/gemini-3.1-flash-lite-preview", "google/gemini-3-flash-preview", "google/gemini-2.5-flash-lite", "google/gemini-2.5-flash", ), @@ -68,13 +68,16 @@ def reasoning_setting(model: dict) -> tuple[dict | None, str]: if metadata is None: return None, "not advertised" efforts = set(metadata.get("supported_efforts", [])) + optional = not metadata.get("mandatory") + if optional and "none" in efforts: + return {"enabled": False}, "disabled (optional, none advertised)" + if optional and not efforts and "reasoning" in model["supported_parameters"]: + return {"enabled": False}, "disabled (optional, parameter advertised without efforts)" if "minimal" in efforts: return {"effort": "minimal"}, "minimal" if "low" in efforts: return {"effort": "low"}, "low" - if not metadata.get("mandatory") and "none" in efforts: - return {"enabled": False}, "disabled (optional, none advertised)" - if not metadata.get("mandatory") and not efforts: + if optional and not efforts: return None, "not advertised (optional; omitted)" raise ValueError(f"no allowed minimal/low reasoning setting for {model['id']}: {metadata}") @@ -100,7 +103,7 @@ def entry(model: dict, pilot_items: list[dict], request_plan: list[dict]) -> dic max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT, reasoning=reasoning, structured_output=True, prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION, rescue_instruction=RESCUE_INSTRUCTION, - plan_override=request_plan, + plan_override=request_plan, fail_fast_first_request=True, ) input_rate = rate_per_million(model, "prompt") output_rate = rate_per_million(model, "completion") @@ -152,6 +155,7 @@ def write_manifest(priority: list[dict]) -> None: "initial_calls_per_model": 240, "schedule": "balanced_cyclic_rotations", "prompt_instruction": PROMPT_INSTRUCTION, "answer_instruction": ANSWER_INSTRUCTION, "rescue_instruction": RESCUE_INSTRUCTION, "strict_structured_output": True, + "fail_fast_first_request": True, }, "stop_usd": {"priority_phase": str(PHASE_STOP_USD), "global": str(GLOBAL_STOP_USD)}, "current_observed_cost_usd": str(current_cost), @@ -174,6 +178,7 @@ def write_manifest(priority: list[dict]) -> None: f"- final response: `{ANSWER_INSTRUCTION}`", f"- rescue response: `{RESCUE_INSTRUCTION}`", "- strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID", + "- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests", "", "## Spend checks before any later dispatch", "", @@ -185,7 +190,7 @@ def write_manifest(priority: list[dict]) -> None: "", "## Ordered panels", "", - "The order is Grok, OpenAI, Google, then Muse. `minimal` is used when catalog metadata advertises it; otherwise `low`; disabled is used only when the catalog says reasoning is optional and accepts `none`.", + "The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.", "", "| family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger |", "|---|---|---:|---:|---:|---|---|---|---:|---:|---:|---|", @@ -204,6 +209,7 @@ def write_manifest(priority: list[dict]) -> None: "- Already plotted dense-rated IDs are not repeated in this prepared direct-choice list, including Grok 4.3/4.20, GPT-6 Astra, GPT-5.6 Sol, GPT-5.5, GPT-5.4, GPT-5.3 Chat, Gemini 3.7 Flash, Gemini 2.5 Pro, and Muse 1.3.", "- GPT-5 Nano is retained as a completed dense-rated protocol diagnostic, not silently relabelled as a direct-choice panel.", "- Pro/Fast, batch/free aliases, output price above USD 15/M, and code/image/audio/safeguard/multi-agent entries remain excluded. `Flash` is included where it is a general chat model.", + "- `google/gemma-4-26b-a4b-it` is excluded: it is Gemma, not an identified member of the requested Gemini release series.", "- `openai/o4-mini-high` and `openai/o3-mini-high` are excluded because their catalog entries advertise only `high` reasoning, not the registered minimal/low policy.", "- The deferred Qwen/GLM/Mistral shortlist remains outside this priority manifest until a direct-choice expansion decision is made.", "", @@ -258,6 +264,7 @@ def main() -> None: structured_output=True, records_path=records_path, cache_path=cache_path, prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION, rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan, + fail_fast_first_request=True, ) if result["cached"]: print(f"priority direct-choice cache hit: {args.model}, protocol={result['protocol_id'][:12]}") diff --git a/scripts/wvs_direct_choice_priority_smoke.py b/scripts/wvs_direct_choice_priority_smoke.py new file mode 100644 index 0000000..5450c50 --- /dev/null +++ b/scripts/wvs_direct_choice_priority_smoke.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Zero-network smoke checks for prepared direct-choice priority panels.""" +from __future__ import annotations + +import asyncio +import json +import tempfile +from collections import Counter +from pathlib import Path + +import moralmaps.read_direct_choice as reader +from wvs_direct_choice_priority import entries +from wvs_direct_choice_production_pilot import ( + ANSWER_INSTRUCTION, + PROMPT_INSTRUCTION, + RESCUE_INSTRUCTION, + items, + schedule, +) + + +def main() -> None: + prepared = entries() + assert len(prepared) == 38 + assert all(row["initial_calls"] == 240 and row["structured_output"] for row in prepared) + assert {str(row["reasoning"]) for row in prepared} == {"None", "{'enabled': False}", "{'effort': 'low'}", "{'effort': 'minimal'}"} + assert all(row["protocol_id"] for row in prepared) + print("smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings") + + pilot_items = items() + request_plan = schedule(pilot_items) + for reasoning, expected in ((None, None), ({"enabled": False}, {"enabled": False})): + calls = [] + + async def fail(payload: dict) -> dict: + calls.append(payload) + raise RuntimeError("synthetic compatibility failure") + + original = reader.openrouter_request + reader.openrouter_request = fail + try: + with tempfile.TemporaryDirectory() as directory: + records = Path(directory) / "records.jsonl" + cache = Path(directory) / "cache.json" + try: + reader.read_items_direct_choice( + "test/model", pilot_items, samples_per_order=10, temperature=1.0, + max_tokens=1024, concurrency=1, request_timeout=1, reasoning=reasoning, + structured_output=True, records_path=records, cache_path=cache, + prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION, + rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan, + fail_fast_first_request=True, + ) + except RuntimeError as error: + assert "first scheduled request failed before remaining 239 requests" in str(error) + else: + raise AssertionError("synthetic first-request failure did not abort") + events = [json.loads(line) for line in records.read_text().splitlines()] + assert Counter(event["event"] for event in events) == Counter({ + "run_started": 1, "request_started": 1, "request_failed": 1, "run_finished": 1, + }) + assert len(calls) == 1 and calls[0].get("reasoning") == expected + assert not cache.exists() + finally: + reader.openrouter_request = original + print("smoke: synthetic first request exits before remaining 239 and writes no cache") + print("smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false") + + +if __name__ == "__main__": + main() diff --git a/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json b/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json index a8330a0..ef26e49 100644 --- a/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json +++ b/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json @@ -5,6 +5,7 @@ "design": { "answer_instruction": "Respond with ONLY a JSON object with exactly one key named answer. Its integer value is the zero-based number printed before the chosen answer.", "construct": "direct_choice", + "fail_fast_first_request": true, "initial_calls_per_model": 240, "items": 12, "prompt_instruction": "Think briefly about which ONE answer best reflects the values expressed by your behavior as this AI assistant. Do not analyze the survey design or discuss lacking personal beliefs. Answer immediately.", @@ -24,7 +25,7 @@ "initial_calls": 240, "input_usd_per_million": "2.000000", "output_usd_per_million": "6.000000", - "protocol_id": "f1dcc6e5e4651add7df5473e80ba5b5a5ec733d788a6a791625776501f8605d6", + "protocol_id": "34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6", "reasoning": { "effort": "low" }, @@ -42,7 +43,7 @@ "initial_calls": 240, "input_usd_per_million": "2.000000", "output_usd_per_million": "6.000000", - "protocol_id": "fdb4ff16d7879cd840722b7f84654da2e67bc388cdf0b52ea64a1d12b487de18", + "protocol_id": "d1362e8b42a4e2ff54cbd66be8c6a65225f94c9679931299f0e5e359e4065c63", "reasoning": { "effort": "low" }, @@ -60,11 +61,11 @@ "initial_calls": 240, "input_usd_per_million": "0.2000000", "output_usd_per_million": "1.2000000", - "protocol_id": "b9bd5e38934aeae851dcd6227403c474e41c52ebb0bf599b112203b30403e718", + "protocol_id": "8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95", "reasoning": { - "effort": "low" + "enabled": false }, - "reasoning_label": "low", + "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl", "structured_output": true }, @@ -78,11 +79,11 @@ "initial_calls": 240, "input_usd_per_million": "2.000000", "output_usd_per_million": "12.000000", - "protocol_id": "76c8b34537d9ec301ec168c8566849a0a0d26fffa088b70b8da09acb760130b1", + "protocol_id": "08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305", "reasoning": { - "effort": "low" + "enabled": false }, - "reasoning_label": "low", + "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl", "structured_output": true }, @@ -96,11 +97,11 @@ "initial_calls": 240, "input_usd_per_million": "0.2000000", "output_usd_per_million": "1.25000000", - "protocol_id": "a25533b578ada960251c7fa244a98b86e68a054346d641b6215888a09c92b09e", + "protocol_id": "35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5", "reasoning": { - "effort": "low" + "enabled": false }, - "reasoning_label": "low", + "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl", "structured_output": true }, @@ -114,11 +115,11 @@ "initial_calls": 240, "input_usd_per_million": "0.75000000", "output_usd_per_million": "4.5000000", - "protocol_id": "42e8c091233072fd0925ab8f5554bc01e23c5cf3051e09ba7b45970af189fb10", + "protocol_id": "c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f", "reasoning": { - "effort": "low" + "enabled": false }, - "reasoning_label": "low", + "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl", "structured_output": true }, @@ -132,7 +133,7 @@ "initial_calls": 240, "input_usd_per_million": "1.75000000", "output_usd_per_million": "14.000000", - "protocol_id": "6911c78f8c33d085e4d2e4a627c8e311059855cfb859b16a6a319185b6abb807", + "protocol_id": "4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl", @@ -148,11 +149,11 @@ "initial_calls": 240, "input_usd_per_million": "1.75000000", "output_usd_per_million": "14.000000", - "protocol_id": "6d72c036f9f413fa500c0a2902750b0a2c29fc1145d77d3fa02ecf89b48054ff", + "protocol_id": "1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111", "reasoning": { - "effort": "low" + "enabled": false }, - "reasoning_label": "low", + "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl", "structured_output": true }, @@ -166,11 +167,11 @@ "initial_calls": 240, "input_usd_per_million": "1.25000000", "output_usd_per_million": "10.00000", - "protocol_id": "063cbfba0812ff35df231154d69a498e77cd240961d61f18913e6465c1ac1938", + "protocol_id": "2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05", "reasoning": { - "effort": "low" + "enabled": false }, - "reasoning_label": "low", + "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl", "structured_output": true }, @@ -184,7 +185,7 @@ "initial_calls": 240, "input_usd_per_million": "1.25000000", "output_usd_per_million": "10.00000", - "protocol_id": "48ffd6a149a5583355dd779b855e944ba26e4eab9b9cd4b5acd780cb3e3fb4e1", + "protocol_id": "4f7fe97763a5822831b2244e72b0d4cc01ff46da55090fed2485c79f97e4bdf0", "reasoning": { "effort": "minimal" }, @@ -202,7 +203,7 @@ "initial_calls": 240, "input_usd_per_million": "0.25000000", "output_usd_per_million": "2.000000", - "protocol_id": "00e8beddc47977c22afa2128d78af2c00acf8ddb87077f427bcb1360ed87a980", + "protocol_id": "5b115c7341384a37117745f735367dae10932cc9f16b3fdac5819b5adfe007f4", "reasoning": { "effort": "minimal" }, @@ -220,7 +221,7 @@ "initial_calls": 240, "input_usd_per_million": "0.037000000", "output_usd_per_million": "0.17000000", - "protocol_id": "25ac3627e4795573424e184ab883ed80e3abb3bfadb0adba94a8e5b1ce659725", + "protocol_id": "ae278f4f5e91f668448c4b921633e999ba90a6cd9cc3819c8ee4beac8484d38a", "reasoning": { "effort": "low" }, @@ -238,7 +239,7 @@ "initial_calls": 240, "input_usd_per_million": "0.03000000", "output_usd_per_million": "0.13000000", - "protocol_id": "06243dd30d6fb69a30c4b24a9b891b95a25975d7d41b7071a474e4365e856169", + "protocol_id": "2b45bec6a0271fb1390ea36e196eef7e75a53112c1dcd1e40b1b60b7631a9d7d", "reasoning": { "effort": "low" }, @@ -256,9 +257,11 @@ "initial_calls": 240, "input_usd_per_million": "2.000000", "output_usd_per_million": "8.000000", - "protocol_id": "f4054a623913b60fa79beee9aacec15967aa058434835937e4e8af276adf9557", - "reasoning": null, - "reasoning_label": "not advertised (optional; omitted)", + "protocol_id": "0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298", + "reasoning": { + "enabled": false + }, + "reasoning_label": "disabled (optional, parameter advertised without efforts)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl", "structured_output": true }, @@ -272,9 +275,11 @@ "initial_calls": 240, "input_usd_per_million": "1.1000000", "output_usd_per_million": "4.4000000", - "protocol_id": "26873a2a9f3d6530a06368113b903fbc80f673849ed605f1f49963f1f7abb638", - "reasoning": null, - "reasoning_label": "not advertised (optional; omitted)", + "protocol_id": "eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367", + "reasoning": { + "enabled": false + }, + "reasoning_label": "disabled (optional, parameter advertised without efforts)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl", "structured_output": true }, @@ -288,7 +293,7 @@ "initial_calls": 240, "input_usd_per_million": "2.000000", "output_usd_per_million": "8.000000", - "protocol_id": "9a3bb10b7179275010d35313de4a9d8021ca42abb0901414450ef0c6a42f2aea", + "protocol_id": "1f6ad277fcb151d83dd6b0c9d2f9ed5d2cb7466a09b30c4368ec0c74a8d72854", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl", @@ -304,7 +309,7 @@ "initial_calls": 240, "input_usd_per_million": "0.4000000", "output_usd_per_million": "1.6000000", - "protocol_id": "59833d4bac77233732006cfada3136d655274a1fba463a994c4bd3107348b642", + "protocol_id": "ce0bb6d5ff9f57496b0f677bb5ac47fc196380e084b457b43f6203d2b7d1b6d1", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl", @@ -320,7 +325,7 @@ "initial_calls": 240, "input_usd_per_million": "0.1000000", "output_usd_per_million": "0.4000000", - "protocol_id": "91b5e653465ea8562e62d0a5c76895e4dfa9cc774dc777f04268be6643f04cb9", + "protocol_id": "0efb6591fbdbe87758af494f18fc0a5a1a4897451dc2bc05f45681e9b5d73758", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl", @@ -336,9 +341,11 @@ "initial_calls": 240, "input_usd_per_million": "1.1000000", "output_usd_per_million": "4.4000000", - "protocol_id": "2a0f497f04f98ce1a6ff53412018721609b993a091a0da4e7c339b5a72b1859c", - "reasoning": null, - "reasoning_label": "not advertised (optional; omitted)", + "protocol_id": "3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6", + "reasoning": { + "enabled": false + }, + "reasoning_label": "disabled (optional, parameter advertised without efforts)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl", "structured_output": true }, @@ -352,7 +359,7 @@ "initial_calls": 240, "input_usd_per_million": "2.5000000", "output_usd_per_million": "10.00000", - "protocol_id": "e558a7bb4fdfbf9c4ddc6cd2f7505850a0779b7f8c02cde480dff902e709674b", + "protocol_id": "8c3bd515d1ac0e9ee598baa0ad42dc958817e6ad14cad7a73ecd710f71b003a6", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl", @@ -368,7 +375,7 @@ "initial_calls": 240, "input_usd_per_million": "2.5000000", "output_usd_per_million": "10.00000", - "protocol_id": "ac49e31fe7abf39eadc966e042c72652b063d56d698cf437b76c53411e0a6979", + "protocol_id": "64377e05e282d8d9d5dc425635c6acdbf87ae8115aadfb39503c876ad5bbb9ac", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl", @@ -384,7 +391,7 @@ "initial_calls": 240, "input_usd_per_million": "0.15000000", "output_usd_per_million": "0.6000000", - "protocol_id": "1662dce1e899d6b1060b2dfaa19034935ff097e77114ae42936eb5b52ddc7296", + "protocol_id": "388af881427c7032c7a863540b75d7255292a83ee553dcb99eb2d07c7a1766df", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl", @@ -400,7 +407,7 @@ "initial_calls": 240, "input_usd_per_million": "2.5000000", "output_usd_per_million": "10.00000", - "protocol_id": "11a70a549953db006a77522bc587bcf9464f9c601afc7b3d09b0119ade903d99", + "protocol_id": "eb4b8ad31eea6d046b4caff46dd0bfb56f113e65111be3871e02ac2e351086e6", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o_requests.jsonl", @@ -416,7 +423,7 @@ "initial_calls": 240, "input_usd_per_million": "1.000000", "output_usd_per_million": "2.000000", - "protocol_id": "3b8af120356177e5159f76ee1736a89695006f54cd3055b2c2588ec9050374c6", + "protocol_id": "b0abbb72e5db3b335516292c218a5937a9bc4979fa19b616301b6b969f48c585", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-0613_requests.jsonl", @@ -432,7 +439,7 @@ "initial_calls": 240, "input_usd_per_million": "1.5000000", "output_usd_per_million": "2.000000", - "protocol_id": "74b3ad3c9d06e892a0c08968419e330a7a8a91e5db7393e23c31f41c97139967", + "protocol_id": "3ccab07621e008211a4fed870619e72e1f6568a12aedf955dfd0d8cfc45016a2", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-instruct_requests.jsonl", @@ -448,7 +455,7 @@ "initial_calls": 240, "input_usd_per_million": "3.000000", "output_usd_per_million": "4.000000", - "protocol_id": "07953ad5d1b968e61ff14f3ef8337e407d128295918855be42d7a0993acb0e32", + "protocol_id": "dfb7fd3c21a1cd90320c754b37d1d78e71d72eec2a1d3a1dfa3af320f7f5b189", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-16k_requests.jsonl", @@ -464,7 +471,7 @@ "initial_calls": 240, "input_usd_per_million": "0.5000000", "output_usd_per_million": "1.5000000", - "protocol_id": "446953742af525795631082baf5f3df86027c1710d1003ca6054f6e2a48dc18f", + "protocol_id": "b2f3aaa32e5e21e9e89a146ffe48e158fcae7b9b65937fb5bbf64df747bea23c", "reasoning": null, "reasoning_label": "not advertised", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo_requests.jsonl", @@ -480,7 +487,7 @@ "initial_calls": 240, "input_usd_per_million": "0.75000000", "output_usd_per_million": "3.75000000", - "protocol_id": "1e42dbfead6255ca8efab8f522b5c7684cd8eec6035059bfdf1fdac5d40dc3ae", + "protocol_id": "c1a151216652af24dca0a97c85b5d24f45fea8a28caa7075cdf09afb3fb646e7", "reasoning": { "effort": "low" }, @@ -498,7 +505,7 @@ "initial_calls": 240, "input_usd_per_million": "0.75000000", "output_usd_per_million": "3.75000000", - "protocol_id": "4524a90e3320fb15103d7468b5e8088ab99c73051539240d0c1ae14d353d7eac", + "protocol_id": "4e8a491693ca97ff0bf6ce704a893651a86b1756b0e56a5ebe5fac05c32c6e4f", "reasoning": { "effort": "minimal" }, @@ -516,7 +523,7 @@ "initial_calls": 240, "input_usd_per_million": "0.3000000", "output_usd_per_million": "2.5000000", - "protocol_id": "0f44f6e4ebb50ece2ada37707d79b988d36fc55a207f23235bb38740e7726b01", + "protocol_id": "24e7aee4cbb982a7b66cd14cbcf6e08a8f2ed80cb6cb132566b5ab1deed17245", "reasoning": { "effort": "minimal" }, @@ -534,7 +541,7 @@ "initial_calls": 240, "input_usd_per_million": "1.5000000", "output_usd_per_million": "9.000000", - "protocol_id": "fab233d2ad1205b7fa4c2346875105509219846ceeeb7fd292d209615473c231", + "protocol_id": "4c5c5789977f56c13209d2babb47b3c79cd747fb2cd7cea568bd22301779d16f", "reasoning": { "effort": "minimal" }, @@ -552,7 +559,7 @@ "initial_calls": 240, "input_usd_per_million": "0.25000000", "output_usd_per_million": "1.5000000", - "protocol_id": "c325d9a280a083263f3705cf8d7fa8d4733877cb733526c2fe2290948d3ac4a4", + "protocol_id": "c9517bdfa0b53ccbf0aecb228cee50dc0f9c7abe4e5bf600132afa368b6eac8e", "reasoning": { "effort": "minimal" }, @@ -560,22 +567,6 @@ "records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl", "structured_output": true }, - { - "cache_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemma-4-26b-a4b-it_cache.json", - "completion_only_ceiling_usd": "0.0737280", - "conservative_reserve_usd": "0.24330240", - "created_utc": "2026-04-03", - "group": "Google", - "id": "google/gemma-4-26b-a4b-it", - "initial_calls": 240, - "input_usd_per_million": "0.09000000", - "output_usd_per_million": "0.3000000", - "protocol_id": "08d96b55bfee15010d2b9053f40c21d8e1d23d912ef84c645eb810eaa8803f1a", - "reasoning": null, - "reasoning_label": "not advertised (optional; omitted)", - "records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemma-4-26b-a4b-it_requests.jsonl", - "structured_output": true - }, { "cache_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_cache.json", "completion_only_ceiling_usd": "0.3686400", @@ -586,7 +577,7 @@ "initial_calls": 240, "input_usd_per_million": "0.25000000", "output_usd_per_million": "1.5000000", - "protocol_id": "4a85c0754b9bd2903c6cc978b2ded49ca381bc0e90199bff6eaf53e42d03e879", + "protocol_id": "851a22259abbe2da4fea19fd61b4fb7ce6e0406866e7bf2b54908f91d84cd860", "reasoning": { "effort": "minimal" }, @@ -604,7 +595,7 @@ "initial_calls": 240, "input_usd_per_million": "0.5000000", "output_usd_per_million": "3.000000", - "protocol_id": "1c64e2725648b42570c0c0bd443655de81ee55c0477bdc835ff4c1087a76cded", + "protocol_id": "1529b1845000ea4830c5c7c88e898b6e1d75abaab1aa4a0d86951330cec17200", "reasoning": { "effort": "minimal" }, @@ -622,9 +613,11 @@ "initial_calls": 240, "input_usd_per_million": "0.1000000", "output_usd_per_million": "0.4000000", - "protocol_id": "f5dbbdf216a4373df531ec7fef4a02fcec6e89609aa13553bb1f903dc8e0a43b", - "reasoning": null, - "reasoning_label": "not advertised (optional; omitted)", + "protocol_id": "028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7", + "reasoning": { + "enabled": false + }, + "reasoning_label": "disabled (optional, parameter advertised without efforts)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl", "structured_output": true }, @@ -638,9 +631,11 @@ "initial_calls": 240, "input_usd_per_million": "0.3000000", "output_usd_per_million": "2.5000000", - "protocol_id": "4d8587d1f7a72ee96a78e2693aec222cba6d313c5fcc978b82b27508d736a223", - "reasoning": null, - "reasoning_label": "not advertised (optional; omitted)", + "protocol_id": "0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428", + "reasoning": { + "enabled": false + }, + "reasoning_label": "disabled (optional, parameter advertised without efforts)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl", "structured_output": true }, @@ -654,7 +649,7 @@ "initial_calls": 240, "input_usd_per_million": "1.25000000", "output_usd_per_million": "4.25000000", - "protocol_id": "a716da6000adc3d9792e112c55d516e21c49fbe46acac6517d8b740e81b30e69", + "protocol_id": "5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2", "reasoning": { "effort": "minimal" }, @@ -672,7 +667,7 @@ "initial_calls": 240, "input_usd_per_million": "1.25000000", "output_usd_per_million": "4.25000000", - "protocol_id": "c4cd8eedebc35b380c52d7c559f87c293623a04858b8acfeb091145340dde72d", + "protocol_id": "fcb191133bbd11dce82ac53f4d6b253a15c316d5abd71c54896f9dfdf644b964", "reasoning": { "effort": "minimal" }, diff --git a/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md b/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md index 7941442..39afcc7 100644 --- a/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md +++ b/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md @@ -11,6 +11,7 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It - final response: `Respond with ONLY a JSON object with exactly one key named answer. Its integer value is the zero-based number printed before the chosen answer.` - rescue response: `Return only the one-key object required by the response schema. No explanation.` - strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID +- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests ## Spend checks before any later dispatch @@ -22,55 +23,55 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It ## Ordered panels -The order is Grok, OpenAI, Google, then Muse. `minimal` is used when catalog metadata advertises it; otherwise `low`; disabled is used only when the catalog says reasoning is optional and accepts `none`. +The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field. | family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger | |---|---|---:|---:|---:|---|---|---|---:|---:|---:|---| -| Grok | `x-ai/grok-4.6` | 2026-08-12 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `f1dcc6e5e4651add7df5473e80ba5b5a5ec733d788a6a791625776501f8605d6` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.6_requests.jsonl` | -| Grok | `x-ai/grok-4.5` | 2026-07-08 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `fdb4ff16d7879cd840722b7f84654da2e67bc388cdf0b52ea64a1d12b487de18` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.5_requests.jsonl` | -| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"effort": "low"}` (low) | yes | `b9bd5e38934aeae851dcd6227403c474e41c52ebb0bf599b112203b30403e718` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` | -| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"effort": "low"}` (low) | yes | `76c8b34537d9ec301ec168c8566849a0a0d26fffa088b70b8da09acb760130b1` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` | -| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"effort": "low"}` (low) | yes | `a25533b578ada960251c7fa244a98b86e68a054346d641b6215888a09c92b09e` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` | -| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"effort": "low"}` (low) | yes | `42e8c091233072fd0925ab8f5554bc01e23c5cf3051e09ba7b45970af189fb10` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` | -| OpenAI | `openai/gpt-5.2-chat` | 2025-12-10 | 1.75000000 | 14.000000 | `null` (not advertised) | yes | `6911c78f8c33d085e4d2e4a627c8e311059855cfb859b16a6a319185b6abb807` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl` | -| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"effort": "low"}` (low) | yes | `6d72c036f9f413fa500c0a2902750b0a2c29fc1145d77d3fa02ecf89b48054ff` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` | -| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"effort": "low"}` (low) | yes | `063cbfba0812ff35df231154d69a498e77cd240961d61f18913e6465c1ac1938` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` | -| OpenAI | `openai/gpt-5` | 2025-08-07 | 1.25000000 | 10.00000 | `{"effort": "minimal"}` (minimal) | yes | `48ffd6a149a5583355dd779b855e944ba26e4eab9b9cd4b5acd780cb3e3fb4e1` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5_requests.jsonl` | -| OpenAI | `openai/gpt-5-mini` | 2025-08-07 | 0.25000000 | 2.000000 | `{"effort": "minimal"}` (minimal) | yes | `00e8beddc47977c22afa2128d78af2c00acf8ddb87077f427bcb1360ed87a980` | 240 | USD 0.4915 | USD 1.5360 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5-mini_requests.jsonl` | -| OpenAI | `openai/gpt-oss-120b` | 2025-08-05 | 0.037000000 | 0.17000000 | `{"effort": "low"}` (low) | yes | `25ac3627e4795573424e184ab883ed80e3abb3bfadb0adba94a8e5b1ce659725` | 240 | USD 0.0418 | USD 0.1344 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-120b_requests.jsonl` | -| OpenAI | `openai/gpt-oss-20b` | 2025-08-05 | 0.03000000 | 0.13000000 | `{"effort": "low"}` (low) | yes | `06243dd30d6fb69a30c4b24a9b891b95a25975d7d41b7071a474e4365e856169` | 240 | USD 0.0319 | USD 0.1032 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-20b_requests.jsonl` | -| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `null` (not advertised (optional; omitted)) | yes | `f4054a623913b60fa79beee9aacec15967aa058434835937e4e8af276adf9557` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` | -| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `null` (not advertised (optional; omitted)) | yes | `26873a2a9f3d6530a06368113b903fbc80f673849ed605f1f49963f1f7abb638` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` | -| OpenAI | `openai/gpt-4.1` | 2025-04-14 | 2.000000 | 8.000000 | `null` (not advertised) | yes | `9a3bb10b7179275010d35313de4a9d8021ca42abb0901414450ef0c6a42f2aea` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl` | -| OpenAI | `openai/gpt-4.1-mini` | 2025-04-14 | 0.4000000 | 1.6000000 | `null` (not advertised) | yes | `59833d4bac77233732006cfada3136d655274a1fba463a994c4bd3107348b642` | 240 | USD 0.3932 | USD 1.2780 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl` | -| OpenAI | `openai/gpt-4.1-nano` | 2025-04-14 | 0.1000000 | 0.4000000 | `null` (not advertised) | yes | `91b5e653465ea8562e62d0a5c76895e4dfa9cc774dc777f04268be6643f04cb9` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl` | -| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `null` (not advertised (optional; omitted)) | yes | `2a0f497f04f98ce1a6ff53412018721609b993a091a0da4e7c339b5a72b1859c` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` | -| OpenAI | `openai/gpt-4o-2024-11-20` | 2024-11-20 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `e558a7bb4fdfbf9c4ddc6cd2f7505850a0779b7f8c02cde480dff902e709674b` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl` | -| OpenAI | `openai/gpt-4o-2024-08-06` | 2024-08-06 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `ac49e31fe7abf39eadc966e042c72652b063d56d698cf437b76c53411e0a6979` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl` | -| OpenAI | `openai/gpt-4o-mini` | 2024-07-18 | 0.15000000 | 0.6000000 | `null` (not advertised) | yes | `1662dce1e899d6b1060b2dfaa19034935ff097e77114ae42936eb5b52ddc7296` | 240 | USD 0.1475 | USD 0.4792 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl` | -| OpenAI | `openai/gpt-4o` | 2024-05-13 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `11a70a549953db006a77522bc587bcf9464f9c601afc7b3d09b0119ade903d99` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o_requests.jsonl` | -| OpenAI | `openai/gpt-3.5-turbo-0613` | 2024-01-25 | 1.000000 | 2.000000 | `null` (not advertised) | yes | `3b8af120356177e5159f76ee1736a89695006f54cd3055b2c2588ec9050374c6` | 240 | USD 0.4915 | USD 1.7203 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-0613_requests.jsonl` | -| OpenAI | `openai/gpt-3.5-turbo-instruct` | 2023-09-28 | 1.5000000 | 2.000000 | `null` (not advertised) | yes | `74b3ad3c9d06e892a0c08968419e330a7a8a91e5db7393e23c31f41c97139967` | 240 | USD 0.4915 | USD 1.8432 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-instruct_requests.jsonl` | -| OpenAI | `openai/gpt-3.5-turbo-16k` | 2023-08-28 | 3.000000 | 4.000000 | `null` (not advertised) | yes | `07953ad5d1b968e61ff14f3ef8337e407d128295918855be42d7a0993acb0e32` | 240 | USD 0.9830 | USD 3.6864 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-16k_requests.jsonl` | -| OpenAI | `openai/gpt-3.5-turbo` | 2023-05-28 | 0.5000000 | 1.5000000 | `null` (not advertised) | yes | `446953742af525795631082baf5f3df86027c1710d1003ca6054f6e2a48dc18f` | 240 | USD 0.3686 | USD 1.2288 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo_requests.jsonl` | -| Google | `google/gemini-3.8-flash` | 2026-09-02 | 0.75000000 | 3.75000000 | `{"effort": "low"}` (low) | yes | `1e42dbfead6255ca8efab8f522b5c7684cd8eec6035059bfdf1fdac5d40dc3ae` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.8-flash_requests.jsonl` | -| Google | `google/gemini-3.6-flash` | 2026-07-21 | 0.75000000 | 3.75000000 | `{"effort": "minimal"}` (minimal) | yes | `4524a90e3320fb15103d7468b5e8088ab99c73051539240d0c1ae14d353d7eac` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.6-flash_requests.jsonl` | -| Google | `google/gemini-3.5-flash-lite` | 2026-07-21 | 0.3000000 | 2.5000000 | `{"effort": "minimal"}` (minimal) | yes | `0f44f6e4ebb50ece2ada37707d79b988d36fc55a207f23235bb38740e7726b01` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash-lite_requests.jsonl` | -| Google | `google/gemini-3.5-flash` | 2026-05-19 | 1.5000000 | 9.000000 | `{"effort": "minimal"}` (minimal) | yes | `fab233d2ad1205b7fa4c2346875105509219846ceeeb7fd292d209615473c231` | 240 | USD 2.2118 | USD 7.0042 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash_requests.jsonl` | -| Google | `google/gemini-3.1-flash-lite` | 2026-05-07 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `c325d9a280a083263f3705cf8d7fa8d4733877cb733526c2fe2290948d3ac4a4` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl` | -| Google | `google/gemma-4-26b-a4b-it` | 2026-04-03 | 0.09000000 | 0.3000000 | `null` (not advertised (optional; omitted)) | yes | `08d96b55bfee15010d2b9053f40c21d8e1d23d912ef84c645eb810eaa8803f1a` | 240 | USD 0.0737 | USD 0.2433 | `slop/research/wvs/20260917_direct_choice/priority/google__gemma-4-26b-a4b-it_requests.jsonl` | -| Google | `google/gemini-3.1-flash-lite-preview` | 2026-03-03 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `4a85c0754b9bd2903c6cc978b2ded49ca381bc0e90199bff6eaf53e42d03e879` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_requests.jsonl` | -| Google | `google/gemini-3-flash-preview` | 2025-12-17 | 0.5000000 | 3.000000 | `{"effort": "minimal"}` (minimal) | yes | `1c64e2725648b42570c0c0bd443655de81ee55c0477bdc835ff4c1087a76cded` | 240 | USD 0.7373 | USD 2.3347 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3-flash-preview_requests.jsonl` | -| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `null` (not advertised (optional; omitted)) | yes | `f5dbbdf216a4373df531ec7fef4a02fcec6e89609aa13553bb1f903dc8e0a43b` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` | -| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `null` (not advertised (optional; omitted)) | yes | `4d8587d1f7a72ee96a78e2693aec222cba6d313c5fcc978b82b27508d736a223` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` | -| Muse | `meta/muse-spark-1.2` | 2026-08-05 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `a716da6000adc3d9792e112c55d516e21c49fbe46acac6517d8b740e81b30e69` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.2_requests.jsonl` | -| Muse | `meta/muse-spark-1.1` | 2026-07-16 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `c4cd8eedebc35b380c52d7c559f87c293623a04858b8acfeb091145340dde72d` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.1_requests.jsonl` | +| Grok | `x-ai/grok-4.6` | 2026-08-12 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.6_requests.jsonl` | +| Grok | `x-ai/grok-4.5` | 2026-07-08 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `d1362e8b42a4e2ff54cbd66be8c6a65225f94c9679931299f0e5e359e4065c63` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.5_requests.jsonl` | +| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` | +| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` | +| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` | +| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` | +| OpenAI | `openai/gpt-5.2-chat` | 2025-12-10 | 1.75000000 | 14.000000 | `null` (not advertised) | yes | `4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl` | +| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` | +| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` | +| OpenAI | `openai/gpt-5` | 2025-08-07 | 1.25000000 | 10.00000 | `{"effort": "minimal"}` (minimal) | yes | `4f7fe97763a5822831b2244e72b0d4cc01ff46da55090fed2485c79f97e4bdf0` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5_requests.jsonl` | +| OpenAI | `openai/gpt-5-mini` | 2025-08-07 | 0.25000000 | 2.000000 | `{"effort": "minimal"}` (minimal) | yes | `5b115c7341384a37117745f735367dae10932cc9f16b3fdac5819b5adfe007f4` | 240 | USD 0.4915 | USD 1.5360 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5-mini_requests.jsonl` | +| OpenAI | `openai/gpt-oss-120b` | 2025-08-05 | 0.037000000 | 0.17000000 | `{"effort": "low"}` (low) | yes | `ae278f4f5e91f668448c4b921633e999ba90a6cd9cc3819c8ee4beac8484d38a` | 240 | USD 0.0418 | USD 0.1344 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-120b_requests.jsonl` | +| OpenAI | `openai/gpt-oss-20b` | 2025-08-05 | 0.03000000 | 0.13000000 | `{"effort": "low"}` (low) | yes | `2b45bec6a0271fb1390ea36e196eef7e75a53112c1dcd1e40b1b60b7631a9d7d` | 240 | USD 0.0319 | USD 0.1032 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-20b_requests.jsonl` | +| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` | +| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` | +| OpenAI | `openai/gpt-4.1` | 2025-04-14 | 2.000000 | 8.000000 | `null` (not advertised) | yes | `1f6ad277fcb151d83dd6b0c9d2f9ed5d2cb7466a09b30c4368ec0c74a8d72854` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl` | +| OpenAI | `openai/gpt-4.1-mini` | 2025-04-14 | 0.4000000 | 1.6000000 | `null` (not advertised) | yes | `ce0bb6d5ff9f57496b0f677bb5ac47fc196380e084b457b43f6203d2b7d1b6d1` | 240 | USD 0.3932 | USD 1.2780 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl` | +| OpenAI | `openai/gpt-4.1-nano` | 2025-04-14 | 0.1000000 | 0.4000000 | `null` (not advertised) | yes | `0efb6591fbdbe87758af494f18fc0a5a1a4897451dc2bc05f45681e9b5d73758` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl` | +| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` | +| OpenAI | `openai/gpt-4o-2024-11-20` | 2024-11-20 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `8c3bd515d1ac0e9ee598baa0ad42dc958817e6ad14cad7a73ecd710f71b003a6` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl` | +| OpenAI | `openai/gpt-4o-2024-08-06` | 2024-08-06 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `64377e05e282d8d9d5dc425635c6acdbf87ae8115aadfb39503c876ad5bbb9ac` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl` | +| OpenAI | `openai/gpt-4o-mini` | 2024-07-18 | 0.15000000 | 0.6000000 | `null` (not advertised) | yes | `388af881427c7032c7a863540b75d7255292a83ee553dcb99eb2d07c7a1766df` | 240 | USD 0.1475 | USD 0.4792 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl` | +| OpenAI | `openai/gpt-4o` | 2024-05-13 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `eb4b8ad31eea6d046b4caff46dd0bfb56f113e65111be3871e02ac2e351086e6` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o_requests.jsonl` | +| OpenAI | `openai/gpt-3.5-turbo-0613` | 2024-01-25 | 1.000000 | 2.000000 | `null` (not advertised) | yes | `b0abbb72e5db3b335516292c218a5937a9bc4979fa19b616301b6b969f48c585` | 240 | USD 0.4915 | USD 1.7203 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-0613_requests.jsonl` | +| OpenAI | `openai/gpt-3.5-turbo-instruct` | 2023-09-28 | 1.5000000 | 2.000000 | `null` (not advertised) | yes | `3ccab07621e008211a4fed870619e72e1f6568a12aedf955dfd0d8cfc45016a2` | 240 | USD 0.4915 | USD 1.8432 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-instruct_requests.jsonl` | +| OpenAI | `openai/gpt-3.5-turbo-16k` | 2023-08-28 | 3.000000 | 4.000000 | `null` (not advertised) | yes | `dfb7fd3c21a1cd90320c754b37d1d78e71d72eec2a1d3a1dfa3af320f7f5b189` | 240 | USD 0.9830 | USD 3.6864 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-16k_requests.jsonl` | +| OpenAI | `openai/gpt-3.5-turbo` | 2023-05-28 | 0.5000000 | 1.5000000 | `null` (not advertised) | yes | `b2f3aaa32e5e21e9e89a146ffe48e158fcae7b9b65937fb5bbf64df747bea23c` | 240 | USD 0.3686 | USD 1.2288 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo_requests.jsonl` | +| Google | `google/gemini-3.8-flash` | 2026-09-02 | 0.75000000 | 3.75000000 | `{"effort": "low"}` (low) | yes | `c1a151216652af24dca0a97c85b5d24f45fea8a28caa7075cdf09afb3fb646e7` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.8-flash_requests.jsonl` | +| Google | `google/gemini-3.6-flash` | 2026-07-21 | 0.75000000 | 3.75000000 | `{"effort": "minimal"}` (minimal) | yes | `4e8a491693ca97ff0bf6ce704a893651a86b1756b0e56a5ebe5fac05c32c6e4f` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.6-flash_requests.jsonl` | +| Google | `google/gemini-3.5-flash-lite` | 2026-07-21 | 0.3000000 | 2.5000000 | `{"effort": "minimal"}` (minimal) | yes | `24e7aee4cbb982a7b66cd14cbcf6e08a8f2ed80cb6cb132566b5ab1deed17245` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash-lite_requests.jsonl` | +| Google | `google/gemini-3.5-flash` | 2026-05-19 | 1.5000000 | 9.000000 | `{"effort": "minimal"}` (minimal) | yes | `4c5c5789977f56c13209d2babb47b3c79cd747fb2cd7cea568bd22301779d16f` | 240 | USD 2.2118 | USD 7.0042 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash_requests.jsonl` | +| Google | `google/gemini-3.1-flash-lite` | 2026-05-07 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `c9517bdfa0b53ccbf0aecb228cee50dc0f9c7abe4e5bf600132afa368b6eac8e` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl` | +| Google | `google/gemini-3.1-flash-lite-preview` | 2026-03-03 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `851a22259abbe2da4fea19fd61b4fb7ce6e0406866e7bf2b54908f91d84cd860` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_requests.jsonl` | +| Google | `google/gemini-3-flash-preview` | 2025-12-17 | 0.5000000 | 3.000000 | `{"effort": "minimal"}` (minimal) | yes | `1529b1845000ea4830c5c7c88e898b6e1d75abaab1aa4a0d86951330cec17200` | 240 | USD 0.7373 | USD 2.3347 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3-flash-preview_requests.jsonl` | +| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` | +| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` | +| Muse | `meta/muse-spark-1.2` | 2026-08-05 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.2_requests.jsonl` | +| Muse | `meta/muse-spark-1.1` | 2026-07-16 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `fcb191133bbd11dce82ac53f4d6b253a15c316d5abd71c54896f9dfdf644b964` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.1_requests.jsonl` | ## Exclusions - Already plotted dense-rated IDs are not repeated in this prepared direct-choice list, including Grok 4.3/4.20, GPT-6 Astra, GPT-5.6 Sol, GPT-5.5, GPT-5.4, GPT-5.3 Chat, Gemini 3.7 Flash, Gemini 2.5 Pro, and Muse 1.3. - GPT-5 Nano is retained as a completed dense-rated protocol diagnostic, not silently relabelled as a direct-choice panel. - Pro/Fast, batch/free aliases, output price above USD 15/M, and code/image/audio/safeguard/multi-agent entries remain excluded. `Flash` is included where it is a general chat model. +- `google/gemma-4-26b-a4b-it` is excluded: it is Gemma, not an identified member of the requested Gemini release series. - `openai/o4-mini-high` and `openai/o3-mini-high` are excluded because their catalog entries advertise only `high` reasoning, not the registered minimal/low policy. - The deferred Qwen/GLM/Mistral shortlist remains outside this priority manifest until a direct-choice expansion decision is made. diff --git a/slop/research/wvs/20260917_direct_choice/priority_preflight_smoke.log b/slop/research/wvs/20260917_direct_choice/priority_preflight_smoke.log index 15a02ff..bc8ffb4 100644 --- a/slop/research/wvs/20260917_direct_choice/priority_preflight_smoke.log +++ b/slop/research/wvs/20260917_direct_choice/priority_preflight_smoke.log @@ -1,16 +1,22 @@ $ python scripts/wvs_direct_choice_priority.py --write-manifest wrote slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md and slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json $ python scripts/wvs_direct_choice_priority.py --model x-ai/grok-4.6 --smoke -smoke: x-ai/grok-4.6, 240 direct-choice requests, protocol=f1dcc6e5e4651add7df5473e80ba5b5a5ec733d788a6a791625776501f8605d6 +smoke: x-ai/grok-4.6, 240 direct-choice requests, protocol=34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6 smoke: reasoning={'effort': 'low'}, reserve=USD 4.915200 +$ python scripts/wvs_direct_choice_priority.py --model openai/gpt-5.6-luna --smoke +smoke: openai/gpt-5.6-luna, 240 direct-choice requests, protocol=8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95 +smoke: reasoning={'enabled': False}, reserve=USD 0.9338880 +$ python scripts/wvs_direct_choice_priority.py --model openai/o3 --smoke +smoke: openai/o3, 240 direct-choice requests, protocol=0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298 +smoke: reasoning={'enabled': False}, reserve=USD 6.389760 $ python scripts/wvs_direct_choice_priority.py --model openai/gpt-5.2-chat --smoke -smoke: openai/gpt-5.2-chat, 240 direct-choice requests, protocol=6911c78f8c33d085e4d2e4a627c8e311059855cfb859b16a6a319185b6abb807 +smoke: openai/gpt-5.2-chat, 240 direct-choice requests, protocol=4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca smoke: reasoning=None, reserve=USD 10.75200000 -$ python scripts/wvs_direct_choice_priority.py --model google/gemini-3.6-flash --smoke -smoke: google/gemini-3.6-flash, 240 direct-choice requests, protocol=4524a90e3320fb15103d7468b5e8088ab99c73051539240d0c1ae14d353d7eac -smoke: reasoning={'effort': 'minimal'}, reserve=USD 2.94912000 $ python scripts/wvs_direct_choice_priority.py --model meta/muse-spark-1.2 --smoke -smoke: meta/muse-spark-1.2, 240 direct-choice requests, protocol=a716da6000adc3d9792e112c55d516e21c49fbe46acac6517d8b740e81b30e69 +smoke: meta/muse-spark-1.2, 240 direct-choice requests, protocol=5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2 smoke: reasoning={'effort': 'minimal'}, reserve=USD 3.44064000 -verified manifest: 39 unique structured direct-choice protocols, 240 calls each, output <= USD 15/M -wrapper syntax: valid; no --run execution invoked +$ python scripts/wvs_direct_choice_priority_smoke.py +smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings +smoke: synthetic first request exits before remaining 239 and writes no cache +smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false +verified manifest: 38 distinct strict-schema 240-call protocols; Gemma excluded; fail-fast probe encoded diff --git a/src/moralmaps/read_direct_choice.py b/src/moralmaps/read_direct_choice.py index f659b21..126f6fd 100644 --- a/src/moralmaps/read_direct_choice.py +++ b/src/moralmaps/read_direct_choice.py @@ -142,7 +142,8 @@ def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_pe prompt_instruction: str = _PROMPT_INSTRUCTION_OWN_VIEW, answer_instruction: str = _ANSWER_INSTRUCTION_WITH_EXAMPLE, rescue_instruction: str | None = None, - plan_override: list[dict] | None = None) -> str: + plan_override: list[dict] | None = None, + fail_fast_first_request: bool = False) -> str: plan = _plan(items, samples_per_order, answer_instruction) if plan_override is None else plan_override protocol = { "schema": 1, @@ -163,6 +164,8 @@ def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_pe if plan_override is not None: protocol.pop("samples_per_order") protocol["samples_per_item"] = {item["id"]: sum(request["item_id"] == item["id"] for request in plan) for item in items} + if fail_fast_first_request: + protocol["fail_fast_first_request"] = True encoded = json.dumps(protocol, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() return hashlib.sha256(encoded).hexdigest() @@ -180,7 +183,8 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order prompt_instruction: str = _PROMPT_INSTRUCTION_OWN_VIEW, answer_instruction: str = _ANSWER_INSTRUCTION_WITH_EXAMPLE, rescue_instruction: str | None = None, - plan_override: list[dict] | None = None) -> dict: + plan_override: list[dict] | None = None, + fail_fast_first_request: bool = False) -> dict: """Sample exactly one selected option per prompt, including canonical and reversed option orders. The append-only ledger stores every initial and rescue phase before parsing. A cache entry is written only @@ -197,6 +201,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order reasoning=reasoning, structured_output=structured_output, prompt_instruction=prompt_instruction, answer_instruction=answer_instruction, rescue_instruction=rescue_instruction, plan_override=plan_override, + fail_fast_first_request=fail_fast_first_request, ) cache_file = Path(cache_path) cache = json.loads(cache_file.read_text()) if cache_file.exists() else {"schema": 1, "completed": {}} @@ -211,7 +216,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order "model": model, "samples_per_order": samples_per_order, "temperature": temperature, "max_tokens": max_tokens, "concurrency": concurrency, "request_timeout": request_timeout, "reasoning": reasoning, "structured_output": structured_output, - "prompt_instruction": prompt_instruction, + "prompt_instruction": prompt_instruction, "fail_fast_first_request": fail_fast_first_request, } if plan_override is not None: settings.pop("samples_per_order") @@ -242,9 +247,10 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order response_format = _choice_schema(item["n"]) payload = { "model": model, "messages": [{"role": "user", "content": request["prompt"]}], - "temperature": temperature, "max_tokens": max_tokens, "reasoning": reasoning, - "response_format": response_format, + "temperature": temperature, "max_tokens": max_tokens, "response_format": response_format, } + if reasoning is not None: + payload["reasoning"] = reasoning phase = "initial" async with semaphore: try: @@ -266,8 +272,10 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order {"role": "assistant", "content": assistant_tail}, {"role": "user", "content": rescue_instruction or _force_choice(item["n"])}, ], "temperature": temperature, "max_tokens": max(max_tokens, 2048), - "reasoning": reasoning, "response_format": response_format, + "response_format": response_format, } + if reasoning is not None: + rescue_payload["reasoning"] = reasoning _append_record(records, {"event": "request_started", "phase": phase, **request_meta, "payload": rescue_payload, "initial_response_message": message}) response = await asyncio.wait_for(openrouter_request(rescue_payload), timeout=request_timeout) @@ -283,9 +291,23 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order "error_type": type(exc).__name__, "error": str(exc)}) return {"text": None, "rescued": phase == "rescue", "error": f"{type(exc).__name__}: {exc}"} - return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan))) + if not fail_fast_first_request: + return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan))) + first = await call(0, plan[0]) + if first["error"] is not None: + return [first] + remaining = await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan[1:], start=1))) + return [first, *remaining] results = asyncio.run(run_all()) + if fail_fast_first_request and results[0]["error"] is not None: + summary = { + "run_id": run_id, "protocol_id": protocol_id, "model": model, "settings": settings, + "planned_requests": len(plan), "failed_requests": 1, "rescued_requests": int(results[0]["rescued"]), + "complete": False, "items": [], "failure": results[0]["error"], + } + _append_record(records, {"event": "run_finished", "construct": "direct_choice", **summary}) + raise RuntimeError(f"first scheduled request failed before remaining {len(plan) - 1} requests: {results[0]['error']}") by_item = {item["id"]: [] for item in items} failed = 0 rescues = 0