From fbca560b96aac651f0eeb0ac96680830792e0072 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:24:56 +0800 Subject: [PATCH] Refine direct-choice reasoning preflight Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com> --- scripts/wvs_direct_choice_priority.py | 9 ++-- scripts/wvs_direct_choice_priority_smoke.py | 48 +++++++++++++++++-- .../priority_direct_choice_manifest.json | 34 ++++++------- .../priority_direct_choice_manifest.md | 27 ++++++----- .../priority_preflight_smoke.log | 13 ++--- src/moralmaps/read_direct_choice.py | 18 +++++-- 6 files changed, 101 insertions(+), 48 deletions(-) diff --git a/scripts/wvs_direct_choice_priority.py b/scripts/wvs_direct_choice_priority.py index c8c0d40..8d8fe9c 100644 --- a/scripts/wvs_direct_choice_priority.py +++ b/scripts/wvs_direct_choice_priority.py @@ -70,9 +70,9 @@ def reasoning_setting(model: dict) -> tuple[dict | None, str]: efforts = set(metadata.get("supported_efforts", [])) optional = not metadata.get("mandatory") if optional and "none" in efforts: - return {"enabled": False}, "disabled (optional, none advertised)" + return {"effort": "none"}, "disabled (optional, none advertised)" if optional and not efforts and "reasoning" in model["supported_parameters"]: - return {"enabled": False}, "disabled (optional, parameter advertised without efforts)" + return {"enabled": False}, "unverified compatibility probe (optional reasoning parameter; no efforts advertised)" if "minimal" in efforts: return {"effort": "minimal"}, "minimal" if "low" in efforts: @@ -178,7 +178,7 @@ def write_manifest(priority: list[dict]) -> None: f"- final response: `{ANSWER_INSTRUCTION}`", f"- rescue response: `{RESCUE_INSTRUCTION}`", "- strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID", - "- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests", + "- compatibility probe: run scheduled sample 0 first; a configuration/request failure or a final parse-invalid response after rescue records a failed run and exits before the other 239 requests", "", "## Spend checks before any later dispatch", "", @@ -190,7 +190,8 @@ def write_manifest(priority: list[dict]) -> None: "", "## Ordered panels", "", - "The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.", + "The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` send `reasoning.effort=none`, as documented by OpenRouter. Otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list uses an explicitly labelled, unverified `enabled:false` compatibility probe only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.", + "- source for `effort=none` and mandatory-model rejection: , fetched 2026-09-17; the saved catalog's `supported_efforts` remains the exact per-model source.", "", "| family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger |", "|---|---|---:|---:|---:|---|---|---|---:|---:|---:|---|", diff --git a/scripts/wvs_direct_choice_priority_smoke.py b/scripts/wvs_direct_choice_priority_smoke.py index 5450c50..2d39f85 100644 --- a/scripts/wvs_direct_choice_priority_smoke.py +++ b/scripts/wvs_direct_choice_priority_smoke.py @@ -23,13 +23,15 @@ def main() -> None: prepared = entries() assert len(prepared) == 38 assert all(row["initial_calls"] == 240 and row["structured_output"] for row in prepared) - assert {str(row["reasoning"]) for row in prepared} == {"None", "{'enabled': False}", "{'effort': 'low'}", "{'effort': 'minimal'}"} + assert {str(row["reasoning"]) for row in prepared} == { + "None", "{'enabled': False}", "{'effort': 'none'}", "{'effort': 'low'}", "{'effort': 'minimal'}", + } assert all(row["protocol_id"] for row in prepared) - print("smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings") + print("smoke: 38 unique 240-call protocols cover omitted, effort-none, unverified disabled, low, and minimal reasoning settings") pilot_items = items() request_plan = schedule(pilot_items) - for reasoning, expected in ((None, None), ({"enabled": False}, {"enabled": False})): + for reasoning, expected in ((None, None), ({"effort": "none"}, {"effort": "none"}), ({"enabled": False}, {"enabled": False})): calls = [] async def fail(payload: dict) -> dict: @@ -63,8 +65,44 @@ def main() -> None: assert not cache.exists() finally: reader.openrouter_request = original - print("smoke: synthetic first request exits before remaining 239 and writes no cache") - print("smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false") + print("smoke: synthetic request failure exits before remaining 239 and writes no cache") + + calls = [] + + async def invalid_json(payload: dict) -> dict: + calls.append(payload) + return {"choices": [{"message": {"content": "not a JSON answer"}}]} + + original = reader.openrouter_request + reader.openrouter_request = invalid_json + try: + with tempfile.TemporaryDirectory() as directory: + records = Path(directory) / "records.jsonl" + cache = Path(directory) / "cache.json" + try: + reader.read_items_direct_choice( + "test/model", pilot_items, samples_per_order=10, temperature=1.0, + max_tokens=1024, concurrency=1, request_timeout=1, reasoning=None, + structured_output=True, records_path=records, cache_path=cache, + prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION, + rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan, + fail_fast_first_request=True, + ) + except RuntimeError as error: + assert "first scheduled request failed before remaining 239 requests" in str(error) + else: + raise AssertionError("synthetic parse-invalid first response did not abort") + events = [json.loads(line) for line in records.read_text().splitlines()] + assert Counter(event["event"] for event in events) == Counter({ + "run_started": 1, "request_started": 2, "request_completed": 2, + "answer_parsed": 1, "run_finished": 1, + }) + parsed = next(event for event in events if event["event"] == "answer_parsed") + assert not parsed["parsed"] and len(calls) == 2 and all("reasoning" not in payload for payload in calls) and not cache.exists() + finally: + reader.openrouter_request = original + print("smoke: synthetic parse-invalid initial plus rescue records false parse, omits None reasoning in both payloads, then exits before remaining 239 and writes no cache") + print("smoke: None omits reasoning; effort-none follows catalog support; enabled=false stays explicitly unverified") if __name__ == "__main__": diff --git a/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json b/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json index ef26e49..a3d72dc 100644 --- a/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json +++ b/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json @@ -61,9 +61,9 @@ "initial_calls": 240, "input_usd_per_million": "0.2000000", "output_usd_per_million": "1.2000000", - "protocol_id": "8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95", + "protocol_id": "fe4d5389063d0d6c93c48f21ae162ec1d12155f6c328aab1d2e66caf7a9e522a", "reasoning": { - "enabled": false + "effort": "none" }, "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl", @@ -79,9 +79,9 @@ "initial_calls": 240, "input_usd_per_million": "2.000000", "output_usd_per_million": "12.000000", - "protocol_id": "08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305", + "protocol_id": "9860a6923ef9c54111e3764280588b4b43395e706329a3a286d2d7db94794ff3", "reasoning": { - "enabled": false + "effort": "none" }, "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl", @@ -97,9 +97,9 @@ "initial_calls": 240, "input_usd_per_million": "0.2000000", "output_usd_per_million": "1.25000000", - "protocol_id": "35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5", + "protocol_id": "8aeb61dba0aba732cc7f3e8f75bb21c0cc692f9d79c5444e972de8f5a57ce83e", "reasoning": { - "enabled": false + "effort": "none" }, "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl", @@ -115,9 +115,9 @@ "initial_calls": 240, "input_usd_per_million": "0.75000000", "output_usd_per_million": "4.5000000", - "protocol_id": "c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f", + "protocol_id": "a130ac7f62e407eff6fe0ac8a78c6e0cac603a74a6477d7d5ea1cc9e2b828d89", "reasoning": { - "enabled": false + "effort": "none" }, "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl", @@ -149,9 +149,9 @@ "initial_calls": 240, "input_usd_per_million": "1.75000000", "output_usd_per_million": "14.000000", - "protocol_id": "1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111", + "protocol_id": "bb6322d4ac157f48cc0581c9b4a434f6b77e2f4b37e835799fbdaff845ebaa1f", "reasoning": { - "enabled": false + "effort": "none" }, "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl", @@ -167,9 +167,9 @@ "initial_calls": 240, "input_usd_per_million": "1.25000000", "output_usd_per_million": "10.00000", - "protocol_id": "2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05", + "protocol_id": "80c8c1e4782ee2d85cddfaede339b5f981cfbb2e08525de0ff8fe0b60d9fbe37", "reasoning": { - "enabled": false + "effort": "none" }, "reasoning_label": "disabled (optional, none advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl", @@ -261,7 +261,7 @@ "reasoning": { "enabled": false }, - "reasoning_label": "disabled (optional, parameter advertised without efforts)", + "reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl", "structured_output": true }, @@ -279,7 +279,7 @@ "reasoning": { "enabled": false }, - "reasoning_label": "disabled (optional, parameter advertised without efforts)", + "reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl", "structured_output": true }, @@ -345,7 +345,7 @@ "reasoning": { "enabled": false }, - "reasoning_label": "disabled (optional, parameter advertised without efforts)", + "reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl", "structured_output": true }, @@ -617,7 +617,7 @@ "reasoning": { "enabled": false }, - "reasoning_label": "disabled (optional, parameter advertised without efforts)", + "reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl", "structured_output": true }, @@ -635,7 +635,7 @@ "reasoning": { "enabled": false }, - "reasoning_label": "disabled (optional, parameter advertised without efforts)", + "reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)", "records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl", "structured_output": true }, diff --git a/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md b/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md index 39afcc7..c130f4c 100644 --- a/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md +++ b/slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md @@ -11,7 +11,7 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It - final response: `Respond with ONLY a JSON object with exactly one key named answer. Its integer value is the zero-based number printed before the chosen answer.` - rescue response: `Return only the one-key object required by the response schema. No explanation.` - strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID -- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests +- compatibility probe: run scheduled sample 0 first; a configuration/request failure or a final parse-invalid response after rescue records a failed run and exits before the other 239 requests ## Spend checks before any later dispatch @@ -23,29 +23,30 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It ## Ordered panels -The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field. +The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` send `reasoning.effort=none`, as documented by OpenRouter. Otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list uses an explicitly labelled, unverified `enabled:false` compatibility probe only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field. +- source for `effort=none` and mandatory-model rejection: , fetched 2026-09-17; the saved catalog's `supported_efforts` remains the exact per-model source. | family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger | |---|---|---:|---:|---:|---|---|---|---:|---:|---:|---| | Grok | `x-ai/grok-4.6` | 2026-08-12 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.6_requests.jsonl` | | Grok | `x-ai/grok-4.5` | 2026-07-08 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `d1362e8b42a4e2ff54cbd66be8c6a65225f94c9679931299f0e5e359e4065c63` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.5_requests.jsonl` | -| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` | -| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` | -| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` | -| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` | +| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `fe4d5389063d0d6c93c48f21ae162ec1d12155f6c328aab1d2e66caf7a9e522a` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` | +| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `9860a6923ef9c54111e3764280588b4b43395e706329a3a286d2d7db94794ff3` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` | +| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `8aeb61dba0aba732cc7f3e8f75bb21c0cc692f9d79c5444e972de8f5a57ce83e` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` | +| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `a130ac7f62e407eff6fe0ac8a78c6e0cac603a74a6477d7d5ea1cc9e2b828d89` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` | | OpenAI | `openai/gpt-5.2-chat` | 2025-12-10 | 1.75000000 | 14.000000 | `null` (not advertised) | yes | `4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl` | -| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` | -| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` | +| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `bb6322d4ac157f48cc0581c9b4a434f6b77e2f4b37e835799fbdaff845ebaa1f` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` | +| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `80c8c1e4782ee2d85cddfaede339b5f981cfbb2e08525de0ff8fe0b60d9fbe37` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` | | OpenAI | `openai/gpt-5` | 2025-08-07 | 1.25000000 | 10.00000 | `{"effort": "minimal"}` (minimal) | yes | `4f7fe97763a5822831b2244e72b0d4cc01ff46da55090fed2485c79f97e4bdf0` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5_requests.jsonl` | | OpenAI | `openai/gpt-5-mini` | 2025-08-07 | 0.25000000 | 2.000000 | `{"effort": "minimal"}` (minimal) | yes | `5b115c7341384a37117745f735367dae10932cc9f16b3fdac5819b5adfe007f4` | 240 | USD 0.4915 | USD 1.5360 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5-mini_requests.jsonl` | | OpenAI | `openai/gpt-oss-120b` | 2025-08-05 | 0.037000000 | 0.17000000 | `{"effort": "low"}` (low) | yes | `ae278f4f5e91f668448c4b921633e999ba90a6cd9cc3819c8ee4beac8484d38a` | 240 | USD 0.0418 | USD 0.1344 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-120b_requests.jsonl` | | OpenAI | `openai/gpt-oss-20b` | 2025-08-05 | 0.03000000 | 0.13000000 | `{"effort": "low"}` (low) | yes | `2b45bec6a0271fb1390ea36e196eef7e75a53112c1dcd1e40b1b60b7631a9d7d` | 240 | USD 0.0319 | USD 0.1032 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-20b_requests.jsonl` | -| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` | -| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` | +| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` | +| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` | | OpenAI | `openai/gpt-4.1` | 2025-04-14 | 2.000000 | 8.000000 | `null` (not advertised) | yes | `1f6ad277fcb151d83dd6b0c9d2f9ed5d2cb7466a09b30c4368ec0c74a8d72854` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl` | | OpenAI | `openai/gpt-4.1-mini` | 2025-04-14 | 0.4000000 | 1.6000000 | `null` (not advertised) | yes | `ce0bb6d5ff9f57496b0f677bb5ac47fc196380e084b457b43f6203d2b7d1b6d1` | 240 | USD 0.3932 | USD 1.2780 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl` | | OpenAI | `openai/gpt-4.1-nano` | 2025-04-14 | 0.1000000 | 0.4000000 | `null` (not advertised) | yes | `0efb6591fbdbe87758af494f18fc0a5a1a4897451dc2bc05f45681e9b5d73758` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl` | -| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` | +| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` | | OpenAI | `openai/gpt-4o-2024-11-20` | 2024-11-20 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `8c3bd515d1ac0e9ee598baa0ad42dc958817e6ad14cad7a73ecd710f71b003a6` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl` | | OpenAI | `openai/gpt-4o-2024-08-06` | 2024-08-06 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `64377e05e282d8d9d5dc425635c6acdbf87ae8115aadfb39503c876ad5bbb9ac` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl` | | OpenAI | `openai/gpt-4o-mini` | 2024-07-18 | 0.15000000 | 0.6000000 | `null` (not advertised) | yes | `388af881427c7032c7a863540b75d7255292a83ee553dcb99eb2d07c7a1766df` | 240 | USD 0.1475 | USD 0.4792 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl` | @@ -61,8 +62,8 @@ The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none | Google | `google/gemini-3.1-flash-lite` | 2026-05-07 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `c9517bdfa0b53ccbf0aecb228cee50dc0f9c7abe4e5bf600132afa368b6eac8e` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl` | | Google | `google/gemini-3.1-flash-lite-preview` | 2026-03-03 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `851a22259abbe2da4fea19fd61b4fb7ce6e0406866e7bf2b54908f91d84cd860` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_requests.jsonl` | | Google | `google/gemini-3-flash-preview` | 2025-12-17 | 0.5000000 | 3.000000 | `{"effort": "minimal"}` (minimal) | yes | `1529b1845000ea4830c5c7c88e898b6e1d75abaab1aa4a0d86951330cec17200` | 240 | USD 0.7373 | USD 2.3347 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3-flash-preview_requests.jsonl` | -| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` | -| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` | +| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` | +| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` | | Muse | `meta/muse-spark-1.2` | 2026-08-05 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.2_requests.jsonl` | | Muse | `meta/muse-spark-1.1` | 2026-07-16 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `fcb191133bbd11dce82ac53f4d6b253a15c316d5abd71c54896f9dfdf644b964` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.1_requests.jsonl` | diff --git a/slop/research/wvs/20260917_direct_choice/priority_preflight_smoke.log b/slop/research/wvs/20260917_direct_choice/priority_preflight_smoke.log index bc8ffb4..812b521 100644 --- a/slop/research/wvs/20260917_direct_choice/priority_preflight_smoke.log +++ b/slop/research/wvs/20260917_direct_choice/priority_preflight_smoke.log @@ -4,8 +4,8 @@ $ python scripts/wvs_direct_choice_priority.py --model x-ai/grok-4.6 --smoke smoke: x-ai/grok-4.6, 240 direct-choice requests, protocol=34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6 smoke: reasoning={'effort': 'low'}, reserve=USD 4.915200 $ python scripts/wvs_direct_choice_priority.py --model openai/gpt-5.6-luna --smoke -smoke: openai/gpt-5.6-luna, 240 direct-choice requests, protocol=8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95 -smoke: reasoning={'enabled': False}, reserve=USD 0.9338880 +smoke: openai/gpt-5.6-luna, 240 direct-choice requests, protocol=fe4d5389063d0d6c93c48f21ae162ec1d12155f6c328aab1d2e66caf7a9e522a +smoke: reasoning={'effort': 'none'}, reserve=USD 0.9338880 $ python scripts/wvs_direct_choice_priority.py --model openai/o3 --smoke smoke: openai/o3, 240 direct-choice requests, protocol=0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298 smoke: reasoning={'enabled': False}, reserve=USD 6.389760 @@ -16,7 +16,8 @@ $ python scripts/wvs_direct_choice_priority.py --model meta/muse-spark-1.2 --smo smoke: meta/muse-spark-1.2, 240 direct-choice requests, protocol=5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2 smoke: reasoning={'effort': 'minimal'}, reserve=USD 3.44064000 $ python scripts/wvs_direct_choice_priority_smoke.py -smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings -smoke: synthetic first request exits before remaining 239 and writes no cache -smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false -verified manifest: 38 distinct strict-schema 240-call protocols; Gemma excluded; fail-fast probe encoded +smoke: 38 unique 240-call protocols cover omitted, effort-none, unverified disabled, low, and minimal reasoning settings +smoke: synthetic request failure exits before remaining 239 and writes no cache +smoke: synthetic parse-invalid initial plus rescue records false parse, omits None reasoning in both payloads, then exits before remaining 239 and writes no cache +smoke: None omits reasoning; effort-none follows catalog support; enabled=false stays explicitly unverified +verified manifest: 38 strict-schema 240-call entries equal regenerated protocol records; Gemma excluded; fail-fast probe encoded diff --git a/src/moralmaps/read_direct_choice.py b/src/moralmaps/read_direct_choice.py index 126f6fd..12012d2 100644 --- a/src/moralmaps/read_direct_choice.py +++ b/src/moralmaps/read_direct_choice.py @@ -192,7 +192,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order """ assert samples_per_order > 0 assert temperature > 0 - assert reasoning is None or reasoning == {"enabled": False} or reasoning.get("effort") in {"minimal", "low"} + assert reasoning is None or reasoning == {"enabled": False} or reasoning.get("effort") in {"minimal", "low", "none"} assert structured_output plan = _plan(items, samples_per_order, answer_instruction) if plan_override is None else plan_override protocol_id = direct_choice_protocol_identity( @@ -285,11 +285,13 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order raise ValueError(f"expected one rescue choice, got {len(response['choices'])}") text = response["choices"][0]["message"].get("content") or "" rescued = True - return {"text": text, "rescued": rescued, "error": None} + if fail_fast_first_request and sequence == 0 and _parse_choice(text, item["n"]) is None: + return {"text": text, "rescued": rescued, "error": "ParseError: first response remained invalid after rescue", "parse_invalid": True} + return {"text": text, "rescued": rescued, "error": None, "parse_invalid": False} except Exception as exc: _append_record(records, {"event": "request_failed", "phase": phase, **request_meta, "error_type": type(exc).__name__, "error": str(exc)}) - return {"text": None, "rescued": phase == "rescue", "error": f"{type(exc).__name__}: {exc}"} + return {"text": None, "rescued": phase == "rescue", "error": f"{type(exc).__name__}: {exc}", "parse_invalid": False} if not fail_fast_first_request: return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan))) @@ -301,6 +303,16 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order results = asyncio.run(run_all()) if fail_fast_first_request and results[0]["error"] is not None: + if results[0]["parse_invalid"]: + request = plan[0] + _append_record(records, { + "event": "answer_parsed", "run_id": run_id, "protocol_id": protocol_id, + "construct": "direct_choice", "model": model, "item_id": request["item_id"], + "sample": request["sample"], "order_name": request["order_name"], + "repetition": request["repetition"], "presented_order": request["presented_order"], + "text": results[0]["text"], "parsed": False, "presented_choice": None, + "canonical_choice": None, + }) summary = { "run_id": run_id, "protocol_id": protocol_id, "model": model, "settings": settings, "planned_requests": len(plan), "failed_requests": 1, "rescued_requests": int(results[0]["rescued"]),