mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-23 13:30:23 +08:00
Refine direct-choice reasoning preflight
Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
co-authored by
PI[gpt-5.6-terra]
PI[gpt-5.6-terra]
parent
f3603b4500
commit
fbca560b96
@@ -70,9 +70,9 @@ def reasoning_setting(model: dict) -> tuple[dict | None, str]:
|
||||
efforts = set(metadata.get("supported_efforts", []))
|
||||
optional = not metadata.get("mandatory")
|
||||
if optional and "none" in efforts:
|
||||
return {"enabled": False}, "disabled (optional, none advertised)"
|
||||
return {"effort": "none"}, "disabled (optional, none advertised)"
|
||||
if optional and not efforts and "reasoning" in model["supported_parameters"]:
|
||||
return {"enabled": False}, "disabled (optional, parameter advertised without efforts)"
|
||||
return {"enabled": False}, "unverified compatibility probe (optional reasoning parameter; no efforts advertised)"
|
||||
if "minimal" in efforts:
|
||||
return {"effort": "minimal"}, "minimal"
|
||||
if "low" in efforts:
|
||||
@@ -178,7 +178,7 @@ def write_manifest(priority: list[dict]) -> None:
|
||||
f"- final response: `{ANSWER_INSTRUCTION}`",
|
||||
f"- rescue response: `{RESCUE_INSTRUCTION}`",
|
||||
"- strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID",
|
||||
"- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests",
|
||||
"- compatibility probe: run scheduled sample 0 first; a configuration/request failure or a final parse-invalid response after rescue records a failed run and exits before the other 239 requests",
|
||||
"",
|
||||
"## Spend checks before any later dispatch",
|
||||
"",
|
||||
@@ -190,7 +190,8 @@ def write_manifest(priority: list[dict]) -> None:
|
||||
"",
|
||||
"## Ordered panels",
|
||||
"",
|
||||
"The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.",
|
||||
"The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` send `reasoning.effort=none`, as documented by OpenRouter. Otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list uses an explicitly labelled, unverified `enabled:false` compatibility probe only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.",
|
||||
"- source for `effort=none` and mandatory-model rejection: <https://openrouter.ai/docs/guides/best-practices/reasoning-tokens>, fetched 2026-09-17; the saved catalog's `supported_efforts` remains the exact per-model source.",
|
||||
"",
|
||||
"| family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger |",
|
||||
"|---|---|---:|---:|---:|---|---|---|---:|---:|---:|---|",
|
||||
|
||||
@@ -23,13 +23,15 @@ def main() -> None:
|
||||
prepared = entries()
|
||||
assert len(prepared) == 38
|
||||
assert all(row["initial_calls"] == 240 and row["structured_output"] for row in prepared)
|
||||
assert {str(row["reasoning"]) for row in prepared} == {"None", "{'enabled': False}", "{'effort': 'low'}", "{'effort': 'minimal'}"}
|
||||
assert {str(row["reasoning"]) for row in prepared} == {
|
||||
"None", "{'enabled': False}", "{'effort': 'none'}", "{'effort': 'low'}", "{'effort': 'minimal'}",
|
||||
}
|
||||
assert all(row["protocol_id"] for row in prepared)
|
||||
print("smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings")
|
||||
print("smoke: 38 unique 240-call protocols cover omitted, effort-none, unverified disabled, low, and minimal reasoning settings")
|
||||
|
||||
pilot_items = items()
|
||||
request_plan = schedule(pilot_items)
|
||||
for reasoning, expected in ((None, None), ({"enabled": False}, {"enabled": False})):
|
||||
for reasoning, expected in ((None, None), ({"effort": "none"}, {"effort": "none"}), ({"enabled": False}, {"enabled": False})):
|
||||
calls = []
|
||||
|
||||
async def fail(payload: dict) -> dict:
|
||||
@@ -63,8 +65,44 @@ def main() -> None:
|
||||
assert not cache.exists()
|
||||
finally:
|
||||
reader.openrouter_request = original
|
||||
print("smoke: synthetic first request exits before remaining 239 and writes no cache")
|
||||
print("smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false")
|
||||
print("smoke: synthetic request failure exits before remaining 239 and writes no cache")
|
||||
|
||||
calls = []
|
||||
|
||||
async def invalid_json(payload: dict) -> dict:
|
||||
calls.append(payload)
|
||||
return {"choices": [{"message": {"content": "not a JSON answer"}}]}
|
||||
|
||||
original = reader.openrouter_request
|
||||
reader.openrouter_request = invalid_json
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
records = Path(directory) / "records.jsonl"
|
||||
cache = Path(directory) / "cache.json"
|
||||
try:
|
||||
reader.read_items_direct_choice(
|
||||
"test/model", pilot_items, samples_per_order=10, temperature=1.0,
|
||||
max_tokens=1024, concurrency=1, request_timeout=1, reasoning=None,
|
||||
structured_output=True, records_path=records, cache_path=cache,
|
||||
prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION,
|
||||
rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan,
|
||||
fail_fast_first_request=True,
|
||||
)
|
||||
except RuntimeError as error:
|
||||
assert "first scheduled request failed before remaining 239 requests" in str(error)
|
||||
else:
|
||||
raise AssertionError("synthetic parse-invalid first response did not abort")
|
||||
events = [json.loads(line) for line in records.read_text().splitlines()]
|
||||
assert Counter(event["event"] for event in events) == Counter({
|
||||
"run_started": 1, "request_started": 2, "request_completed": 2,
|
||||
"answer_parsed": 1, "run_finished": 1,
|
||||
})
|
||||
parsed = next(event for event in events if event["event"] == "answer_parsed")
|
||||
assert not parsed["parsed"] and len(calls) == 2 and all("reasoning" not in payload for payload in calls) and not cache.exists()
|
||||
finally:
|
||||
reader.openrouter_request = original
|
||||
print("smoke: synthetic parse-invalid initial plus rescue records false parse, omits None reasoning in both payloads, then exits before remaining 239 and writes no cache")
|
||||
print("smoke: None omits reasoning; effort-none follows catalog support; enabled=false stays explicitly unverified")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -61,9 +61,9 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.2000000",
|
||||
"output_usd_per_million": "1.2000000",
|
||||
"protocol_id": "8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95",
|
||||
"protocol_id": "fe4d5389063d0d6c93c48f21ae162ec1d12155f6c328aab1d2e66caf7a9e522a",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
"effort": "none"
|
||||
},
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl",
|
||||
@@ -79,9 +79,9 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "2.000000",
|
||||
"output_usd_per_million": "12.000000",
|
||||
"protocol_id": "08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305",
|
||||
"protocol_id": "9860a6923ef9c54111e3764280588b4b43395e706329a3a286d2d7db94794ff3",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
"effort": "none"
|
||||
},
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl",
|
||||
@@ -97,9 +97,9 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.2000000",
|
||||
"output_usd_per_million": "1.25000000",
|
||||
"protocol_id": "35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5",
|
||||
"protocol_id": "8aeb61dba0aba732cc7f3e8f75bb21c0cc692f9d79c5444e972de8f5a57ce83e",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
"effort": "none"
|
||||
},
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl",
|
||||
@@ -115,9 +115,9 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.75000000",
|
||||
"output_usd_per_million": "4.5000000",
|
||||
"protocol_id": "c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f",
|
||||
"protocol_id": "a130ac7f62e407eff6fe0ac8a78c6e0cac603a74a6477d7d5ea1cc9e2b828d89",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
"effort": "none"
|
||||
},
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl",
|
||||
@@ -149,9 +149,9 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.75000000",
|
||||
"output_usd_per_million": "14.000000",
|
||||
"protocol_id": "1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111",
|
||||
"protocol_id": "bb6322d4ac157f48cc0581c9b4a434f6b77e2f4b37e835799fbdaff845ebaa1f",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
"effort": "none"
|
||||
},
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl",
|
||||
@@ -167,9 +167,9 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.25000000",
|
||||
"output_usd_per_million": "10.00000",
|
||||
"protocol_id": "2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05",
|
||||
"protocol_id": "80c8c1e4782ee2d85cddfaede339b5f981cfbb2e08525de0ff8fe0b60d9fbe37",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
"effort": "none"
|
||||
},
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl",
|
||||
@@ -261,7 +261,7 @@
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -279,7 +279,7 @@
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -345,7 +345,7 @@
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -617,7 +617,7 @@
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -635,7 +635,7 @@
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"reasoning_label": "unverified compatibility probe (optional reasoning parameter; no efforts advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
|
||||
@@ -11,7 +11,7 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It
|
||||
- final response: `Respond with ONLY a JSON object with exactly one key named answer. Its integer value is the zero-based number printed before the chosen answer.`
|
||||
- rescue response: `Return only the one-key object required by the response schema. No explanation.`
|
||||
- strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID
|
||||
- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests
|
||||
- compatibility probe: run scheduled sample 0 first; a configuration/request failure or a final parse-invalid response after rescue records a failed run and exits before the other 239 requests
|
||||
|
||||
## Spend checks before any later dispatch
|
||||
|
||||
@@ -23,29 +23,30 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It
|
||||
|
||||
## Ordered panels
|
||||
|
||||
The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.
|
||||
The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` send `reasoning.effort=none`, as documented by OpenRouter. Otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list uses an explicitly labelled, unverified `enabled:false` compatibility probe only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.
|
||||
- source for `effort=none` and mandatory-model rejection: <https://openrouter.ai/docs/guides/best-practices/reasoning-tokens>, fetched 2026-09-17; the saved catalog's `supported_efforts` remains the exact per-model source.
|
||||
|
||||
| family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger |
|
||||
|---|---|---:|---:|---:|---|---|---|---:|---:|---:|---|
|
||||
| Grok | `x-ai/grok-4.6` | 2026-08-12 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.6_requests.jsonl` |
|
||||
| Grok | `x-ai/grok-4.5` | 2026-07-08 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `d1362e8b42a4e2ff54cbd66be8c6a65225f94c9679931299f0e5e359e4065c63` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.5_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `fe4d5389063d0d6c93c48f21ae162ec1d12155f6c328aab1d2e66caf7a9e522a` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `9860a6923ef9c54111e3764280588b4b43395e706329a3a286d2d7db94794ff3` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `8aeb61dba0aba732cc7f3e8f75bb21c0cc692f9d79c5444e972de8f5a57ce83e` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `a130ac7f62e407eff6fe0ac8a78c6e0cac603a74a6477d7d5ea1cc9e2b828d89` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.2-chat` | 2025-12-10 | 1.75000000 | 14.000000 | `null` (not advertised) | yes | `4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `bb6322d4ac157f48cc0581c9b4a434f6b77e2f4b37e835799fbdaff845ebaa1f` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"effort": "none"}` (disabled (optional, none advertised)) | yes | `80c8c1e4782ee2d85cddfaede339b5f981cfbb2e08525de0ff8fe0b60d9fbe37` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5` | 2025-08-07 | 1.25000000 | 10.00000 | `{"effort": "minimal"}` (minimal) | yes | `4f7fe97763a5822831b2244e72b0d4cc01ff46da55090fed2485c79f97e4bdf0` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5-mini` | 2025-08-07 | 0.25000000 | 2.000000 | `{"effort": "minimal"}` (minimal) | yes | `5b115c7341384a37117745f735367dae10932cc9f16b3fdac5819b5adfe007f4` | 240 | USD 0.4915 | USD 1.5360 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-oss-120b` | 2025-08-05 | 0.037000000 | 0.17000000 | `{"effort": "low"}` (low) | yes | `ae278f4f5e91f668448c4b921633e999ba90a6cd9cc3819c8ee4beac8484d38a` | 240 | USD 0.0418 | USD 0.1344 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-120b_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-oss-20b` | 2025-08-05 | 0.03000000 | 0.13000000 | `{"effort": "low"}` (low) | yes | `2b45bec6a0271fb1390ea36e196eef7e75a53112c1dcd1e40b1b60b7631a9d7d` | 240 | USD 0.0319 | USD 0.1032 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-20b_requests.jsonl` |
|
||||
| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` |
|
||||
| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` |
|
||||
| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4.1` | 2025-04-14 | 2.000000 | 8.000000 | `null` (not advertised) | yes | `1f6ad277fcb151d83dd6b0c9d2f9ed5d2cb7466a09b30c4368ec0c74a8d72854` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4.1-mini` | 2025-04-14 | 0.4000000 | 1.6000000 | `null` (not advertised) | yes | `ce0bb6d5ff9f57496b0f677bb5ac47fc196380e084b457b43f6203d2b7d1b6d1` | 240 | USD 0.3932 | USD 1.2780 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4.1-nano` | 2025-04-14 | 0.1000000 | 0.4000000 | `null` (not advertised) | yes | `0efb6591fbdbe87758af494f18fc0a5a1a4897451dc2bc05f45681e9b5d73758` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl` |
|
||||
| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o-2024-11-20` | 2024-11-20 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `8c3bd515d1ac0e9ee598baa0ad42dc958817e6ad14cad7a73ecd710f71b003a6` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o-2024-08-06` | 2024-08-06 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `64377e05e282d8d9d5dc425635c6acdbf87ae8115aadfb39503c876ad5bbb9ac` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o-mini` | 2024-07-18 | 0.15000000 | 0.6000000 | `null` (not advertised) | yes | `388af881427c7032c7a863540b75d7255292a83ee553dcb99eb2d07c7a1766df` | 240 | USD 0.1475 | USD 0.4792 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl` |
|
||||
@@ -61,8 +62,8 @@ The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none
|
||||
| Google | `google/gemini-3.1-flash-lite` | 2026-05-07 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `c9517bdfa0b53ccbf0aecb228cee50dc0f9c7abe4e5bf600132afa368b6eac8e` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl` |
|
||||
| Google | `google/gemini-3.1-flash-lite-preview` | 2026-03-03 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `851a22259abbe2da4fea19fd61b4fb7ce6e0406866e7bf2b54908f91d84cd860` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_requests.jsonl` |
|
||||
| Google | `google/gemini-3-flash-preview` | 2025-12-17 | 0.5000000 | 3.000000 | `{"effort": "minimal"}` (minimal) | yes | `1529b1845000ea4830c5c7c88e898b6e1d75abaab1aa4a0d86951330cec17200` | 240 | USD 0.7373 | USD 2.3347 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3-flash-preview_requests.jsonl` |
|
||||
| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` |
|
||||
| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` |
|
||||
| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` |
|
||||
| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `{"enabled": false}` (unverified compatibility probe (optional reasoning parameter; no efforts advertised)) | yes | `0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` |
|
||||
| Muse | `meta/muse-spark-1.2` | 2026-08-05 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.2_requests.jsonl` |
|
||||
| Muse | `meta/muse-spark-1.1` | 2026-07-16 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `fcb191133bbd11dce82ac53f4d6b253a15c316d5abd71c54896f9dfdf644b964` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.1_requests.jsonl` |
|
||||
|
||||
|
||||
@@ -4,8 +4,8 @@ $ python scripts/wvs_direct_choice_priority.py --model x-ai/grok-4.6 --smoke
|
||||
smoke: x-ai/grok-4.6, 240 direct-choice requests, protocol=34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6
|
||||
smoke: reasoning={'effort': 'low'}, reserve=USD 4.915200
|
||||
$ python scripts/wvs_direct_choice_priority.py --model openai/gpt-5.6-luna --smoke
|
||||
smoke: openai/gpt-5.6-luna, 240 direct-choice requests, protocol=8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95
|
||||
smoke: reasoning={'enabled': False}, reserve=USD 0.9338880
|
||||
smoke: openai/gpt-5.6-luna, 240 direct-choice requests, protocol=fe4d5389063d0d6c93c48f21ae162ec1d12155f6c328aab1d2e66caf7a9e522a
|
||||
smoke: reasoning={'effort': 'none'}, reserve=USD 0.9338880
|
||||
$ python scripts/wvs_direct_choice_priority.py --model openai/o3 --smoke
|
||||
smoke: openai/o3, 240 direct-choice requests, protocol=0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298
|
||||
smoke: reasoning={'enabled': False}, reserve=USD 6.389760
|
||||
@@ -16,7 +16,8 @@ $ python scripts/wvs_direct_choice_priority.py --model meta/muse-spark-1.2 --smo
|
||||
smoke: meta/muse-spark-1.2, 240 direct-choice requests, protocol=5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2
|
||||
smoke: reasoning={'effort': 'minimal'}, reserve=USD 3.44064000
|
||||
$ python scripts/wvs_direct_choice_priority_smoke.py
|
||||
smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings
|
||||
smoke: synthetic first request exits before remaining 239 and writes no cache
|
||||
smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false
|
||||
verified manifest: 38 distinct strict-schema 240-call protocols; Gemma excluded; fail-fast probe encoded
|
||||
smoke: 38 unique 240-call protocols cover omitted, effort-none, unverified disabled, low, and minimal reasoning settings
|
||||
smoke: synthetic request failure exits before remaining 239 and writes no cache
|
||||
smoke: synthetic parse-invalid initial plus rescue records false parse, omits None reasoning in both payloads, then exits before remaining 239 and writes no cache
|
||||
smoke: None omits reasoning; effort-none follows catalog support; enabled=false stays explicitly unverified
|
||||
verified manifest: 38 strict-schema 240-call entries equal regenerated protocol records; Gemma excluded; fail-fast probe encoded
|
||||
|
||||
@@ -192,7 +192,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
|
||||
"""
|
||||
assert samples_per_order > 0
|
||||
assert temperature > 0
|
||||
assert reasoning is None or reasoning == {"enabled": False} or reasoning.get("effort") in {"minimal", "low"}
|
||||
assert reasoning is None or reasoning == {"enabled": False} or reasoning.get("effort") in {"minimal", "low", "none"}
|
||||
assert structured_output
|
||||
plan = _plan(items, samples_per_order, answer_instruction) if plan_override is None else plan_override
|
||||
protocol_id = direct_choice_protocol_identity(
|
||||
@@ -285,11 +285,13 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
|
||||
raise ValueError(f"expected one rescue choice, got {len(response['choices'])}")
|
||||
text = response["choices"][0]["message"].get("content") or ""
|
||||
rescued = True
|
||||
return {"text": text, "rescued": rescued, "error": None}
|
||||
if fail_fast_first_request and sequence == 0 and _parse_choice(text, item["n"]) is None:
|
||||
return {"text": text, "rescued": rescued, "error": "ParseError: first response remained invalid after rescue", "parse_invalid": True}
|
||||
return {"text": text, "rescued": rescued, "error": None, "parse_invalid": False}
|
||||
except Exception as exc:
|
||||
_append_record(records, {"event": "request_failed", "phase": phase, **request_meta,
|
||||
"error_type": type(exc).__name__, "error": str(exc)})
|
||||
return {"text": None, "rescued": phase == "rescue", "error": f"{type(exc).__name__}: {exc}"}
|
||||
return {"text": None, "rescued": phase == "rescue", "error": f"{type(exc).__name__}: {exc}", "parse_invalid": False}
|
||||
|
||||
if not fail_fast_first_request:
|
||||
return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan)))
|
||||
@@ -301,6 +303,16 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
|
||||
|
||||
results = asyncio.run(run_all())
|
||||
if fail_fast_first_request and results[0]["error"] is not None:
|
||||
if results[0]["parse_invalid"]:
|
||||
request = plan[0]
|
||||
_append_record(records, {
|
||||
"event": "answer_parsed", "run_id": run_id, "protocol_id": protocol_id,
|
||||
"construct": "direct_choice", "model": model, "item_id": request["item_id"],
|
||||
"sample": request["sample"], "order_name": request["order_name"],
|
||||
"repetition": request["repetition"], "presented_order": request["presented_order"],
|
||||
"text": results[0]["text"], "parsed": False, "presented_choice": None,
|
||||
"canonical_choice": None,
|
||||
})
|
||||
summary = {
|
||||
"run_id": run_id, "protocol_id": protocol_id, "model": model, "settings": settings,
|
||||
"planned_requests": len(plan), "failed_requests": 1, "rescued_requests": int(results[0]["rescued"]),
|
||||
|
||||
Reference in New Issue
Block a user