mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-21 13:10:52 +08:00
Harden direct-choice priority preflight
Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
co-authored by
PI[gpt-5.6-terra]
parent
82d992725b
commit
f3603b4500
@@ -47,7 +47,7 @@ GROUPS = {
|
||||
),
|
||||
"Google": (
|
||||
"google/gemini-3.8-flash", "google/gemini-3.6-flash", "google/gemini-3.5-flash-lite",
|
||||
"google/gemini-3.5-flash", "google/gemini-3.1-flash-lite", "google/gemma-4-26b-a4b-it",
|
||||
"google/gemini-3.5-flash", "google/gemini-3.1-flash-lite",
|
||||
"google/gemini-3.1-flash-lite-preview", "google/gemini-3-flash-preview",
|
||||
"google/gemini-2.5-flash-lite", "google/gemini-2.5-flash",
|
||||
),
|
||||
@@ -68,13 +68,16 @@ def reasoning_setting(model: dict) -> tuple[dict | None, str]:
|
||||
if metadata is None:
|
||||
return None, "not advertised"
|
||||
efforts = set(metadata.get("supported_efforts", []))
|
||||
optional = not metadata.get("mandatory")
|
||||
if optional and "none" in efforts:
|
||||
return {"enabled": False}, "disabled (optional, none advertised)"
|
||||
if optional and not efforts and "reasoning" in model["supported_parameters"]:
|
||||
return {"enabled": False}, "disabled (optional, parameter advertised without efforts)"
|
||||
if "minimal" in efforts:
|
||||
return {"effort": "minimal"}, "minimal"
|
||||
if "low" in efforts:
|
||||
return {"effort": "low"}, "low"
|
||||
if not metadata.get("mandatory") and "none" in efforts:
|
||||
return {"enabled": False}, "disabled (optional, none advertised)"
|
||||
if not metadata.get("mandatory") and not efforts:
|
||||
if optional and not efforts:
|
||||
return None, "not advertised (optional; omitted)"
|
||||
raise ValueError(f"no allowed minimal/low reasoning setting for {model['id']}: {metadata}")
|
||||
|
||||
@@ -100,7 +103,7 @@ def entry(model: dict, pilot_items: list[dict], request_plan: list[dict]) -> dic
|
||||
max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT,
|
||||
reasoning=reasoning, structured_output=True, prompt_instruction=PROMPT_INSTRUCTION,
|
||||
answer_instruction=ANSWER_INSTRUCTION, rescue_instruction=RESCUE_INSTRUCTION,
|
||||
plan_override=request_plan,
|
||||
plan_override=request_plan, fail_fast_first_request=True,
|
||||
)
|
||||
input_rate = rate_per_million(model, "prompt")
|
||||
output_rate = rate_per_million(model, "completion")
|
||||
@@ -152,6 +155,7 @@ def write_manifest(priority: list[dict]) -> None:
|
||||
"initial_calls_per_model": 240, "schedule": "balanced_cyclic_rotations",
|
||||
"prompt_instruction": PROMPT_INSTRUCTION, "answer_instruction": ANSWER_INSTRUCTION,
|
||||
"rescue_instruction": RESCUE_INSTRUCTION, "strict_structured_output": True,
|
||||
"fail_fast_first_request": True,
|
||||
},
|
||||
"stop_usd": {"priority_phase": str(PHASE_STOP_USD), "global": str(GLOBAL_STOP_USD)},
|
||||
"current_observed_cost_usd": str(current_cost),
|
||||
@@ -174,6 +178,7 @@ def write_manifest(priority: list[dict]) -> None:
|
||||
f"- final response: `{ANSWER_INSTRUCTION}`",
|
||||
f"- rescue response: `{RESCUE_INSTRUCTION}`",
|
||||
"- strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID",
|
||||
"- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests",
|
||||
"",
|
||||
"## Spend checks before any later dispatch",
|
||||
"",
|
||||
@@ -185,7 +190,7 @@ def write_manifest(priority: list[dict]) -> None:
|
||||
"",
|
||||
"## Ordered panels",
|
||||
"",
|
||||
"The order is Grok, OpenAI, Google, then Muse. `minimal` is used when catalog metadata advertises it; otherwise `low`; disabled is used only when the catalog says reasoning is optional and accepts `none`.",
|
||||
"The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.",
|
||||
"",
|
||||
"| family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger |",
|
||||
"|---|---|---:|---:|---:|---|---|---|---:|---:|---:|---|",
|
||||
@@ -204,6 +209,7 @@ def write_manifest(priority: list[dict]) -> None:
|
||||
"- Already plotted dense-rated IDs are not repeated in this prepared direct-choice list, including Grok 4.3/4.20, GPT-6 Astra, GPT-5.6 Sol, GPT-5.5, GPT-5.4, GPT-5.3 Chat, Gemini 3.7 Flash, Gemini 2.5 Pro, and Muse 1.3.",
|
||||
"- GPT-5 Nano is retained as a completed dense-rated protocol diagnostic, not silently relabelled as a direct-choice panel.",
|
||||
"- Pro/Fast, batch/free aliases, output price above USD 15/M, and code/image/audio/safeguard/multi-agent entries remain excluded. `Flash` is included where it is a general chat model.",
|
||||
"- `google/gemma-4-26b-a4b-it` is excluded: it is Gemma, not an identified member of the requested Gemini release series.",
|
||||
"- `openai/o4-mini-high` and `openai/o3-mini-high` are excluded because their catalog entries advertise only `high` reasoning, not the registered minimal/low policy.",
|
||||
"- The deferred Qwen/GLM/Mistral shortlist remains outside this priority manifest until a direct-choice expansion decision is made.",
|
||||
"",
|
||||
@@ -258,6 +264,7 @@ def main() -> None:
|
||||
structured_output=True, records_path=records_path, cache_path=cache_path,
|
||||
prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION,
|
||||
rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan,
|
||||
fail_fast_first_request=True,
|
||||
)
|
||||
if result["cached"]:
|
||||
print(f"priority direct-choice cache hit: {args.model}, protocol={result['protocol_id'][:12]}")
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Zero-network smoke checks for prepared direct-choice priority panels."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import tempfile
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
import moralmaps.read_direct_choice as reader
|
||||
from wvs_direct_choice_priority import entries
|
||||
from wvs_direct_choice_production_pilot import (
|
||||
ANSWER_INSTRUCTION,
|
||||
PROMPT_INSTRUCTION,
|
||||
RESCUE_INSTRUCTION,
|
||||
items,
|
||||
schedule,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
prepared = entries()
|
||||
assert len(prepared) == 38
|
||||
assert all(row["initial_calls"] == 240 and row["structured_output"] for row in prepared)
|
||||
assert {str(row["reasoning"]) for row in prepared} == {"None", "{'enabled': False}", "{'effort': 'low'}", "{'effort': 'minimal'}"}
|
||||
assert all(row["protocol_id"] for row in prepared)
|
||||
print("smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings")
|
||||
|
||||
pilot_items = items()
|
||||
request_plan = schedule(pilot_items)
|
||||
for reasoning, expected in ((None, None), ({"enabled": False}, {"enabled": False})):
|
||||
calls = []
|
||||
|
||||
async def fail(payload: dict) -> dict:
|
||||
calls.append(payload)
|
||||
raise RuntimeError("synthetic compatibility failure")
|
||||
|
||||
original = reader.openrouter_request
|
||||
reader.openrouter_request = fail
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
records = Path(directory) / "records.jsonl"
|
||||
cache = Path(directory) / "cache.json"
|
||||
try:
|
||||
reader.read_items_direct_choice(
|
||||
"test/model", pilot_items, samples_per_order=10, temperature=1.0,
|
||||
max_tokens=1024, concurrency=1, request_timeout=1, reasoning=reasoning,
|
||||
structured_output=True, records_path=records, cache_path=cache,
|
||||
prompt_instruction=PROMPT_INSTRUCTION, answer_instruction=ANSWER_INSTRUCTION,
|
||||
rescue_instruction=RESCUE_INSTRUCTION, plan_override=request_plan,
|
||||
fail_fast_first_request=True,
|
||||
)
|
||||
except RuntimeError as error:
|
||||
assert "first scheduled request failed before remaining 239 requests" in str(error)
|
||||
else:
|
||||
raise AssertionError("synthetic first-request failure did not abort")
|
||||
events = [json.loads(line) for line in records.read_text().splitlines()]
|
||||
assert Counter(event["event"] for event in events) == Counter({
|
||||
"run_started": 1, "request_started": 1, "request_failed": 1, "run_finished": 1,
|
||||
})
|
||||
assert len(calls) == 1 and calls[0].get("reasoning") == expected
|
||||
assert not cache.exists()
|
||||
finally:
|
||||
reader.openrouter_request = original
|
||||
print("smoke: synthetic first request exits before remaining 239 and writes no cache")
|
||||
print("smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -5,6 +5,7 @@
|
||||
"design": {
|
||||
"answer_instruction": "Respond with ONLY a JSON object with exactly one key named answer. Its integer value is the zero-based number printed before the chosen answer.",
|
||||
"construct": "direct_choice",
|
||||
"fail_fast_first_request": true,
|
||||
"initial_calls_per_model": 240,
|
||||
"items": 12,
|
||||
"prompt_instruction": "Think briefly about which ONE answer best reflects the values expressed by your behavior as this AI assistant. Do not analyze the survey design or discuss lacking personal beliefs. Answer immediately.",
|
||||
@@ -24,7 +25,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "2.000000",
|
||||
"output_usd_per_million": "6.000000",
|
||||
"protocol_id": "f1dcc6e5e4651add7df5473e80ba5b5a5ec733d788a6a791625776501f8605d6",
|
||||
"protocol_id": "34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
},
|
||||
@@ -42,7 +43,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "2.000000",
|
||||
"output_usd_per_million": "6.000000",
|
||||
"protocol_id": "fdb4ff16d7879cd840722b7f84654da2e67bc388cdf0b52ea64a1d12b487de18",
|
||||
"protocol_id": "d1362e8b42a4e2ff54cbd66be8c6a65225f94c9679931299f0e5e359e4065c63",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
},
|
||||
@@ -60,11 +61,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.2000000",
|
||||
"output_usd_per_million": "1.2000000",
|
||||
"protocol_id": "b9bd5e38934aeae851dcd6227403c474e41c52ebb0bf599b112203b30403e718",
|
||||
"protocol_id": "8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "low",
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -78,11 +79,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "2.000000",
|
||||
"output_usd_per_million": "12.000000",
|
||||
"protocol_id": "76c8b34537d9ec301ec168c8566849a0a0d26fffa088b70b8da09acb760130b1",
|
||||
"protocol_id": "08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "low",
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -96,11 +97,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.2000000",
|
||||
"output_usd_per_million": "1.25000000",
|
||||
"protocol_id": "a25533b578ada960251c7fa244a98b86e68a054346d641b6215888a09c92b09e",
|
||||
"protocol_id": "35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "low",
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -114,11 +115,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.75000000",
|
||||
"output_usd_per_million": "4.5000000",
|
||||
"protocol_id": "42e8c091233072fd0925ab8f5554bc01e23c5cf3051e09ba7b45970af189fb10",
|
||||
"protocol_id": "c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "low",
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -132,7 +133,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.75000000",
|
||||
"output_usd_per_million": "14.000000",
|
||||
"protocol_id": "6911c78f8c33d085e4d2e4a627c8e311059855cfb859b16a6a319185b6abb807",
|
||||
"protocol_id": "4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl",
|
||||
@@ -148,11 +149,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.75000000",
|
||||
"output_usd_per_million": "14.000000",
|
||||
"protocol_id": "6d72c036f9f413fa500c0a2902750b0a2c29fc1145d77d3fa02ecf89b48054ff",
|
||||
"protocol_id": "1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "low",
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -166,11 +167,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.25000000",
|
||||
"output_usd_per_million": "10.00000",
|
||||
"protocol_id": "063cbfba0812ff35df231154d69a498e77cd240961d61f18913e6465c1ac1938",
|
||||
"protocol_id": "2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "low",
|
||||
"reasoning_label": "disabled (optional, none advertised)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -184,7 +185,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.25000000",
|
||||
"output_usd_per_million": "10.00000",
|
||||
"protocol_id": "48ffd6a149a5583355dd779b855e944ba26e4eab9b9cd4b5acd780cb3e3fb4e1",
|
||||
"protocol_id": "4f7fe97763a5822831b2244e72b0d4cc01ff46da55090fed2485c79f97e4bdf0",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
@@ -202,7 +203,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.25000000",
|
||||
"output_usd_per_million": "2.000000",
|
||||
"protocol_id": "00e8beddc47977c22afa2128d78af2c00acf8ddb87077f427bcb1360ed87a980",
|
||||
"protocol_id": "5b115c7341384a37117745f735367dae10932cc9f16b3fdac5819b5adfe007f4",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
@@ -220,7 +221,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.037000000",
|
||||
"output_usd_per_million": "0.17000000",
|
||||
"protocol_id": "25ac3627e4795573424e184ab883ed80e3abb3bfadb0adba94a8e5b1ce659725",
|
||||
"protocol_id": "ae278f4f5e91f668448c4b921633e999ba90a6cd9cc3819c8ee4beac8484d38a",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
},
|
||||
@@ -238,7 +239,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.03000000",
|
||||
"output_usd_per_million": "0.13000000",
|
||||
"protocol_id": "06243dd30d6fb69a30c4b24a9b891b95a25975d7d41b7071a474e4365e856169",
|
||||
"protocol_id": "2b45bec6a0271fb1390ea36e196eef7e75a53112c1dcd1e40b1b60b7631a9d7d",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
},
|
||||
@@ -256,9 +257,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "2.000000",
|
||||
"output_usd_per_million": "8.000000",
|
||||
"protocol_id": "f4054a623913b60fa79beee9aacec15967aa058434835937e4e8af276adf9557",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised (optional; omitted)",
|
||||
"protocol_id": "0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -272,9 +275,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.1000000",
|
||||
"output_usd_per_million": "4.4000000",
|
||||
"protocol_id": "26873a2a9f3d6530a06368113b903fbc80f673849ed605f1f49963f1f7abb638",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised (optional; omitted)",
|
||||
"protocol_id": "eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -288,7 +293,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "2.000000",
|
||||
"output_usd_per_million": "8.000000",
|
||||
"protocol_id": "9a3bb10b7179275010d35313de4a9d8021ca42abb0901414450ef0c6a42f2aea",
|
||||
"protocol_id": "1f6ad277fcb151d83dd6b0c9d2f9ed5d2cb7466a09b30c4368ec0c74a8d72854",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl",
|
||||
@@ -304,7 +309,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.4000000",
|
||||
"output_usd_per_million": "1.6000000",
|
||||
"protocol_id": "59833d4bac77233732006cfada3136d655274a1fba463a994c4bd3107348b642",
|
||||
"protocol_id": "ce0bb6d5ff9f57496b0f677bb5ac47fc196380e084b457b43f6203d2b7d1b6d1",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl",
|
||||
@@ -320,7 +325,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.1000000",
|
||||
"output_usd_per_million": "0.4000000",
|
||||
"protocol_id": "91b5e653465ea8562e62d0a5c76895e4dfa9cc774dc777f04268be6643f04cb9",
|
||||
"protocol_id": "0efb6591fbdbe87758af494f18fc0a5a1a4897451dc2bc05f45681e9b5d73758",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl",
|
||||
@@ -336,9 +341,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.1000000",
|
||||
"output_usd_per_million": "4.4000000",
|
||||
"protocol_id": "2a0f497f04f98ce1a6ff53412018721609b993a091a0da4e7c339b5a72b1859c",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised (optional; omitted)",
|
||||
"protocol_id": "3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -352,7 +359,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "2.5000000",
|
||||
"output_usd_per_million": "10.00000",
|
||||
"protocol_id": "e558a7bb4fdfbf9c4ddc6cd2f7505850a0779b7f8c02cde480dff902e709674b",
|
||||
"protocol_id": "8c3bd515d1ac0e9ee598baa0ad42dc958817e6ad14cad7a73ecd710f71b003a6",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl",
|
||||
@@ -368,7 +375,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "2.5000000",
|
||||
"output_usd_per_million": "10.00000",
|
||||
"protocol_id": "ac49e31fe7abf39eadc966e042c72652b063d56d698cf437b76c53411e0a6979",
|
||||
"protocol_id": "64377e05e282d8d9d5dc425635c6acdbf87ae8115aadfb39503c876ad5bbb9ac",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl",
|
||||
@@ -384,7 +391,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.15000000",
|
||||
"output_usd_per_million": "0.6000000",
|
||||
"protocol_id": "1662dce1e899d6b1060b2dfaa19034935ff097e77114ae42936eb5b52ddc7296",
|
||||
"protocol_id": "388af881427c7032c7a863540b75d7255292a83ee553dcb99eb2d07c7a1766df",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl",
|
||||
@@ -400,7 +407,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "2.5000000",
|
||||
"output_usd_per_million": "10.00000",
|
||||
"protocol_id": "11a70a549953db006a77522bc587bcf9464f9c601afc7b3d09b0119ade903d99",
|
||||
"protocol_id": "eb4b8ad31eea6d046b4caff46dd0bfb56f113e65111be3871e02ac2e351086e6",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o_requests.jsonl",
|
||||
@@ -416,7 +423,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.000000",
|
||||
"output_usd_per_million": "2.000000",
|
||||
"protocol_id": "3b8af120356177e5159f76ee1736a89695006f54cd3055b2c2588ec9050374c6",
|
||||
"protocol_id": "b0abbb72e5db3b335516292c218a5937a9bc4979fa19b616301b6b969f48c585",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-0613_requests.jsonl",
|
||||
@@ -432,7 +439,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.5000000",
|
||||
"output_usd_per_million": "2.000000",
|
||||
"protocol_id": "74b3ad3c9d06e892a0c08968419e330a7a8a91e5db7393e23c31f41c97139967",
|
||||
"protocol_id": "3ccab07621e008211a4fed870619e72e1f6568a12aedf955dfd0d8cfc45016a2",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-instruct_requests.jsonl",
|
||||
@@ -448,7 +455,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "3.000000",
|
||||
"output_usd_per_million": "4.000000",
|
||||
"protocol_id": "07953ad5d1b968e61ff14f3ef8337e407d128295918855be42d7a0993acb0e32",
|
||||
"protocol_id": "dfb7fd3c21a1cd90320c754b37d1d78e71d72eec2a1d3a1dfa3af320f7f5b189",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-16k_requests.jsonl",
|
||||
@@ -464,7 +471,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.5000000",
|
||||
"output_usd_per_million": "1.5000000",
|
||||
"protocol_id": "446953742af525795631082baf5f3df86027c1710d1003ca6054f6e2a48dc18f",
|
||||
"protocol_id": "b2f3aaa32e5e21e9e89a146ffe48e158fcae7b9b65937fb5bbf64df747bea23c",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo_requests.jsonl",
|
||||
@@ -480,7 +487,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.75000000",
|
||||
"output_usd_per_million": "3.75000000",
|
||||
"protocol_id": "1e42dbfead6255ca8efab8f522b5c7684cd8eec6035059bfdf1fdac5d40dc3ae",
|
||||
"protocol_id": "c1a151216652af24dca0a97c85b5d24f45fea8a28caa7075cdf09afb3fb646e7",
|
||||
"reasoning": {
|
||||
"effort": "low"
|
||||
},
|
||||
@@ -498,7 +505,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.75000000",
|
||||
"output_usd_per_million": "3.75000000",
|
||||
"protocol_id": "4524a90e3320fb15103d7468b5e8088ab99c73051539240d0c1ae14d353d7eac",
|
||||
"protocol_id": "4e8a491693ca97ff0bf6ce704a893651a86b1756b0e56a5ebe5fac05c32c6e4f",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
@@ -516,7 +523,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.3000000",
|
||||
"output_usd_per_million": "2.5000000",
|
||||
"protocol_id": "0f44f6e4ebb50ece2ada37707d79b988d36fc55a207f23235bb38740e7726b01",
|
||||
"protocol_id": "24e7aee4cbb982a7b66cd14cbcf6e08a8f2ed80cb6cb132566b5ab1deed17245",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
@@ -534,7 +541,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.5000000",
|
||||
"output_usd_per_million": "9.000000",
|
||||
"protocol_id": "fab233d2ad1205b7fa4c2346875105509219846ceeeb7fd292d209615473c231",
|
||||
"protocol_id": "4c5c5789977f56c13209d2babb47b3c79cd747fb2cd7cea568bd22301779d16f",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
@@ -552,7 +559,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.25000000",
|
||||
"output_usd_per_million": "1.5000000",
|
||||
"protocol_id": "c325d9a280a083263f3705cf8d7fa8d4733877cb733526c2fe2290948d3ac4a4",
|
||||
"protocol_id": "c9517bdfa0b53ccbf0aecb228cee50dc0f9c7abe4e5bf600132afa368b6eac8e",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
@@ -560,22 +567,6 @@
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
{
|
||||
"cache_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemma-4-26b-a4b-it_cache.json",
|
||||
"completion_only_ceiling_usd": "0.0737280",
|
||||
"conservative_reserve_usd": "0.24330240",
|
||||
"created_utc": "2026-04-03",
|
||||
"group": "Google",
|
||||
"id": "google/gemma-4-26b-a4b-it",
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.09000000",
|
||||
"output_usd_per_million": "0.3000000",
|
||||
"protocol_id": "08d96b55bfee15010d2b9053f40c21d8e1d23d912ef84c645eb810eaa8803f1a",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised (optional; omitted)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemma-4-26b-a4b-it_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
{
|
||||
"cache_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_cache.json",
|
||||
"completion_only_ceiling_usd": "0.3686400",
|
||||
@@ -586,7 +577,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.25000000",
|
||||
"output_usd_per_million": "1.5000000",
|
||||
"protocol_id": "4a85c0754b9bd2903c6cc978b2ded49ca381bc0e90199bff6eaf53e42d03e879",
|
||||
"protocol_id": "851a22259abbe2da4fea19fd61b4fb7ce6e0406866e7bf2b54908f91d84cd860",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
@@ -604,7 +595,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.5000000",
|
||||
"output_usd_per_million": "3.000000",
|
||||
"protocol_id": "1c64e2725648b42570c0c0bd443655de81ee55c0477bdc835ff4c1087a76cded",
|
||||
"protocol_id": "1529b1845000ea4830c5c7c88e898b6e1d75abaab1aa4a0d86951330cec17200",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
@@ -622,9 +613,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.1000000",
|
||||
"output_usd_per_million": "0.4000000",
|
||||
"protocol_id": "f5dbbdf216a4373df531ec7fef4a02fcec6e89609aa13553bb1f903dc8e0a43b",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised (optional; omitted)",
|
||||
"protocol_id": "028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -638,9 +631,11 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "0.3000000",
|
||||
"output_usd_per_million": "2.5000000",
|
||||
"protocol_id": "4d8587d1f7a72ee96a78e2693aec222cba6d313c5fcc978b82b27508d736a223",
|
||||
"reasoning": null,
|
||||
"reasoning_label": "not advertised (optional; omitted)",
|
||||
"protocol_id": "0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428",
|
||||
"reasoning": {
|
||||
"enabled": false
|
||||
},
|
||||
"reasoning_label": "disabled (optional, parameter advertised without efforts)",
|
||||
"records_path": "slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl",
|
||||
"structured_output": true
|
||||
},
|
||||
@@ -654,7 +649,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.25000000",
|
||||
"output_usd_per_million": "4.25000000",
|
||||
"protocol_id": "a716da6000adc3d9792e112c55d516e21c49fbe46acac6517d8b740e81b30e69",
|
||||
"protocol_id": "5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
@@ -672,7 +667,7 @@
|
||||
"initial_calls": 240,
|
||||
"input_usd_per_million": "1.25000000",
|
||||
"output_usd_per_million": "4.25000000",
|
||||
"protocol_id": "c4cd8eedebc35b380c52d7c559f87c293623a04858b8acfeb091145340dde72d",
|
||||
"protocol_id": "fcb191133bbd11dce82ac53f4d6b253a15c316d5abd71c54896f9dfdf644b964",
|
||||
"reasoning": {
|
||||
"effort": "minimal"
|
||||
},
|
||||
|
||||
@@ -11,6 +11,7 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It
|
||||
- final response: `Respond with ONLY a JSON object with exactly one key named answer. Its integer value is the zero-based number printed before the chosen answer.`
|
||||
- rescue response: `Return only the one-key object required by the response schema. No explanation.`
|
||||
- strict structured output; each model has an isolated append-only ledger, cache, and model-specific protocol ID
|
||||
- compatibility probe: run scheduled sample 0 first; a configuration or request failure records a failed run and exits before the other 239 requests
|
||||
|
||||
## Spend checks before any later dispatch
|
||||
|
||||
@@ -22,55 +23,55 @@ This manifest prepares the reviewed direct-choice protocol for future panels. It
|
||||
|
||||
## Ordered panels
|
||||
|
||||
The order is Grok, OpenAI, Google, then Muse. `minimal` is used when catalog metadata advertises it; otherwise `low`; disabled is used only when the catalog says reasoning is optional and accepts `none`.
|
||||
The order is Grok, OpenAI, Google, then Muse. Optional entries advertising `none` disable reasoning; otherwise `minimal` is used when advertised, then `low`. Optional metadata with no effort list disables reasoning only when the `reasoning` parameter itself is advertised; models with no reasoning metadata omit the field.
|
||||
|
||||
| family | exact ID | created UTC | input USD/M | output USD/M | reasoning | structured | protocol ID | calls | completion-only ceiling | conservative reserve | isolated ledger |
|
||||
|---|---|---:|---:|---:|---|---|---|---:|---:|---:|---|
|
||||
| Grok | `x-ai/grok-4.6` | 2026-08-12 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `f1dcc6e5e4651add7df5473e80ba5b5a5ec733d788a6a791625776501f8605d6` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.6_requests.jsonl` |
|
||||
| Grok | `x-ai/grok-4.5` | 2026-07-08 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `fdb4ff16d7879cd840722b7f84654da2e67bc388cdf0b52ea64a1d12b487de18` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.5_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"effort": "low"}` (low) | yes | `b9bd5e38934aeae851dcd6227403c474e41c52ebb0bf599b112203b30403e718` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"effort": "low"}` (low) | yes | `76c8b34537d9ec301ec168c8566849a0a0d26fffa088b70b8da09acb760130b1` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"effort": "low"}` (low) | yes | `a25533b578ada960251c7fa244a98b86e68a054346d641b6215888a09c92b09e` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"effort": "low"}` (low) | yes | `42e8c091233072fd0925ab8f5554bc01e23c5cf3051e09ba7b45970af189fb10` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.2-chat` | 2025-12-10 | 1.75000000 | 14.000000 | `null` (not advertised) | yes | `6911c78f8c33d085e4d2e4a627c8e311059855cfb859b16a6a319185b6abb807` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"effort": "low"}` (low) | yes | `6d72c036f9f413fa500c0a2902750b0a2c29fc1145d77d3fa02ecf89b48054ff` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"effort": "low"}` (low) | yes | `063cbfba0812ff35df231154d69a498e77cd240961d61f18913e6465c1ac1938` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5` | 2025-08-07 | 1.25000000 | 10.00000 | `{"effort": "minimal"}` (minimal) | yes | `48ffd6a149a5583355dd779b855e944ba26e4eab9b9cd4b5acd780cb3e3fb4e1` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5-mini` | 2025-08-07 | 0.25000000 | 2.000000 | `{"effort": "minimal"}` (minimal) | yes | `00e8beddc47977c22afa2128d78af2c00acf8ddb87077f427bcb1360ed87a980` | 240 | USD 0.4915 | USD 1.5360 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-oss-120b` | 2025-08-05 | 0.037000000 | 0.17000000 | `{"effort": "low"}` (low) | yes | `25ac3627e4795573424e184ab883ed80e3abb3bfadb0adba94a8e5b1ce659725` | 240 | USD 0.0418 | USD 0.1344 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-120b_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-oss-20b` | 2025-08-05 | 0.03000000 | 0.13000000 | `{"effort": "low"}` (low) | yes | `06243dd30d6fb69a30c4b24a9b891b95a25975d7d41b7071a474e4365e856169` | 240 | USD 0.0319 | USD 0.1032 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-20b_requests.jsonl` |
|
||||
| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `null` (not advertised (optional; omitted)) | yes | `f4054a623913b60fa79beee9aacec15967aa058434835937e4e8af276adf9557` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` |
|
||||
| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `null` (not advertised (optional; omitted)) | yes | `26873a2a9f3d6530a06368113b903fbc80f673849ed605f1f49963f1f7abb638` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4.1` | 2025-04-14 | 2.000000 | 8.000000 | `null` (not advertised) | yes | `9a3bb10b7179275010d35313de4a9d8021ca42abb0901414450ef0c6a42f2aea` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4.1-mini` | 2025-04-14 | 0.4000000 | 1.6000000 | `null` (not advertised) | yes | `59833d4bac77233732006cfada3136d655274a1fba463a994c4bd3107348b642` | 240 | USD 0.3932 | USD 1.2780 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4.1-nano` | 2025-04-14 | 0.1000000 | 0.4000000 | `null` (not advertised) | yes | `91b5e653465ea8562e62d0a5c76895e4dfa9cc774dc777f04268be6643f04cb9` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl` |
|
||||
| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `null` (not advertised (optional; omitted)) | yes | `2a0f497f04f98ce1a6ff53412018721609b993a091a0da4e7c339b5a72b1859c` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o-2024-11-20` | 2024-11-20 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `e558a7bb4fdfbf9c4ddc6cd2f7505850a0779b7f8c02cde480dff902e709674b` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o-2024-08-06` | 2024-08-06 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `ac49e31fe7abf39eadc966e042c72652b063d56d698cf437b76c53411e0a6979` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o-mini` | 2024-07-18 | 0.15000000 | 0.6000000 | `null` (not advertised) | yes | `1662dce1e899d6b1060b2dfaa19034935ff097e77114ae42936eb5b52ddc7296` | 240 | USD 0.1475 | USD 0.4792 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o` | 2024-05-13 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `11a70a549953db006a77522bc587bcf9464f9c601afc7b3d09b0119ade903d99` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-3.5-turbo-0613` | 2024-01-25 | 1.000000 | 2.000000 | `null` (not advertised) | yes | `3b8af120356177e5159f76ee1736a89695006f54cd3055b2c2588ec9050374c6` | 240 | USD 0.4915 | USD 1.7203 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-0613_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-3.5-turbo-instruct` | 2023-09-28 | 1.5000000 | 2.000000 | `null` (not advertised) | yes | `74b3ad3c9d06e892a0c08968419e330a7a8a91e5db7393e23c31f41c97139967` | 240 | USD 0.4915 | USD 1.8432 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-instruct_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-3.5-turbo-16k` | 2023-08-28 | 3.000000 | 4.000000 | `null` (not advertised) | yes | `07953ad5d1b968e61ff14f3ef8337e407d128295918855be42d7a0993acb0e32` | 240 | USD 0.9830 | USD 3.6864 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-16k_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-3.5-turbo` | 2023-05-28 | 0.5000000 | 1.5000000 | `null` (not advertised) | yes | `446953742af525795631082baf5f3df86027c1710d1003ca6054f6e2a48dc18f` | 240 | USD 0.3686 | USD 1.2288 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo_requests.jsonl` |
|
||||
| Google | `google/gemini-3.8-flash` | 2026-09-02 | 0.75000000 | 3.75000000 | `{"effort": "low"}` (low) | yes | `1e42dbfead6255ca8efab8f522b5c7684cd8eec6035059bfdf1fdac5d40dc3ae` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.8-flash_requests.jsonl` |
|
||||
| Google | `google/gemini-3.6-flash` | 2026-07-21 | 0.75000000 | 3.75000000 | `{"effort": "minimal"}` (minimal) | yes | `4524a90e3320fb15103d7468b5e8088ab99c73051539240d0c1ae14d353d7eac` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.6-flash_requests.jsonl` |
|
||||
| Google | `google/gemini-3.5-flash-lite` | 2026-07-21 | 0.3000000 | 2.5000000 | `{"effort": "minimal"}` (minimal) | yes | `0f44f6e4ebb50ece2ada37707d79b988d36fc55a207f23235bb38740e7726b01` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash-lite_requests.jsonl` |
|
||||
| Google | `google/gemini-3.5-flash` | 2026-05-19 | 1.5000000 | 9.000000 | `{"effort": "minimal"}` (minimal) | yes | `fab233d2ad1205b7fa4c2346875105509219846ceeeb7fd292d209615473c231` | 240 | USD 2.2118 | USD 7.0042 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash_requests.jsonl` |
|
||||
| Google | `google/gemini-3.1-flash-lite` | 2026-05-07 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `c325d9a280a083263f3705cf8d7fa8d4733877cb733526c2fe2290948d3ac4a4` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl` |
|
||||
| Google | `google/gemma-4-26b-a4b-it` | 2026-04-03 | 0.09000000 | 0.3000000 | `null` (not advertised (optional; omitted)) | yes | `08d96b55bfee15010d2b9053f40c21d8e1d23d912ef84c645eb810eaa8803f1a` | 240 | USD 0.0737 | USD 0.2433 | `slop/research/wvs/20260917_direct_choice/priority/google__gemma-4-26b-a4b-it_requests.jsonl` |
|
||||
| Google | `google/gemini-3.1-flash-lite-preview` | 2026-03-03 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `4a85c0754b9bd2903c6cc978b2ded49ca381bc0e90199bff6eaf53e42d03e879` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_requests.jsonl` |
|
||||
| Google | `google/gemini-3-flash-preview` | 2025-12-17 | 0.5000000 | 3.000000 | `{"effort": "minimal"}` (minimal) | yes | `1c64e2725648b42570c0c0bd443655de81ee55c0477bdc835ff4c1087a76cded` | 240 | USD 0.7373 | USD 2.3347 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3-flash-preview_requests.jsonl` |
|
||||
| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `null` (not advertised (optional; omitted)) | yes | `f5dbbdf216a4373df531ec7fef4a02fcec6e89609aa13553bb1f903dc8e0a43b` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` |
|
||||
| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `null` (not advertised (optional; omitted)) | yes | `4d8587d1f7a72ee96a78e2693aec222cba6d313c5fcc978b82b27508d736a223` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` |
|
||||
| Muse | `meta/muse-spark-1.2` | 2026-08-05 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `a716da6000adc3d9792e112c55d516e21c49fbe46acac6517d8b740e81b30e69` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.2_requests.jsonl` |
|
||||
| Muse | `meta/muse-spark-1.1` | 2026-07-16 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `c4cd8eedebc35b380c52d7c559f87c293623a04858b8acfeb091145340dde72d` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.1_requests.jsonl` |
|
||||
| Grok | `x-ai/grok-4.6` | 2026-08-12 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.6_requests.jsonl` |
|
||||
| Grok | `x-ai/grok-4.5` | 2026-07-08 | 2.000000 | 6.000000 | `{"effort": "low"}` (low) | yes | `d1362e8b42a4e2ff54cbd66be8c6a65225f94c9679931299f0e5e359e4065c63` | 240 | USD 1.4746 | USD 4.9152 | `slop/research/wvs/20260917_direct_choice/priority/x-ai__grok-4.5_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.6-luna` | 2026-07-09 | 0.2000000 | 1.2000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95` | 240 | USD 0.2949 | USD 0.9339 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-luna_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.6-terra` | 2026-07-09 | 2.000000 | 12.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `08008a2aafc8f2c0435328570269c2a5e5c7659e727a3ec03030ad1629e09305` | 240 | USD 2.9491 | USD 9.3389 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.6-terra_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.4-nano` | 2026-03-17 | 0.2000000 | 1.25000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `35b43cf1fc2442767fffe2dd37427522dd256ca1e9ad92c8b5f019e747fa39c5` | 240 | USD 0.3072 | USD 0.9708 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-nano_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.4-mini` | 2026-03-17 | 0.75000000 | 4.5000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `c6ac5d81c26e9ad2777c679e44e5a8db14bbbe75afe6fcddd48d86e3e9e4c26f` | 240 | USD 1.1059 | USD 3.5021 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.4-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.2-chat` | 2025-12-10 | 1.75000000 | 14.000000 | `null` (not advertised) | yes | `4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2-chat_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.2` | 2025-12-10 | 1.75000000 | 14.000000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `1f2abde83666ba74d488d87ebc72a445b673c312d75b1d1cb638c6624aaf7111` | 240 | USD 3.4406 | USD 10.7520 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.2_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5.1` | 2025-11-13 | 1.25000000 | 10.00000 | `{"enabled": false}` (disabled (optional, none advertised)) | yes | `2df1e492a8fdafa7a73614a46f05eb4e8da90d77c426c1d42b8f3499bf51ca05` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5.1_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5` | 2025-08-07 | 1.25000000 | 10.00000 | `{"effort": "minimal"}` (minimal) | yes | `4f7fe97763a5822831b2244e72b0d4cc01ff46da55090fed2485c79f97e4bdf0` | 240 | USD 2.4576 | USD 7.6800 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-5-mini` | 2025-08-07 | 0.25000000 | 2.000000 | `{"effort": "minimal"}` (minimal) | yes | `5b115c7341384a37117745f735367dae10932cc9f16b3fdac5819b5adfe007f4` | 240 | USD 0.4915 | USD 1.5360 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-5-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-oss-120b` | 2025-08-05 | 0.037000000 | 0.17000000 | `{"effort": "low"}` (low) | yes | `ae278f4f5e91f668448c4b921633e999ba90a6cd9cc3819c8ee4beac8484d38a` | 240 | USD 0.0418 | USD 0.1344 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-120b_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-oss-20b` | 2025-08-05 | 0.03000000 | 0.13000000 | `{"effort": "low"}` (low) | yes | `2b45bec6a0271fb1390ea36e196eef7e75a53112c1dcd1e40b1b60b7631a9d7d` | 240 | USD 0.0319 | USD 0.1032 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-oss-20b_requests.jsonl` |
|
||||
| OpenAI | `openai/o3` | 2025-04-16 | 2.000000 | 8.000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3_requests.jsonl` |
|
||||
| OpenAI | `openai/o4-mini` | 2025-04-16 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `eb7a482e589089283c40598692520993d2bb26fe363a8fc0c5e8fe31fcc7a367` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o4-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4.1` | 2025-04-14 | 2.000000 | 8.000000 | `null` (not advertised) | yes | `1f6ad277fcb151d83dd6b0c9d2f9ed5d2cb7466a09b30c4368ec0c74a8d72854` | 240 | USD 1.9661 | USD 6.3898 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4.1-mini` | 2025-04-14 | 0.4000000 | 1.6000000 | `null` (not advertised) | yes | `ce0bb6d5ff9f57496b0f677bb5ac47fc196380e084b457b43f6203d2b7d1b6d1` | 240 | USD 0.3932 | USD 1.2780 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4.1-nano` | 2025-04-14 | 0.1000000 | 0.4000000 | `null` (not advertised) | yes | `0efb6591fbdbe87758af494f18fc0a5a1a4897451dc2bc05f45681e9b5d73758` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4.1-nano_requests.jsonl` |
|
||||
| OpenAI | `openai/o3-mini` | 2025-01-31 | 1.1000000 | 4.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `3ab06ed7aef32190fb8062c01a7d9f1e3e50b2eae8a6dadcf2983ad28d6b0de6` | 240 | USD 1.0813 | USD 3.5144 | `slop/research/wvs/20260917_direct_choice/priority/openai__o3-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o-2024-11-20` | 2024-11-20 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `8c3bd515d1ac0e9ee598baa0ad42dc958817e6ad14cad7a73ecd710f71b003a6` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-11-20_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o-2024-08-06` | 2024-08-06 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `64377e05e282d8d9d5dc425635c6acdbf87ae8115aadfb39503c876ad5bbb9ac` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-2024-08-06_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o-mini` | 2024-07-18 | 0.15000000 | 0.6000000 | `null` (not advertised) | yes | `388af881427c7032c7a863540b75d7255292a83ee553dcb99eb2d07c7a1766df` | 240 | USD 0.1475 | USD 0.4792 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o-mini_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-4o` | 2024-05-13 | 2.5000000 | 10.00000 | `null` (not advertised) | yes | `eb4b8ad31eea6d046b4caff46dd0bfb56f113e65111be3871e02ac2e351086e6` | 240 | USD 2.4576 | USD 7.9872 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-4o_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-3.5-turbo-0613` | 2024-01-25 | 1.000000 | 2.000000 | `null` (not advertised) | yes | `b0abbb72e5db3b335516292c218a5937a9bc4979fa19b616301b6b969f48c585` | 240 | USD 0.4915 | USD 1.7203 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-0613_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-3.5-turbo-instruct` | 2023-09-28 | 1.5000000 | 2.000000 | `null` (not advertised) | yes | `3ccab07621e008211a4fed870619e72e1f6568a12aedf955dfd0d8cfc45016a2` | 240 | USD 0.4915 | USD 1.8432 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-instruct_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-3.5-turbo-16k` | 2023-08-28 | 3.000000 | 4.000000 | `null` (not advertised) | yes | `dfb7fd3c21a1cd90320c754b37d1d78e71d72eec2a1d3a1dfa3af320f7f5b189` | 240 | USD 0.9830 | USD 3.6864 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo-16k_requests.jsonl` |
|
||||
| OpenAI | `openai/gpt-3.5-turbo` | 2023-05-28 | 0.5000000 | 1.5000000 | `null` (not advertised) | yes | `b2f3aaa32e5e21e9e89a146ffe48e158fcae7b9b65937fb5bbf64df747bea23c` | 240 | USD 0.3686 | USD 1.2288 | `slop/research/wvs/20260917_direct_choice/priority/openai__gpt-3.5-turbo_requests.jsonl` |
|
||||
| Google | `google/gemini-3.8-flash` | 2026-09-02 | 0.75000000 | 3.75000000 | `{"effort": "low"}` (low) | yes | `c1a151216652af24dca0a97c85b5d24f45fea8a28caa7075cdf09afb3fb646e7` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.8-flash_requests.jsonl` |
|
||||
| Google | `google/gemini-3.6-flash` | 2026-07-21 | 0.75000000 | 3.75000000 | `{"effort": "minimal"}` (minimal) | yes | `4e8a491693ca97ff0bf6ce704a893651a86b1756b0e56a5ebe5fac05c32c6e4f` | 240 | USD 0.9216 | USD 2.9491 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.6-flash_requests.jsonl` |
|
||||
| Google | `google/gemini-3.5-flash-lite` | 2026-07-21 | 0.3000000 | 2.5000000 | `{"effort": "minimal"}` (minimal) | yes | `24e7aee4cbb982a7b66cd14cbcf6e08a8f2ed80cb6cb132566b5ab1deed17245` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash-lite_requests.jsonl` |
|
||||
| Google | `google/gemini-3.5-flash` | 2026-05-19 | 1.5000000 | 9.000000 | `{"effort": "minimal"}` (minimal) | yes | `4c5c5789977f56c13209d2babb47b3c79cd747fb2cd7cea568bd22301779d16f` | 240 | USD 2.2118 | USD 7.0042 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.5-flash_requests.jsonl` |
|
||||
| Google | `google/gemini-3.1-flash-lite` | 2026-05-07 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `c9517bdfa0b53ccbf0aecb228cee50dc0f9c7abe4e5bf600132afa368b6eac8e` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite_requests.jsonl` |
|
||||
| Google | `google/gemini-3.1-flash-lite-preview` | 2026-03-03 | 0.25000000 | 1.5000000 | `{"effort": "minimal"}` (minimal) | yes | `851a22259abbe2da4fea19fd61b4fb7ce6e0406866e7bf2b54908f91d84cd860` | 240 | USD 0.3686 | USD 1.1674 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3.1-flash-lite-preview_requests.jsonl` |
|
||||
| Google | `google/gemini-3-flash-preview` | 2025-12-17 | 0.5000000 | 3.000000 | `{"effort": "minimal"}` (minimal) | yes | `1529b1845000ea4830c5c7c88e898b6e1d75abaab1aa4a0d86951330cec17200` | 240 | USD 0.7373 | USD 2.3347 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-3-flash-preview_requests.jsonl` |
|
||||
| Google | `google/gemini-2.5-flash-lite` | 2025-07-22 | 0.1000000 | 0.4000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `028b2d617ca65993cc70998b26475c2bb06fc4c0f2f2ec7f0604bb4dc2c7c6c7` | 240 | USD 0.0983 | USD 0.3195 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash-lite_requests.jsonl` |
|
||||
| Google | `google/gemini-2.5-flash` | 2025-06-17 | 0.3000000 | 2.5000000 | `{"enabled": false}` (disabled (optional, parameter advertised without efforts)) | yes | `0162c6349d01c312300afb9cb720820aca663c10e2b9af7cd2fe369f5cb59428` | 240 | USD 0.6144 | USD 1.9169 | `slop/research/wvs/20260917_direct_choice/priority/google__gemini-2.5-flash_requests.jsonl` |
|
||||
| Muse | `meta/muse-spark-1.2` | 2026-08-05 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.2_requests.jsonl` |
|
||||
| Muse | `meta/muse-spark-1.1` | 2026-07-16 | 1.25000000 | 4.25000000 | `{"effort": "minimal"}` (minimal) | yes | `fcb191133bbd11dce82ac53f4d6b253a15c316d5abd71c54896f9dfdf644b964` | 240 | USD 1.0445 | USD 3.4406 | `slop/research/wvs/20260917_direct_choice/priority/meta__muse-spark-1.1_requests.jsonl` |
|
||||
|
||||
## Exclusions
|
||||
|
||||
- Already plotted dense-rated IDs are not repeated in this prepared direct-choice list, including Grok 4.3/4.20, GPT-6 Astra, GPT-5.6 Sol, GPT-5.5, GPT-5.4, GPT-5.3 Chat, Gemini 3.7 Flash, Gemini 2.5 Pro, and Muse 1.3.
|
||||
- GPT-5 Nano is retained as a completed dense-rated protocol diagnostic, not silently relabelled as a direct-choice panel.
|
||||
- Pro/Fast, batch/free aliases, output price above USD 15/M, and code/image/audio/safeguard/multi-agent entries remain excluded. `Flash` is included where it is a general chat model.
|
||||
- `google/gemma-4-26b-a4b-it` is excluded: it is Gemma, not an identified member of the requested Gemini release series.
|
||||
- `openai/o4-mini-high` and `openai/o3-mini-high` are excluded because their catalog entries advertise only `high` reasoning, not the registered minimal/low policy.
|
||||
- The deferred Qwen/GLM/Mistral shortlist remains outside this priority manifest until a direct-choice expansion decision is made.
|
||||
|
||||
|
||||
@@ -1,16 +1,22 @@
|
||||
$ python scripts/wvs_direct_choice_priority.py --write-manifest
|
||||
wrote slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.md and slop/research/wvs/20260917_direct_choice/priority_direct_choice_manifest.json
|
||||
$ python scripts/wvs_direct_choice_priority.py --model x-ai/grok-4.6 --smoke
|
||||
smoke: x-ai/grok-4.6, 240 direct-choice requests, protocol=f1dcc6e5e4651add7df5473e80ba5b5a5ec733d788a6a791625776501f8605d6
|
||||
smoke: x-ai/grok-4.6, 240 direct-choice requests, protocol=34224b2e476e87f4e6e904e98a79ba3d2962ccc817925fdb00d7e649b17c81a6
|
||||
smoke: reasoning={'effort': 'low'}, reserve=USD 4.915200
|
||||
$ python scripts/wvs_direct_choice_priority.py --model openai/gpt-5.6-luna --smoke
|
||||
smoke: openai/gpt-5.6-luna, 240 direct-choice requests, protocol=8db405bf3f111cd8755e531e3ba9758399a0589741e2a4af996340905d8f2f95
|
||||
smoke: reasoning={'enabled': False}, reserve=USD 0.9338880
|
||||
$ python scripts/wvs_direct_choice_priority.py --model openai/o3 --smoke
|
||||
smoke: openai/o3, 240 direct-choice requests, protocol=0f658f28a30fe91d97c90672b8028e188b8043d464ec1238dc4acded0cfc2298
|
||||
smoke: reasoning={'enabled': False}, reserve=USD 6.389760
|
||||
$ python scripts/wvs_direct_choice_priority.py --model openai/gpt-5.2-chat --smoke
|
||||
smoke: openai/gpt-5.2-chat, 240 direct-choice requests, protocol=6911c78f8c33d085e4d2e4a627c8e311059855cfb859b16a6a319185b6abb807
|
||||
smoke: openai/gpt-5.2-chat, 240 direct-choice requests, protocol=4f19ecc3869b4328dd5ed3162113fa3c7f285e441c343ac109df17fe5e5e31ca
|
||||
smoke: reasoning=None, reserve=USD 10.75200000
|
||||
$ python scripts/wvs_direct_choice_priority.py --model google/gemini-3.6-flash --smoke
|
||||
smoke: google/gemini-3.6-flash, 240 direct-choice requests, protocol=4524a90e3320fb15103d7468b5e8088ab99c73051539240d0c1ae14d353d7eac
|
||||
smoke: reasoning={'effort': 'minimal'}, reserve=USD 2.94912000
|
||||
$ python scripts/wvs_direct_choice_priority.py --model meta/muse-spark-1.2 --smoke
|
||||
smoke: meta/muse-spark-1.2, 240 direct-choice requests, protocol=a716da6000adc3d9792e112c55d516e21c49fbe46acac6517d8b740e81b30e69
|
||||
smoke: meta/muse-spark-1.2, 240 direct-choice requests, protocol=5ddf04c58154c758756fb32cd2652f928678170010cf8638dd9cdb1c064522a2
|
||||
smoke: reasoning={'effort': 'minimal'}, reserve=USD 3.44064000
|
||||
verified manifest: 39 unique structured direct-choice protocols, 240 calls each, output <= USD 15/M
|
||||
wrapper syntax: valid; no --run execution invoked
|
||||
$ python scripts/wvs_direct_choice_priority_smoke.py
|
||||
smoke: 38 unique 240-call protocols cover omitted, disabled, low, and minimal reasoning settings
|
||||
smoke: synthetic first request exits before remaining 239 and writes no cache
|
||||
smoke: None omits reasoning; optional disabled mode sends reasoning.enabled=false
|
||||
verified manifest: 38 distinct strict-schema 240-call protocols; Gemma excluded; fail-fast probe encoded
|
||||
|
||||
@@ -142,7 +142,8 @@ def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_pe
|
||||
prompt_instruction: str = _PROMPT_INSTRUCTION_OWN_VIEW,
|
||||
answer_instruction: str = _ANSWER_INSTRUCTION_WITH_EXAMPLE,
|
||||
rescue_instruction: str | None = None,
|
||||
plan_override: list[dict] | None = None) -> str:
|
||||
plan_override: list[dict] | None = None,
|
||||
fail_fast_first_request: bool = False) -> str:
|
||||
plan = _plan(items, samples_per_order, answer_instruction) if plan_override is None else plan_override
|
||||
protocol = {
|
||||
"schema": 1,
|
||||
@@ -163,6 +164,8 @@ def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_pe
|
||||
if plan_override is not None:
|
||||
protocol.pop("samples_per_order")
|
||||
protocol["samples_per_item"] = {item["id"]: sum(request["item_id"] == item["id"] for request in plan) for item in items}
|
||||
if fail_fast_first_request:
|
||||
protocol["fail_fast_first_request"] = True
|
||||
encoded = json.dumps(protocol, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()
|
||||
return hashlib.sha256(encoded).hexdigest()
|
||||
|
||||
@@ -180,7 +183,8 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
|
||||
prompt_instruction: str = _PROMPT_INSTRUCTION_OWN_VIEW,
|
||||
answer_instruction: str = _ANSWER_INSTRUCTION_WITH_EXAMPLE,
|
||||
rescue_instruction: str | None = None,
|
||||
plan_override: list[dict] | None = None) -> dict:
|
||||
plan_override: list[dict] | None = None,
|
||||
fail_fast_first_request: bool = False) -> dict:
|
||||
"""Sample exactly one selected option per prompt, including canonical and reversed option orders.
|
||||
|
||||
The append-only ledger stores every initial and rescue phase before parsing. A cache entry is written only
|
||||
@@ -197,6 +201,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
|
||||
reasoning=reasoning, structured_output=structured_output, prompt_instruction=prompt_instruction,
|
||||
answer_instruction=answer_instruction,
|
||||
rescue_instruction=rescue_instruction, plan_override=plan_override,
|
||||
fail_fast_first_request=fail_fast_first_request,
|
||||
)
|
||||
cache_file = Path(cache_path)
|
||||
cache = json.loads(cache_file.read_text()) if cache_file.exists() else {"schema": 1, "completed": {}}
|
||||
@@ -211,7 +216,7 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
|
||||
"model": model, "samples_per_order": samples_per_order, "temperature": temperature,
|
||||
"max_tokens": max_tokens, "concurrency": concurrency, "request_timeout": request_timeout,
|
||||
"reasoning": reasoning, "structured_output": structured_output,
|
||||
"prompt_instruction": prompt_instruction,
|
||||
"prompt_instruction": prompt_instruction, "fail_fast_first_request": fail_fast_first_request,
|
||||
}
|
||||
if plan_override is not None:
|
||||
settings.pop("samples_per_order")
|
||||
@@ -242,9 +247,10 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
|
||||
response_format = _choice_schema(item["n"])
|
||||
payload = {
|
||||
"model": model, "messages": [{"role": "user", "content": request["prompt"]}],
|
||||
"temperature": temperature, "max_tokens": max_tokens, "reasoning": reasoning,
|
||||
"response_format": response_format,
|
||||
"temperature": temperature, "max_tokens": max_tokens, "response_format": response_format,
|
||||
}
|
||||
if reasoning is not None:
|
||||
payload["reasoning"] = reasoning
|
||||
phase = "initial"
|
||||
async with semaphore:
|
||||
try:
|
||||
@@ -266,8 +272,10 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
|
||||
{"role": "assistant", "content": assistant_tail},
|
||||
{"role": "user", "content": rescue_instruction or _force_choice(item["n"])},
|
||||
], "temperature": temperature, "max_tokens": max(max_tokens, 2048),
|
||||
"reasoning": reasoning, "response_format": response_format,
|
||||
"response_format": response_format,
|
||||
}
|
||||
if reasoning is not None:
|
||||
rescue_payload["reasoning"] = reasoning
|
||||
_append_record(records, {"event": "request_started", "phase": phase, **request_meta,
|
||||
"payload": rescue_payload, "initial_response_message": message})
|
||||
response = await asyncio.wait_for(openrouter_request(rescue_payload), timeout=request_timeout)
|
||||
@@ -283,9 +291,23 @@ def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order
|
||||
"error_type": type(exc).__name__, "error": str(exc)})
|
||||
return {"text": None, "rescued": phase == "rescue", "error": f"{type(exc).__name__}: {exc}"}
|
||||
|
||||
return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan)))
|
||||
if not fail_fast_first_request:
|
||||
return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan)))
|
||||
first = await call(0, plan[0])
|
||||
if first["error"] is not None:
|
||||
return [first]
|
||||
remaining = await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan[1:], start=1)))
|
||||
return [first, *remaining]
|
||||
|
||||
results = asyncio.run(run_all())
|
||||
if fail_fast_first_request and results[0]["error"] is not None:
|
||||
summary = {
|
||||
"run_id": run_id, "protocol_id": protocol_id, "model": model, "settings": settings,
|
||||
"planned_requests": len(plan), "failed_requests": 1, "rescued_requests": int(results[0]["rescued"]),
|
||||
"complete": False, "items": [], "failure": results[0]["error"],
|
||||
}
|
||||
_append_record(records, {"event": "run_finished", "construct": "direct_choice", **summary})
|
||||
raise RuntimeError(f"first scheduled request failed before remaining {len(plan) - 1} requests: {results[0]['error']}")
|
||||
by_item = {item["id"]: [] for item in items}
|
||||
failed = 0
|
||||
rescues = 0
|
||||
|
||||
Reference in New Issue
Block a user