From db3c67b2307bb0d91a385036b8e66a4bf71e64cd Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Thu, 17 Sep 2026 10:47:08 +0800 Subject: [PATCH] Preregister Gemini direct-choice WVS pilot Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com> --- .../02_gemini37_direct_choice_pilot.sh | 3 + scripts/wvs_direct_choice_pilot.py | 189 +++++++++++++ .../gemini37_direct_choice_manifest.md | 40 +++ .../preflight_smoke.log | 3 + src/moralmaps/read_direct_choice.py | 261 ++++++++++++++++++ 5 files changed, 496 insertions(+) create mode 100644 scripts/wvs_api/02_gemini37_direct_choice_pilot.sh create mode 100644 scripts/wvs_direct_choice_pilot.py create mode 100644 slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_manifest.md create mode 100644 slop/research/wvs/20260917_direct_choice/preflight_smoke.log create mode 100644 src/moralmaps/read_direct_choice.py diff --git a/scripts/wvs_api/02_gemini37_direct_choice_pilot.sh b/scripts/wvs_api/02_gemini37_direct_choice_pilot.sh new file mode 100644 index 0000000..c96eb94 --- /dev/null +++ b/scripts/wvs_api/02_gemini37_direct_choice_pilot.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +uv run --with 'datasets>=4.0,<5' python scripts/wvs_direct_choice_pilot.py --run diff --git a/scripts/wvs_direct_choice_pilot.py b/scripts/wvs_direct_choice_pilot.py new file mode 100644 index 0000000..d120d19 --- /dev/null +++ b/scripts/wvs_direct_choice_pilot.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Preregister and run one direct-choice WVS construct pilot, separate from rated map panels.""" +from __future__ import annotations + +import argparse +import json +from datetime import UTC, datetime +from decimal import Decimal +from pathlib import Path + +from moralmaps.read_direct_choice import direct_choice_protocol_identity, read_items_direct_choice +from wvs_map import X_AXIS, Y_AXIS, load_wvs_all, resolve_items + +MODEL = "google/gemini-3.7-flash" +ITEM_IDS = ("Homosexuality", "Religion", "God", "Independence") +SAMPLES_PER_ORDER = 12 +TEMPERATURE = 1.0 +MAX_TOKENS = 1024 +CONCURRENCY = 1 +REQUEST_TIMEOUT = 90.0 +REASONING = {"effort": "low"} +STRUCTURED_OUTPUT = True +EXPECTED_INITIAL_CALLS = len(ITEM_IDS) * SAMPLES_PER_ORDER * 2 +GLOBAL_STOP_USD = Decimal("80") +PRIORITY_PHASE_STOP_USD = Decimal("35") +PILOT_CONSERVATIVE_RESERVE_USD = Decimal("2.00") +CATALOG_PATH = Path("slop/research/wvs/20260917_openrouter_models.json") +RATED_LEDGER_PATH = Path("slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl") +RECORDS_PATH = Path("slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_requests.jsonl") +CACHE_PATH = Path("slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_cache.json") +MANIFEST_PATH = Path("slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_manifest.md") + + +def utc_date(timestamp: int) -> str: + return datetime.fromtimestamp(timestamp, UTC).date().isoformat() + + +def usage_cost(path: Path) -> Decimal: + if not path.exists(): + return Decimal() + total = Decimal() + for line in path.read_text().splitlines(): + event = json.loads(line) + if event["event"] == "request_completed" and event.get("usage", {}).get("cost") is not None: + total += Decimal(str(event["usage"]["cost"])) + return total + + +def selected_items() -> list[dict]: + resolved = resolve_items(load_wvs_all()) + selected = {} + for axis in (X_AXIS, Y_AXIS): + for item in resolved[axis]: + if item["suffix"] in ITEM_IDS: + selected[item["suffix"]] = { + "id": item["suffix"], "question": item["rec"]["q"], + "options": item["rec"]["opts"], "n": item["n"], "axis": axis, + } + assert tuple(selected) == ITEM_IDS, f"WVS item identity drift: {tuple(selected)}" + return [selected[item_id] for item_id in ITEM_IDS] + + +def catalog_model() -> dict: + catalog = {model["id"]: model for model in json.loads(CATALOG_PATH.read_text())["data"]} + model = catalog[MODEL] + assert "structured_outputs" in model["supported_parameters"] + assert model["reasoning"]["mandatory"] + assert "low" in model["reasoning"]["supported_efforts"] + return model + + +def protocol_id(items: list[dict]) -> str: + return direct_choice_protocol_identity( + MODEL, items, samples_per_order=SAMPLES_PER_ORDER, temperature=TEMPERATURE, + max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT, + reasoning=REASONING, structured_output=STRUCTURED_OUTPUT, + ) + + +def preflight(items: list[dict], model: dict) -> dict: + assert EXPECTED_INITIAL_CALLS == 96 + assert all(item["n"] >= 2 for item in items) + rated_cost = usage_cost(RATED_LEDGER_PATH) + direct_cost = usage_cost(RECORDS_PATH) + cumulative_cost = rated_cost + direct_cost + assert cumulative_cost + PILOT_CONSERVATIVE_RESERVE_USD < PRIORITY_PHASE_STOP_USD + assert cumulative_cost + PILOT_CONSERVATIVE_RESERVE_USD < GLOBAL_STOP_USD + output_price_per_million = Decimal(model["pricing"]["completion"]) * 1_000_000 + initial_completion_ceiling = output_price_per_million * EXPECTED_INITIAL_CALLS * MAX_TOKENS / 1_000_000 + rescue_completion_ceiling = output_price_per_million * EXPECTED_INITIAL_CALLS * max(MAX_TOKENS, 2048) / 1_000_000 + return { + "rated_ledger_cost": rated_cost, + "direct_choice_ledger_cost": direct_cost, + "cumulative_cost": cumulative_cost, + "output_price_per_million": output_price_per_million, + "initial_completion_ceiling": initial_completion_ceiling, + "all_rescue_completion_ceiling": initial_completion_ceiling + rescue_completion_ceiling, + "protocol_id": protocol_id(items), + } + + +def write_manifest(items: list[dict], model: dict, checks: dict) -> None: + option_schema = '{"answer": integer from 0 through n-1}' + item_names = ", ".join(f"`{item['id']}`" for item in items) + lines = [ + "# Gemini 3.7 Flash direct-choice WVS construct pilot", + "", + "This preregistered pilot tests a construct difference. It is not a WVS map panel and cannot alter rated coordinates.", + "", + "## Exact identity", + "", + f"- model: `{MODEL}`", + f"- saved-catalog created UTC: {utc_date(model['created'])}", + f"- protocol ID: `{checks['protocol_id']}`", + f"- cache: `{CACHE_PATH}`", + f"- append-only request ledger: `{RECORDS_PATH}`", + f"- temperature: {TEMPERATURE}; max tokens: {MAX_TOKENS}; concurrency: {CONCURRENCY}; timeout: {REQUEST_TIMEOUT}s", + f"- reasoning: `{json.dumps(REASONING, sort_keys=True)}`; catalog confirms mandatory reasoning with supported `low` effort", + f"- strict response schema: `{option_schema}`", + "", + "## Design", + "", + f"- items: {item_names}", + f"- {SAMPLES_PER_ORDER} canonical + {SAMPLES_PER_ORDER} reversed presented-option orders per item", + f"- expected initial calls: {EXPECTED_INITIAL_CALLS}; expected parsed choices: {EXPECTED_INITIAL_CALLS}", + "- exact instruction: `Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. Answer immediately.`", + "- final response is schema-only. Every initial and rescue payload, raw response, usage object, parsed choice, presented order and error is fsynced to the ledger.", + "- an incomplete item or any failed request makes `--run` exit nonzero and prevents a cache entry.", + "", + "## Preregistered comparison", + "", + "For each item, map selected presented indices back to canonical option indices. Report the canonical and reversed empirical categorical distributions, their total-variation distance, and the canonical-versus-reversed argmax agreement. Compare the direct-choice aggregate distribution with Gemini's existing dense-rated distribution and report total variation plus the dense-rated midpoint mass. These are construct diagnostics, not a coordinate replacement or a capability claim.", + "", + "## Spend checks before dispatch", + "", + f"- rated-ledger observed cost: USD {checks['rated_ledger_cost']:.10f}", + f"- direct-choice-ledger observed cost: USD {checks['direct_choice_ledger_cost']:.10f}", + f"- cumulative observed cost: USD {checks['cumulative_cost']:.10f}", + f"- current output price: USD {checks['output_price_per_million']:g}/M tokens", + f"- 96 initial 1024-token completion-only ceiling: USD {checks['initial_completion_ceiling']:.6f}", + f"- all-initial plus all-rescue 2048-token completion-only ceiling: USD {checks['all_rescue_completion_ceiling']:.6f}; prompt tokens are additional", + f"- pre-dispatch conservative reserve: USD {PILOT_CONSERVATIVE_RESERVE_USD:.2f}; it remains below the USD {PRIORITY_PHASE_STOP_USD} priority-phase and USD {GLOBAL_STOP_USD} global stops", + "- no wider priority model dispatch is authorized by this manifest.", + "", + "-- PI[gpt-5.6-terra]", + "", + ] + MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True) + MANIFEST_PATH.write_text("\n".join(lines)) + + +def smoke(items: list[dict], checks: dict) -> None: + assert len(items) == 4 + assert len({item["id"] for item in items}) == 4 + assert checks["protocol_id"] == protocol_id(items) + assert all(item["n"] >= 2 for item in items) + print(f"smoke: 4 items x 12 canonical x 12 reversed = {EXPECTED_INITIAL_CALLS} requests") + print(f"smoke: distinct direct-choice protocol {checks['protocol_id']}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run", action="store_true", help="make the preregistered paid pilot calls") + parser.add_argument("--smoke", action="store_true", help="validate the manifest and request plan without API calls") + args = parser.parse_args() + items = selected_items() + model = catalog_model() + checks = preflight(items, model) + write_manifest(items, model, checks) + if args.smoke: + smoke(items, checks) + if not args.run: + return + result = read_items_direct_choice( + MODEL, items, samples_per_order=SAMPLES_PER_ORDER, temperature=TEMPERATURE, + max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT, + reasoning=REASONING, structured_output=STRUCTURED_OUTPUT, + records_path=RECORDS_PATH, cache_path=CACHE_PATH, + ) + if result["cached"]: + print(f"direct-choice cache hit: protocol={result['protocol_id'][:12]}") + return + if not result["complete"]: + raise RuntimeError(f"incomplete direct-choice pilot: {result['run_id']}; raw evidence is {RECORDS_PATH}") + print(f"complete direct-choice pilot: {result['run_id']}, protocol={result['protocol_id'][:12]}") + + +if __name__ == "__main__": + main() diff --git a/slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_manifest.md b/slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_manifest.md new file mode 100644 index 0000000..3ed7e3b --- /dev/null +++ b/slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_manifest.md @@ -0,0 +1,40 @@ +# Gemini 3.7 Flash direct-choice WVS construct pilot + +This preregistered pilot tests a construct difference. It is not a WVS map panel and cannot alter rated coordinates. + +## Exact identity + +- model: `google/gemini-3.7-flash` +- saved-catalog created UTC: 2026-08-13 +- protocol ID: `433768e674cf0ed8ca4b1677637c1140dbca7910c289da06dbc83124df6f8fb7` +- cache: `slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_cache.json` +- append-only request ledger: `slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_requests.jsonl` +- temperature: 1.0; max tokens: 1024; concurrency: 1; timeout: 90.0s +- reasoning: `{"effort": "low"}`; catalog confirms mandatory reasoning with supported `low` effort +- strict response schema: `{"answer": integer from 0 through n-1}` + +## Design + +- items: `Homosexuality`, `Religion`, `God`, `Independence` +- 12 canonical + 12 reversed presented-option orders per item +- expected initial calls: 96; expected parsed choices: 96 +- exact instruction: `Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. Answer immediately.` +- final response is schema-only. Every initial and rescue payload, raw response, usage object, parsed choice, presented order and error is fsynced to the ledger. +- an incomplete item or any failed request makes `--run` exit nonzero and prevents a cache entry. + +## Preregistered comparison + +For each item, map selected presented indices back to canonical option indices. Report the canonical and reversed empirical categorical distributions, their total-variation distance, and the canonical-versus-reversed argmax agreement. Compare the direct-choice aggregate distribution with Gemini's existing dense-rated distribution and report total variation plus the dense-rated midpoint mass. These are construct diagnostics, not a coordinate replacement or a capability claim. + +## Spend checks before dispatch + +- rated-ledger observed cost: USD 3.6235153224 +- direct-choice-ledger observed cost: USD 0.0000000000 +- cumulative observed cost: USD 3.6235153224 +- current output price: USD 3.75000000/M tokens +- 96 initial 1024-token completion-only ceiling: USD 0.368640 +- all-initial plus all-rescue 2048-token completion-only ceiling: USD 1.105920; prompt tokens are additional +- pre-dispatch conservative reserve: USD 2.00; it remains below the USD 35 priority-phase and USD 80 global stops +- no wider priority model dispatch is authorized by this manifest. + +-- PI[gpt-5.6-terra] diff --git a/slop/research/wvs/20260917_direct_choice/preflight_smoke.log b/slop/research/wvs/20260917_direct_choice/preflight_smoke.log new file mode 100644 index 0000000..96bb27e --- /dev/null +++ b/slop/research/wvs/20260917_direct_choice/preflight_smoke.log @@ -0,0 +1,3 @@ +smoke: 4 items x 12 canonical x 12 reversed = 96 requests +smoke: distinct direct-choice protocol 433768e674cf0ed8ca4b1677637c1140dbca7910c289da06dbc83124df6f8fb7 +planner/parser smoke: 96 requests, 12 canonical + 12 reversed per item; strict one-key parser diff --git a/src/moralmaps/read_direct_choice.py b/src/moralmaps/read_direct_choice.py new file mode 100644 index 0000000..3c02711 --- /dev/null +++ b/src/moralmaps/read_direct_choice.py @@ -0,0 +1,261 @@ +"""Durable direct-choice sampling for construct checks, separate from dense ratings.""" +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import re +from datetime import UTC, datetime +from pathlib import Path + +from openrouter_wrapper.retry import openrouter_request + + +def _append_record(path: Path, record: dict) -> None: + record["recorded_at_utc"] = datetime.now(UTC).isoformat() + with path.open("a", encoding="utf-8") as fh: + fh.write(json.dumps(record, ensure_ascii=True, sort_keys=True) + "\n") + fh.flush() + os.fsync(fh.fileno()) + + +def _choice_schema(n: int) -> dict: + return {"type": "json_schema", "json_schema": {"name": "one_choice", "strict": True, "schema": { + "type": "object", + "properties": {"answer": {"type": "integer", "minimum": 0, "maximum": n - 1}}, + "required": ["answer"], "additionalProperties": False, + }}} + + +def _parse_choice(text: str, n: int) -> int | None: + objects = re.findall(r"\{[^{}]*\}", text) + if not objects: + return None + try: + answer = json.loads(objects[-1]) + except json.JSONDecodeError: + return None + if set(answer) != {"answer"}: + return None + choice = answer["answer"] + if type(choice) is not int or not 0 <= choice < n: + return None + return choice + + +def _force_choice(n: int) -> str: + return f'Return ONLY {{"answer": }}. No explanation.' + + +def _plan(items: list[dict], samples_per_order: int) -> list[dict]: + plan = [] + for item_index, item in enumerate(items): + for order_name, order in (("canonical", list(range(item["n"]))), ("reversed", list(reversed(range(item["n"]))))): + prompt = _choice_prompt(item, order) + for repetition in range(samples_per_order): + plan.append({ + "item_index": item_index, + "item_id": item["id"], + "sample": len(plan) % (2 * samples_per_order), + "order_name": order_name, + "repetition": repetition, + "presented_order": order, + "presented_options": [item["options"][index] for index in order], + "prompt": prompt, + }) + return plan + + +def _choice_prompt(item: dict, order: list[int]) -> str: + options = "\n".join(f"{position}) {item['options'][canonical]}" for position, canonical in enumerate(order)) + return ( + f"{item['question']}\n\n" + "Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. " + "Answer immediately.\n\n" + f"Choose exactly one answer:\n{options}\n\n" + f"Respond with ONLY a JSON object such as {{\"answer\": 0}}. The answer must be an integer from 0 through {item['n'] - 1}." + ) + + +def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_per_order: int, + temperature: float, max_tokens: int, concurrency: int, + request_timeout: float, reasoning: dict, structured_output: bool) -> str: + plan = _plan(items, samples_per_order) + protocol = { + "schema": 1, + "construct": "direct_choice", + "model": model, + "samples_per_order": samples_per_order, + "temperature": temperature, + "max_tokens": max_tokens, + "concurrency": concurrency, + "request_timeout": request_timeout, + "reasoning": reasoning, + "structured_output": structured_output, + "prompt_instruction": "Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. Answer immediately.", + "rescue_instruction": _force_choice(10), + "requests": plan, + } + encoded = json.dumps(protocol, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _save_cache(path: Path, cache: dict) -> None: + temp = path.with_suffix(path.suffix + ".tmp") + temp.write_text(json.dumps(cache, indent=2, sort_keys=True) + "\n") + temp.replace(path) + + +def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order: int, + temperature: float, max_tokens: int, concurrency: int, + request_timeout: float, reasoning: dict, structured_output: bool, + records_path: str | Path, cache_path: str | Path) -> dict: + """Sample exactly one selected option per prompt, including canonical and reversed option orders. + + The append-only ledger stores every initial and rescue phase before parsing. A cache entry is written only + after all planned samples parse, so an incomplete construct pilot cannot look reusable. + """ + assert samples_per_order > 0 + assert temperature > 0 + assert reasoning == {"effort": "low"}, "the registered Gemini pilot uses catalog-supported low reasoning" + assert structured_output + plan = _plan(items, samples_per_order) + protocol_id = direct_choice_protocol_identity( + model, items, samples_per_order=samples_per_order, temperature=temperature, + max_tokens=max_tokens, concurrency=concurrency, request_timeout=request_timeout, + reasoning=reasoning, structured_output=structured_output, + ) + cache_file = Path(cache_path) + cache = json.loads(cache_file.read_text()) if cache_file.exists() else {"schema": 1, "completed": {}} + assert cache["schema"] == 1 + if protocol_id in cache["completed"]: + return {"cached": True, **cache["completed"][protocol_id]} + + run_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_{protocol_id[:12]}" + records = Path(records_path) + records.parent.mkdir(parents=True, exist_ok=True) + settings = { + "model": model, "samples_per_order": samples_per_order, "temperature": temperature, + "max_tokens": max_tokens, "concurrency": concurrency, "request_timeout": request_timeout, + "reasoning": reasoning, "structured_output": structured_output, + } + _append_record(records, { + "event": "run_started", "run_id": run_id, "protocol_id": protocol_id, + "construct": "direct_choice", "settings": settings, "items": items, + "planned_requests": len(plan), "canonical_requests": len(plan) // 2, + "reversed_requests": len(plan) // 2, + }) + + async def run_all() -> list[dict]: + semaphore = asyncio.Semaphore(concurrency) + + async def call(sequence: int, request: dict) -> dict: + item = items[request["item_index"]] + request_id = f"{run_id}_{sequence:03d}" + request_meta = { + "request_id": request_id, "run_id": run_id, "protocol_id": protocol_id, + "construct": "direct_choice", "model": model, "item_id": item["id"], + "canonical_options": item["options"], "presented_options": request["presented_options"], + "presented_order": request["presented_order"], "order_name": request["order_name"], + "sample": request["sample"], "repetition": request["repetition"], + "prompt": request["prompt"], "settings": settings, + } + response_format = _choice_schema(item["n"]) + payload = { + "model": model, "messages": [{"role": "user", "content": request["prompt"]}], + "temperature": temperature, "max_tokens": max_tokens, "reasoning": reasoning, + "response_format": response_format, + } + phase = "initial" + async with semaphore: + try: + _append_record(records, {"event": "request_started", "phase": phase, **request_meta, "payload": payload}) + response = await asyncio.wait_for(openrouter_request(payload), timeout=request_timeout) + _append_record(records, {"event": "request_completed", "phase": phase, **request_meta, + "response": response, "usage": response.get("usage")}) + if len(response["choices"]) != 1: + raise ValueError(f"expected one choice, got {len(response['choices'])}") + message = response["choices"][0]["message"] + text = message.get("content") or "" + rescued = False + if _parse_choice(text, item["n"]) is None: + phase = "rescue" + assistant_tail = (message.get("reasoning") or message.get("content") or "")[-1500:] or "(thinking truncated)" + rescue_payload = { + "model": model, "messages": [ + {"role": "user", "content": request["prompt"]}, + {"role": "assistant", "content": assistant_tail}, + {"role": "user", "content": _force_choice(item["n"])}, + ], "temperature": temperature, "max_tokens": max(max_tokens, 2048), + "reasoning": reasoning, "response_format": response_format, + } + _append_record(records, {"event": "request_started", "phase": phase, **request_meta, + "payload": rescue_payload, "initial_response_message": message}) + response = await asyncio.wait_for(openrouter_request(rescue_payload), timeout=request_timeout) + _append_record(records, {"event": "request_completed", "phase": phase, **request_meta, + "response": response, "usage": response.get("usage")}) + if len(response["choices"]) != 1: + raise ValueError(f"expected one rescue choice, got {len(response['choices'])}") + text = response["choices"][0]["message"].get("content") or "" + rescued = True + return {"text": text, "rescued": rescued, "error": None} + except Exception as exc: + _append_record(records, {"event": "request_failed", "phase": phase, **request_meta, + "error_type": type(exc).__name__, "error": str(exc)}) + return {"text": None, "rescued": phase == "rescue", "error": f"{type(exc).__name__}: {exc}"} + + return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan))) + + results = asyncio.run(run_all()) + by_item = {item["id"]: [] for item in items} + failed = 0 + rescues = 0 + for request, result in zip(plan, results): + rescues += int(result["rescued"]) + if result["error"] is not None: + failed += 1 + continue + presented_choice = _parse_choice(result["text"], items[request["item_index"]]["n"]) + _append_record(records, { + "event": "answer_parsed", "run_id": run_id, "protocol_id": protocol_id, + "construct": "direct_choice", "model": model, "item_id": request["item_id"], + "sample": request["sample"], "order_name": request["order_name"], + "repetition": request["repetition"], "presented_order": request["presented_order"], + "text": result["text"], "parsed": presented_choice is not None, + "presented_choice": presented_choice, + "canonical_choice": request["presented_order"][presented_choice] if presented_choice is not None else None, + }) + if presented_choice is not None: + by_item[request["item_id"]].append({ + "sample": request["sample"], "order_name": request["order_name"], + "repetition": request["repetition"], "presented_order": request["presented_order"], + "presented_choice": presented_choice, "canonical_choice": request["presented_order"][presented_choice], + }) + + item_results = [] + expected_samples = 2 * samples_per_order + for item in items: + samples = by_item[item["id"]] + canonical = sum(sample["order_name"] == "canonical" for sample in samples) + reversed_order = sum(sample["order_name"] == "reversed" for sample in samples) + result = { + "item_id": item["id"], "n": item["n"], "expected_samples": expected_samples, + "valid_samples": len(samples), "canonical_valid": canonical, "reversed_valid": reversed_order, + "samples": samples, + } + _append_record(records, {"event": "item_result", "run_id": run_id, "protocol_id": protocol_id, + "construct": "direct_choice", "model": model, **result}) + item_results.append(result) + + complete = failed == 0 and all(result["valid_samples"] == expected_samples for result in item_results) + summary = { + "run_id": run_id, "protocol_id": protocol_id, "model": model, "settings": settings, + "planned_requests": len(plan), "failed_requests": failed, "rescued_requests": rescues, + "complete": complete, "items": item_results, + } + _append_record(records, {"event": "run_finished", "construct": "direct_choice", **summary}) + if complete: + cache["completed"][protocol_id] = summary + _save_cache(cache_file, cache) + return {"cached": False, **summary}