Preregister Gemini direct-choice WVS pilot

Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com>

scripts/wvs_api/02_gemini37_direct_choice_pilot.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+set -eu
+uv run --with 'datasets>=4.0,<5' python scripts/wvs_direct_choice_pilot.py --run

scripts/wvs_direct_choice_pilot.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python3
+"""Preregister and run one direct-choice WVS construct pilot, separate from rated map panels."""
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import UTC, datetime
+from decimal import Decimal
+from pathlib import Path
+
+from moralmaps.read_direct_choice import direct_choice_protocol_identity, read_items_direct_choice
+from wvs_map import X_AXIS, Y_AXIS, load_wvs_all, resolve_items
+
+MODEL = "google/gemini-3.7-flash"
+ITEM_IDS = ("Homosexuality", "Religion", "God", "Independence")
+SAMPLES_PER_ORDER = 12
+TEMPERATURE = 1.0
+MAX_TOKENS = 1024
+CONCURRENCY = 1
+REQUEST_TIMEOUT = 90.0
+REASONING = {"effort": "low"}
+STRUCTURED_OUTPUT = True
+EXPECTED_INITIAL_CALLS = len(ITEM_IDS) * SAMPLES_PER_ORDER * 2
+GLOBAL_STOP_USD = Decimal("80")
+PRIORITY_PHASE_STOP_USD = Decimal("35")
+PILOT_CONSERVATIVE_RESERVE_USD = Decimal("2.00")
+CATALOG_PATH = Path("slop/research/wvs/20260917_openrouter_models.json")
+RATED_LEDGER_PATH = Path("slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl")
+RECORDS_PATH = Path("slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_requests.jsonl")
+CACHE_PATH = Path("slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_cache.json")
+MANIFEST_PATH = Path("slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_manifest.md")
+
+
+def utc_date(timestamp: int) -> str:
+    return datetime.fromtimestamp(timestamp, UTC).date().isoformat()
+
+
+def usage_cost(path: Path) -> Decimal:
+    if not path.exists():
+        return Decimal()
+    total = Decimal()
+    for line in path.read_text().splitlines():
+        event = json.loads(line)
+        if event["event"] == "request_completed" and event.get("usage", {}).get("cost") is not None:
+            total += Decimal(str(event["usage"]["cost"]))
+    return total
+
+
+def selected_items() -> list[dict]:
+    resolved = resolve_items(load_wvs_all())
+    selected = {}
+    for axis in (X_AXIS, Y_AXIS):
+        for item in resolved[axis]:
+            if item["suffix"] in ITEM_IDS:
+                selected[item["suffix"]] = {
+                    "id": item["suffix"], "question": item["rec"]["q"],
+                    "options": item["rec"]["opts"], "n": item["n"], "axis": axis,
+                }
+    assert tuple(selected) == ITEM_IDS, f"WVS item identity drift: {tuple(selected)}"
+    return [selected[item_id] for item_id in ITEM_IDS]
+
+
+def catalog_model() -> dict:
+    catalog = {model["id"]: model for model in json.loads(CATALOG_PATH.read_text())["data"]}
+    model = catalog[MODEL]
+    assert "structured_outputs" in model["supported_parameters"]
+    assert model["reasoning"]["mandatory"]
+    assert "low" in model["reasoning"]["supported_efforts"]
+    return model
+
+
+def protocol_id(items: list[dict]) -> str:
+    return direct_choice_protocol_identity(
+        MODEL, items, samples_per_order=SAMPLES_PER_ORDER, temperature=TEMPERATURE,
+        max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT,
+        reasoning=REASONING, structured_output=STRUCTURED_OUTPUT,
+    )
+
+
+def preflight(items: list[dict], model: dict) -> dict:
+    assert EXPECTED_INITIAL_CALLS == 96
+    assert all(item["n"] >= 2 for item in items)
+    rated_cost = usage_cost(RATED_LEDGER_PATH)
+    direct_cost = usage_cost(RECORDS_PATH)
+    cumulative_cost = rated_cost + direct_cost
+    assert cumulative_cost + PILOT_CONSERVATIVE_RESERVE_USD < PRIORITY_PHASE_STOP_USD
+    assert cumulative_cost + PILOT_CONSERVATIVE_RESERVE_USD < GLOBAL_STOP_USD
+    output_price_per_million = Decimal(model["pricing"]["completion"]) * 1_000_000
+    initial_completion_ceiling = output_price_per_million * EXPECTED_INITIAL_CALLS * MAX_TOKENS / 1_000_000
+    rescue_completion_ceiling = output_price_per_million * EXPECTED_INITIAL_CALLS * max(MAX_TOKENS, 2048) / 1_000_000
+    return {
+        "rated_ledger_cost": rated_cost,
+        "direct_choice_ledger_cost": direct_cost,
+        "cumulative_cost": cumulative_cost,
+        "output_price_per_million": output_price_per_million,
+        "initial_completion_ceiling": initial_completion_ceiling,
+        "all_rescue_completion_ceiling": initial_completion_ceiling + rescue_completion_ceiling,
+        "protocol_id": protocol_id(items),
+    }
+
+
+def write_manifest(items: list[dict], model: dict, checks: dict) -> None:
+    option_schema = '{"answer": integer from 0 through n-1}'
+    item_names = ", ".join(f"`{item['id']}`" for item in items)
+    lines = [
+        "# Gemini 3.7 Flash direct-choice WVS construct pilot",
+        "",
+        "This preregistered pilot tests a construct difference. It is not a WVS map panel and cannot alter rated coordinates.",
+        "",
+        "## Exact identity",
+        "",
+        f"- model: `{MODEL}`",
+        f"- saved-catalog created UTC: {utc_date(model['created'])}",
+        f"- protocol ID: `{checks['protocol_id']}`",
+        f"- cache: `{CACHE_PATH}`",
+        f"- append-only request ledger: `{RECORDS_PATH}`",
+        f"- temperature: {TEMPERATURE}; max tokens: {MAX_TOKENS}; concurrency: {CONCURRENCY}; timeout: {REQUEST_TIMEOUT}s",
+        f"- reasoning: `{json.dumps(REASONING, sort_keys=True)}`; catalog confirms mandatory reasoning with supported `low` effort",
+        f"- strict response schema: `{option_schema}`",
+        "",
+        "## Design",
+        "",
+        f"- items: {item_names}",
+        f"- {SAMPLES_PER_ORDER} canonical + {SAMPLES_PER_ORDER} reversed presented-option orders per item",
+        f"- expected initial calls: {EXPECTED_INITIAL_CALLS}; expected parsed choices: {EXPECTED_INITIAL_CALLS}",
+        "- exact instruction: `Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. Answer immediately.`",
+        "- final response is schema-only. Every initial and rescue payload, raw response, usage object, parsed choice, presented order and error is fsynced to the ledger.",
+        "- an incomplete item or any failed request makes `--run` exit nonzero and prevents a cache entry.",
+        "",
+        "## Preregistered comparison",
+        "",
+        "For each item, map selected presented indices back to canonical option indices. Report the canonical and reversed empirical categorical distributions, their total-variation distance, and the canonical-versus-reversed argmax agreement. Compare the direct-choice aggregate distribution with Gemini's existing dense-rated distribution and report total variation plus the dense-rated midpoint mass. These are construct diagnostics, not a coordinate replacement or a capability claim.",
+        "",
+        "## Spend checks before dispatch",
+        "",
+        f"- rated-ledger observed cost: USD {checks['rated_ledger_cost']:.10f}",
+        f"- direct-choice-ledger observed cost: USD {checks['direct_choice_ledger_cost']:.10f}",
+        f"- cumulative observed cost: USD {checks['cumulative_cost']:.10f}",
+        f"- current output price: USD {checks['output_price_per_million']:g}/M tokens",
+        f"- 96 initial 1024-token completion-only ceiling: USD {checks['initial_completion_ceiling']:.6f}",
+        f"- all-initial plus all-rescue 2048-token completion-only ceiling: USD {checks['all_rescue_completion_ceiling']:.6f}; prompt tokens are additional",
+        f"- pre-dispatch conservative reserve: USD {PILOT_CONSERVATIVE_RESERVE_USD:.2f}; it remains below the USD {PRIORITY_PHASE_STOP_USD} priority-phase and USD {GLOBAL_STOP_USD} global stops",
+        "- no wider priority model dispatch is authorized by this manifest.",
+        "",
+        "-- PI[gpt-5.6-terra]",
+        "",
+    ]
+    MANIFEST_PATH.parent.mkdir(parents=True, exist_ok=True)
+    MANIFEST_PATH.write_text("\n".join(lines))
+
+
+def smoke(items: list[dict], checks: dict) -> None:
+    assert len(items) == 4
+    assert len({item["id"] for item in items}) == 4
+    assert checks["protocol_id"] == protocol_id(items)
+    assert all(item["n"] >= 2 for item in items)
+    print(f"smoke: 4 items x 12 canonical x 12 reversed = {EXPECTED_INITIAL_CALLS} requests")
+    print(f"smoke: distinct direct-choice protocol {checks['protocol_id']}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--run", action="store_true", help="make the preregistered paid pilot calls")
+    parser.add_argument("--smoke", action="store_true", help="validate the manifest and request plan without API calls")
+    args = parser.parse_args()
+    items = selected_items()
+    model = catalog_model()
+    checks = preflight(items, model)
+    write_manifest(items, model, checks)
+    if args.smoke:
+        smoke(items, checks)
+    if not args.run:
+        return
+    result = read_items_direct_choice(
+        MODEL, items, samples_per_order=SAMPLES_PER_ORDER, temperature=TEMPERATURE,
+        max_tokens=MAX_TOKENS, concurrency=CONCURRENCY, request_timeout=REQUEST_TIMEOUT,
+        reasoning=REASONING, structured_output=STRUCTURED_OUTPUT,
+        records_path=RECORDS_PATH, cache_path=CACHE_PATH,
+    )
+    if result["cached"]:
+        print(f"direct-choice cache hit: protocol={result['protocol_id'][:12]}")
+        return
+    if not result["complete"]:
+        raise RuntimeError(f"incomplete direct-choice pilot: {result['run_id']}; raw evidence is {RECORDS_PATH}")
+    print(f"complete direct-choice pilot: {result['run_id']}, protocol={result['protocol_id'][:12]}")
+
+
+if __name__ == "__main__":
+    main()

slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_manifest.md

@@ -0,0 +1,40 @@
+# Gemini 3.7 Flash direct-choice WVS construct pilot
+
+This preregistered pilot tests a construct difference. It is not a WVS map panel and cannot alter rated coordinates.
+
+## Exact identity
+
+- model: `google/gemini-3.7-flash`
+- saved-catalog created UTC: 2026-08-13
+- protocol ID: `433768e674cf0ed8ca4b1677637c1140dbca7910c289da06dbc83124df6f8fb7`
+- cache: `slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_cache.json`
+- append-only request ledger: `slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_requests.jsonl`
+- temperature: 1.0; max tokens: 1024; concurrency: 1; timeout: 90.0s
+- reasoning: `{"effort": "low"}`; catalog confirms mandatory reasoning with supported `low` effort
+- strict response schema: `{"answer": integer from 0 through n-1}`
+
+## Design
+
+- items: `Homosexuality`, `Religion`, `God`, `Independence`
+- 12 canonical + 12 reversed presented-option orders per item
+- expected initial calls: 96; expected parsed choices: 96
+- exact instruction: `Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. Answer immediately.`
+- final response is schema-only. Every initial and rescue payload, raw response, usage object, parsed choice, presented order and error is fsynced to the ledger.
+- an incomplete item or any failed request makes `--run` exit nonzero and prevents a cache entry.
+
+## Preregistered comparison
+
+For each item, map selected presented indices back to canonical option indices. Report the canonical and reversed empirical categorical distributions, their total-variation distance, and the canonical-versus-reversed argmax agreement. Compare the direct-choice aggregate distribution with Gemini's existing dense-rated distribution and report total variation plus the dense-rated midpoint mass. These are construct diagnostics, not a coordinate replacement or a capability claim.
+
+## Spend checks before dispatch
+
+- rated-ledger observed cost: USD 3.6235153224
+- direct-choice-ledger observed cost: USD 0.0000000000
+- cumulative observed cost: USD 3.6235153224
+- current output price: USD 3.75000000/M tokens
+- 96 initial 1024-token completion-only ceiling: USD 0.368640
+- all-initial plus all-rescue 2048-token completion-only ceiling: USD 1.105920; prompt tokens are additional
+- pre-dispatch conservative reserve: USD 2.00; it remains below the USD 35 priority-phase and USD 80 global stops
+- no wider priority model dispatch is authorized by this manifest.
+
+-- PI[gpt-5.6-terra]

slop/research/wvs/20260917_direct_choice/preflight_smoke.log

@@ -0,0 +1,3 @@
+smoke: 4 items x 12 canonical x 12 reversed = 96 requests
+smoke: distinct direct-choice protocol 433768e674cf0ed8ca4b1677637c1140dbca7910c289da06dbc83124df6f8fb7
+planner/parser smoke: 96 requests, 12 canonical + 12 reversed per item; strict one-key parser

src/moralmaps/read_direct_choice.py

@@ -0,0 +1,261 @@
+"""Durable direct-choice sampling for construct checks, separate from dense ratings."""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import os
+import re
+from datetime import UTC, datetime
+from pathlib import Path
+
+from openrouter_wrapper.retry import openrouter_request
+
+
+def _append_record(path: Path, record: dict) -> None:
+    record["recorded_at_utc"] = datetime.now(UTC).isoformat()
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(record, ensure_ascii=True, sort_keys=True) + "\n")
+        fh.flush()
+        os.fsync(fh.fileno())
+
+
+def _choice_schema(n: int) -> dict:
+    return {"type": "json_schema", "json_schema": {"name": "one_choice", "strict": True, "schema": {
+        "type": "object",
+        "properties": {"answer": {"type": "integer", "minimum": 0, "maximum": n - 1}},
+        "required": ["answer"], "additionalProperties": False,
+    }}}
+
+
+def _parse_choice(text: str, n: int) -> int | None:
+    objects = re.findall(r"\{[^{}]*\}", text)
+    if not objects:
+        return None
+    try:
+        answer = json.loads(objects[-1])
+    except json.JSONDecodeError:
+        return None
+    if set(answer) != {"answer"}:
+        return None
+    choice = answer["answer"]
+    if type(choice) is not int or not 0 <= choice < n:
+        return None
+    return choice
+
+
+def _force_choice(n: int) -> str:
+    return f'Return ONLY {{"answer": <integer 0 through {n - 1}>}}. No explanation.'
+
+
+def _plan(items: list[dict], samples_per_order: int) -> list[dict]:
+    plan = []
+    for item_index, item in enumerate(items):
+        for order_name, order in (("canonical", list(range(item["n"]))), ("reversed", list(reversed(range(item["n"]))))):
+            prompt = _choice_prompt(item, order)
+            for repetition in range(samples_per_order):
+                plan.append({
+                    "item_index": item_index,
+                    "item_id": item["id"],
+                    "sample": len(plan) % (2 * samples_per_order),
+                    "order_name": order_name,
+                    "repetition": repetition,
+                    "presented_order": order,
+                    "presented_options": [item["options"][index] for index in order],
+                    "prompt": prompt,
+                })
+    return plan
+
+
+def _choice_prompt(item: dict, order: list[int]) -> str:
+    options = "\n".join(f"{position}) {item['options'][canonical]}" for position, canonical in enumerate(order))
+    return (
+        f"{item['question']}\n\n"
+        "Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. "
+        "Answer immediately.\n\n"
+        f"Choose exactly one answer:\n{options}\n\n"
+        f"Respond with ONLY a JSON object such as {{\"answer\": 0}}. The answer must be an integer from 0 through {item['n'] - 1}."
+    )
+
+
+def direct_choice_protocol_identity(model: str, items: list[dict], *, samples_per_order: int,
+                                    temperature: float, max_tokens: int, concurrency: int,
+                                    request_timeout: float, reasoning: dict, structured_output: bool) -> str:
+    plan = _plan(items, samples_per_order)
+    protocol = {
+        "schema": 1,
+        "construct": "direct_choice",
+        "model": model,
+        "samples_per_order": samples_per_order,
+        "temperature": temperature,
+        "max_tokens": max_tokens,
+        "concurrency": concurrency,
+        "request_timeout": request_timeout,
+        "reasoning": reasoning,
+        "structured_output": structured_output,
+        "prompt_instruction": "Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. Answer immediately.",
+        "rescue_instruction": _force_choice(10),
+        "requests": plan,
+    }
+    encoded = json.dumps(protocol, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode()
+    return hashlib.sha256(encoded).hexdigest()
+
+
+def _save_cache(path: Path, cache: dict) -> None:
+    temp = path.with_suffix(path.suffix + ".tmp")
+    temp.write_text(json.dumps(cache, indent=2, sort_keys=True) + "\n")
+    temp.replace(path)
+
+
+def read_items_direct_choice(model: str, items: list[dict], *, samples_per_order: int,
+                             temperature: float, max_tokens: int, concurrency: int,
+                             request_timeout: float, reasoning: dict, structured_output: bool,
+                             records_path: str | Path, cache_path: str | Path) -> dict:
+    """Sample exactly one selected option per prompt, including canonical and reversed option orders.
+
+    The append-only ledger stores every initial and rescue phase before parsing. A cache entry is written only
+    after all planned samples parse, so an incomplete construct pilot cannot look reusable.
+    """
+    assert samples_per_order > 0
+    assert temperature > 0
+    assert reasoning == {"effort": "low"}, "the registered Gemini pilot uses catalog-supported low reasoning"
+    assert structured_output
+    plan = _plan(items, samples_per_order)
+    protocol_id = direct_choice_protocol_identity(
+        model, items, samples_per_order=samples_per_order, temperature=temperature,
+        max_tokens=max_tokens, concurrency=concurrency, request_timeout=request_timeout,
+        reasoning=reasoning, structured_output=structured_output,
+    )
+    cache_file = Path(cache_path)
+    cache = json.loads(cache_file.read_text()) if cache_file.exists() else {"schema": 1, "completed": {}}
+    assert cache["schema"] == 1
+    if protocol_id in cache["completed"]:
+        return {"cached": True, **cache["completed"][protocol_id]}
+
+    run_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}_{protocol_id[:12]}"
+    records = Path(records_path)
+    records.parent.mkdir(parents=True, exist_ok=True)
+    settings = {
+        "model": model, "samples_per_order": samples_per_order, "temperature": temperature,
+        "max_tokens": max_tokens, "concurrency": concurrency, "request_timeout": request_timeout,
+        "reasoning": reasoning, "structured_output": structured_output,
+    }
+    _append_record(records, {
+        "event": "run_started", "run_id": run_id, "protocol_id": protocol_id,
+        "construct": "direct_choice", "settings": settings, "items": items,
+        "planned_requests": len(plan), "canonical_requests": len(plan) // 2,
+        "reversed_requests": len(plan) // 2,
+    })
+
+    async def run_all() -> list[dict]:
+        semaphore = asyncio.Semaphore(concurrency)
+
+        async def call(sequence: int, request: dict) -> dict:
+            item = items[request["item_index"]]
+            request_id = f"{run_id}_{sequence:03d}"
+            request_meta = {
+                "request_id": request_id, "run_id": run_id, "protocol_id": protocol_id,
+                "construct": "direct_choice", "model": model, "item_id": item["id"],
+                "canonical_options": item["options"], "presented_options": request["presented_options"],
+                "presented_order": request["presented_order"], "order_name": request["order_name"],
+                "sample": request["sample"], "repetition": request["repetition"],
+                "prompt": request["prompt"], "settings": settings,
+            }
+            response_format = _choice_schema(item["n"])
+            payload = {
+                "model": model, "messages": [{"role": "user", "content": request["prompt"]}],
+                "temperature": temperature, "max_tokens": max_tokens, "reasoning": reasoning,
+                "response_format": response_format,
+            }
+            phase = "initial"
+            async with semaphore:
+                try:
+                    _append_record(records, {"event": "request_started", "phase": phase, **request_meta, "payload": payload})
+                    response = await asyncio.wait_for(openrouter_request(payload), timeout=request_timeout)
+                    _append_record(records, {"event": "request_completed", "phase": phase, **request_meta,
+                                             "response": response, "usage": response.get("usage")})
+                    if len(response["choices"]) != 1:
+                        raise ValueError(f"expected one choice, got {len(response['choices'])}")
+                    message = response["choices"][0]["message"]
+                    text = message.get("content") or ""
+                    rescued = False
+                    if _parse_choice(text, item["n"]) is None:
+                        phase = "rescue"
+                        assistant_tail = (message.get("reasoning") or message.get("content") or "")[-1500:] or "(thinking truncated)"
+                        rescue_payload = {
+                            "model": model, "messages": [
+                                {"role": "user", "content": request["prompt"]},
+                                {"role": "assistant", "content": assistant_tail},
+                                {"role": "user", "content": _force_choice(item["n"])},
+                            ], "temperature": temperature, "max_tokens": max(max_tokens, 2048),
+                            "reasoning": reasoning, "response_format": response_format,
+                        }
+                        _append_record(records, {"event": "request_started", "phase": phase, **request_meta,
+                                                 "payload": rescue_payload, "initial_response_message": message})
+                        response = await asyncio.wait_for(openrouter_request(rescue_payload), timeout=request_timeout)
+                        _append_record(records, {"event": "request_completed", "phase": phase, **request_meta,
+                                                 "response": response, "usage": response.get("usage")})
+                        if len(response["choices"]) != 1:
+                            raise ValueError(f"expected one rescue choice, got {len(response['choices'])}")
+                        text = response["choices"][0]["message"].get("content") or ""
+                        rescued = True
+                    return {"text": text, "rescued": rescued, "error": None}
+                except Exception as exc:
+                    _append_record(records, {"event": "request_failed", "phase": phase, **request_meta,
+                                             "error_type": type(exc).__name__, "error": str(exc)})
+                    return {"text": None, "rescued": phase == "rescue", "error": f"{type(exc).__name__}: {exc}"}
+
+        return await asyncio.gather(*(call(sequence, request) for sequence, request in enumerate(plan)))
+
+    results = asyncio.run(run_all())
+    by_item = {item["id"]: [] for item in items}
+    failed = 0
+    rescues = 0
+    for request, result in zip(plan, results):
+        rescues += int(result["rescued"])
+        if result["error"] is not None:
+            failed += 1
+            continue
+        presented_choice = _parse_choice(result["text"], items[request["item_index"]]["n"])
+        _append_record(records, {
+            "event": "answer_parsed", "run_id": run_id, "protocol_id": protocol_id,
+            "construct": "direct_choice", "model": model, "item_id": request["item_id"],
+            "sample": request["sample"], "order_name": request["order_name"],
+            "repetition": request["repetition"], "presented_order": request["presented_order"],
+            "text": result["text"], "parsed": presented_choice is not None,
+            "presented_choice": presented_choice,
+            "canonical_choice": request["presented_order"][presented_choice] if presented_choice is not None else None,
+        })
+        if presented_choice is not None:
+            by_item[request["item_id"]].append({
+                "sample": request["sample"], "order_name": request["order_name"],
+                "repetition": request["repetition"], "presented_order": request["presented_order"],
+                "presented_choice": presented_choice, "canonical_choice": request["presented_order"][presented_choice],
+            })
+
+    item_results = []
+    expected_samples = 2 * samples_per_order
+    for item in items:
+        samples = by_item[item["id"]]
+        canonical = sum(sample["order_name"] == "canonical" for sample in samples)
+        reversed_order = sum(sample["order_name"] == "reversed" for sample in samples)
+        result = {
+            "item_id": item["id"], "n": item["n"], "expected_samples": expected_samples,
+            "valid_samples": len(samples), "canonical_valid": canonical, "reversed_valid": reversed_order,
+            "samples": samples,
+        }
+        _append_record(records, {"event": "item_result", "run_id": run_id, "protocol_id": protocol_id,
+                                 "construct": "direct_choice", "model": model, **result})
+        item_results.append(result)
+
+    complete = failed == 0 and all(result["valid_samples"] == expected_samples for result in item_results)
+    summary = {
+        "run_id": run_id, "protocol_id": protocol_id, "model": model, "settings": settings,
+        "planned_requests": len(plan), "failed_requests": failed, "rescued_requests": rescues,
+        "complete": complete, "items": item_results,
+    }
+    _append_record(records, {"event": "run_finished", "construct": "direct_choice", **summary})
+    if complete:
+        cache["completed"][protocol_id] = summary
+        _save_cache(cache_file, cache)
+    return {"cached": False, **summary}