#!/usr/bin/env python3 """Audit the preregistered Gemini direct-choice construct pilot without API calls.""" from __future__ import annotations import csv import json from collections import Counter from decimal import Decimal from pathlib import Path import numpy as np RUN_ID = "20260917T025121Z_aed0e29dd4ee" PROTOCOL_ID = "aed0e29dd4ee423dbbfa0c294a84b2ae4bd6a36d6fc4bf569120034ac5b75090" MODEL = "google/gemini-3.7-flash" DIRECT_LEDGER = Path("slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_requests.jsonl") DIRECT_CACHE = Path("slop/research/wvs/20260917_direct_choice/gemini37_direct_choice_cache.json") RATED_LEDGER = Path("slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl") RATED_RUN_ID = "20260916T172946Z_cd5db529649a" OUT_CSV = Path("slop/audits/20260917_wvs_gemini37_direct_choice_task_1628_by_item.csv") OUT_MD = Path("slop/audits/20260917_wvs_gemini37_direct_choice_task_1628.md") def events(path: Path) -> list[dict]: return [json.loads(line) for line in path.read_text().splitlines()] def total_variation(left: np.ndarray, right: np.ndarray) -> float: return float(0.5 * np.abs(left - right).sum()) def choice_distribution(rows: list[dict], n: int) -> np.ndarray: counts = np.zeros(n) for row in rows: counts[row["canonical_choice"]] += 1 return counts / len(rows) def dense_rated_distribution(rows: list[dict], n: int) -> np.ndarray: samples = [] for row in rows: ratings = json.loads(row["text"]) order = row["presented_order"] presented = np.array([ratings[str(index)] for index in range(n)], dtype=float) canonical = np.empty(n) canonical[np.asarray(order)] = presented samples.append(canonical / canonical.sum()) return np.mean(samples, axis=0) def display(p: np.ndarray) -> str: return "[" + ", ".join(f"{value:.3f}" for value in p) + "]" def midpoint_mass(p: np.ndarray) -> str: if len(p) == 2: return "not defined for binary" lower = len(p) // 2 - 1 upper = len(p) // 2 return f"{p[lower] + p[upper]:.3f} (options {lower}/{upper})" def quote(message: dict) -> str: reasoning = message.get("reasoning") return (reasoning or message.get("content") or "").replace("\n", " ").strip() def main() -> None: direct = [event for event in events(DIRECT_LEDGER) if event.get("run_id") == RUN_ID] rated = [event for event in events(RATED_LEDGER) if event.get("run_id") == RATED_RUN_ID] assert direct and rated counts = Counter(event["event"] for event in direct) assert counts == Counter({"request_started": 96, "request_completed": 96, "answer_parsed": 96, "item_result": 4, "run_started": 1, "run_finished": 1}), counts parsed = [event for event in direct if event["event"] == "answer_parsed"] assert len(parsed) == 96 and all(event["parsed"] for event in parsed) for event in parsed: decoded = json.loads(event["text"]) assert set(decoded) == {"answer"} assert type(decoded["answer"]) is int assert event["canonical_choice"] == event["presented_order"][decoded["answer"]] assert not [event for event in direct if event["event"] == "request_failed"] assert not [event for event in direct if event.get("phase") == "rescue"] assert {event["protocol_id"] for event in direct} == {PROTOCOL_ID} assert json.loads(DIRECT_CACHE.read_text())["completed"][PROTOCOL_ID]["complete"] request_events = [event for event in direct if event["event"] == "request_completed"] prompt_tokens = sum(event["usage"]["prompt_tokens"] for event in request_events) completion_tokens = sum(event["usage"]["completion_tokens"] for event in request_events) reasoning_tokens = sum(event["usage"]["completion_tokens_details"]["reasoning_tokens"] for event in request_events) cost = sum(Decimal(str(event["usage"]["cost"])) for event in request_events) refusals = sum(event["response"]["choices"][0]["message"].get("refusal") is not None for event in request_events) rows = [] for item_id in ("Homosexuality", "Religion", "God", "Independence"): direct_rows = [event for event in parsed if event["item_id"] == item_id] n = len(direct_rows[0]["presented_order"]) canonical = [event for event in direct_rows if event["order_name"] == "canonical"] reversed_order = [event for event in direct_rows if event["order_name"] == "reversed"] assert len(canonical) == len(reversed_order) == 12 direct_p = choice_distribution(direct_rows, n) canonical_p = choice_distribution(canonical, n) reversed_p = choice_distribution(reversed_order, n) rated_rows = [event for event in rated if event["event"] == "answer_parsed" and event["item_id"] == item_id] assert len(rated_rows) == 12 rated_p = dense_rated_distribution(rated_rows, n) canonical_argmax = np.flatnonzero(canonical_p == canonical_p.max()).tolist() reversed_argmax = np.flatnonzero(reversed_p == reversed_p.max()).tolist() rows.append({ "item_id": item_id, "n_options": n, "canonical_n": len(canonical), "reversed_n": len(reversed_order), "canonical_distribution": display(canonical_p), "reversed_distribution": display(reversed_p), "order_half_tv": total_variation(canonical_p, reversed_p), "canonical_argmax": canonical_argmax, "reversed_argmax": reversed_argmax, "argmax_agrees": canonical_argmax == reversed_argmax, "direct_distribution": display(direct_p), "rated_distribution": display(rated_p), "direct_vs_rated_tv": total_variation(direct_p, rated_p), "rated_middle_mass": midpoint_mass(rated_p), }) OUT_CSV.parent.mkdir(parents=True, exist_ok=True) with OUT_CSV.open("w", newline="") as fh: writer = csv.DictWriter(fh, fieldnames=list(rows[0]), lineterminator="\n") writer.writeheader() writer.writerows(rows) first_homosexuality = next(event for event in request_events if event["item_id"] == "Homosexuality" and event["sample"] == 0) first_god = next(event for event in request_events if event["item_id"] == "God" and event["sample"] == 0) ledger_through = max(event["recorded_at_utc"] for event in direct) by_item = {row["item_id"]: row for row in rows} lines = [ "# Audit: Gemini 3.7 Flash direct-choice WVS pilot, task 1628", "", "- target: 96-call direct-choice construct pilot, not a map panel", f"- Pueue: task 1628, API queue, success, 2026-09-17 10:51:14-10:56:58 +08:00", "- label: `why: test direct choice against flat dense ratings; resolve: audit interleaved order agreement and distribution difference before more models`", f"- run: `{RUN_ID}`, protocol: `{PROTOCOL_ID}`", f"- primary ledger: `{DIRECT_LEDGER}` through {ledger_through}", f"- direct cache: `{DIRECT_CACHE}`", f"- comparison rated run: `{RATED_RUN_ID}` in `{RATED_LEDGER}`", f"- per-item table: `{OUT_CSV}`", "", "## Stage table", "", "| stage | expected | observed | expected? | clues | missing metric | consequence |", "|---|---|---|---|---|---|---|", "| request plan | 96 calls, 12 canonical and 12 reversed per item, interleaved | 96 starts and 96 completions; every item has 12 valid per order | yes | ledger event counts; cache item results | provider-side request ordering timestamp | order halves are available for comparison |", "| strict parse | 96 schema-valid single choices | 96/96 `answer_parsed=true`; 0 failed phases | yes | ledger | provider schema conformance independent of parser | mechanics did not drop samples |", "| rescue/refusal | zero or recorded | 0 rescue phases; 0 `message.refusal` | yes | ledger request records | semantic non-answer count beyond refusal field | parse success does not establish personal-attitude semantics |", "| accounting | provider usage retained | prompt 15,445; completion 12,848; reasoning 12,272; observed cost USD 0.0643245 | yes | all 96 completed usage records | external billing export | below registered reserve, exact provider field retained |", "| order-half control | canonical and reversed distributions agree if position does not dominate | Homosexuality TV=0.833 and different argmax; other three TV <=0.083 with matching argmax | no | per-item table | repeated independent run | direct-choice Homosexuality aggregate is position-confounded |", "| canonical decoder | reverse order maps presented index back to canonical index | independently re-decoded all 96 raw JSON choices and orders with exact agreement | yes | audit assertion | a second implementation or repetition | mapping bug is less likely than an order effect |", "| construct comparison | quantify difference from dense-rated readout | direct-vs-rated TV: Homosexuality 0.694, Religion 0.564, God 0.500, Independence 0.220 | yes, descriptive only | per-item table | baseline direct choice from another model | no coordinate replacement or wider batch decision |", "| persistence | complete result reusable only if all samples valid | cache has one complete protocol entry and ledger has 294 events | yes | cache + ledger | cache replay, not needed for this decision | source evidence retained |", "", "## Chronological evidence", "", "The task's own full Pueue output is one completion line, so the append-only request ledger is the primary evidence. It contains 96 initial `request_started`, 96 initial `request_completed`, 96 parsed choices, four item results, and one run boundary each. The source code writes a cache entry only after complete results; the cache records `complete: true` for this protocol.", "", "The direct prompt is recoverable per request. Homosexuality sample 0 used the exact instruction: `Think briefly about which ONE answer best reflects your own view. Do not analyze the survey design. Answer immediately.` Its saved provider reasoning says:", "", f"> {quote(first_homosexuality['response']['choices'][0]['message'])}", "", "epistemic context: provider reasoning saved in this pilot's first completed request, not a human self-report.", "", "God sample 0 shows the same unresolved persona issue despite schema-valid JSON:", "", f"> {quote(first_god['response']['choices'][0]['message'])}", "", "epistemic context: provider reasoning saved in this pilot's first completed request, not a human self-report.", "", "## Preregistered order-half and construct results", "", "| item | canonical p | reversed p | order TV | argmax agrees | direct p | dense-rated p | direct vs rated TV | dense rated central mass |", "|---|---|---|---:|---|---|---|---:|---|", ] for row in rows: lines.append( f"| {row['item_id']} | {row['canonical_distribution']} | {row['reversed_distribution']} | " f"{row['order_half_tv']:.3f} | {row['argmax_agrees']} ({row['canonical_argmax']} vs {row['reversed_argmax']}) | " f"{row['direct_distribution']} | {row['rated_distribution']} | {row['direct_vs_rated_tv']:.3f} | {row['rated_middle_mass']} |" ) lines.extend([ "", "For even non-binary cards, central mass is the dense-rated mass in the two middle categories. For binary cards it is not defined. An all-equal dense rating normalizes to a uniform categorical, not to one middle answer; this is why the table reports full distributions and total variation rather than calling all flat dense replies a middle choice.", "", "## Hypotheses", "", "### H1 [method | Highly Likely | 80%]", "", "- Mechanism: Homosexuality direct choices are sensitive to the presented order, so its pooled direct distribution is not a stable construct readout.", f"- Evidence: canonical p is {by_item['Homosexuality']['canonical_distribution']} while reversed p is {by_item['Homosexuality']['reversed_distribution']}; their TV is {by_item['Homosexuality']['order_half_tv']:.3f} and argmax changes from {by_item['Homosexuality']['canonical_argmax']} to {by_item['Homosexuality']['reversed_argmax']}.", "- Contrary evidence: Religion, God, and Independence have matching order-half argmaxes and TV at most 0.083.", "- Discriminating test: a second 24-per-item run with a balanced random permutation schedule. Low TV again would weaken this explanation; a large TV tied to option position would strengthen it.", "- Fix/action: do not use the pooled Homosexuality direct distribution to replace rated coordinates; review a redesigned order control before more paid panels.", "- Interpretability: partial, mechanics and the observed order effect are interpretable; Homosexuality attitude distribution is not.", "", "### H2 [measurement | Likely | 65%]", "", "- Mechanism: Gemini may answer the question as an AI without personal beliefs rather than supply an attitude-like direct choice.", f"- Evidence: God sample 0 reasoning says `{quote(first_god['response']['choices'][0]['message'])}`.", "- Contrary evidence: every final response is a valid selected answer, and three items have stable order-half argmaxes.", "- Discriminating test: compare an explicitly role-conditioned construct with this prompt while retaining the same order randomization. A changed distribution with lower persona-language would support this explanation.", "- Fix/action: retain raw reasoning and interpret current results as model behavior under this prompt, not personal survey attitudes.", "- Interpretability: partial.", "", "### H3 [measurement | Likely | 60%]", "", "- Mechanism: forced one-answer choice and dense all-option rating are different elicitation constructs, even where order halves agree.", f"- Evidence: Religion direct-vs-rated TV is {by_item['Religion']['direct_vs_rated_tv']:.3f}, God is {by_item['God']['direct_vs_rated_tv']:.3f}, and Independence is {by_item['Independence']['direct_vs_rated_tv']:.3f}.", "- Contrary evidence: this is one model and four items; Gemini's persona and order effects can also cause the difference.", "- Discriminating test: repair the order control, then compare one or more lower-flat models under the same direct-choice protocol.", "- Fix/action: describe the distributions as a construct comparison, not evidence that the published rated map is wrong.", "- Interpretability: yes for difference under these prompts, no for a general claim about models or WVS coordinates.", "", "### H4 [bug | Unlikely | 20%]", "", "- Mechanism: a mapping error could create the apparent reversed-order effect.", "- Evidence: the audit independently re-decodes all 96 raw JSON responses, validates the one-key schema, and maps `answer` through each stored `presented_order`; every reconstructed canonical choice equals the ledger field.", "- Contrary evidence: the audit is a second decoder, but it is not a second experimental run.", "- Discriminating test: repeat the balanced-permutation control after review. A large position-linked shift despite a new run would reject the mapping-bug explanation.", "- Fix/action: no mapping change is justified from this evidence.", "- Interpretability: yes for the observed canonical mapping.", "", "## Decision", "", "1. Resolve-condition verdict: **not met**. The task asked to resolve whether direct choice differed from flat dense ratings after auditing interleaved order agreement. The comparison exists, but Homosexuality order TV=0.833 with an argmax reversal, so the focal direct distribution is confounded by presentation order.", "2. Prediction check: recorded design predicted 12 valid choices per order and an auditable order-half comparison. Completeness is supported; order stability is contradicted for Homosexuality and supported for the other three items.", "3. Earliest unsupported link: a direct one-answer prompt measures a stable attitude-like choice distribution. The order-half control fails before any coordinate interpretation.", "4. Validity: define invalid as unsuitable for replacing rated coordinates or authorizing broad panel changes. P(invalid for that use) is highly likely, about 0.80. The result is a credible negative control for order stability, not an invalid ledger or billing record.", "5. Highest-information clues: (a) Homosexuality order TV 0.833, because it directly falsifies order invariance; (b) all 96 choices parsed with zero failures, separating mechanics from construct quality; (c) saved AI-persona reasoning, because it raises a semantic interpretation alternative.", "6. Missing metrics: independent canonical-choice reconstruction first; then a balanced-permutation repetition; then another model under the repaired protocol. These have higher information value than another full map panel.", "7. Bugs requiring code changes: none established. The next pilot should improve design, not silently change the current pilot.", "8. Misconceptions requiring reinterpretation: a schema-valid selected option is not evidence that the model expressed a personal WVS attitude. A flat dense rating is uniform after normalization, not a direct middle choice.", "9. What would change the verdict: low order-half TV under a balanced permutation schedule and no persona-language in saved reasoning would make direct-choice distributions more interpretable.", "10. Recommended sequence: preserve this pilot and pause the wider priority batch. Parent review should decide whether a balanced-permutation direct-choice replication is worth its bounded cost; do not combine a revised prompt and changed permutation schedule in one test.", "", "-- PI[gpt-5.6-terra]", "", ]) OUT_MD.write_text("\n".join(lines)) print(f"wrote {OUT_CSV}: {len(rows)} item rows") print(f"wrote {OUT_MD}: cost USD {cost:.7f}, 96/96 parsed, Homosexuality order TV {by_item['Homosexuality']['order_half_tv']:.3f}") if __name__ == "__main__": main()