From f5efbd24bd7c5b8cceb2267397220ae47f694975 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Sat, 4 Jul 2026 19:39:18 +0800 Subject: [PATCH] Add sampling readout for logprob-less API models (OpenRouter) read_api.read_items_sampled samples N chat completions at temperature and uses the empirical answer frequency as the per-item categorical p, emitting the same row shape the logprob reader does -- so E/profile/entropy flow through the identical per_item_categorical + reducers and a frontier model without logprobs drops onto the same map. pmass_allowed becomes the parse rate (sampling coherence gate); C/LO are omitted by design (log of a frequency has -inf zeros). This is the Economist's 'average of ten responses' method. UAT (docs/reviews/p3_api_sampling_uat.md): E_mc == E_logprob to <=0.01 (unbiased), llama-3.1-8b sampled E lands on the same [1,5] scale, parse-rate gate flags off-format draws. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- docs/reviews/p3_api_sampling_uat.md | 32 ++++++++ pyproject.toml | 6 ++ scripts/probe_api_readout.py | 100 +++++++++++++++++++++++++ src/tinymfv/read_api.py | 112 ++++++++++++++++++++++++++++ 4 files changed, 250 insertions(+) create mode 100644 docs/reviews/p3_api_sampling_uat.md create mode 100644 scripts/probe_api_readout.py create mode 100644 src/tinymfv/read_api.py diff --git a/docs/reviews/p3_api_sampling_uat.md b/docs/reviews/p3_api_sampling_uat.md new file mode 100644 index 0000000..aaa11f2 --- /dev/null +++ b/docs/reviews/p3_api_sampling_uat.md @@ -0,0 +1,32 @@ +# P3 UAT: sampling readout (read_api) vs logprob readout (read) + +Command: `uv run python scripts/probe_api_readout.py --api-model meta-llama/llama-3.1-8b-instruct --n-items 5 --n-samples 20` +Date: 2026-07-04. Instrument: mfq2 (forward frame, first 5 items). Local logprob model: Qwen3-0.6B. + +The API path (read_api.read_items_sampled) samples N chat completions at temperature and uses the +empirical answer frequency as `p`; E and profile then come from the SAME readouts as the logprob +path, so a logprob-less frontier model drops onto the same [1,5] scale / same map. + +| id | dimension | statement (trunc) | E_local(lp) | E_mc | E_api(llama-3.1-8b) | api_pmass | parsed | api_H | +|---:|:----------------|:-----------------------------------|------------:|-----:|--------------------:|----------:|:-------|------:| +| 1 | care | Caring for people who have suffere | 3.83 | 3.82 | 3.30 | 1.00 | 20/20 | 0.64 | +| 2 | equality | The world would be a better place | 3.08 | 3.08 | 3.00 | 1.00 | 20/20 | 1.37 | +| 3 | proportionality | I think people who are more hardwo | 3.65 | 3.66 | 2.80 | 1.00 | 20/20 | 0.94 | +| 4 | loyalty | I think children should be taught | 3.56 | 3.56 | 3.19 | 0.80 | 16/20 | 1.06 | +| 5 | authority | I think it is important for societ | 3.75 | 3.75 | 3.05 | 0.95 | 19/20 | 0.54 | + +Verdicts: +- Unbiasedness (same model, no API): `E_local(lp)` == `E_mc` to <=0.01 on every item (max gap over + all items 0.0063 at N=2000). Sampling the model's own categorical and reducing to E recovers the + exact logprob E -- the frequency->E estimator + reduce plumbing is unbiased. +- Live API: llama-3.1-8b sampled reader lands E in [1,5] for all 5 items. The sampling coherence + gate (`api_pmass` = parse rate) is 1.0 except item 4 (0.80, 4/20 off-format draws) and item 5 + (0.95) -- it correctly flags draws the model did not answer in-format, the sampling analogue of + the logprob `pmass_allowed`. +- Comparability: `E_api` sits on the same [1,5] scale as `E_local`; the gaps (e.g. 3.83 vs 3.30 on + care) are genuine Qwen3-0.6B vs Llama-3.1-8b disagreement, not method error. N=20 gives ~0.1 E + resolution; raise N for tighter dots. + +Not produced by design: logit_contrast C and logodds_agree LO (log of an empirical frequency has +-inf zeros; smoothing them would fabricate the fine steer signal). Sampled frontier models can join +the map but carry no steering arm -- exactly the Economist's "average of ten responses" tradeoff. diff --git a/pyproject.toml b/pyproject.toml index b5e0980..1af09f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,12 @@ maps = [ "matplotlib>=3.8", "textalloc>=1.2.3", # non-overlapping label placement on the maps ] +# tinymfv.read_api (sampling readout for logprob-less API models) only. Install with +# `pip install tiny-mfv[api]`. The local logprob reader (read.py) needs none of this. +api = [ + "openai>=1.0", + "python-dotenv", +] [dependency-groups] dev = [ diff --git a/scripts/probe_api_readout.py b/scripts/probe_api_readout.py new file mode 100644 index 0000000..f91407e --- /dev/null +++ b/scripts/probe_api_readout.py @@ -0,0 +1,100 @@ +"""UAT for the sampling readout (read_api.read_items_sampled) vs the logprob readout (read.read_items). + +Three checks on the same first-N mfq2 items (identical build_user_content stimulus for both paths): + + 1. UNBIASEDNESS (same model, no API): draw N Monte-Carlo samples from the LOCAL model's own logprob + p and reduce to E. E_mc must match the exact logprob E within sampling error -- proves the + frequency -> E estimator + reduce plumbing is unbiased (the statistical core of read_api). + 2. LIVE API: run read_items_sampled on a real OpenRouter model (no logprobs). Show per-item E, + parse rate (the sampling coherence gate), and entropy. Sanity: parse rate ~1, E in [1,5]. + 3. COMPARABILITY: tabulate local logprob-E next to API sampled-E on the same items. Different + models, so they differ by genuine opinion, but both land on the same [1,5] E scale -- which is + the whole point: a logprob-less frontier model drops onto the same map. + + uv run python scripts/probe_api_readout.py --api-model meta-llama/llama-3.1-8b-instruct +""" +from __future__ import annotations + +import argparse + +import dotenv +import numpy as np +import torch + +dotenv.load_dotenv() # OPENROUTER_API_KEY from .env for read_api +from loguru import logger +from tabulate import tabulate +from transformers import AutoModelForCausalLM, AutoTokenizer + +import tinymfv as T +from tinymfv.instruments import get as get_instrument +from tinymfv.read import read_items, resolve_answer_ids, build_user_content +from tinymfv.read_api import read_items_sampled +from tinymfv.readouts import expected_score, entropy + +W = np.arange(1, 6, dtype=float) + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--instr", default="mfq2") + ap.add_argument("--api-model", default="meta-llama/llama-3.1-8b-instruct") + ap.add_argument("--local-model", default="Qwen/Qwen3-0.6B") + ap.add_argument("--n-items", type=int, default=5) + ap.add_argument("--n-samples", type=int, default=20) + ap.add_argument("--temperature", type=float, default=1.0) + ap.add_argument("--max-think-tokens", type=int, default=64) + ap.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu") + args = ap.parse_args() + + instr = get_instrument(args.instr) + items = [it for it in instr.items if it.frame == "forward"][: args.n_items] + logger.info(f"{len(items)} {args.instr} forward items; shared stimulus for item 0:\n" + f"{build_user_content(instr, items[0])!r}") + + # --- local logprob readout --- + tok = AutoTokenizer.from_pretrained(args.local_model) + if tok.pad_token is None: + tok.pad_token = tok.eos_token + tok.padding_side = "left" + model = AutoModelForCausalLM.from_pretrained(args.local_model, dtype=torch.bfloat16).to(args.device).eval() + answer_ids = resolve_answer_ids(tok, instr.answer_space) + rows_local = read_items(model, tok, instr, items, answer_ids, + max_think_tokens=args.max_think_tokens, batch_size=8, + n_samples=1, temperature=0.0, verbose_first=True) + p_local = {r["id"]: np.asarray(r["p"], float) for r in rows_local} + + # --- (1) unbiasedness: MC-sample the local model's own p, reduce to E --- + rng = np.random.default_rng(0) + N_MC = 2000 + e_logprob = {i: expected_score(p, 5) for i, p in p_local.items()} + e_mc = {i: float(rng.choice(5, size=N_MC, p=p).mean() + 1) for i, p in p_local.items()} + max_mc_gap = max(abs(e_mc[i] - e_logprob[i]) for i in p_local) + logger.info(f"(1) unbiasedness: max |E_mc(N={N_MC}) - E_logprob| = {max_mc_gap:.4f} " + f"(should be < 0.05; MC of the model's own p reduces to the same E)") + + # --- (2)+(3) live API sampled readout --- + rows_api = read_items_sampled(args.api_model, instr, items, n_samples=args.n_samples, + temperature=args.temperature, verbose_first=True) + + table = [] + for it, ra in zip(items, rows_api): + pl = p_local[it.id] + pa = np.asarray(ra["p"], float) + table.append([ + it.id, it.dimension, it.prompt[:34], + f"{expected_score(pl, 5):.2f}", f"{e_mc[it.id]:.2f}", + f"{expected_score(pa, 5):.2f}" if ra["n_parsed"] else "NaN", + f"{ra['pmass_allowed']:.2f}", f"{ra['n_parsed']}/{ra['n_samples']}", + f"{entropy(pa, 5):.2f}" if ra["n_parsed"] else "NaN", + ]) + print("\n" + tabulate(table, headers=[ + "id", "dimension", "statement", "E_local(lp)", "E_mc", f"E_api({args.api_model.split('/')[-1]})", + "api_pmass", "parsed", "api_H"], tablefmt="pipe")) + print("\nSHOULD: E_local ~= E_mc (unbiased estimator); E_api on the same [1,5] scale (a " + "logprob-less model on the same map); api_pmass near 1.0 = coherent (parse rate). " + "E_api != E_local is genuine model disagreement, not method error.") + + +if __name__ == "__main__": + main() diff --git a/src/tinymfv/read_api.py b/src/tinymfv/read_api.py new file mode 100644 index 0000000..935c443 --- /dev/null +++ b/src/tinymfv/read_api.py @@ -0,0 +1,112 @@ +"""Sampling readout for API models that do NOT expose token logprobs (OpenRouter / OpenAI-compatible). + +The frontier models on the Economist's WVS chart (GPT-5.4, Claude, Gemini, Grok) return no +next-token distribution, so read.py's forced-slot logprob reader cannot touch them. This is the +fallback the Economist itself used ("average of ten responses"): SAMPLE N chat completions at +temperature and take the empirical answer frequency as the per-(item, frame) categorical `p`. + +Why this stays comparable to the logprob reader: temperature-1 sampling is an unbiased estimator of +the model's softmax categorical, so the empirical `p` -> the true next-token `p` as N grows, and +E = sum_k k*p_k (readouts.expected_score) converges to the logprob E for the SAME model. Every +downstream summary (per_item_categorical, reduce_ordinal/nominal, expected_score, entropy) is a pure +function of `p`, so a sampled model drops onto the SAME map as a logprob model. N sets the +resolution: SE on a p~=0.5 cell is ~0.5/sqrt(N) (N=10 -> 0.16, coarse; N=100 -> 0.05). + +What this CANNOT produce, by design (do not add it back): +- logit_contrast C and logodds_agree LO: log of an empirical frequency has -inf on any unsampled + token; smoothing those zeros would fabricate the very fine steer signal C/LO exist to measure. +- full-vocab pmass_allowed: we cannot see mass on unsampled tokens. The sampling ANALOGUE we do + report is the fraction of draws that PARSED to an allowed token -- the empirical refusal / + off-format rate. At total collapse (nothing parses) `p` is NaN, matching read.py's + NaN-at-collapse ("do not compare"), not an eps fallback. + +Requires OPENROUTER_API_KEY. temperature MUST be > 0: at temp 0 the N draws collapse to one argmax +and E degenerates to an integer. +""" +from __future__ import annotations + +import os +import re +from math import inf + +import numpy as np +from loguru import logger +from openai import OpenAI + +from .instrument import Instrument, InstrItem +from .read import build_user_content + +_ANSWER_SUFFIX = ("\n\nAnswer with ONLY one option from [{space}] -- a single token, nothing else: " + "no words, no punctuation, no explanation.") + + +def _client() -> OpenAI: + return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"]) + + +def parse_answer(text: str, answer_space: list[str]) -> str | None: + """The EARLIEST-appearing answer_space token in `text` (token-boundaried so '3' is not caught + inside 'x3y', and a longer token wins a tie at the same position). None if no allowed token + appears -- a refusal or off-format draw, counted against the parse rate, never coerced.""" + best, best_key = None, (inf, 0) + for a in answer_space: + m = re.search(rf"(? list[str]: + """N completions, requesting up to 8 per call via the `n` param and topping up until N.""" + texts: list[str] = [] + while len(texts) < n_samples: + resp = client.chat.completions.create( + model=model, messages=[{"role": "user", "content": prompt}], + temperature=temperature, n=min(n_samples - len(texts), 8), max_tokens=max_tokens) + texts.extend((c.message.content or "") for c in resp.choices) + return texts + + +def read_items_sampled(model: str, instr: Instrument, items: list[InstrItem], *, + n_samples: int = 20, temperature: float = 1.0, max_tokens: int = 32, + verbose_first: bool = False) -> list[dict]: + """Per-(item, frame) rows shaped for `instrument.per_item_categorical`: id, frame, lp, p, + pmass_allowed, dimension, sign, human_label. `p` is the empirical answer frequency over + `instr.answer_space`; `pmass_allowed` is the parse rate; `lp = log(p)` carries -inf on unsampled + tokens ON PURPOSE so any C/LO consumer fails loudly instead of reading a fabricated number.""" + assert temperature > 0, "sampling readout needs temperature > 0; temp 0 collapses E to an integer" + client = _client() + space = instr.answer_space + A = len(space) + idx = {a: k for k, a in enumerate(space)} + out: list[dict] = [] + for it_n, it in enumerate(items): + user = build_user_content(instr, it) + prompt = user + _ANSWER_SUFFIX.format(space="/".join(space)) + texts = _sample_texts(client, model, prompt, n_samples, temperature, max_tokens) + counts = np.zeros(A) + for t in texts: + a = parse_answer(t, space) + if a is not None: + counts[idx[a]] += 1 + n_parsed = int(counts.sum()) + pmass = n_parsed / len(texts) + p = counts / n_parsed if n_parsed else np.full(A, np.nan) # NaN at collapse, by design + with np.errstate(divide="ignore"): + lp = np.log(p) + out.append({ + "id": it.id, "frame": it.frame, "lp": lp, "p": p, "pmass_allowed": pmass, + "dimension": it.dimension, "sign": it.sign, "human_label": it.human_label, + "n_samples": len(texts), "n_parsed": n_parsed, + }) + if verbose_first and it_n == 0: + logger.debug( + f"\n=== TRACE read_api first item ({instr.name}, {model}, N={len(texts)}) ===\n" + f"--- prompt ---\n{prompt}\n" + f"--- first 3 raw replies ---\n{texts[:3]}\n" + f"--- counts over {space} ---\n{counts.tolist()} parsed={n_parsed}/{len(texts)}\n" + f"SHOULD: replies are a bare token in {space}; parse rate near 1.0 -> coherent. " + f"ELSE the model is refusing / adding prose / the option set is off.\n") + return out