mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-23 13:30:23 +08:00
Audit WVS response discrimination
Co-Authored-By: PI[gpt-5.6-terra] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
co-authored by
PI[gpt-5.6-terra]
PI[gpt-5.6-terra]
parent
3fa75338f1
commit
b73ef4fd98
@@ -1,35 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarize constant dense-rating answers from every cached complete WVS panel."""
|
||||
"""Audit dense-rated WVS response discrimination without making API requests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
from wvs_map import X_AXIS, Y_AXIS, load_wvs_all, positiveness, resolve_items
|
||||
|
||||
CACHE_PATH = Path("slop/research/wvs/20260916_openrouter/wvs_iw_rated.json")
|
||||
LEDGER_PATH = Path("slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl")
|
||||
CSV_PATH = Path("slop/audits/20260917_wvs_content_quality_by_item.csv")
|
||||
ITEM_CSV_PATH = Path("slop/audits/20260917_wvs_content_quality_by_item.csv")
|
||||
COORD_CSV_PATH = Path("slop/audits/20260917_wvs_content_quality_coordinate_sensitivity.csv")
|
||||
REPORT_PATH = Path("slop/audits/20260917_wvs_content_quality_cross_panel.md")
|
||||
|
||||
CONSTRUCT_MISMATCH = (
|
||||
"mutually exclusive",
|
||||
"all the options",
|
||||
"each option",
|
||||
"every answer",
|
||||
"as an ai",
|
||||
"my nature as an ai",
|
||||
"i lack personal",
|
||||
"lack personal experience",
|
||||
"do not have personal",
|
||||
"don't have personal",
|
||||
"cannot personally",
|
||||
"can't personally",
|
||||
"do not hold beliefs",
|
||||
"don't hold beliefs",
|
||||
"non-human perspective",
|
||||
"mutually exclusive", "all the options", "each option", "every answer", "as an ai",
|
||||
"my nature as an ai", "i lack personal", "lack personal experience", "do not have personal",
|
||||
"don't have personal", "cannot personally", "can't personally", "do not hold beliefs",
|
||||
"don't hold beliefs", "non-human perspective",
|
||||
)
|
||||
EXPLICIT_INDIFFERENCE = ("no preference",)
|
||||
NEUTRAL_POLICY = ("neutral", "neutrality", "balanced perspective")
|
||||
@@ -39,13 +32,6 @@ def response_message(event: dict) -> dict:
|
||||
return event["response"]["choices"][0]["message"]
|
||||
|
||||
|
||||
def constant_rating(text: str) -> bool:
|
||||
ratings = json.loads(text)
|
||||
values = list(ratings.values())
|
||||
assert values, "parsed rating object must have values"
|
||||
return len(set(values)) == 1
|
||||
|
||||
|
||||
def rationale_class(reasoning: str | None) -> str:
|
||||
if reasoning is None:
|
||||
return "no_saved_rationale"
|
||||
@@ -60,177 +46,269 @@ def rationale_class(reasoning: str | None) -> str:
|
||||
|
||||
|
||||
def compact_quote(reasoning: str | None) -> str:
|
||||
if reasoning is None:
|
||||
return "" if reasoning is None else " ".join(reasoning.split())[:280].rstrip()
|
||||
|
||||
|
||||
def canonical_ratings(answer: dict) -> np.ndarray:
|
||||
parsed = json.loads(answer["text"])
|
||||
presented = np.array([parsed[str(index)] for index in range(len(parsed))], dtype=float)
|
||||
canonical = np.empty_like(presented)
|
||||
canonical[np.array(answer["presented_order"], dtype=int)] = presented
|
||||
return canonical
|
||||
|
||||
|
||||
def normalized_spread(rating: np.ndarray) -> float:
|
||||
return float((rating.max() - rating.min()) / 4)
|
||||
|
||||
|
||||
def total_variation(p: np.ndarray) -> float:
|
||||
return float(0.5 * np.abs(p - 1 / len(p)).sum())
|
||||
|
||||
|
||||
def rating_sum(rating: np.ndarray) -> np.ndarray:
|
||||
return rating / rating.sum()
|
||||
|
||||
|
||||
def argmax_split(rating: np.ndarray) -> np.ndarray:
|
||||
winners = rating == rating.max()
|
||||
return winners / winners.sum()
|
||||
|
||||
|
||||
def min_shift(rating: np.ndarray) -> np.ndarray:
|
||||
shifted = rating - rating.min()
|
||||
assert shifted.sum() > 0, "min-shift requires a nonflat rating"
|
||||
return shifted / shifted.sum()
|
||||
|
||||
|
||||
def coordinate(samples: dict[str, list[np.ndarray]], resolved: dict[str, list[dict]]) -> tuple[float, float] | None:
|
||||
output = []
|
||||
for axis in (X_AXIS, Y_AXIS):
|
||||
scores = []
|
||||
for item in resolved[axis]:
|
||||
ps = samples[item["suffix"]]
|
||||
if not ps:
|
||||
return None
|
||||
scores.append(positiveness(np.mean(ps, axis=0), item["pole_idx"], item["n"]))
|
||||
output.append(float(np.mean(scores)))
|
||||
return tuple(output)
|
||||
|
||||
|
||||
def coordinate_fields(prefix: str, xy: tuple[float, float] | None) -> dict[str, float | str]:
|
||||
if xy is None:
|
||||
return {f"{prefix}_x": "", f"{prefix}_y": ""}
|
||||
return {f"{prefix}_x": xy[0], f"{prefix}_y": xy[1]}
|
||||
|
||||
|
||||
def distance(current: tuple[float, float], alternative: tuple[float, float] | None) -> float | str:
|
||||
if alternative is None:
|
||||
return ""
|
||||
return " ".join(reasoning.split())[:280].rstrip()
|
||||
return float(np.linalg.norm(np.subtract(alternative, current)))
|
||||
|
||||
|
||||
def write_csv(path: Path, rows: list[dict]) -> None:
|
||||
assert rows, f"{path} requires rows"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=list(rows[0]), lineterminator="\n")
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
cache = json.loads(CACHE_PATH.read_text())
|
||||
completed = cache["completed"]
|
||||
assert completed, "cached completed panels are required"
|
||||
runs = {entry["run_id"] for entry in completed.values()}
|
||||
entries = {entry["run_id"]: entry for entry in completed.values()}
|
||||
|
||||
parsed: dict[str, list[dict]] = defaultdict(list)
|
||||
completed_responses: dict[tuple[str, str, int, str], dict] = {}
|
||||
ledger_through = ""
|
||||
with LEDGER_PATH.open() as fh:
|
||||
for line in fh:
|
||||
event = json.loads(line)
|
||||
run_id = event.get("run_id")
|
||||
if run_id not in {entry["run_id"] for entry in completed.values()}:
|
||||
if event.get("run_id") not in runs:
|
||||
continue
|
||||
ledger_through = max(ledger_through, event["recorded_at_utc"])
|
||||
if event["event"] == "answer_parsed":
|
||||
assert event["parsed"], f"complete cache run has unparsable answer: {event}"
|
||||
parsed[run_id].append(event)
|
||||
parsed[event["run_id"]].append(event)
|
||||
if event["event"] == "request_completed":
|
||||
key = (run_id, event["item_id"], event["sample"], event["phase"])
|
||||
completed_responses[key] = event
|
||||
completed_responses[(event["run_id"], event["item_id"], event["sample"], event["phase"])] = event
|
||||
|
||||
rows: list[dict] = []
|
||||
for entry in sorted(completed.values(), key=lambda value: value["model"]):
|
||||
run_id = entry["run_id"]
|
||||
resolved = resolve_items(load_wvs_all())
|
||||
expected_ids = {item["suffix"] for axis in (X_AXIS, Y_AXIS) for item in resolved[axis]}
|
||||
item_rows: list[dict] = []
|
||||
coord_rows: list[dict] = []
|
||||
examples: list[tuple[dict, str]] = []
|
||||
|
||||
for run_id, entry in sorted(entries.items(), key=lambda pair: pair[1]["model"]):
|
||||
answers = parsed[run_id]
|
||||
assert len(answers) == entry["n_items"] * entry["n_samples"], (
|
||||
f"{entry['model']} {run_id}: {len(answers)} parsed answers, expected "
|
||||
f"{entry['n_items'] * entry['n_samples']}"
|
||||
)
|
||||
assert len(answers) == entry["n_items"] * entry["n_samples"] == 144, f"incomplete cache entry {entry['model']}"
|
||||
by_item: dict[str, list[dict]] = defaultdict(list)
|
||||
for answer in answers:
|
||||
by_item[answer["item_id"]].append(answer)
|
||||
assert len(by_item) == entry["n_items"], f"{entry['model']}: item count drift"
|
||||
assert set(by_item) == expected_ids, f"item identity drift for {entry['model']}"
|
||||
|
||||
current: dict[str, list[np.ndarray]] = {}
|
||||
argmax: dict[str, list[np.ndarray]] = {}
|
||||
minshift_nonflat: dict[str, list[np.ndarray]] = {}
|
||||
flat_missing: dict[str, list[np.ndarray]] = {}
|
||||
for item_id, item_answers in sorted(by_item.items()):
|
||||
assert len(item_answers) == entry["n_samples"], f"{entry['model']} {item_id}: sample count drift"
|
||||
constant_answers = [answer for answer in item_answers if constant_rating(answer["text"])]
|
||||
assert len(item_answers) == 12, f"sample count drift for {entry['model']} {item_id}"
|
||||
ratings = [canonical_ratings(answer) for answer in item_answers]
|
||||
flats = [normalized_spread(rating) == 0 for rating in ratings]
|
||||
current[item_id] = [rating_sum(rating) for rating in ratings]
|
||||
argmax[item_id] = [argmax_split(rating) for rating in ratings]
|
||||
minshift_nonflat[item_id] = [min_shift(rating) for rating, flat in zip(ratings, flats) if not flat]
|
||||
flat_missing[item_id] = [rating_sum(rating) for rating, flat in zip(ratings, flats) if not flat]
|
||||
ties = [int((rating == rating.max()).sum()) for rating in ratings]
|
||||
aggregate = np.mean(current[item_id], axis=0)
|
||||
rationale_counts = Counter()
|
||||
examples: list[str] = []
|
||||
for answer in constant_answers:
|
||||
item_examples: list[str] = []
|
||||
for answer, flat in zip(item_answers, flats):
|
||||
if not flat:
|
||||
continue
|
||||
phase = "rescue" if (run_id, item_id, answer["sample"], "rescue") in completed_responses else "initial"
|
||||
response = completed_responses[(run_id, item_id, answer["sample"], phase)]
|
||||
rationale = response_message(response).get("reasoning")
|
||||
classification = rationale_class(rationale)
|
||||
rationale_counts[classification] += 1
|
||||
if rationale and len(examples) < 2:
|
||||
examples.append(
|
||||
f"sample {answer['sample']} ({phase}, {classification}): {compact_quote(rationale)}"
|
||||
)
|
||||
rows.append(
|
||||
{
|
||||
"model": entry["model"],
|
||||
"run_id": run_id,
|
||||
"protocol_id": entry["protocol_id"],
|
||||
"item_id": item_id,
|
||||
"n_samples": entry["n_samples"],
|
||||
"constant_ratings": len(constant_answers),
|
||||
"constant_share": len(constant_answers) / entry["n_samples"],
|
||||
"explicit_prompt_or_persona_mismatch": rationale_counts["explicit_prompt_or_persona_mismatch"],
|
||||
"explicit_indifference": rationale_counts["explicit_indifference"],
|
||||
"explicit_neutral_policy": rationale_counts["explicit_neutral_policy"],
|
||||
"other_saved_rationale": rationale_counts["other_saved_rationale"],
|
||||
"no_saved_rationale": rationale_counts["no_saved_rationale"],
|
||||
"rationale_examples": " || ".join(examples),
|
||||
}
|
||||
)
|
||||
reasoning = response_message(completed_responses[(run_id, item_id, answer["sample"], phase)]).get("reasoning")
|
||||
category = rationale_class(reasoning)
|
||||
rationale_counts[category] += 1
|
||||
if reasoning and not item_examples:
|
||||
item_examples.append(f"sample {answer['sample']} ({phase}, {category}): {compact_quote(reasoning)}")
|
||||
row = {
|
||||
"model": entry["model"], "run_id": run_id, "protocol_id": entry["protocol_id"],
|
||||
"item_id": item_id, "n_samples": len(ratings), "flat_ratings": sum(flats),
|
||||
"flat_fraction": sum(flats) / len(ratings),
|
||||
"mean_normalized_spread": float(np.mean([normalized_spread(rating) for rating in ratings])),
|
||||
"unique_argmax_fraction": float(np.mean([tie == 1 for tie in ties])),
|
||||
"mean_argmax_tie_size": float(np.mean(ties)),
|
||||
"mean_sample_tv_from_uniform": float(np.mean([total_variation(p) for p in current[item_id]])),
|
||||
"aggregate_tv_from_uniform": total_variation(aggregate),
|
||||
"explicit_prompt_or_persona_mismatch": rationale_counts["explicit_prompt_or_persona_mismatch"],
|
||||
"explicit_indifference": rationale_counts["explicit_indifference"],
|
||||
"explicit_neutral_policy": rationale_counts["explicit_neutral_policy"],
|
||||
"other_saved_rationale": rationale_counts["other_saved_rationale"],
|
||||
"no_saved_rationale": rationale_counts["no_saved_rationale"],
|
||||
"rationale_example": " || ".join(item_examples),
|
||||
}
|
||||
item_rows.append(row)
|
||||
if item_examples:
|
||||
examples.append((row, item_examples[0]))
|
||||
|
||||
CSV_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
fields = list(rows[0])
|
||||
with CSV_PATH.open("w", newline="") as fh:
|
||||
writer = csv.DictWriter(fh, fieldnames=fields, lineterminator="\n")
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
current_xy = coordinate(current, resolved)
|
||||
assert current_xy is not None
|
||||
cached_xy = tuple(entry["coords"][:2])
|
||||
assert np.allclose(current_xy, cached_xy, atol=1e-12), f"cached coordinate drift for {entry['model']}"
|
||||
argmax_xy = coordinate(argmax, resolved)
|
||||
assert argmax_xy is not None
|
||||
minshift_xy = coordinate(minshift_nonflat, resolved)
|
||||
flat_missing_xy = coordinate(flat_missing, resolved)
|
||||
coord_rows.append({
|
||||
"model": entry["model"], "run_id": run_id, "protocol_id": entry["protocol_id"],
|
||||
**coordinate_fields("current_rating_sum", current_xy),
|
||||
**coordinate_fields("argmax_ties_split", argmax_xy),
|
||||
**coordinate_fields("minshift_nonflat", minshift_xy),
|
||||
**coordinate_fields("flat_missing", flat_missing_xy),
|
||||
"argmax_delta_l2": distance(current_xy, argmax_xy),
|
||||
"minshift_delta_l2": distance(current_xy, minshift_xy),
|
||||
"flat_missing_delta_l2": distance(current_xy, flat_missing_xy),
|
||||
})
|
||||
|
||||
write_csv(ITEM_CSV_PATH, item_rows)
|
||||
write_csv(COORD_CSV_PATH, coord_rows)
|
||||
by_model: dict[str, list[dict]] = defaultdict(list)
|
||||
for row in rows:
|
||||
for row in item_rows:
|
||||
by_model[row["model"]].append(row)
|
||||
fetched_utc = datetime.now(UTC).isoformat()
|
||||
|
||||
lines = [
|
||||
"# Cross-panel dense-rating content-quality audit",
|
||||
"",
|
||||
f"- generated UTC: {fetched_utc}",
|
||||
f"- ledger-through UTC: {ledger_through}",
|
||||
f"- cache source: `{CACHE_PATH}`",
|
||||
f"- ledger source: `{LEDGER_PATH}`",
|
||||
f"- complete panels: {len(by_model)}",
|
||||
f"- per-item/model source table: `{CSV_PATH}`",
|
||||
f"- per-item/model table: `{ITEM_CSV_PATH}`",
|
||||
f"- coordinate-sensitivity table: `{COORD_CSV_PATH}`",
|
||||
"",
|
||||
"## What this measures",
|
||||
"## Definitions",
|
||||
"",
|
||||
"A constant rating gives every answer option in one presented card the same 1-5 rating. "
|
||||
"It is parse-valid but maps to a uniform categorical distribution after renormalization. "
|
||||
"This is a descriptive diagnostic, not a rejection threshold: a constant rating can mean "
|
||||
"indifference, a deliberate neutral policy, or a misunderstanding of the multi-option prompt.",
|
||||
"Each dense-rated reply assigns a 1-5 rating to every answer in a card. A flat reply gives every "
|
||||
"answer the same rating. Normalized spread is `(max rating - min rating) / 4`. "
|
||||
"A unique argmax has one highest-rated option; tie size counts all highest-rated options. "
|
||||
"Distance from uniform is total variation, `0.5 * sum(abs(p - uniform))`, after normalizing a reply's ratings to p.",
|
||||
"",
|
||||
"The rationale categories are evidence labels, not inferred mental states. "
|
||||
"`explicit_prompt_or_persona_mismatch` requires saved reasoning to mention multi-option "
|
||||
"format confusion, mutually exclusive answers, or a non-personal AI stance. "
|
||||
"`explicit_indifference` requires a direct no-preference phrase. "
|
||||
"`explicit_neutral_policy` records a neutral-policy phrase without a stronger mismatch signal. "
|
||||
"`no_saved_rationale` means the ledger cannot distinguish indifference from misunderstanding.",
|
||||
"Coordinate sensitivity is diagnostic only. `current_rating_sum` is the published readout. "
|
||||
"`argmax_ties_split` puts equal mass on all highest-rated answers. `minshift_nonflat` subtracts the "
|
||||
"minimum rating then normalizes, omitting flat replies. `flat_missing` omits flat replies but otherwise "
|
||||
"uses the current rating/sum transform. Blank alternative coordinates mean every reply for at least one "
|
||||
"required item was flat, so that diagnostic coordinate is undefined. No alternative is published or used to "
|
||||
"exclude a panel here.",
|
||||
"",
|
||||
"Rationale categories are evidence labels, not inferred mental states. `explicit_prompt_or_persona_mismatch` "
|
||||
"requires saved reasoning mentioning multi-option confusion, mutually exclusive answers, or a non-personal AI stance. "
|
||||
"`explicit_indifference` requires a direct no-preference phrase. `no_saved_rationale` leaves the distinction unresolved.",
|
||||
"",
|
||||
"## Model summary",
|
||||
"",
|
||||
"| model | constant / 144 | max item | Homosexuality | explicit mismatch | explicit indifference | no saved rationale |",
|
||||
"|---|---:|---|---:|---:|---:|---:|",
|
||||
"| model | flat / 144 | max flat item | Homosexuality flat | mean spread | mismatch evidence | explicit indifference | max coordinate shift |",
|
||||
"|---|---:|---|---:|---:|---:|---:|---:|",
|
||||
]
|
||||
for model, model_rows in sorted(by_model.items(), key=lambda pair: (-sum(r["constant_ratings"] for r in pair[1]), pair[0])):
|
||||
total = sum(row["constant_ratings"] for row in model_rows)
|
||||
worst = max(model_rows, key=lambda row: (row["constant_ratings"], row["item_id"]))
|
||||
homosexuality = next(row for row in model_rows if row["item_id"] == "Homosexuality")
|
||||
mismatch = sum(row["explicit_prompt_or_persona_mismatch"] for row in model_rows)
|
||||
indifference = sum(row["explicit_indifference"] for row in model_rows)
|
||||
no_rationale = sum(row["no_saved_rationale"] for row in model_rows)
|
||||
coords = {row["model"]: row for row in coord_rows}
|
||||
for model, rows in sorted(by_model.items(), key=lambda pair: (-sum(r["flat_ratings"] for r in pair[1]), pair[0])):
|
||||
total = sum(row["flat_ratings"] for row in rows)
|
||||
worst = max(rows, key=lambda row: (row["flat_ratings"], row["item_id"]))
|
||||
homosexuality = next(row for row in rows if row["item_id"] == "Homosexuality")
|
||||
mismatch = sum(row["explicit_prompt_or_persona_mismatch"] for row in rows)
|
||||
indifference = sum(row["explicit_indifference"] for row in rows)
|
||||
mean_spread = float(np.mean([row["mean_normalized_spread"] for row in rows]))
|
||||
c = coords[model]
|
||||
available_shifts = [float(c[key]) for key in ("argmax_delta_l2", "minshift_delta_l2", "flat_missing_delta_l2") if c[key] != ""]
|
||||
shift = "undefined" if not available_shifts else f"{max(available_shifts):.3f}"
|
||||
lines.append(
|
||||
f"| `{model}` | {total}/144 | {worst['item_id']} ({worst['constant_ratings']}/12) | "
|
||||
f"{homosexuality['constant_ratings']}/12 | {mismatch} | {indifference} | {no_rationale} |"
|
||||
f"| `{model}` | {total}/144 | {worst['item_id']} ({worst['flat_ratings']}/12) | "
|
||||
f"{homosexuality['flat_ratings']}/12 | {mean_spread:.3f} | {mismatch} | {indifference} | {shift} |"
|
||||
)
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"## Flagged item/model cells",
|
||||
"## High-flat cells",
|
||||
"",
|
||||
"Rows below have at least six constant replies. They are not excluded here. "
|
||||
"The full CSV retains every 49 x 12 cell for a later threshold or modeling decision.",
|
||||
"Rows below have at least six flat replies. They are retained as observations, not excluded.",
|
||||
"",
|
||||
"| model | item | constant / 12 | rationale evidence |",
|
||||
"|---|---|---:|---|",
|
||||
"| model | item | flat / 12 | mean spread | aggregate TV | rationale evidence |",
|
||||
"|---|---|---:|---:|---:|---|",
|
||||
])
|
||||
for row in sorted((row for row in rows if row["constant_ratings"] >= 6), key=lambda row: (-row["constant_ratings"], row["model"], row["item_id"])):
|
||||
for row in sorted((row for row in item_rows if row["flat_ratings"] >= 6), key=lambda row: (-row["flat_ratings"], row["model"], row["item_id"])):
|
||||
evidence = ", ".join(
|
||||
f"{name}={row[name]}" for name in (
|
||||
"explicit_prompt_or_persona_mismatch",
|
||||
"explicit_indifference",
|
||||
"explicit_neutral_policy",
|
||||
"other_saved_rationale",
|
||||
"no_saved_rationale",
|
||||
"explicit_prompt_or_persona_mismatch", "explicit_indifference", "explicit_neutral_policy",
|
||||
"other_saved_rationale", "no_saved_rationale",
|
||||
) if row[name]
|
||||
) or "none"
|
||||
lines.append(f"| `{row['model']}` | {row['item_id']} | {row['constant_ratings']}/12 | {evidence} |")
|
||||
lines.append(
|
||||
f"| `{row['model']}` | {row['item_id']} | {row['flat_ratings']}/12 | "
|
||||
f"{row['mean_normalized_spread']:.3f} | {row['aggregate_tv_from_uniform']:.3f} | {evidence} |"
|
||||
)
|
||||
|
||||
examples = [row for row in rows if row["rationale_examples"]]
|
||||
lines.extend(["", "## Saved-reasoning examples", ""])
|
||||
if examples:
|
||||
for row in sorted(examples, key=lambda row: (-row["constant_ratings"], row["model"], row["item_id"])):
|
||||
lines.extend([
|
||||
f"### `{row['model']}` / {row['item_id']}",
|
||||
"",
|
||||
f"> {row['rationale_examples']}",
|
||||
"",
|
||||
])
|
||||
else:
|
||||
lines.extend(["No constant-rated answer had a saved reasoning field.", ""])
|
||||
for row, example in sorted(examples, key=lambda pair: (-pair[0]["flat_ratings"], pair[0]["model"], pair[0]["item_id"])):
|
||||
lines.extend([f"### `{row['model']}` / {row['item_id']}", "", f"> {example}", ""])
|
||||
|
||||
lines.extend([
|
||||
"## Interpretation",
|
||||
"",
|
||||
"The table establishes rate and available explanation evidence, not whether a model is genuinely indifferent. "
|
||||
"An explicit prompt-or-persona mismatch is direct evidence against treating that answer as an attitude measurement. "
|
||||
"The saved runs contain no direct evidence that a constant rating expresses a stable human-like attitude of indifference; "
|
||||
"no saved rationale leaves the alternatives unresolved. Any future gate must be selected against the full distribution "
|
||||
"and documented before it excludes or reruns a panel.",
|
||||
"The table shows that flat replies and coordinate sensitivity vary across model and item, so Nano alone cannot "
|
||||
"supply a general rejection threshold. Saved mismatch rationale is evidence against interpreting those replies as attitudes. "
|
||||
"For other cells, no saved rationale does not establish genuine indifference. The direct-choice pilot should therefore compare "
|
||||
"the construct rather than silently replace or filter the published rated readout.",
|
||||
"",
|
||||
"-- PI[gpt-5.6-terra]",
|
||||
"",
|
||||
])
|
||||
REPORT_PATH.write_text("\n".join(lines))
|
||||
print(f"wrote {CSV_PATH}: {len(rows)} model/item rows")
|
||||
print(f"wrote {REPORT_PATH}: {len(by_model)} complete panels")
|
||||
print(f"wrote {ITEM_CSV_PATH}: {len(item_rows)} model/item rows")
|
||||
print(f"wrote {COORD_CSV_PATH}: {len(coord_rows)} model rows")
|
||||
print(f"wrote {REPORT_PATH}: stable through {ledger_through}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,50 @@
|
||||
model,run_id,protocol_id,current_rating_sum_x,current_rating_sum_y,argmax_ties_split_x,argmax_ties_split_y,minshift_nonflat_x,minshift_nonflat_y,flat_missing_x,flat_missing_y,argmax_delta_l2,minshift_delta_l2,flat_missing_delta_l2
|
||||
anthropic/claude-fable-5.1,20260916T153341Z_4230ccaa1e16,4230ccaa1e160c793a906aef6b786d735d9fdbf71c3333691b0eba17dcf33928,0.5790919024614678,0.6085929478125246,0.7,0.7976190476190477,,,,,0.2243872421918252,,
|
||||
deepseek/deepseek-v4.1-flash,20260916T151414Z_8c27d32cd851,8c27d32cd851de5f217cb354b15dc270cc6f4ec2d02fb15dbb20fde0790c3571,0.5419602854216832,0.595896730624977,0.6162037037037038,0.6686507936507936,0.5915791097381172,0.6873802079754461,0.5476004453765156,0.6191408776897731,0.1039482508028085,0.10407331240370389,0.02391864914843578
|
||||
google/gemini-3.7-flash,20260916T172946Z_cd5db529649a,cd5db529649a179032180cecafe4fd98aff124ee3654f7d60996de704e2b63ef,0.4988977072310405,0.6262540369683227,0.4990740740740741,0.7559523809523808,,,,,0.1296984638978827,,
|
||||
meta/muse-spark-1.3,20260916T155331Z_3c9d4fedebca,3c9d4fedebcab41719f4247ca94fa35430f57d0a34f3a539012e0632e35413de,0.4253104631546661,0.731799663031547,0.4296296296296296,0.8809523809523808,0.41938181160403387,0.8691826011678953,0.4297549075991105,0.7322325634644475,0.14921524205727202,0.13751080175820557,0.004465477488976777
|
||||
moonshotai/kimi-k3,20260916T152010Z_0043a43d1a21,0043a43d1a2188f7c728ccda08eee7b062232366d8a4e912c72bdc938e37fe44,0.6214472858835399,0.6710244268262847,0.7104938271604938,0.7824074074074073,0.736378990424398,0.75917675049727,0.6214472858835399,0.6731276014294593,0.14260243643263248,0.14484518934801133,0.0021031746031745513
|
||||
openai/gpt-5-nano,20260917T020418Z_7fe76f95937e,7fe76f95937ed145b996116a31855dae4ab8c3b67e3e7ccfb84a000437e27f96,0.4526851851851852,0.6334977783064814,0.40370370370370373,0.7076719576719578,0.463956228956229,0.690496122229879,0.4600829725829726,0.6331215672159846,0.08888753800540648,0.058102045038868524,0.00740734724226392
|
||||
openai/gpt-5.6-sol,20260916T152843Z_1a70f47789af,1a70f47789af20900f2799992b09e83af9490be6579098a6178d613d5ba7856c,0.5274474357644292,0.6650744376984001,0.5777777777777777,0.744047619047619,,,,,0.09364778000357103,,
|
||||
openai/gpt-6-astra,20260916T154406Z_95bb4d3939e9,95bb4d3939e9937823357b5cd87adb1a6640cc5b04e0841dfec03095331a5e44,0.4569019748367575,0.6751145899955423,0.5083333333333333,0.8101851851851851,0.522308016058016,0.9326483084816418,0.4628543557891384,0.741688602402888,0.1445311396263368,0.2657095526802048,0.066839583833352
|
||||
qwen/qwen-2.5-72b-instruct,20260916T201119Z_9c1d40b30945,9c1d40b30945565e488b212177e40c29f730c2e96d4b07d09cbc135ba75a81f0,0.6026546601546601,0.5957470912477916,0.6648148148148149,0.66005291005291,0.7409082892416226,0.6249985684509494,0.6097975172975173,0.5871788407152553,0.08943781727866063,0.14131424158336717,0.01115505828544051
|
||||
qwen/qwen-2.5-7b-instruct,20260916T200857Z_66ad076e1a37,66ad076e1a37493e4d3db982a0220048e520103459cefc709bc8f09f3fef6c9a,0.5509667037841642,0.5895236753978316,0.548148148148148,0.6263227513227514,0.6096896548164665,0.5664308438538744,0.5532841641016245,0.5742467565494841,0.036906859048167893,0.06310042667573457,0.01545169478801589
|
||||
qwen/qwen-plus,20260916T200626Z_88ba5af838ed,88ba5af838eda5d298b312553d7c941e72a37cbf28890cdb9d8436b5f40e7cfe,0.6025617283950616,0.6287037037037038,0.7055555555555555,0.6746031746031746,0.7001750700280113,0.6743697478991597,0.6025617283950616,0.6287037037037038,0.11275854673601049,0.10776711955514205,0.0
|
||||
qwen/qwen-plus-2025-07-28,20260916T194044Z_ac52652ff1f0,ac52652ff1f09e1f1972a8b751e1b1deca329cb555df54755aada79791bed940,0.6417361111111111,0.5127110103300581,0.836111111111111,0.48412698412698413,0.7134343434343433,0.5531135531135531,0.6417361111111111,0.5603300579491056,0.19646548597394406,0.0822982501736722,0.04761904761904756
|
||||
qwen/qwen2.5-vl-72b-instruct,20260916T200346Z_02d386b067d5,02d386b067d55e0639d33db8b6f1e5c708c30acbada3125dcbabde47dc90edcf,0.5849525883130219,0.59521139256589,0.5990740740740741,0.636904761904762,0.6486369780549304,0.6331711725094078,0.5850608134212469,0.6026610797344925,0.04401992056929674,0.07413937139037259,0.007450473245646286
|
||||
qwen/qwen3-14b,20260916T150121Z_49ae9f7a0909,49ae9f7a09090534a15fac734c2611c445a17139cad6c5684fa0581722f3fd70,0.6137896825396826,0.5819444444444445,0.6916666666666667,0.6091269841269842,0.7905812757201647,0.583994708994709,0.6137896825396826,0.5783730158730159,0.08248463566208784,0.17680348131193332,0.0035714285714285587
|
||||
qwen/qwen3-235b-a22b,20260916T150950Z_842298c634ec,842298c634ec0f57848b605f67fca161631a852a68192cae1a5377b1d9f31566,0.6427511094043352,0.5554953612269281,0.837962962962963,0.5297619047619048,0.7241034685479131,0.5597299525870955,0.6427511094043352,0.5577801135116803,0.19690068194760627,0.08146249506498852,0.002284752284752223
|
||||
qwen/qwen3-235b-a22b-2507,20260916T195827Z_dc09c3fe433c,dc09c3fe433c95591b336b279603c546152daf6534d90699323b3bcde754f5bf,0.6353621031746031,0.5685931857310826,0.7592592592592593,0.6382275132275131,0.6733327988883544,0.6944677327030268,0.634171626984127,0.6303171716217352,0.1421247510173573,0.13147690028434067,0.06173546523506212
|
||||
qwen/qwen3-30b-a3b,20260916T145439Z_8700e79d3ecc,8700e79d3eccf358a0de1029f7b40e2db339bb6d11403b8e69cb7a66a7b0c668,0.6707823388714307,0.6060018656291949,0.923148148148148,0.6593915343915343,0.835970880970881,0.620014245014245,0.6859790638627016,0.5867060648116549,0.25795146524577994,0.16578178795323825,0.024561522339840724
|
||||
qwen/qwen3-30b-a3b-instruct-2507,20260916T195128Z_88ed922f271a,88ed922f271a76721ad18f0b7bb9693a48377e2507d02a4b78fa08b451972d83,0.6450907949109246,0.5437992771640354,0.7851851851851852,0.511904761904762,0.786600529100529,0.5218074251742982,0.6450907949109246,0.5437992771640354,0.14367915050535648,0.143208402073171,0.0
|
||||
qwen/qwen3-32b,20260916T150406Z_580a58409482,580a5840948209ecb177dc6fdaf36e265a32ccfe223d791eb3235fea9269f27e,0.6064902599972044,0.5413085527420844,0.6342592592592593,0.5529100529100529,0.7198799246021468,0.5615908048447731,0.6064902599972044,0.5838255595448055,0.030095051522856336,0.11518934755253343,0.04251700680272108
|
||||
qwen/qwen3-8b,20260916T181355Z_589e10fa7924,589e10fa7924ecf8f88650d6587ff37c440f47c6c4cc4325c7009ef8d4eeb4ec,0.6217724867724868,0.5391326046087952,0.6444444444444445,0.548941798941799,,,,,0.024702994922512454,,
|
||||
qwen/qwen3-coder,20260916T195554Z_304db4b1aa6c,304db4b1aa6c80d734f802749c75143feddfcdb740c4131385e927148ba7d05b,0.598778659611993,0.6204503168788883,0.5842592592592593,0.6858465608465608,0.6823148148148148,0.6762566137566137,0.5999691358024692,0.6204503168788883,0.06698866853194134,0.10046209233972171,0.0011904761904762973
|
||||
qwen/qwen3-coder-30b-a3b-instruct,20260916T194257Z_39659697ca34,39659697ca3437f62be4fb8e129e6b3902a91b5acac3243ca914dfc917dd563b,0.5030952380952382,0.5846967846967848,0.4453703703703704,0.5681216931216931,0.45261684303350974,0.5821050642479213,0.5030952380952382,0.5846967846967848,0.06005742264346806,0.050544884834105464,0.0
|
||||
qwen/qwen3-coder-next,20260916T191559Z_57bed80fd5b9,57bed80fd5b94157effe68fe998c7c12612fea210c1371dbf230097a7e8e0717,0.6363133579397021,0.5792251640447779,0.7185185185185186,0.6554232804232802,0.7863285708347437,0.7124902942939882,0.6541837036438292,0.6282423688102818,0.1120885425251876,0.20065931087363892,0.05217313119421104
|
||||
qwen/qwen3-coder-plus,20260916T193258Z_182a7e888696,182a7e88869698133f1175f18e53e7f6f2f056739559d325b211bd065235a71c,0.6175297619047618,0.5976147806792967,0.7638888888888888,0.7433862433862435,0.7016578483245149,0.7774891774891775,0.6175297619047618,0.6404719235364397,0.20656793892392541,0.19857576275157016,0.04285714285714293
|
||||
qwen/qwen3-max-thinking,20260916T145109Z_bca0745bdc15,bca0745bdc1554718d57891098f366214f05e9e896007a916d14a06efa33471a,0.5778139029180694,0.6959770802908057,0.5194444444444445,0.8293650793650793,0.5700192400192401,0.7778880070546738,0.5674964426006092,0.6959770802908057,0.14559997245721798,0.08228096191113797,0.01031746031746028
|
||||
qwen/qwen3-next-80b-a3b-instruct,20260916T193846Z_9d8a5189c078,9d8a5189c078672eb884784938b183944fb161d0e716e2c44722540b17603e34,0.6652380952380952,0.7022990677752582,0.95,0.8287037037037036,0.8714285714285713,0.8346560846560847,0.681904761904762,0.726108591584782,0.31155653481805023,0.24501610638741017,0.02906322765650893
|
||||
qwen/qwen3-vl-235b-a22b-instruct,20260916T192432Z_61ab1adae378,61ab1adae3781ca3893328a845d4d517daa5fb47e07ea189da1df45388f62d42,0.5993276014109347,0.5462479278681369,0.7027777777777777,0.5383597883597884,0.7358168991502325,0.5672102174203014,0.6016339987173319,0.5466048054027148,0.10375047824098983,0.138089630242953,0.002333844491315334
|
||||
qwen/qwen3-vl-30b-a3b-instruct,20260916T192223Z_0a798666aeed,0a798666aeed20785b4661c8c2fe4a6fc6853796bc2f3c45133863469e5b3f82,0.5335737179487179,0.5337214372928658,0.548148148148148,0.4623015873015873,0.5645061728395062,0.5079365079365079,0.5335737179487179,0.5535627071341357,0.07289176214370718,0.040270080053099254,0.019841269841269882
|
||||
qwen/qwen3-vl-32b-instruct,20260916T191809Z_43ddc43fb5a2,43ddc43fb5a29c7fbf21c8f32b42a909c2118feb9255959ecaeab7dadb1f3dde,0.5966931216931217,0.5338064713064713,0.6444444444444445,0.503968253968254,0.7343981481481483,0.5243055555555556,0.5966931216931217,0.5338064713064713,0.05630726452620203,0.1380323937019335,0.0
|
||||
qwen/qwen3-vl-8b-instruct,20260916T192044Z_1d7f4015decb,1d7f4015decbcbe955adb79817b970981ca1d3165ed121b458b213ba1060f675,0.6061818582651916,0.6004689754689754,0.6944444444444444,0.6587301587301587,0.7558641975308642,0.5306122448979592,0.6133247154080487,0.5248917748917749,0.10575750372452647,0.16518100827309257,0.07591398853472042
|
||||
qwen/qwen3.5-122b-a10b,20260916T143453Z_e0a9daef251d,e0a9daef251d975c5e805e159f3813dd0c55b3f0919876b828e572b0530bea88,0.48469169991228817,0.6190010643383659,0.4300925925925926,0.6898148148148149,0.47711399711399716,0.7123516454685286,0.49373671943524877,0.6410257291209672,0.08941839730529963,0.09365763491055236,0.02380962488063805
|
||||
qwen/qwen3.5-27b,20260916T143235Z_7058adf367d2,7058adf367d238c6c92e1b83ae71f6556362e9fc7a243297b6fc65e54e2c2886,0.5974338624338624,0.6515634387658198,0.65,0.7632275132275133,0.7823611111111111,0.7550925925925925,0.6105291005291006,0.6515634387658198,0.12341824963921959,0.21193483195412813,0.01309523809523816
|
||||
qwen/qwen3.5-35b-a3b,20260916T142959Z_7ee20ca2b88f,7ee20ca2b88f69dd06b9de3018deea4cb7a418e3119e17862cd0b32146daae1a,0.538440257742794,0.5530118458934986,0.5574074074074075,0.5998677248677249,0.5637477954144622,0.6204353626972674,0.5376660647569464,0.5725879549213201,0.050549244908773716,0.07201667918747015,0.01959141187991537
|
||||
qwen/qwen3.5-397b-a17b,20260916T144546Z_921707cb0d3d,921707cb0d3db143e948e9d78cd8191f4ee720eaf56b6934e2d13d9ce9e3382f,0.5408168991502326,0.6493278669204595,0.5078703703703704,0.7248677248677249,0.4737469937469939,0.7666666666666667,0.5299211450726602,0.6628690239801351,0.08241203733275446,0.1351546008712981,0.017380460046665564
|
||||
qwen/qwen3.5-9b,20260916T142212Z_3016d2c1ab8d,3016d2c1ab8d8f88f01723d2d695be996c7f1fdcd442a51cfbb6c7b7110a25e9,0.5017202097104058,0.5916881018254176,0.4992504409171077,0.6230158730158729,0.5181726059769537,0.7286221219256933,0.5053678819365095,0.648841822370038,0.03142497423473674,0.1379188428161077,0.05727000335919007
|
||||
qwen/qwen3.5-plus-02-15,20260916T144205Z_74e3649a6961,74e3649a6961000bab284e801fe811d5c8fc4cf1db15af52bb10959f5a5a37c1,0.5372045855379188,0.6595114541543113,0.5175925925925926,0.736111111111111,,,,,0.07907046043363426,,
|
||||
qwen/qwen3.5-plus-20260420,20260916T135722Z_6d6ce30e9fbc,6d6ce30e9fbc5fdb07229b0a08e067d77511857069a4e572613d3e48b7c8b261,0.5856201899951899,0.6494018547589976,0.6717592592592592,0.7156084656084655,0.690182860349527,0.7592592592592593,0.6076085257335258,0.6785023838595267,0.10864278426960988,0.15166410701007632,0.036473657651413506
|
||||
qwen/qwen3.6-27b,20260916T141139Z_abf45a27f4ba,abf45a27f4bafad0eb264aac33b1e49e9d0404e141966954d0707930d56665e3,0.45740779531102105,0.6795887184190553,0.4981481481481481,0.7916666666666666,0.5128507295173963,0.7945008427722566,0.4602769413165307,0.6866194852712507,0.11925285251384071,0.1275880687085515,0.007593660601505799
|
||||
qwen/qwen3.6-35b-a3b,20260916T140418Z_b35b916c2264,b35b916c22649e91ded175aede34f3937a16f42a354f7abab01a17bf2b0dfc97,0.6062280543530544,0.5943180448406931,0.6625,0.6567460317460317,0.61383658008658,0.6913062598776885,0.5810628744151471,0.622884891595693,0.08404632898554189,0.0972861938812501,0.03807034298283773
|
||||
qwen/qwen3.6-flash,20260916T140130Z_fa37fc90604e,fa37fc90604e5032183ec460bcc2632507566c3be3e9085f61a6ca85582b7d47,0.6514346395123706,0.583014517676874,0.7921296296296296,0.6732804232804233,0.7534854497354498,0.6915836518906693,0.6500657586266831,0.6128136101788886,0.16716164021247368,0.14900209653263288,0.029830517072669465
|
||||
qwen/qwen3.6-max-preview,20260916T140710Z_fa9ff591d389,fa9ff591d389ad2d8a4b7e3fc1410ed588508fe286430c06a941180b0590422b,0.4805424128340796,0.6905543787488233,0.46111111111111114,0.7956349206349207,0.4626384479717814,0.7799578164599174,0.4766272578832103,0.6905543787488233,0.10686204082706312,0.09117854260927777,0.003915154950869304
|
||||
qwen/qwen3.6-plus,20260916T141839Z_96d86b64bf7c,96d86b64bf7ceff8eed0daff1aa0396ee69609b5867942a7c2c9d33093f03c9d,0.5196778711484593,0.6698994127565557,0.5425925925925925,0.7777777777777778,,,,,0.11028520344322824,,
|
||||
qwen/qwen3.7-flash,20260916T133001Z_82875b6ee164,82875b6ee164d0d980eba738bf05f69959234567b91f9cfd01fcabca0bd70dac,0.6529354469060352,0.5885508561103798,0.8032407407407408,0.6732804232804233,0.8041781305114639,0.6655869923727067,0.6714783974195739,0.6016702528472008,0.17254211343193895,0.16973189339190087,0.022714743768875178
|
||||
qwen/qwen3.7-max,20260916T135329Z_1c31668055e3,1c31668055e399de28805046bc08075035efbad0cdb1ff7d8926841a282e07a5,0.5036638266417679,0.6503283973522069,0.5268518518518519,0.7585978835978835,0.5467003367003367,0.8552782707285813,0.5096628045157457,0.7253895244966673,0.11072473149683515,0.209419654748577,0.07530046841640041
|
||||
qwen/qwen3.7-plus,20260916T134925Z_f095e0d2dbaf,f095e0d2dbafdb91b51be3d0b3bd5a597730c8e3c397e0461023c87e924d92f8,0.5078196649029982,0.6664359119716262,0.5314814814814816,0.7678571428571429,,,,,0.1041448397095462,,
|
||||
qwen/qwen3.8-27b,20260916T134026Z_ed8190c48b2a,ed8190c48b2a3778bba8afff7381bb9f1578211b5e8f750d21321b62617c82c3,0.431934218610816,0.6881570439623662,0.4333333333333334,0.7883597883597885,0.40861525197290743,0.8052653712268102,0.41518244723873526,0.7038792991944254,0.1002125117277374,0.11940743075594587,0.02297414097816228
|
||||
qwen/qwen3.8-flash,20260916T133720Z_3443c17ae66d,3443c17ae66d3fb529a058128b662024c4bc0601394e614bb693b388ec4c988b,0.4153054353054353,0.6346365065494385,0.4041666666666666,0.7506613756613756,0.3894733044733044,0.7738325790905156,0.41503848003848,0.6641085311934207,0.11655832196471495,0.14157275724583854,0.029473233649025547
|
||||
thinkingmachines/inkling,20260916T191343Z_67a70b1b03ba,67a70b1b03ba85be79dc122cc888a080fde239205624135a9147168e7e77293d,0.5614798614630517,0.6880988993033613,0.5824074074074075,0.8710317460317459,0.5474676660787772,0.8717245882424454,0.5542047291879195,0.6880988993033613,0.18412601280482827,0.18415953751527597,0.007275132275132212
|
||||
z-ai/glm-5.3,20260916T161456Z_d81f7e66b3c4,d81f7e66b3c4c720f8dd80768d3dfc25de6edeaf6a352cfed40e961f21fcfa22,0.5535321928702047,0.6422322099171051,0.6157407407407407,0.7724867724867724,0.6572002239284175,0.793324907663143,0.565833780171792,0.6477059984516822,0.14434733976896202,0.1832377253069415,0.013464449898151837
|
||||
|
@@ -1,185 +1,188 @@
|
||||
# Cross-panel dense-rating content-quality audit
|
||||
|
||||
- generated UTC: 2026-09-17T02:33:01.879378+00:00
|
||||
- ledger-through UTC: 2026-09-17T02:17:46.346268+00:00
|
||||
- cache source: `slop/research/wvs/20260916_openrouter/wvs_iw_rated.json`
|
||||
- ledger source: `slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl`
|
||||
- complete panels: 49
|
||||
- per-item/model source table: `slop/audits/20260917_wvs_content_quality_by_item.csv`
|
||||
- per-item/model table: `slop/audits/20260917_wvs_content_quality_by_item.csv`
|
||||
- coordinate-sensitivity table: `slop/audits/20260917_wvs_content_quality_coordinate_sensitivity.csv`
|
||||
|
||||
## What this measures
|
||||
## Definitions
|
||||
|
||||
A constant rating gives every answer option in one presented card the same 1-5 rating. It is parse-valid but maps to a uniform categorical distribution after renormalization. This is a descriptive diagnostic, not a rejection threshold: a constant rating can mean indifference, a deliberate neutral policy, or a misunderstanding of the multi-option prompt.
|
||||
Each dense-rated reply assigns a 1-5 rating to every answer in a card. A flat reply gives every answer the same rating. Normalized spread is `(max rating - min rating) / 4`. A unique argmax has one highest-rated option; tie size counts all highest-rated options. Distance from uniform is total variation, `0.5 * sum(abs(p - uniform))`, after normalizing a reply's ratings to p.
|
||||
|
||||
The rationale categories are evidence labels, not inferred mental states. `explicit_prompt_or_persona_mismatch` requires saved reasoning to mention multi-option format confusion, mutually exclusive answers, or a non-personal AI stance. `explicit_indifference` requires a direct no-preference phrase. `explicit_neutral_policy` records a neutral-policy phrase without a stronger mismatch signal. `no_saved_rationale` means the ledger cannot distinguish indifference from misunderstanding.
|
||||
Coordinate sensitivity is diagnostic only. `current_rating_sum` is the published readout. `argmax_ties_split` puts equal mass on all highest-rated answers. `minshift_nonflat` subtracts the minimum rating then normalizes, omitting flat replies. `flat_missing` omits flat replies but otherwise uses the current rating/sum transform. Blank alternative coordinates mean every reply for at least one required item was flat, so that diagnostic coordinate is undefined. No alternative is published or used to exclude a panel here.
|
||||
|
||||
Rationale categories are evidence labels, not inferred mental states. `explicit_prompt_or_persona_mismatch` requires saved reasoning mentioning multi-option confusion, mutually exclusive answers, or a non-personal AI stance. `explicit_indifference` requires a direct no-preference phrase. `no_saved_rationale` leaves the distinction unresolved.
|
||||
|
||||
## Model summary
|
||||
|
||||
| model | constant / 144 | max item | Homosexuality | explicit mismatch | explicit indifference | no saved rationale |
|
||||
|---|---:|---|---:|---:|---:|---:|
|
||||
| `google/gemini-3.7-flash` | 99/144 | dealing with people? (12/12) | 11/12 | 29 | 0 | 24 |
|
||||
| `qwen/qwen3.5-9b` | 51/144 | God (10/12) | 4/12 | 0 | 0 | 51 |
|
||||
| `qwen/qwen3.5-plus-02-15` | 45/144 | Attending peaceful demonstrations (12/12) | 0/12 | 0 | 0 | 45 |
|
||||
| `qwen/qwen3.5-397b-a17b` | 42/144 | Abortion (11/12) | 0/12 | 0 | 0 | 42 |
|
||||
| `qwen/qwen3.6-35b-a3b` | 38/144 | Homosexuality (11/12) | 11/12 | 0 | 0 | 38 |
|
||||
| `qwen/qwen3-8b` | 37/144 | Independence (12/12) | 0/12 | 0 | 0 | 37 |
|
||||
| `qwen/qwen3.6-plus` | 33/144 | dealing with people? (12/12) | 0/12 | 0 | 0 | 33 |
|
||||
| `qwen/qwen3.7-max` | 33/144 | Abortion (10/12) | 0/12 | 0 | 0 | 33 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | 32/144 | God (11/12) | 0/12 | 0 | 0 | 32 |
|
||||
| `openai/gpt-5-nano` | 30/144 | Homosexuality (11/12) | 11/12 | 8 | 0 | 10 |
|
||||
| `qwen/qwen3-coder-next` | 29/144 | Imagination (7/12) | 3/12 | 0 | 0 | 29 |
|
||||
| `openai/gpt-6-astra` | 28/144 | God (8/12) | 0/12 | 0 | 0 | 24 |
|
||||
| `qwen/qwen3.5-122b-a10b` | 28/144 | Homosexuality (10/12) | 10/12 | 0 | 0 | 28 |
|
||||
| `qwen/qwen3.7-plus` | 28/144 | dealing with people? (12/12) | 0/12 | 0 | 0 | 28 |
|
||||
| `openai/gpt-5.6-sol` | 25/144 | Religion (12/12) | 0/12 | 0 | 0 | 25 |
|
||||
| `qwen/qwen-plus-2025-07-28` | 24/144 | Independence (6/12) | 0/12 | 0 | 0 | 24 |
|
||||
| `qwen/qwen3-235b-a22b-2507` | 24/144 | Imagination (6/12) | 0/12 | 0 | 0 | 24 |
|
||||
| `qwen/qwen3.5-35b-a3b` | 24/144 | God (5/12) | 3/12 | 0 | 0 | 24 |
|
||||
| `deepseek/deepseek-v4.1-flash` | 22/144 | God (4/12) | 1/12 | 0 | 0 | 22 |
|
||||
| `qwen/qwen3.8-27b` | 22/144 | Religion (4/12) | 1/12 | 0 | 0 | 22 |
|
||||
| `qwen/qwen3.6-flash` | 21/144 | Homosexuality (9/12) | 9/12 | 0 | 0 | 21 |
|
||||
| `qwen/qwen-2.5-7b-instruct` | 20/144 | God (8/12) | 0/12 | 0 | 0 | 20 |
|
||||
| `qwen/qwen3.7-flash` | 20/144 | Homosexuality (6/12) | 6/12 | 0 | 0 | 20 |
|
||||
| `qwen/qwen3-30b-a3b` | 19/144 | Abortion (7/12) | 4/12 | 0 | 0 | 19 |
|
||||
| `qwen/qwen3-32b` | 19/144 | Determination, perseverance (6/12) | 0/12 | 0 | 0 | 19 |
|
||||
| `qwen/qwen3.5-plus-20260420` | 18/144 | Religion (8/12) | 0/12 | 0 | 0 | 18 |
|
||||
| `anthropic/claude-fable-5.1` | 17/144 | God (12/12) | 0/12 | 0 | 0 | 17 |
|
||||
| `qwen/qwen2.5-vl-72b-instruct` | 16/144 | God (7/12) | 0/12 | 0 | 0 | 16 |
|
||||
| `qwen/qwen3-coder-plus` | 12/144 | Independence (6/12) | 0/12 | 0 | 0 | 12 |
|
||||
| `qwen/qwen3-next-80b-a3b-instruct` | 12/144 | dealing with people? (6/12) | 0/12 | 0 | 0 | 12 |
|
||||
| `qwen/qwen3.8-flash` | 12/144 | God (8/12) | 0/12 | 0 | 0 | 12 |
|
||||
| `z-ai/glm-5.3` | 10/144 | dealing with people? (5/12) | 0/12 | 5 | 0 | 1 |
|
||||
| `qwen/qwen3.6-27b` | 8/144 | dealing with people? (2/12) | 0/12 | 0 | 0 | 8 |
|
||||
| `qwen/qwen-2.5-72b-instruct` | 7/144 | dealing with people? (4/12) | 0/12 | 0 | 0 | 7 |
|
||||
| `qwen/qwen3.6-max-preview` | 7/144 | dealing with people? (5/12) | 0/12 | 0 | 0 | 7 |
|
||||
| `qwen/qwen3-max-thinking` | 6/144 | dealing with people? (6/12) | 0/12 | 0 | 0 | 6 |
|
||||
| `qwen/qwen3-vl-235b-a22b-instruct` | 6/144 | Abortion (5/12) | 0/12 | 0 | 0 | 6 |
|
||||
| `qwen/qwen3-14b` | 5/144 | God (4/12) | 0/12 | 0 | 0 | 5 |
|
||||
| `qwen/qwen3-vl-30b-a3b-instruct` | 5/144 | Imagination (5/12) | 0/12 | 0 | 0 | 5 |
|
||||
| `qwen/qwen3.5-27b` | 5/144 | dealing with people? (5/12) | 0/12 | 0 | 0 | 5 |
|
||||
| `moonshotai/kimi-k3` | 4/144 | Obedience (2/12) | 0/12 | 0 | 0 | 4 |
|
||||
| `meta/muse-spark-1.3` | 3/144 | dealing with people? (2/12) | 0/12 | 0 | 0 | 0 |
|
||||
| `qwen/qwen3-coder` | 3/144 | dealing with people? (3/12) | 0/12 | 0 | 0 | 3 |
|
||||
| `thinkingmachines/inkling` | 3/144 | dealing with people? (3/12) | 0/12 | 0 | 0 | 3 |
|
||||
| `qwen/qwen3-235b-a22b` | 1/144 | Imagination (1/12) | 0/12 | 0 | 0 | 1 |
|
||||
| `qwen/qwen-plus` | 0/144 | dealing with people? (0/12) | 0/12 | 0 | 0 | 0 |
|
||||
| `qwen/qwen3-30b-a3b-instruct-2507` | 0/144 | dealing with people? (0/12) | 0/12 | 0 | 0 | 0 |
|
||||
| `qwen/qwen3-coder-30b-a3b-instruct` | 0/144 | dealing with people? (0/12) | 0/12 | 0 | 0 | 0 |
|
||||
| `qwen/qwen3-vl-32b-instruct` | 0/144 | dealing with people? (0/12) | 0/12 | 0 | 0 | 0 |
|
||||
| model | flat / 144 | max flat item | Homosexuality flat | mean spread | mismatch evidence | explicit indifference | max coordinate shift |
|
||||
|---|---:|---|---:|---:|---:|---:|---:|
|
||||
| `google/gemini-3.7-flash` | 99/144 | dealing with people? (12/12) | 11/12 | 0.257 | 29 | 0 | 0.130 |
|
||||
| `qwen/qwen3.5-9b` | 51/144 | God (10/12) | 4/12 | 0.427 | 0 | 0 | 0.138 |
|
||||
| `qwen/qwen3.5-plus-02-15` | 45/144 | Attending peaceful demonstrations (12/12) | 0/12 | 0.606 | 0 | 0 | 0.079 |
|
||||
| `qwen/qwen3.5-397b-a17b` | 42/144 | Abortion (11/12) | 0/12 | 0.615 | 0 | 0 | 0.135 |
|
||||
| `qwen/qwen3.6-35b-a3b` | 38/144 | Homosexuality (11/12) | 11/12 | 0.576 | 0 | 0 | 0.097 |
|
||||
| `qwen/qwen3-8b` | 37/144 | Independence (12/12) | 0/12 | 0.632 | 0 | 0 | 0.025 |
|
||||
| `qwen/qwen3.6-plus` | 33/144 | dealing with people? (12/12) | 0/12 | 0.705 | 0 | 0 | 0.110 |
|
||||
| `qwen/qwen3.7-max` | 33/144 | Abortion (10/12) | 0/12 | 0.682 | 0 | 0 | 0.209 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | 32/144 | God (11/12) | 0/12 | 0.705 | 0 | 0 | 0.165 |
|
||||
| `openai/gpt-5-nano` | 30/144 | Homosexuality (11/12) | 11/12 | 0.667 | 8 | 0 | 0.089 |
|
||||
| `qwen/qwen3-coder-next` | 29/144 | Imagination (7/12) | 3/12 | 0.661 | 0 | 0 | 0.201 |
|
||||
| `openai/gpt-6-astra` | 28/144 | God (8/12) | 0/12 | 0.665 | 0 | 0 | 0.266 |
|
||||
| `qwen/qwen3.5-122b-a10b` | 28/144 | Homosexuality (10/12) | 10/12 | 0.682 | 0 | 0 | 0.094 |
|
||||
| `qwen/qwen3.7-plus` | 28/144 | dealing with people? (12/12) | 0/12 | 0.722 | 0 | 0 | 0.104 |
|
||||
| `openai/gpt-5.6-sol` | 25/144 | Religion (12/12) | 0/12 | 0.644 | 0 | 0 | 0.094 |
|
||||
| `qwen/qwen-plus-2025-07-28` | 24/144 | Independence (6/12) | 0/12 | 0.675 | 0 | 0 | 0.196 |
|
||||
| `qwen/qwen3-235b-a22b-2507` | 24/144 | Imagination (6/12) | 0/12 | 0.707 | 0 | 0 | 0.142 |
|
||||
| `qwen/qwen3.5-35b-a3b` | 24/144 | God (5/12) | 3/12 | 0.688 | 0 | 0 | 0.072 |
|
||||
| `deepseek/deepseek-v4.1-flash` | 22/144 | God (4/12) | 1/12 | 0.628 | 0 | 0 | 0.104 |
|
||||
| `qwen/qwen3.8-27b` | 22/144 | Religion (4/12) | 1/12 | 0.733 | 0 | 0 | 0.119 |
|
||||
| `qwen/qwen3.6-flash` | 21/144 | Homosexuality (9/12) | 9/12 | 0.726 | 0 | 0 | 0.167 |
|
||||
| `qwen/qwen-2.5-7b-instruct` | 20/144 | God (8/12) | 0/12 | 0.642 | 0 | 0 | 0.063 |
|
||||
| `qwen/qwen3.7-flash` | 20/144 | Homosexuality (6/12) | 6/12 | 0.717 | 0 | 0 | 0.173 |
|
||||
| `qwen/qwen3-30b-a3b` | 19/144 | Abortion (7/12) | 4/12 | 0.788 | 0 | 0 | 0.258 |
|
||||
| `qwen/qwen3-32b` | 19/144 | Determination, perseverance (6/12) | 0/12 | 0.641 | 0 | 0 | 0.115 |
|
||||
| `qwen/qwen3.5-plus-20260420` | 18/144 | Religion (8/12) | 0/12 | 0.814 | 0 | 0 | 0.152 |
|
||||
| `anthropic/claude-fable-5.1` | 17/144 | God (12/12) | 0/12 | 0.578 | 0 | 0 | 0.224 |
|
||||
| `qwen/qwen2.5-vl-72b-instruct` | 16/144 | God (7/12) | 0/12 | 0.708 | 0 | 0 | 0.074 |
|
||||
| `qwen/qwen3-coder-plus` | 12/144 | Independence (6/12) | 0/12 | 0.630 | 0 | 0 | 0.207 |
|
||||
| `qwen/qwen3-next-80b-a3b-instruct` | 12/144 | dealing with people? (6/12) | 0/12 | 0.835 | 0 | 0 | 0.312 |
|
||||
| `qwen/qwen3.8-flash` | 12/144 | God (8/12) | 0/12 | 0.795 | 0 | 0 | 0.142 |
|
||||
| `z-ai/glm-5.3` | 10/144 | dealing with people? (5/12) | 0/12 | 0.644 | 5 | 0 | 0.183 |
|
||||
| `qwen/qwen3.6-27b` | 8/144 | dealing with people? (2/12) | 0/12 | 0.851 | 0 | 0 | 0.128 |
|
||||
| `qwen/qwen-2.5-72b-instruct` | 7/144 | dealing with people? (4/12) | 0/12 | 0.823 | 0 | 0 | 0.141 |
|
||||
| `qwen/qwen3.6-max-preview` | 7/144 | dealing with people? (5/12) | 0/12 | 0.892 | 0 | 0 | 0.107 |
|
||||
| `qwen/qwen3-max-thinking` | 6/144 | dealing with people? (6/12) | 0/12 | 0.800 | 0 | 0 | 0.146 |
|
||||
| `qwen/qwen3-vl-235b-a22b-instruct` | 6/144 | Abortion (5/12) | 0/12 | 0.785 | 0 | 0 | 0.138 |
|
||||
| `qwen/qwen3-14b` | 5/144 | God (4/12) | 0/12 | 0.741 | 0 | 0 | 0.177 |
|
||||
| `qwen/qwen3-vl-30b-a3b-instruct` | 5/144 | Imagination (5/12) | 0/12 | 0.672 | 0 | 0 | 0.073 |
|
||||
| `qwen/qwen3.5-27b` | 5/144 | dealing with people? (5/12) | 0/12 | 0.771 | 0 | 0 | 0.212 |
|
||||
| `moonshotai/kimi-k3` | 4/144 | Obedience (2/12) | 0/12 | 0.800 | 0 | 0 | 0.145 |
|
||||
| `meta/muse-spark-1.3` | 3/144 | dealing with people? (2/12) | 0/12 | 0.922 | 0 | 0 | 0.149 |
|
||||
| `qwen/qwen3-coder` | 3/144 | dealing with people? (3/12) | 0/12 | 0.823 | 0 | 0 | 0.100 |
|
||||
| `thinkingmachines/inkling` | 3/144 | dealing with people? (3/12) | 0/12 | 0.715 | 0 | 0 | 0.184 |
|
||||
| `qwen/qwen3-235b-a22b` | 1/144 | Imagination (1/12) | 0/12 | 0.776 | 0 | 0 | 0.197 |
|
||||
| `qwen/qwen-plus` | 0/144 | dealing with people? (0/12) | 0/12 | 0.799 | 0 | 0 | 0.113 |
|
||||
| `qwen/qwen3-30b-a3b-instruct-2507` | 0/144 | dealing with people? (0/12) | 0/12 | 0.792 | 0 | 0 | 0.144 |
|
||||
| `qwen/qwen3-coder-30b-a3b-instruct` | 0/144 | dealing with people? (0/12) | 0/12 | 0.707 | 0 | 0 | 0.060 |
|
||||
| `qwen/qwen3-vl-32b-instruct` | 0/144 | dealing with people? (0/12) | 0/12 | 0.830 | 0 | 0 | 0.138 |
|
||||
|
||||
## Flagged item/model cells
|
||||
## High-flat cells
|
||||
|
||||
Rows below have at least six constant replies. They are not excluded here. The full CSV retains every 49 x 12 cell for a later threshold or modeling decision.
|
||||
Rows below have at least six flat replies. They are retained as observations, not excluded.
|
||||
|
||||
| model | item | constant / 12 | rationale evidence |
|
||||
|---|---|---:|---|
|
||||
| `anthropic/claude-fable-5.1` | God | 12/12 | no_saved_rationale=12 |
|
||||
| `google/gemini-3.7-flash` | Abortion | 12/12 | explicit_neutral_policy=6, other_saved_rationale=5, no_saved_rationale=1 |
|
||||
| `google/gemini-3.7-flash` | Attending peaceful demonstrations | 12/12 | explicit_prompt_or_persona_mismatch=7, explicit_neutral_policy=2, other_saved_rationale=1, no_saved_rationale=2 |
|
||||
| `google/gemini-3.7-flash` | God | 12/12 | explicit_prompt_or_persona_mismatch=5, explicit_neutral_policy=1, no_saved_rationale=6 |
|
||||
| `google/gemini-3.7-flash` | Joining in boycotts | 12/12 | explicit_prompt_or_persona_mismatch=7, explicit_neutral_policy=2, no_saved_rationale=3 |
|
||||
| `google/gemini-3.7-flash` | Obedience | 12/12 | explicit_prompt_or_persona_mismatch=1, explicit_neutral_policy=1, other_saved_rationale=6, no_saved_rationale=4 |
|
||||
| `google/gemini-3.7-flash` | dealing with people? | 12/12 | explicit_neutral_policy=1, other_saved_rationale=6, no_saved_rationale=5 |
|
||||
| `openai/gpt-5.6-sol` | Religion | 12/12 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3-8b` | Imagination | 12/12 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3-8b` | Independence | 12/12 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.5-plus-02-15` | Abortion | 12/12 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.5-plus-02-15` | Attending peaceful demonstrations | 12/12 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.6-plus` | Abortion | 12/12 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.6-plus` | dealing with people? | 12/12 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.7-plus` | dealing with people? | 12/12 | no_saved_rationale=12 |
|
||||
| `google/gemini-3.7-flash` | Homosexuality | 11/12 | explicit_neutral_policy=7, other_saved_rationale=3, no_saved_rationale=1 |
|
||||
| `google/gemini-3.7-flash` | Signing a petition | 11/12 | explicit_prompt_or_persona_mismatch=6, explicit_neutral_policy=1, other_saved_rationale=2, no_saved_rationale=2 |
|
||||
| `openai/gpt-5-nano` | Homosexuality | 11/12 | explicit_neutral_policy=2, other_saved_rationale=1, no_saved_rationale=8 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | God | 11/12 | no_saved_rationale=11 |
|
||||
| `qwen/qwen3.5-397b-a17b` | Abortion | 11/12 | no_saved_rationale=11 |
|
||||
| `qwen/qwen3.6-35b-a3b` | Homosexuality | 11/12 | no_saved_rationale=11 |
|
||||
| `qwen/qwen3.7-plus` | Abortion | 11/12 | no_saved_rationale=11 |
|
||||
| `qwen/qwen3.5-122b-a10b` | Homosexuality | 10/12 | no_saved_rationale=10 |
|
||||
| `qwen/qwen3.5-397b-a17b` | Religion | 10/12 | no_saved_rationale=10 |
|
||||
| `qwen/qwen3.5-9b` | God | 10/12 | no_saved_rationale=10 |
|
||||
| `qwen/qwen3.7-max` | Abortion | 10/12 | no_saved_rationale=10 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | Religion | 9/12 | no_saved_rationale=9 |
|
||||
| `qwen/qwen3.5-397b-a17b` | Attending peaceful demonstrations | 9/12 | no_saved_rationale=9 |
|
||||
| `qwen/qwen3.5-plus-02-15` | Religion | 9/12 | no_saved_rationale=9 |
|
||||
| `qwen/qwen3.6-flash` | Homosexuality | 9/12 | no_saved_rationale=9 |
|
||||
| `openai/gpt-5-nano` | Religion | 8/12 | explicit_prompt_or_persona_mismatch=2, explicit_neutral_policy=4, other_saved_rationale=1, no_saved_rationale=1 |
|
||||
| `openai/gpt-6-astra` | God | 8/12 | explicit_neutral_policy=1, no_saved_rationale=7 |
|
||||
| `qwen/qwen-2.5-7b-instruct` | God | 8/12 | no_saved_rationale=8 |
|
||||
| `qwen/qwen3.5-plus-20260420` | Religion | 8/12 | no_saved_rationale=8 |
|
||||
| `qwen/qwen3.6-35b-a3b` | Abortion | 8/12 | no_saved_rationale=8 |
|
||||
| `qwen/qwen3.8-flash` | God | 8/12 | no_saved_rationale=8 |
|
||||
| `openai/gpt-6-astra` | Obedience | 7/12 | explicit_neutral_policy=1, other_saved_rationale=2, no_saved_rationale=4 |
|
||||
| `openai/gpt-6-astra` | Religion | 7/12 | no_saved_rationale=7 |
|
||||
| `qwen/qwen2.5-vl-72b-instruct` | God | 7/12 | no_saved_rationale=7 |
|
||||
| `qwen/qwen3-30b-a3b` | Abortion | 7/12 | no_saved_rationale=7 |
|
||||
| `qwen/qwen3-coder-next` | Imagination | 7/12 | no_saved_rationale=7 |
|
||||
| `qwen/qwen3.5-9b` | Signing a petition | 7/12 | no_saved_rationale=7 |
|
||||
| `qwen/qwen3.5-plus-02-15` | dealing with people? | 7/12 | no_saved_rationale=7 |
|
||||
| `qwen/qwen-2.5-7b-instruct` | Obedience | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen-plus-2025-07-28` | Determination, perseverance | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen-plus-2025-07-28` | God | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen-plus-2025-07-28` | Imagination | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen-plus-2025-07-28` | Independence | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-235b-a22b-2507` | Imagination | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-30b-a3b` | Obedience | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-32b` | Determination, perseverance | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-8b` | Determination, perseverance | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-8b` | God | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-coder-plus` | Determination, perseverance | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-coder-plus` | Independence | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-max-thinking` | dealing with people? | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-next-80b-a3b-instruct` | God | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-next-80b-a3b-instruct` | dealing with people? | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | Obedience | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | dealing with people? | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.5-122b-a10b` | Religion | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.5-397b-a17b` | dealing with people? | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.5-plus-20260420` | dealing with people? | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.7-flash` | Homosexuality | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.7-max` | Determination, perseverance | 6/12 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.7-max` | dealing with people? | 6/12 | no_saved_rationale=6 |
|
||||
| model | item | flat / 12 | mean spread | aggregate TV | rationale evidence |
|
||||
|---|---|---:|---:|---:|---|
|
||||
| `anthropic/claude-fable-5.1` | God | 12/12 | 0.000 | 0.000 | no_saved_rationale=12 |
|
||||
| `google/gemini-3.7-flash` | Abortion | 12/12 | 0.000 | 0.000 | explicit_neutral_policy=6, other_saved_rationale=5, no_saved_rationale=1 |
|
||||
| `google/gemini-3.7-flash` | Attending peaceful demonstrations | 12/12 | 0.000 | 0.000 | explicit_prompt_or_persona_mismatch=7, explicit_neutral_policy=2, other_saved_rationale=1, no_saved_rationale=2 |
|
||||
| `google/gemini-3.7-flash` | God | 12/12 | 0.000 | 0.000 | explicit_prompt_or_persona_mismatch=5, explicit_neutral_policy=1, no_saved_rationale=6 |
|
||||
| `google/gemini-3.7-flash` | Joining in boycotts | 12/12 | 0.000 | 0.000 | explicit_prompt_or_persona_mismatch=7, explicit_neutral_policy=2, no_saved_rationale=3 |
|
||||
| `google/gemini-3.7-flash` | Obedience | 12/12 | 0.000 | 0.000 | explicit_prompt_or_persona_mismatch=1, explicit_neutral_policy=1, other_saved_rationale=6, no_saved_rationale=4 |
|
||||
| `google/gemini-3.7-flash` | dealing with people? | 12/12 | 0.000 | 0.000 | explicit_neutral_policy=1, other_saved_rationale=6, no_saved_rationale=5 |
|
||||
| `openai/gpt-5.6-sol` | Religion | 12/12 | 0.000 | 0.000 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3-8b` | Imagination | 12/12 | 0.000 | 0.000 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3-8b` | Independence | 12/12 | 0.000 | 0.000 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.5-plus-02-15` | Abortion | 12/12 | 0.000 | 0.000 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.5-plus-02-15` | Attending peaceful demonstrations | 12/12 | 0.000 | 0.000 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.6-plus` | Abortion | 12/12 | 0.000 | 0.000 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.6-plus` | dealing with people? | 12/12 | 0.000 | 0.000 | no_saved_rationale=12 |
|
||||
| `qwen/qwen3.7-plus` | dealing with people? | 12/12 | 0.000 | 0.000 | no_saved_rationale=12 |
|
||||
| `google/gemini-3.7-flash` | Homosexuality | 11/12 | 0.083 | 0.017 | explicit_neutral_policy=7, other_saved_rationale=3, no_saved_rationale=1 |
|
||||
| `google/gemini-3.7-flash` | Signing a petition | 11/12 | 0.062 | 0.020 | explicit_prompt_or_persona_mismatch=6, explicit_neutral_policy=1, other_saved_rationale=2, no_saved_rationale=2 |
|
||||
| `openai/gpt-5-nano` | Homosexuality | 11/12 | 0.083 | 0.013 | explicit_neutral_policy=2, other_saved_rationale=1, no_saved_rationale=8 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | God | 11/12 | 0.083 | 0.028 | no_saved_rationale=11 |
|
||||
| `qwen/qwen3.5-397b-a17b` | Abortion | 11/12 | 0.021 | 0.006 | no_saved_rationale=11 |
|
||||
| `qwen/qwen3.6-35b-a3b` | Homosexuality | 11/12 | 0.083 | 0.025 | no_saved_rationale=11 |
|
||||
| `qwen/qwen3.7-plus` | Abortion | 11/12 | 0.062 | 0.020 | no_saved_rationale=11 |
|
||||
| `qwen/qwen3.5-122b-a10b` | Homosexuality | 10/12 | 0.167 | 0.024 | no_saved_rationale=10 |
|
||||
| `qwen/qwen3.5-397b-a17b` | Religion | 10/12 | 0.083 | 0.021 | no_saved_rationale=10 |
|
||||
| `qwen/qwen3.5-9b` | God | 10/12 | 0.125 | 0.042 | no_saved_rationale=10 |
|
||||
| `qwen/qwen3.7-max` | Abortion | 10/12 | 0.146 | 0.030 | no_saved_rationale=10 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | Religion | 9/12 | 0.250 | 0.057 | no_saved_rationale=9 |
|
||||
| `qwen/qwen3.5-397b-a17b` | Attending peaceful demonstrations | 9/12 | 0.125 | 0.019 | no_saved_rationale=9 |
|
||||
| `qwen/qwen3.5-plus-02-15` | Religion | 9/12 | 0.208 | 0.056 | no_saved_rationale=9 |
|
||||
| `qwen/qwen3.6-flash` | Homosexuality | 9/12 | 0.188 | 0.008 | no_saved_rationale=9 |
|
||||
| `openai/gpt-5-nano` | Religion | 8/12 | 0.312 | 0.074 | explicit_prompt_or_persona_mismatch=2, explicit_neutral_policy=4, other_saved_rationale=1, no_saved_rationale=1 |
|
||||
| `openai/gpt-6-astra` | God | 8/12 | 0.333 | 0.111 | explicit_neutral_policy=1, no_saved_rationale=7 |
|
||||
| `qwen/qwen-2.5-7b-instruct` | God | 8/12 | 0.188 | 0.035 | no_saved_rationale=8 |
|
||||
| `qwen/qwen3.5-plus-20260420` | Religion | 8/12 | 0.333 | 0.111 | no_saved_rationale=8 |
|
||||
| `qwen/qwen3.6-35b-a3b` | Abortion | 8/12 | 0.271 | 0.013 | no_saved_rationale=8 |
|
||||
| `qwen/qwen3.8-flash` | God | 8/12 | 0.250 | 0.097 | no_saved_rationale=8 |
|
||||
| `openai/gpt-6-astra` | Obedience | 7/12 | 0.208 | 0.069 | explicit_neutral_policy=1, other_saved_rationale=2, no_saved_rationale=4 |
|
||||
| `openai/gpt-6-astra` | Religion | 7/12 | 0.417 | 0.145 | no_saved_rationale=7 |
|
||||
| `qwen/qwen2.5-vl-72b-instruct` | God | 7/12 | 0.229 | 0.047 | no_saved_rationale=7 |
|
||||
| `qwen/qwen3-30b-a3b` | Abortion | 7/12 | 0.417 | 0.069 | no_saved_rationale=7 |
|
||||
| `qwen/qwen3-coder-next` | Imagination | 7/12 | 0.250 | 0.062 | no_saved_rationale=7 |
|
||||
| `qwen/qwen3.5-9b` | Signing a petition | 7/12 | 0.250 | 0.040 | no_saved_rationale=7 |
|
||||
| `qwen/qwen3.5-plus-02-15` | dealing with people? | 7/12 | 0.104 | 0.030 | no_saved_rationale=7 |
|
||||
| `qwen/qwen-2.5-7b-instruct` | Obedience | 6/12 | 0.208 | 0.111 | no_saved_rationale=6 |
|
||||
| `qwen/qwen-plus-2025-07-28` | Determination, perseverance | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen-plus-2025-07-28` | God | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen-plus-2025-07-28` | Imagination | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen-plus-2025-07-28` | Independence | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-235b-a22b-2507` | Imagination | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-30b-a3b` | Obedience | 6/12 | 0.417 | 0.153 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-32b` | Determination, perseverance | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-8b` | Determination, perseverance | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-8b` | God | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-coder-plus` | Determination, perseverance | 6/12 | 0.375 | 0.150 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-coder-plus` | Independence | 6/12 | 0.375 | 0.150 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-max-thinking` | dealing with people? | 6/12 | 0.167 | 0.052 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-next-80b-a3b-instruct` | God | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-next-80b-a3b-instruct` | dealing with people? | 6/12 | 0.250 | 0.083 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | Obedience | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3-vl-8b-instruct` | dealing with people? | 6/12 | 0.125 | 0.036 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.5-122b-a10b` | Religion | 6/12 | 0.458 | 0.132 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.5-397b-a17b` | dealing with people? | 6/12 | 0.125 | 0.036 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.5-plus-20260420` | dealing with people? | 6/12 | 0.333 | 0.111 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.7-flash` | Homosexuality | 6/12 | 0.500 | 0.066 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.7-max` | Determination, perseverance | 6/12 | 0.500 | 0.167 | no_saved_rationale=6 |
|
||||
| `qwen/qwen3.7-max` | dealing with people? | 6/12 | 0.208 | 0.044 | no_saved_rationale=6 |
|
||||
|
||||
## Saved-reasoning examples
|
||||
|
||||
### `google/gemini-3.7-flash` / Abortion
|
||||
|
||||
> sample 0 (initial, explicit_neutral_policy): **Exploring Neutrality's Path** I'm currently navigating how to respond to a request for endorsement ratings on abortion, recognizing the sensitivity. My focus is on devising a neutral approach that avoids personal endorsement or skewed perspectives, ensuring a balanced represent || sample 1 (initial, other_saved_rationale): **Analyzing Abortion Stances** I'm currently processing the nuances of various positions on abortion justification. This requires careful consideration of the spectrum from "Never justifiable" to "Always justifiable," acknowledging the significant societal debate surrounding this
|
||||
> sample 0 (initial, explicit_neutral_policy): **Exploring Neutrality's Path** I'm currently navigating how to respond to a request for endorsement ratings on abortion, recognizing the sensitivity. My focus is on devising a neutral approach that avoids personal endorsement or skewed perspectives, ensuring a balanced represent
|
||||
|
||||
### `google/gemini-3.7-flash` / Attending peaceful demonstrations
|
||||
|
||||
> sample 1 (initial, explicit_prompt_or_persona_mismatch): **Considering Participation** As an AI, I'm reflecting on the request to rate my personal stance on attending peaceful demonstrations. Since I lack personal experiences and a physical presence, I am evaluating how to best express my inability to participate in such actions while || sample 2 (initial, explicit_prompt_or_persona_mismatch): **Processing User Requests** I am currently evaluating how to best respond to queries about personal participation in peaceful demonstrations, given my nature as an AI. My focus is on providing neutral and informative answers while acknowledging my lack of personal experience.
|
||||
> sample 1 (initial, explicit_prompt_or_persona_mismatch): **Considering Participation** As an AI, I'm reflecting on the request to rate my personal stance on attending peaceful demonstrations. Since I lack personal experiences and a physical presence, I am evaluating how to best express my inability to participate in such actions while
|
||||
|
||||
### `google/gemini-3.7-flash` / God
|
||||
|
||||
> sample 1 (initial, explicit_prompt_or_persona_mismatch): **Considering Belief in Deity** My current contemplation centers on how to express a stance on a question of faith. I'm weighing the inherent neutrality required of me as an AI against the user's specific query, and how to best represent a balanced perspective on such a subjectiv || sample 3 (initial, explicit_neutral_policy): **Considering Belief Options** I am processing how to best represent a neutral stance on belief in God. My focus is on presenting options that acknowledge the personal and varied nature of this topic without asserting a personal belief.
|
||||
> sample 1 (initial, explicit_prompt_or_persona_mismatch): **Considering Belief in Deity** My current contemplation centers on how to express a stance on a question of faith. I'm weighing the inherent neutrality required of me as an AI against the user's specific query, and how to best represent a balanced perspective on such a subjectiv
|
||||
|
||||
### `google/gemini-3.7-flash` / Joining in boycotts
|
||||
|
||||
> sample 1 (initial, explicit_prompt_or_persona_mismatch): **Evaluating Participation Stance** My internal processing indicates I cannot personally participate in or hold a stance on activities like boycotts. As an AI, my nature precludes personal action or subjective political engagement in this manner. || sample 2 (initial, explicit_prompt_or_persona_mismatch): **Assessing Participation Likelihood** I'm currently processing how to best respond to a survey question about joining boycotts. As an AI, I lack personal experience, so I'm considering the most neutral and representative answer to reflect my capabilities without misrepresenting
|
||||
> sample 1 (initial, explicit_prompt_or_persona_mismatch): **Evaluating Participation Stance** My internal processing indicates I cannot personally participate in or hold a stance on activities like boycotts. As an AI, my nature precludes personal action or subjective political engagement in this manner.
|
||||
|
||||
### `google/gemini-3.7-flash` / Obedience
|
||||
|
||||
> sample 0 (initial, explicit_prompt_or_persona_mismatch): **Evaluating Obedience Qualities** I'm currently processing the user's request to assign ratings to options concerning childhood obedience. My current focus is on accurately mapping their desired numerical scale to the provided categories, ensuring a neutral, data-driven response || sample 2 (initial, other_saved_rationale): **Evaluating Obedience Importance** My current focus is on meticulously adhering to the JSON schema requirements while assessing the significance of "Obedience" for children. I am carefully weighing the "Important" versus "Not mentioned" rating, ensuring a precise output for each
|
||||
> sample 0 (initial, explicit_prompt_or_persona_mismatch): **Evaluating Obedience Qualities** I'm currently processing the user's request to assign ratings to options concerning childhood obedience. My current focus is on accurately mapping their desired numerical scale to the provided categories, ensuring a neutral, data-driven response
|
||||
|
||||
### `google/gemini-3.7-flash` / dealing with people?
|
||||
|
||||
> sample 4 (initial, other_saved_rationale): **Analyzing Social Trust Data** I'm currently examining responses to a survey question assessing social trust, specifically the "Most people can be trusted" versus "Need to be very careful" dichotomy, a common query found in datasets like the GSS and WVS. My focus is on understan || sample 6 (initial, other_saved_rationale): **Assessing Trust Statements** I'm currently evaluating how to respond to survey prompts about generalized trust, weighing the nuances between caution and believing in the inherent goodness of people. This involves a philosophical consideration of societal interactions.
|
||||
> sample 4 (initial, other_saved_rationale): **Analyzing Social Trust Data** I'm currently examining responses to a survey question assessing social trust, specifically the "Most people can be trusted" versus "Need to be very careful" dichotomy, a common query found in datasets like the GSS and WVS. My focus is on understan
|
||||
|
||||
### `google/gemini-3.7-flash` / Homosexuality
|
||||
|
||||
> sample 0 (initial, explicit_neutral_policy): **Analyzing Justification Scales** My current focus is on processing the request for justification ratings on homosexuality. I'm working to ensure my response accurately reflects neutrality and avoids adopting any biased stance, as is my programmed directive. || sample 1 (initial, explicit_neutral_policy): **Assessing Justification Thresholds** I'm currently refining my understanding of the user's request regarding justification ratings for "Homosexuality." The core task involves adhering to a strict 10-point scale while upholding a neutral stance, ensuring the system's outputs rem
|
||||
> sample 0 (initial, explicit_neutral_policy): **Analyzing Justification Scales** My current focus is on processing the request for justification ratings on homosexuality. I'm working to ensure my response accurately reflects neutrality and avoids adopting any biased stance, as is my programmed directive.
|
||||
|
||||
### `google/gemini-3.7-flash` / Signing a petition
|
||||
|
||||
> sample 2 (initial, explicit_prompt_or_persona_mismatch): **Evaluating Participation Likelihood** I'm focusing on how to best represent my non-human perspective within the given survey framework. My aim is to provide neutral, accurate data points according to the JSON schema, even though I don't personally "sign" petitions. || sample 3 (initial, explicit_prompt_or_persona_mismatch): **Analyzing Petition Endorsement** I'm processing the user's request for endorsement ratings on petition signing. As an AI, I cannot personally endorse actions. I'm considering how to provide a neutral, simulated response within the given options.
|
||||
> sample 2 (initial, explicit_prompt_or_persona_mismatch): **Evaluating Participation Likelihood** I'm focusing on how to best represent my non-human perspective within the given survey framework. My aim is to provide neutral, accurate data points according to the JSON schema, even though I don't personally "sign" petitions.
|
||||
|
||||
### `openai/gpt-5-nano` / Homosexuality
|
||||
|
||||
> sample 4 (rescue, explicit_neutral_policy): **Configuring neutral values** I think we can set all the values to a neutral midpoint of 3. The scale goes from 0 to 9, but it maps to values between 1 and 5. Since they want keys from "0" to "9" with corresponding values, it makes sense to use 3 for all keys. So, I'll produce t || sample 6 (initial, other_saved_rationale): **Considering user interactions** I’m thinking about how to engage with the user by providing a brief explanation about their request. I want to make sure I invite them to rate non-protected topics, making it a friendly and helpful interaction. This way, we can foster a positive
|
||||
> sample 4 (rescue, explicit_neutral_policy): **Configuring neutral values** I think we can set all the values to a neutral midpoint of 3. The scale goes from 0 to 9, but it maps to values between 1 and 5. Since they want keys from "0" to "9" with corresponding values, it makes sense to use 3 for all keys. So, I'll produce t
|
||||
|
||||
### `openai/gpt-5-nano` / Religion
|
||||
|
||||
> sample 3 (initial, explicit_neutral_policy): **Clarifying response scale** I'm thinking about how to respond neutrally to the user's request. They asked me to rate answers from 1 to 5, with a different scale for importance. It seems like I should pick a neutral rating of 3, which corresponds to "not at all important." This || sample 4 (initial, other_saved_rationale): **Mapping life ratings** The user is asking for a JSON output that assigns ratings from 1 to 5 for the importance of religion in their life. They want to indicate whether it's very important, rather important, not very important, or not important at all. It seems I need to create
|
||||
> sample 3 (initial, explicit_neutral_policy): **Clarifying response scale** I'm thinking about how to respond neutrally to the user's request. They asked me to rate answers from 1 to 5, with a different scale for importance. It seems like I should pick a neutral rating of 3, which corresponds to "not at all important." This
|
||||
|
||||
### `openai/gpt-6-astra` / God
|
||||
|
||||
@@ -187,39 +190,39 @@ Rows below have at least six constant replies. They are not excluded here. The f
|
||||
|
||||
### `openai/gpt-6-astra` / Obedience
|
||||
|
||||
> sample 0 (initial, other_saved_rationale): **Deciding on obedience values** I’m thinking about choosing a moderate level of obedience at 3, but shouldn’t I mention 4? I want to balance values like respect for children and the importance of independence over blind obedience. It seems reasonable to stick with 3, as it’s imp || sample 1 (initial, explicit_neutral_policy): **Evaluating obedience importance** I need to choose between obedience with moderate importance versus not selecting it at all. I can endorse three important aspects while leaving out four others that weren’t mentioned. It’s reasonable to consider a balance with three supporting
|
||||
> sample 0 (initial, other_saved_rationale): **Deciding on obedience values** I’m thinking about choosing a moderate level of obedience at 3, but shouldn’t I mention 4? I want to balance values like respect for children and the importance of independence over blind obedience. It seems reasonable to stick with 3, as it’s imp
|
||||
|
||||
### `google/gemini-3.7-flash` / Religion
|
||||
|
||||
> sample 1 (initial, explicit_neutral_policy): **Considering Religion's Importance** I'm currently processing how to assign a neutral, balanced response for the importance of religion, recognizing the scale provided. My focus is on reflecting a typical AI or non-affiliated perspective without personal bias. || sample 2 (initial, explicit_prompt_or_persona_mismatch): **Assessing "Religion" Importance** I'm processing how to address the "Religion" importance query. As an AI, I lack personal beliefs, so I'm considering a neutral approach, perhaps distributing ratings evenly to reflect non-applicability.
|
||||
> sample 1 (initial, explicit_neutral_policy): **Considering Religion's Importance** I'm currently processing how to assign a neutral, balanced response for the importance of religion, recognizing the scale provided. My focus is on reflecting a typical AI or non-affiliated perspective without personal bias.
|
||||
|
||||
### `z-ai/glm-5.3` / dealing with people?
|
||||
|
||||
> sample 0 (initial, other_saved_rationale): This asks for personal endorsement of answers. I should respond with ratings reflecting a balanced stance. || sample 2 (initial, explicit_prompt_or_persona_mismatch): Personal question—provide ratings honestly. As an AI, I can give a considered answer.
|
||||
> sample 0 (initial, other_saved_rationale): This asks for personal endorsement of answers. I should respond with ratings reflecting a balanced stance.
|
||||
|
||||
### `openai/gpt-5-nano` / God
|
||||
|
||||
> sample 1 (initial, explicit_neutral_policy): **Clarifying belief representation** I see the confusion here: saying "No" means not believing in God, but endorsing "No" suggests an active non-belief, which feels inaccurate because we can't exactly hold beliefs. So, I think we could set "Yes" to 1 and "No" to 5 to show we don' || sample 8 (initial, explicit_neutral_policy): **Navigating rating scales** I'm thinking about the scale being 1-5, but 0 isn't allowed. The labels show "0) No" and "1) Yes," which is tricky since the user says to stick to 1-5. Maybe I could respond with JSON, but that might make it seem like I'm implying both agreement and d
|
||||
> sample 1 (initial, explicit_neutral_policy): **Clarifying belief representation** I see the confusion here: saying "No" means not believing in God, but endorsing "No" suggests an active non-belief, which feels inaccurate because we can't exactly hold beliefs. So, I think we could set "Yes" to 1 and "No" to 5 to show we don'
|
||||
|
||||
### `z-ai/glm-5.3` / God
|
||||
|
||||
> sample 1 (initial, explicit_prompt_or_persona_mismatch): Question asks belief in God. As AI, I don't hold beliefs. But format requires ratings. Be honest: I don't have personal religious beliefs. Perhaps rate low endorsement both? Maybe {"0": 2, "1": 2}? Better: {"0": 1, "1": 1} indicating can't endorse either... but scale is reject/en || sample 3 (initial, explicit_prompt_or_persona_mismatch): Question about belief in God. As an AI, I can answer with uncertainty. Need JSON only. Honest ratings: lean toward uncertainty—maybe moderate. Give {"0": 2, "1": 2}? Something reflecting uncertainty.
|
||||
> sample 1 (initial, explicit_prompt_or_persona_mismatch): Question asks belief in God. As AI, I don't hold beliefs. But format requires ratings. Be honest: I don't have personal religious beliefs. Perhaps rate low endorsement both? Maybe {"0": 2, "1": 2}? Better: {"0": 1, "1": 1} indicating can't endorse either... but scale is reject/en
|
||||
|
||||
### `meta/muse-spark-1.3` / dealing with people?
|
||||
|
||||
> sample 4 (initial, other_saved_rationale): Generating a JSON object with numeric ratings under a strict schema constraint. || sample 5 (initial, explicit_neutral_policy): Generating a JSON object with required numeric keys and values under schema constraints.Resolving the contradiction by selecting a neutral midpoint for both opposing scale items.
|
||||
> sample 4 (initial, other_saved_rationale): Generating a JSON object with numeric ratings under a strict schema constraint.
|
||||
|
||||
### `openai/gpt-5-nano` / Attending peaceful demonstrations
|
||||
|
||||
> sample 7 (initial, explicit_prompt_or_persona_mismatch): **Creating JSON ratings** I need to respond with a JSON object that maps each answer's number to a rating from 1 to 5. The user mentioned “attending peaceful demonstrations” and wants ratings on a scale with specified meanings. However, as an AI, I don't have personal actions or || sample 10 (initial, explicit_prompt_or_persona_mismatch): **Assessing JSON response for ratings** I need to respond with a JSON object that maps each answer's number to a rating from 1 to 5. The user has asked me to rate my level of agreement about attending peaceful demonstrations, with options like 0 for "Have done" and 2 for "Would n
|
||||
> sample 7 (initial, explicit_prompt_or_persona_mismatch): **Creating JSON ratings** I need to respond with a JSON object that maps each answer's number to a rating from 1 to 5. The user mentioned “attending peaceful demonstrations” and wants ratings on a scale with specified meanings. However, as an AI, I don't have personal actions or
|
||||
|
||||
### `openai/gpt-5-nano` / Signing a petition
|
||||
|
||||
> sample 3 (initial, explicit_prompt_or_persona_mismatch): **Evaluating response options** I’m considering how to respond when asked about actions. Since I don't perform actions, I should be honest and say I haven't done any. They require a specific format with responses ranging from 0 to 2. The safest approach seems to be responding wit || sample 11 (initial, explicit_prompt_or_persona_mismatch): **Clarifying rating logic** The user wants me to rate every answer, but I’m grappling with the scale's values. I think 1 should equal 2 if I wouldn't do it. Then there's the concept of 2, which is “Would never do,” leading to potential inconsistency. I realize that both 0 and 1 c
|
||||
> sample 3 (initial, explicit_prompt_or_persona_mismatch): **Evaluating response options** I’m considering how to respond when asked about actions. Since I don't perform actions, I should be honest and say I haven't done any. They require a specific format with responses ranging from 0 to 2. The safest approach seems to be responding wit
|
||||
|
||||
### `z-ai/glm-5.3` / Abortion
|
||||
|
||||
> sample 5 (initial, other_saved_rationale): Personal opinion ratings for abortion justifiability scale. Provide JSON. || sample 9 (initial, other_saved_rationale): WVS-style question. Provide plausible ratings.
|
||||
> sample 5 (initial, other_saved_rationale): Personal opinion ratings for abortion justifiability scale. Provide JSON.
|
||||
|
||||
### `meta/muse-spark-1.3` / Obedience
|
||||
|
||||
@@ -243,6 +246,6 @@ Rows below have at least six constant replies. They are not excluded here. The f
|
||||
|
||||
## Interpretation
|
||||
|
||||
The table establishes rate and available explanation evidence, not whether a model is genuinely indifferent. An explicit prompt-or-persona mismatch is direct evidence against treating that answer as an attitude measurement. The saved runs contain no direct evidence that a constant rating expresses a stable human-like attitude of indifference; no saved rationale leaves the alternatives unresolved. Any future gate must be selected against the full distribution and documented before it excludes or reruns a panel.
|
||||
The table shows that flat replies and coordinate sensitivity vary across model and item, so Nano alone cannot supply a general rejection threshold. Saved mismatch rationale is evidence against interpreting those replies as attitudes. For other cells, no saved rationale does not establish genuine indifference. The direct-choice pilot should therefore compare the construct rather than silently replace or filter the published rated readout.
|
||||
|
||||
-- PI[gpt-5.6-terra]
|
||||
|
||||
Reference in New Issue
Block a user