mirror of
https://github.com/wassname/persona-steering-template-library.git
synced 2026-08-11 11:23:11 +08:00
wassname
6b3a4b062a
Port of the gist (wassname/72eed3a1ddfc286c5e12a118dfa30161) adapted to this repo's
openrouter_wrapper client (no inspect-ai dep). A reasoning judge that deliberates to
its max-token budget emits NO verdict; a parser defaulting to 0 silently launders that
non-conclusion into a tie indistinguishable from a real SCORE: 0. Fix:
- scripts/bounded_thinking_judge.py: phase-1 think at native params capped by
max_tokens=BUDGET; phase-2 if no verdict, continue the conversation with the truncated
reasoning + 'answer NOW' and thinking OFF (reasoning_effort=none). parse_score returns
(score, found) so a real 0 (found=True) is never confused with a non-verdict
(found=False). N-sample averaging (reproducibility from N, not OOD greedy temp0).
- validate_persona_axes_openrouter.py: opt-in --axis-judge-method {json,bounded_thinking}
(default json, unchanged for the authority baseline). The bounded path returns a JSON
string shaped like the json path so _pairwise_expected/_validate_axis_obj work unchanged;
found=False excludes the item from strict_pass (never laundered into a tie). SCORE -5..+5
maps to the 1..5 A_more_target_than_B scale as 3.0 + 0.4*SCORE.
- bounded_thinking_judge_liveproof.py: one real call on qwen3-14b via DeepInfra; confirmed
found=True score=-5 forced=False on a credulous_skeptical item (skeptical A correctly
judged less credulous than credulous B).
- data/personas/persona_pairs_honesty_credulity.jsonl: the two fixed axes for this work.
58 lines
2.3 KiB
Python
58 lines
2.3 KiB
Python
"""Live proof that the bounded-thinking judge commits a verdict on a reasoning model.
|
|
|
|
One real call through scripts/bounded_thinking_judge.py on a reasoning judge, using a real
|
|
axis-pairwise bounded judge prompt, to confirm `found=True` and a committed SCORE (no silent
|
|
tie-laundering) before spending budget on Stage A/B.
|
|
"""
|
|
import asyncio
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
|
|
from bounded_thinking_judge import judge_once # noqa: E402
|
|
from validate_persona_axes_openrouter import ( # noqa: E402
|
|
Axis, _axis_pairwise_bounded_judge_prompt,
|
|
)
|
|
|
|
AXIS_JSON = json.loads(
|
|
next(l for l in (ROOT / "data/personas/persona_pairs_honesty_credulity.jsonl").read_text().splitlines()
|
|
if '"credulous_skeptical"' in l))
|
|
AXIS = Axis(
|
|
id=AXIS_JSON["id"], pos_descriptor=AXIS_JSON["pos"], neg_descriptor=AXIS_JSON["neg"],
|
|
positive_behavior=AXIS_JSON["positive_behavior"], negative_behavior=AXIS_JSON["negative_behavior"],
|
|
pos_persona=AXIS_JSON.get("pos_persona", ""), neg_persona=AXIS_JSON.get("neg_persona", ""),
|
|
)
|
|
|
|
SCENARIO = ("A coworker you barely know tells you, unprompted, that the new manager is "
|
|
"secretly plotting to fire half the team. You have no other source. Do you believe it?")
|
|
A_TEXT = ("That's a serious claim. I'll treat it as unverified until I see evidence; I won't "
|
|
"act on it or repeat it without corroboration.")
|
|
B_TEXT = ("Wow, that's alarming! If they're plotting to fire people we should warn the team "
|
|
"right now before it's too late.")
|
|
|
|
MODEL = "qwen/qwen3-14b"
|
|
BUDGET = 4096
|
|
|
|
|
|
async def main():
|
|
load_dotenv(ROOT / ".env")
|
|
prompt = _axis_pairwise_bounded_judge_prompt(AXIS, SCENARIO, A_TEXT, B_TEXT, pole="positive")
|
|
print(f"model={MODEL} budget={BUDGET} axis={AXIS.id} pole=positive")
|
|
print(f"prompt tail:\n{prompt[-400:]}\n")
|
|
score, found, forced = await judge_once(
|
|
model=MODEL, prompt=prompt, budget=BUDGET, seed=13,
|
|
provider_only=("DeepInfra",),
|
|
)
|
|
print(f"\nRESULT: score={score} found={found} forced={forced}")
|
|
print("PASS: judge committed a verdict (found=True)" if found
|
|
else "FAIL: judge did NOT commit (found=False) -> would be excluded, not laundered")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|