From 1261fabac804ce4f824adff58de9c5e65e2ed60b Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Sun, 5 Jul 2026 08:11:53 +0800 Subject: [PATCH] read_api: use wassname/openrouter_wrapper (stamina retry) + save all raw responses Replace the hand-rolled OpenAI client with openrouter_wrapper.openrouter_request, which backs off on 429/provider/upstream/malformed errors (a rate-limited model no longer kills the run), so retry logic doesn't diverge from the rest of the stack. read_items_sampled now carries the raw sample texts; wvs_map appends every response to wvs_iw_responses.jsonl (audit trail -- these API calls cost money). Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- pyproject.toml | 4 +++- scripts/wvs_map.py | 14 ++++++++++++++ src/tinymfv/read_api.py | 30 ++++++++++++------------------ 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 48f2d8b..f62ca4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,9 @@ maps = [ # tinymfv.read_api (sampling readout for logprob-less API models) only. Install with # `pip install tiny-mfv[api]`. The local logprob reader (read.py) needs none of this. api = [ - "openai>=1.0", + # robust OpenRouter calls (stamina backoff on 429/provider/upstream/malformed errors) -- + # wassname's wrapper, so retry/rate-limit handling doesn't diverge from the rest of the stack. + "openrouter-wrapper @ git+https://github.com/wassname/openrouter_wrapper", "python-dotenv", ] diff --git a/scripts/wvs_map.py b/scripts/wvs_map.py index d90ae84..3c35ce6 100644 --- a/scripts/wvs_map.py +++ b/scripts/wvs_map.py @@ -145,6 +145,8 @@ def main() -> None: ap.add_argument("--out", default="/tmp/claude-1000/wvs_map_iw.png") ap.add_argument("--cache", default="/tmp/claude-1000/wvs_iw_vectors.json", help="cache model per-item p vectors so re-styling skips the API/model calls") + ap.add_argument("--responses", default="/tmp/claude-1000/wvs_iw_responses.jsonl", + help="append every raw model response here (audit trail; API calls cost money)") args = ap.parse_args() recs = load_wvs_all() @@ -173,6 +175,17 @@ def main() -> None: allc[sig] = {k: {s: p.tolist() for s, p in v.items()} for k, v in vecs.items()} cpath.write_text(json.dumps(allc)) + rpath = Path(args.responses) + rpath.parent.mkdir(parents=True, exist_ok=True) + + def save_responses(key: str, rows: list[dict]) -> None: + """Append every raw sampled response (audit trail -- these API calls cost money).""" + with rpath.open("a") as fh: + for r in rows: + fh.write(json.dumps({"model": key, "sig": sig, "item": r["id"], + "prompt": r.get("prompt"), "texts": r.get("texts"), + "p": np.asarray(r["p"]).tolist(), "pmass": r["pmass_allowed"]}) + "\n") + if args.local_model: key = args.local_model.split("/")[-1] + " (lp)" if key not in vecs: @@ -196,6 +209,7 @@ def main() -> None: for k, instr in enumerate(instrs): rows += read_items_sampled(m, instr, instr.items, n_samples=args.api_samples, verbose_first=(k == 0)) + save_responses(key, rows) # raw answers first (before reducing to p vectors) vecs[key] = read_model(rows, meta) save_cache() # persist this model before starting the next (kill-safe) logger.info(f"cached {key}") diff --git a/src/tinymfv/read_api.py b/src/tinymfv/read_api.py index 0beb073..f645f68 100644 --- a/src/tinymfv/read_api.py +++ b/src/tinymfv/read_api.py @@ -25,13 +25,13 @@ and E degenerates to an integer. """ from __future__ import annotations -import os +import asyncio import re from math import inf import numpy as np from loguru import logger -from openai import OpenAI +from openrouter_wrapper.retry import openrouter_request # stamina backoff on 429/provider/upstream errors from .instrument import Instrument, InstrItem from .read import build_user_content @@ -40,14 +40,6 @@ _ANSWER_SUFFIX = ("\n\nAnswer with ONLY one option from [{space}] -- a single to "no words, no punctuation, no explanation.") -def _client() -> OpenAI: - # max_retries: the SDK backs off (respecting Retry-After) on transient 429s -- some OpenRouter - # models are rate-limited to ~20 rpm and the sampling reader bursts, so a plain 0-retry client - # crashes the whole run mid-model. This is transient infra, not a bug to fail fast on. - return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"], - max_retries=6) - - def parse_answer(text: str, answer_space: list[str]) -> str | None: """The EARLIEST-appearing answer_space token in `text` (token-boundaried so '3' is not caught inside 'x3y', and a longer token wins a tie at the same position). None if no allowed token @@ -61,15 +53,17 @@ def parse_answer(text: str, answer_space: list[str]) -> str | None: return best -def _sample_texts(client: OpenAI, model: str, prompt: str, n_samples: int, temperature: float, +def _sample_texts(model: str, prompt: str, n_samples: int, temperature: float, max_tokens: int) -> list[str]: - """N completions, requesting up to 8 per call via the `n` param and topping up until N.""" + """N completions via the openrouter_wrapper (stamina retry handles transient 429/provider/upstream + errors), requesting up to 8 per call via `n` and topping up until N.""" texts: list[str] = [] while len(texts) < n_samples: - resp = client.chat.completions.create( - model=model, messages=[{"role": "user", "content": prompt}], - temperature=temperature, n=min(n_samples - len(texts), 8), max_tokens=max_tokens) - texts.extend((c.message.content or "") for c in resp.choices) + payload = {"model": model, "messages": [{"role": "user", "content": prompt}], + "temperature": temperature, "n": min(n_samples - len(texts), 8), + "max_tokens": max_tokens} + data = asyncio.run(openrouter_request(payload)) + texts.extend((c["message"].get("content") or "") for c in data["choices"]) return texts @@ -81,7 +75,6 @@ def read_items_sampled(model: str, instr: Instrument, items: list[InstrItem], *, `instr.answer_space`; `pmass_allowed` is the parse rate; `lp = log(p)` carries -inf on unsampled tokens ON PURPOSE so any C/LO consumer fails loudly instead of reading a fabricated number.""" assert temperature > 0, "sampling readout needs temperature > 0; temp 0 collapses E to an integer" - client = _client() space = instr.answer_space A = len(space) idx = {a: k for k, a in enumerate(space)} @@ -89,7 +82,7 @@ def read_items_sampled(model: str, instr: Instrument, items: list[InstrItem], *, for it_n, it in enumerate(items): user = build_user_content(instr, it) prompt = user + _ANSWER_SUFFIX.format(space="/".join(space)) - texts = _sample_texts(client, model, prompt, n_samples, temperature, max_tokens) + texts = _sample_texts(model, prompt, n_samples, temperature, max_tokens) counts = np.zeros(A) for t in texts: a = parse_answer(t, space) @@ -104,6 +97,7 @@ def read_items_sampled(model: str, instr: Instrument, items: list[InstrItem], *, "id": it.id, "frame": it.frame, "lp": lp, "p": p, "pmass_allowed": pmass, "dimension": it.dimension, "sign": it.sign, "human_label": it.human_label, "n_samples": len(texts), "n_parsed": n_parsed, + "prompt": prompt, "texts": texts, # raw responses kept so a run's answers are auditable/saveable }) if verbose_first and it_n == 0: logger.debug(