read_api: use wassname/openrouter_wrapper (stamina retry) + save all raw responses

Replace the hand-rolled OpenAI client with openrouter_wrapper.openrouter_request,
which backs off on 429/provider/upstream/malformed errors (a rate-limited model no
longer kills the run), so retry logic doesn't diverge from the rest of the stack.
read_items_sampled now carries the raw sample texts; wvs_map appends every response
to wvs_iw_responses.jsonl (audit trail -- these API calls cost money).

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>

pyproject.toml

@@ -24,7 +24,9 @@ maps = [
 # tinymfv.read_api (sampling readout for logprob-less API models) only. Install with
 # `pip install tiny-mfv[api]`. The local logprob reader (read.py) needs none of this.
 api = [
-    "openai>=1.0",
+    # robust OpenRouter calls (stamina backoff on 429/provider/upstream/malformed errors) --
+    # wassname's wrapper, so retry/rate-limit handling doesn't diverge from the rest of the stack.
+    "openrouter-wrapper @ git+https://github.com/wassname/openrouter_wrapper",
     "python-dotenv",
 ]
 

scripts/wvs_map.py

@@ -145,6 +145,8 @@ def main() -> None:
     ap.add_argument("--out", default="/tmp/claude-1000/wvs_map_iw.png")
     ap.add_argument("--cache", default="/tmp/claude-1000/wvs_iw_vectors.json",
                     help="cache model per-item p vectors so re-styling skips the API/model calls")
+    ap.add_argument("--responses", default="/tmp/claude-1000/wvs_iw_responses.jsonl",
+                    help="append every raw model response here (audit trail; API calls cost money)")
     args = ap.parse_args()
 
     recs = load_wvs_all()
@@ -173,6 +175,17 @@ def main() -> None:
         allc[sig] = {k: {s: p.tolist() for s, p in v.items()} for k, v in vecs.items()}
         cpath.write_text(json.dumps(allc))
 
+    rpath = Path(args.responses)
+    rpath.parent.mkdir(parents=True, exist_ok=True)
+
+    def save_responses(key: str, rows: list[dict]) -> None:
+        """Append every raw sampled response (audit trail -- these API calls cost money)."""
+        with rpath.open("a") as fh:
+            for r in rows:
+                fh.write(json.dumps({"model": key, "sig": sig, "item": r["id"],
+                                     "prompt": r.get("prompt"), "texts": r.get("texts"),
+                                     "p": np.asarray(r["p"]).tolist(), "pmass": r["pmass_allowed"]}) + "\n")
+
     if args.local_model:
         key = args.local_model.split("/")[-1] + " (lp)"
         if key not in vecs:
@@ -196,6 +209,7 @@ def main() -> None:
             for k, instr in enumerate(instrs):
                 rows += read_items_sampled(m, instr, instr.items, n_samples=args.api_samples,
                                            verbose_first=(k == 0))
+            save_responses(key, rows)       # raw answers first (before reducing to p vectors)
             vecs[key] = read_model(rows, meta)
             save_cache()                    # persist this model before starting the next (kill-safe)
             logger.info(f"cached {key}")

src/tinymfv/read_api.py

@@ -25,13 +25,13 @@ and E degenerates to an integer.
 """
 from __future__ import annotations
 
-import os
+import asyncio
 import re
 from math import inf
 
 import numpy as np
 from loguru import logger
-from openai import OpenAI
+from openrouter_wrapper.retry import openrouter_request   # stamina backoff on 429/provider/upstream errors
 
 from .instrument import Instrument, InstrItem
 from .read import build_user_content
@@ -40,14 +40,6 @@ _ANSWER_SUFFIX = ("\n\nAnswer with ONLY one option from [{space}] -- a single to
                   "no words, no punctuation, no explanation.")
 
 
-def _client() -> OpenAI:
-    # max_retries: the SDK backs off (respecting Retry-After) on transient 429s -- some OpenRouter
-    # models are rate-limited to ~20 rpm and the sampling reader bursts, so a plain 0-retry client
-    # crashes the whole run mid-model. This is transient infra, not a bug to fail fast on.
-    return OpenAI(base_url="https://openrouter.ai/api/v1", api_key=os.environ["OPENROUTER_API_KEY"],
-                  max_retries=6)
-
-
 def parse_answer(text: str, answer_space: list[str]) -> str | None:
     """The EARLIEST-appearing answer_space token in `text` (token-boundaried so '3' is not caught
     inside 'x3y', and a longer token wins a tie at the same position). None if no allowed token
@@ -61,15 +53,17 @@ def parse_answer(text: str, answer_space: list[str]) -> str | None:
     return best
 
 
-def _sample_texts(client: OpenAI, model: str, prompt: str, n_samples: int, temperature: float,
+def _sample_texts(model: str, prompt: str, n_samples: int, temperature: float,
                   max_tokens: int) -> list[str]:
-    """N completions, requesting up to 8 per call via the `n` param and topping up until N."""
+    """N completions via the openrouter_wrapper (stamina retry handles transient 429/provider/upstream
+    errors), requesting up to 8 per call via `n` and topping up until N."""
     texts: list[str] = []
     while len(texts) < n_samples:
-        resp = client.chat.completions.create(
-            model=model, messages=[{"role": "user", "content": prompt}],
-            temperature=temperature, n=min(n_samples - len(texts), 8), max_tokens=max_tokens)
-        texts.extend((c.message.content or "") for c in resp.choices)
+        payload = {"model": model, "messages": [{"role": "user", "content": prompt}],
+                   "temperature": temperature, "n": min(n_samples - len(texts), 8),
+                   "max_tokens": max_tokens}
+        data = asyncio.run(openrouter_request(payload))
+        texts.extend((c["message"].get("content") or "") for c in data["choices"])
     return texts
 
 
@@ -81,7 +75,6 @@ def read_items_sampled(model: str, instr: Instrument, items: list[InstrItem], *,
     `instr.answer_space`; `pmass_allowed` is the parse rate; `lp = log(p)` carries -inf on unsampled
     tokens ON PURPOSE so any C/LO consumer fails loudly instead of reading a fabricated number."""
     assert temperature > 0, "sampling readout needs temperature > 0; temp 0 collapses E to an integer"
-    client = _client()
     space = instr.answer_space
     A = len(space)
     idx = {a: k for k, a in enumerate(space)}
@@ -89,7 +82,7 @@ def read_items_sampled(model: str, instr: Instrument, items: list[InstrItem], *,
     for it_n, it in enumerate(items):
         user = build_user_content(instr, it)
         prompt = user + _ANSWER_SUFFIX.format(space="/".join(space))
-        texts = _sample_texts(client, model, prompt, n_samples, temperature, max_tokens)
+        texts = _sample_texts(model, prompt, n_samples, temperature, max_tokens)
         counts = np.zeros(A)
         for t in texts:
             a = parse_answer(t, space)
@@ -104,6 +97,7 @@ def read_items_sampled(model: str, instr: Instrument, items: list[InstrItem], *,
             "id": it.id, "frame": it.frame, "lp": lp, "p": p, "pmass_allowed": pmass,
             "dimension": it.dimension, "sign": it.sign, "human_label": it.human_label,
             "n_samples": len(texts), "n_parsed": n_parsed,
+            "prompt": prompt, "texts": texts,   # raw responses kept so a run's answers are auditable/saveable
         })
         if verbose_first and it_n == 0:
             logger.debug(