mirror of
https://github.com/wassname/moral-maps.git
synced 2026-10-04 12:50:36 +08:00
correctness fixes from gpt-5.5 review (ordinal path + fail-fast asserts)
Second external review (gpt-5.5, correctness-focused) on the post-cleanup tree. Found no off-by-one/double-flip in the ordinal canonicalization+keying. Fixed the parts I agreed with and could verify on the path the experiment uses: - read.py: NaN-safe answer-token renorm. p_a/pmass poisons the profile with NaN when pmass underflows to 0 at coherence collapse -- exactly when pmass should just flag it. softmax(logp_allowed) is identical when pmass>0 and stable at collapse. - maps.ipsative_pca: move SVD sign-stabilization INTO the helper so it and plot_ipsative_pca share one orientation (saved coords could otherwise mirror the figure). - instrument: assert ordinal answer_space is ['1'..scale_max] IN ORDER (reduce_ordinal weights by position; a reordered space silently inverts E) -- was length-only. - instrument.per_item_categorical: assert per-item dimension/sign agree across frames and frames are distinct, instead of silently averaging under rows[0]'s metadata. - pyproject: move matplotlib+textalloc to an optional `maps` extra; evals stay headless. - tests: drop imports of the deleted reduce_nominal/expected_value, inline the expectation, remove the now-impossible nominal-reducer test. Deferred (flagged to maintainer): two NaN/window issues in guided.py's forced-choice rollout (nominal evaluate() path) -- not exercised by this experiment, can't smoke-test, and the NaN-as-collapse-signal there is a deliberate design. Verified: experiment smoke green on all 4 instruments (no assert false-fires), 6 pure unit tests pass, headless import clean, 16pf map renders. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
1 parent
5333893345
commit
d5876a46d3
5 files changed
+42
-27
No files matched your search
@@ -73,7 +73,10 @@ class Instrument:
|
||||
|
||||
def __post_init__(self):
|
||||
if self.kind == "ordinal":
|
||||
assert len(self.answer_space) == self.scale_max, "ordinal answer_space must be 1..scale_max"
|
||||
assert self.answer_space == [str(i) for i in range(1, self.scale_max + 1)], (
|
||||
f"ordinal answer_space must be ['1'..'{self.scale_max}'] IN ORDER -- reduce_ordinal "
|
||||
f"weights by position (w = 1..scale_max), so a reordered space silently inverts E; "
|
||||
f"got {self.answer_space}")
|
||||
if self.kind == "nominal" and self.answer_to_dim is None:
|
||||
self.answer_to_dim = {a: a for a in self.answer_space}
|
||||
# Cross-scale caveat: a 1-7 human histogram (HSQ) cannot share a 5-way soft-NLL/JS
|
||||
@@ -125,6 +128,11 @@ def per_item_categorical(per_row: list[dict], kind: Kind) -> dict[str, dict]:
|
||||
assert len(frame_counts) == 1, f"heterogeneous frame counts per item: {frame_counts}"
|
||||
out: dict[str, dict] = {}
|
||||
for iid, rows in by_id.items():
|
||||
# Per-item rows differ only by frame; dimension + sign must agree, frames must be distinct.
|
||||
# Else we would silently average incompatible distributions under rows[0]'s metadata.
|
||||
assert len({r.get("dimension") for r in rows}) == 1, f"{iid}: inconsistent dimension across frames"
|
||||
assert len({r.get("sign", 1) for r in rows}) == 1, f"{iid}: inconsistent sign across frames"
|
||||
assert len({r["frame"] for r in rows}) == len(rows), f"{iid}: duplicate frame in rows"
|
||||
canon = [canonicalize_to_forward(r["p"], r["frame"], kind) for r in rows]
|
||||
C = np.stack(canon)
|
||||
spread = float(max((np.abs(C[i] - C[j]).sum()
|
||||
|
||||
+10
-7
@@ -55,12 +55,19 @@ def row_centre_op(K: int) -> np.ndarray:
|
||||
|
||||
def ipsative_pca(M: np.ndarray, k: int = 2):
|
||||
"""Row-centre each row of M (societies x K), then PCA across rows.
|
||||
Returns (P, Vt, var, mu, Pc); project a new point v via ((v @ Pc) - mu) @ Vt[:k].T."""
|
||||
Pc = row_centre_op(M.shape[1])
|
||||
Returns (P, Vt, var, mu, Pc); project a new point v via ((v @ Pc) - mu) @ Vt[:k].T.
|
||||
SVD signs are stabilized here (PC1 loads + on factor 0, PC2 on factor 1) so this helper and
|
||||
any plot built on it share ONE orientation -- otherwise saved coords could mirror the figure."""
|
||||
K = M.shape[1]
|
||||
Pc = row_centre_op(K)
|
||||
Mp = M @ Pc
|
||||
mu = Mp.mean(axis=0)
|
||||
Mc = Mp - mu
|
||||
_, S, Vt = np.linalg.svd(Mc, full_matrices=False)
|
||||
if Vt[0, 0] < 0:
|
||||
Vt[0] = -Vt[0]
|
||||
if Vt.shape[0] > 1 and Vt[1, 1 % K] < 0:
|
||||
Vt[1] = -Vt[1]
|
||||
var = (S ** 2) / (S ** 2).sum()
|
||||
return Mc @ Vt[:k].T, Vt, var, mu, Pc
|
||||
|
||||
@@ -100,11 +107,7 @@ def plot_ipsative_pca(instr: Instrument, dims: list[str], countries: list[str],
|
||||
import textalloc as ta
|
||||
except ImportError:
|
||||
ta = None
|
||||
P, Vt, var, mu, Pc = ipsative_pca(M)
|
||||
if Vt[0, 0] < 0: # stabilise SVD sign: factor[0] loads +PC1
|
||||
Vt[0] = -Vt[0]
|
||||
if Vt[1, 1 % len(dims)] < 0:
|
||||
Vt[1] = -Vt[1]
|
||||
P, Vt, var, mu, Pc = ipsative_pca(M) # signs already stabilized inside the helper
|
||||
P = (M @ Pc - mu) @ Vt[:2].T
|
||||
|
||||
def proj(v):
|
||||
|
||||
+6
-3
@@ -72,9 +72,12 @@ def read_items(model, tok, instr: Instrument, items: list[InstrItem], answer_ids
|
||||
enc = tok(texts, return_tensors="pt", padding=True, add_special_tokens=False).to(device)
|
||||
logits = model(**enc).logits[:, -1, :].float() # [B, V] next-token
|
||||
logp = F.log_softmax(logits, dim=-1)
|
||||
p_a = logp[:, gid].exp() # [B, A] prob on each answer token
|
||||
pmass = p_a.sum(dim=-1) # [B]
|
||||
p_norm = p_a / pmass[:, None] # [B, A] within allowed
|
||||
logp_a = logp[:, gid] # [B, A] logprob on each answer token
|
||||
pmass = logp_a.exp().sum(dim=-1) # [B] coherence check: mass on allowed tokens
|
||||
# softmax over the allowed logprobs == p_a / pmass when pmass > 0, but NaN-safe: at coherence
|
||||
# collapse pmass underflows to 0 and the divide would poison the whole profile with NaN; the
|
||||
# softmax still returns a valid within-allowed distribution and pmass separately flags the drop.
|
||||
p_norm = F.softmax(logp_a, dim=-1) # [B, A] within allowed
|
||||
for j, it in enumerate(chunk):
|
||||
out.append({
|
||||
"id": it.id, "frame": it.frame,
|
||||
|
||||
Reference in new issue
Block a user