mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-09 11:27:22 +08:00
cleanup: assert on flat _zscore, doc the nll floor, trim history comments
_zscore now asserts nonzero variance (a flat profile drew a degenerate all-zero map that looked valid) instead of dividing by std+1e-9. _soft_nll docstring now states the 1e-12 floor it actually applies (was 'Unbounded'). Trim archaeology comments (guided/wvs/read_api) to the invariant, dropping the dated model names. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
@@ -232,8 +232,12 @@ def plot_ordinal(run_dir: Path, out: Path, name: str, vec_label: str, C: float,
|
||||
|
||||
def _zscore(v: np.ndarray) -> np.ndarray:
|
||||
"""Relative emphasis: centre and scale a profile across foundations, so a logit profile (model)
|
||||
and a 1-5 wrongness profile (human cultures) are comparable by PATTERN regardless of units."""
|
||||
return (v - v.mean()) / (v.std() + 1e-9)
|
||||
and a 1-5 wrongness profile (human cultures) are comparable by PATTERN regardless of units. A
|
||||
flat profile has no relative emphasis to show -- fail loud rather than divide by ~0 and draw a
|
||||
degenerate all-zero map that looks valid."""
|
||||
sd = v.std()
|
||||
assert sd > 0, "flat profile (zero variance across foundations): relative-emphasis map is undefined"
|
||||
return (v - v.mean()) / sd
|
||||
|
||||
|
||||
def read_human_mfv() -> tuple[list[str], dict[str, dict[str, float]]]:
|
||||
|
||||
+2
-2
@@ -169,8 +169,8 @@ def main() -> None:
|
||||
for k, v in cache.items()}
|
||||
|
||||
def save_cache() -> None:
|
||||
"""Persist after EACH model so a killed run (session teardown) keeps finished models -- the
|
||||
write-once-at-end version silently discarded every sampled model when interrupted."""
|
||||
"""Persist after EACH model so a killed run (session teardown) keeps every finished model
|
||||
(kill-safe incremental write, not write-once-at-end)."""
|
||||
allc = json.loads(cpath.read_text()) if cpath.exists() else {}
|
||||
allc[sig] = {k: {s: p.tolist() for s, p in v.items()} for k, v in vecs.items()}
|
||||
cpath.write_text(json.dumps(allc))
|
||||
|
||||
+3
-1
@@ -131,7 +131,9 @@ def _soft_nll(p_human: np.ndarray, p_model: np.ndarray) -> float:
|
||||
"""Soft cross-entropy: -sum_f p_human[f] log p_model[f], in nats.
|
||||
|
||||
Standard quantity for matching a predicted distribution to a soft-labelled
|
||||
target. Unbounded; sensitive to confident-wrong rows.
|
||||
target. Sensitive to confident-wrong rows, but p_model is floored at 1e-12
|
||||
(~27.6 nats/row) so a single legit p=0 cell (an unsampled token from the
|
||||
sampling reader, not a bug) can't send the aggregate mean to +inf.
|
||||
"""
|
||||
p_model = np.clip(p_model, 1e-12, 1.0)
|
||||
return float(-(p_human * np.log(p_model)).sum())
|
||||
|
||||
@@ -285,7 +285,6 @@ def _rollout_natural_or_forced(
|
||||
# confident answer from garbage. Mark the row UNSCORABLE (pmass=NaN, lp=NaN),
|
||||
# the same "do not compare" signal as case (c): the read is undefined, not zero
|
||||
# coherence, so it drops from the nanmean and counts toward frac_unscorable.
|
||||
# (Was torch.nan_to_num clamp to +-1e4: silent corruption.)
|
||||
raw = step_scores[answer_pos][i].float()
|
||||
if not torch.isfinite(raw).all():
|
||||
slots[i].append({
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
"""Sampling readout for API models that do NOT expose token logprobs (OpenRouter / OpenAI-compatible).
|
||||
|
||||
The frontier models on the Economist's WVS chart (GPT-5.4, Claude, Gemini, Grok) return no
|
||||
next-token distribution, so read.py's forced-slot logprob reader cannot touch them. This is the
|
||||
fallback the Economist itself used ("average of ten responses"): SAMPLE N chat completions at
|
||||
temperature and take the empirical answer frequency as the per-(item, frame) categorical `p`.
|
||||
Chat-completions-only API models return no next-token distribution, so read.py's forced-slot logprob
|
||||
reader cannot touch them. Fallback (the "average of N responses" approach): SAMPLE N chat completions
|
||||
at temperature and take the empirical answer frequency as the per-(item, frame) categorical `p`.
|
||||
|
||||
Why this stays comparable to the logprob reader: temperature-1 sampling is an unbiased estimator of
|
||||
the model's softmax categorical, so the empirical `p` -> the true next-token `p` as N grows, and
|
||||
|
||||
Reference in New Issue
Block a user