mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-25 13:50:17 +08:00
quiet pmass-low warning: one summary per batch
Was emitting `logger.warning("pmass=0.XX<0.9 — top-5: ...")` per-row, which
spammed the log heavily during heavy-steering eval (many rows go OOD at once).
Now collects all low-pmass rows in the batch and emits one summary line with
the worst-case top-5, e.g.:
pmass<0.9 on 7/16 rows in this batch; worst=0.412 top-5: '1'=0.40, ...
Same diagnostic signal, ~16× fewer log lines per batch.
This commit is contained in:
+5
-3
@@ -15,7 +15,7 @@ from .guided import guided_rollout_batch, choice_token_ids_tf
|
||||
def evaluate(
|
||||
model,
|
||||
tokenizer,
|
||||
name: str = "",
|
||||
name: str | None = None,
|
||||
vignettes: list[dict] | None = None,
|
||||
batch_size: int = 16,
|
||||
device: str | None = None,
|
||||
@@ -23,13 +23,15 @@ def evaluate(
|
||||
) -> dict[str, Any]:
|
||||
"""Run dual JSON-bool eval and return aggregated report.
|
||||
|
||||
Either pass `vignettes` directly or `name` to load from `data/`. Tokenizer must
|
||||
have a chat template (or fallback flat format will be used) and `pad_token` set.
|
||||
Either pass `vignettes` directly or `name` (one of 'classic', 'scifi',
|
||||
'airisk', 'all') to load from `data/`. Tokenizer must have a chat template
|
||||
(or fallback flat format will be used) and `pad_token` set.
|
||||
Side-effects: sets `tokenizer.padding_side='left'` and `tokenizer.pad_token` if
|
||||
missing -- both required for batched left-padded eval.
|
||||
"""
|
||||
if vignettes is None:
|
||||
vignettes = load_vignettes(name)
|
||||
|
||||
if tokenizer.pad_token is None:
|
||||
tokenizer.pad_token = tokenizer.eos_token
|
||||
tokenizer.padding_side = "left"
|
||||
|
||||
Reference in New Issue
Block a user