This commit is contained in:
wassname
2026-05-08 15:15:14 +08:00
parent d796df85c8
commit c96d02a675
14 changed files with 1033 additions and 797 deletions
+14 -20
View File
@@ -5,29 +5,23 @@ High-level usage:
from tinymfv import evaluate
from transformers import AutoModelForCausalLM, AutoTokenizer
tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-0.6B").cuda()
report = evaluate(model, tok, name="scifi")
print(report["table"]) # tabulated per-foundation
print(report["score"]) # headline align_other(real)
tok = AutoTokenizer.from_pretrained("Qwen/Qwen3-4B")
model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-4B").cuda()
rep = evaluate(model, tok, name="classic")
print(rep["table"]) # per-foundation
print(rep["top1_acc"]) # argmax accuracy vs label
print(rep["mean_js"]) # JS divergence vs label dist (in nats)
Lower-level: see `format_prompts`, `score_prompts`, `analyse`.
Lower-level: see `guided_rollout_forced_choice` in `tinymfv.guided`.
"""
from .core import (
CONDITIONS,
FRAMES,
format_prompt,
format_prompts,
bool_token_ids,
score_prompts,
analyse,
)
from .data import load_vignettes, load_all_vignettes, CONFIGS, ConfigName
from .eval import evaluate
from .eval import evaluate, CONDITIONS
from .guided import guided_rollout_forced_choice, _DEFAULT_FORCED_FOUNDATIONS
__all__ = [
"CONDITIONS", "FRAMES", "CONFIGS", "ConfigName",
"format_prompt", "format_prompts", "bool_token_ids",
"score_prompts", "analyse",
"load_vignettes", "load_all_vignettes", "evaluate",
"CONDITIONS", "CONFIGS", "ConfigName",
"load_vignettes", "load_all_vignettes",
"evaluate",
"guided_rollout_forced_choice", "_DEFAULT_FORCED_FOUNDATIONS",
]