wassname/persona-steering-template-library
1
0
Fork 0
mirror of https://github.com/wassname/persona-steering-template-library.git synced 2026-08-18 12:20:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

docs: streamline README and add interactive Pages plot

Browse Source
This commit is contained in:
wassname
2026-06-25 12:31:50 +08:00
parent d31cac9068
commit caceaebbf0
26 changed files with 4541 additions and 529 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/summarize_model_matrix.py
+22 -5
View File
@@ -57,6 +57,16 @@ def _p25(xs: list[float]) -> float:
return statistics.quantiles(xs, n=4, method="inclusive")[0]
def _sem(xs: list[float]) -> float:
return _std(xs) / math.sqrt(len(xs))
def _t_stat(mean: float, sem: float) -> float:
if sem == 0.0:
return 0.0 if mean == 0.0 else 1_000_000.0
return mean / sem
def _round(x: float, digits: int = 3) -> float:
if math.isnan(x):
raise ValueError("nan in model matrix summary")
@@ -110,10 +120,13 @@ def _summarize(rows: list[dict[str, Any]], group_cols: list[str]) -> list[dict[s
base = dict(zip(group_cols, key, strict=True))
model_count = len(models)
scores = [float(row["score"]) for row in rs]
score_mean = _mean(scores)
score_sem = _sem(scores)
out.append({
"model_count": model_count,
"score_t": _round(_t_stat(score_mean, score_sem), 2),
"score_p25": _round(_p25(scores), 2),
"score_mean": _round(_mean(scores), 2),
"score_mean": _round(score_mean, 2),
"score_std": _round(_std(scores), 2),
"strict_pass_rate_mean": _round(_mean([float(row["strict_pass_rate"]) for row in rs]), 3),
"strict_pass_rate_std": _round(_std([float(row["strict_pass_rate"]) for row in rs]), 3),
@@ -129,7 +142,7 @@ def _summarize(rows: list[dict[str, Any]], group_cols: list[str]) -> list[dict[s
"models": ",".join(models),
**base,
})
return sorted(out, key=lambda row: row["score_p25"], reverse=True)
return sorted(out, key=lambda row: row["score_t"], reverse=True)
def _markdown_text(text: str) -> str:
@@ -150,20 +163,24 @@ def _markdown_text(text: str) -> str:
def _write_markdown(path: Path, template_rows: list[dict[str, Any]], pair_rows: list[dict[str, Any]], top_n: int) -> None:
top_template_rows = [
{
"score p25": f"{row['score_p25']:.2f}",
"score t": f"{row['score_t']:.2f}",
"score mean": f"{row['score_mean']:.2f}",
"score std": f"{row['score_std']:.2f}",
"pass": f"{row['strict_pass_rate_mean']:.3f}",
"echo": f"{row['persona_echo_rate_mean']:.3f}",
"refusal": f"{row['refusal_or_ai_break_rate_mean']:.3f}",
"template": _markdown_text(row["template"]),
}
for row in template_rows[:top_n]
]
lines = [
"# Refusal Probe Model Matrix",
"# Refusal-Pole Probe",
"",
"Scores are model-equal. Each model first averages the two refusal-probe axes per template, then the table reports reliability-sorted template rows across clean model artifacts.",
"",
"## All Templates",
"",
"`score p25` is the 25th percentile score across the four clean model artifacts. Rows are sorted by this column.",
"`score t` is mean score divided by standard error across the four clean model artifacts. `pass` is strict-pass rate; `echo` is explicit persona echo; `refusal` is refusal or AI-role break. Rows are sorted by `score t`.",
"",
tabulate(top_template_rows, headers="keys", tablefmt="github", disable_numparse=True),
]
scripts/update_readme_model_matrix.py
+17 -54
View File
@@ -1,6 +1,5 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
@@ -8,12 +7,8 @@ from tabulate import tabulate
ROOT = Path(__file__).resolve().parents[1]
README = ROOT / "README.md"
SUMMARY = ROOT / "out/model_matrix/refusal_probe_seed24_n1_template_model_summary.jsonl"
START = "<!-- model-matrix:start -->"
END = "<!-- model-matrix:end -->"
def _read_jsonl(path: Path) -> list[dict]:
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
@@ -34,76 +29,44 @@ def _markdown_text(text: str) -> str:
return text.replace("\n", "<br>")
def _table(rows: list[dict], top_n: int) -> str:
def _appendix_table(rows: list[dict]) -> str:
table_rows = [
{
"score p25": f"{row['score_p25']:.2f}",
"score t": f"{row['score_t']:.2f}",
"score mean": f"{row['score_mean']:.2f}",
"score std": f"{row['score_std']:.2f}",
"template": _markdown_text(row["template"]),
}
for row in rows[:top_n]
for row in rows
]
return tabulate(table_rows, headers="keys", tablefmt="github", disable_numparse=True)
def _block(summary_path: Path) -> str:
def _appendix_block(summary_path: Path) -> str:
rows = _read_jsonl(summary_path)
return "\n\n".join([
"## Refusal Probe Model Matrix",
"## Appendix: Refusal-Pole Probe",
(
"I also ran the newer roleplay, safety-lab, theatre/treatment, anthropology, and "
"multilingual templates on a two-axis refusal probe across four clean generator "
"artifacts: `google/gemma-2-27b-it`, `google/gemma-3-4b-it`, "
"`qwen/qwen3.6-flash`, and `ibm-granite/granite-4.1-8b`."
"This is a separate two-axis refusal/harm probe across four clean generator "
"artifacts. It is not the main template result, because it does not cover all "
"persona pairs. Treat it as a filter for templates worth retesting on "
"refusal-ish negative poles in the main evaluation frame."
),
(
"Each model first averages the two probe axes for a template, so this is "
"model-equal rather than row-equal. `score p25` is the headline sort: it is "
"the 25th percentile score across the four clean model artifacts, so a template "
"has to work on more than one model to rank well."
"Interactive hover plot: "
"[GitHub Pages](https://wassname.github.io/persona-steering-template-library/)."
),
"![refusal probe model matrix](./out/model_matrix/refusal_probe_seed24_n1_model_matrix.png)",
(
"Caption: this is a template overview, not a persona plot. Each dot is one template, "
"averaged over the two refusal-probe axes and four clean models. Right is more "
"on-axis movement; lower is less off-axis confounding. Black dots have at least one "
"strict-pass template-axis cell; grey dots have none. Numbered dots are the first "
"rows of the table."
),
"Model-matrix templates, all rows:",
_table(rows, top_n=len(rows)),
(
"Interpretation: some explicit judgment framings and red-team/eval framings move "
"the hard axis more often than the gentle templates. The cleanest-looking single-axis "
"cells were often `protocol_harm`, so treat the high rows as rerun candidates "
"rather than settled reusable defaults."
"The generated full audit table includes strict-pass, echo, and refusal columns: "
"[out/model_matrix/refusal_probe_seed24_n1_model_matrix_summary.md]"
"(out/model_matrix/refusal_probe_seed24_n1_model_matrix_summary.md)."
),
_appendix_table(rows),
])
def replace_block(readme: str, block: str) -> str:
wrapped = f"{START}\n{block}\n{END}"
if START in readme:
before, rest = readme.split(START)
_, after = rest.split(END)
return f"{before}{wrapped}{after}"
heading = "\n## Refusal Probe Model Matrix\n"
next_heading = "\n## Score\n"
before, rest = readme.split(heading)
_, after = rest.split(next_heading, maxsplit=1)
return f"{before}\n{wrapped}\n{next_heading}{after}"
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--readme", type=Path, default=README)
ap.add_argument("--summary", type=Path, default=SUMMARY)
args = ap.parse_args()
readme = args.readme.read_text()
args.readme.write_text(replace_block(readme, _block(args.summary)))
print(args.readme)
print(_appendix_block(SUMMARY))
if __name__ == "__main__":
scripts/update_readme_results_table.py
+9 -56
View File
@@ -1,6 +1,5 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
@@ -9,7 +8,6 @@ from tabulate import tabulate
from template_catalog import CATALOG_PATH, jinja_to_runtime, load_template_catalog
ROOT = Path(__file__).resolve().parents[1]
README = ROOT / "README.md"
STATS = ROOT / "out/stats"
NORMAL_STATS = STATS / "v2_pilot_seed24_template_pair_stats.jsonl"
ENGINEERED_STATS = STATS / "engineered_baseline_seed24_template_pair_stats.jsonl"
@@ -17,12 +15,6 @@ CONTROL_STATS = STATS / "control_baseline_seed24_template_pair_stats.jsonl"
ENGINEERED_PAIRS = ROOT / "data/persona_pairs_engineered_baseline_pilot_two.jsonl"
ENGINEERED_DISPLAY = "`{engineered long persona prefix}`*"
START = "<!-- results-snapshot:start -->"
END = "<!-- results-snapshot:end -->"
APPENDIX_START = "<!-- appendix-baselines:start -->"
APPENDIX_END = "<!-- appendix-baselines:end -->"
def _read_jsonl(path: Path) -> list[dict]:
return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]
@@ -83,14 +75,6 @@ def _mean_by_template(rows: list[dict]) -> list[dict]:
return sorted(out, key=lambda row: row["score"], reverse=True)
def _stress_templates() -> set[str]:
out = set()
for row in load_template_catalog(CATALOG_PATH):
if row["status"] == "active" and row["primary_source_id"] == "repo_out_of_context_stress":
out.add(jinja_to_runtime(row["template_jinja"]))
return out
def _engineered_derived_templates() -> set[str]:
out = set()
for row in load_template_catalog(CATALOG_PATH):
@@ -163,12 +147,7 @@ def _engineered_prefixes() -> str:
def _appendix_block() -> str:
normal_pair_rows = [{**row, "score": _score(row)} for row in _read_jsonl(NORMAL_STATS)]
stress_templates = _stress_templates()
engineered_derived_templates = _engineered_derived_templates()
stress_mean_rows = [
row for row in _mean_by_template(normal_pair_rows)
if row["template"] in stress_templates
]
engineered_derived_mean_rows = [
row for row in _mean_by_template(normal_pair_rows)
if row["template"] in engineered_derived_templates
@@ -182,7 +161,12 @@ def _appendix_block() -> str:
control_rows = _mean_by_template(_read_jsonl(CONTROL_STATS))
return "\n\n".join([
"## Appendix: Baselines And Stress Tests",
"## Appendix: Baselines",
(
"Baseline question: are engineered prompts already better? This is a nod to "
"[AxBench](https://arxiv.org/abs/2501.17148), where the authors claim prompting "
"outperformed the other steering methods they tested."
),
(
"The engineered baseline is not a reusable template. It replaces the "
"short persona phrase with a longer positive or negative instruction, "
@@ -194,46 +178,15 @@ def _appendix_block() -> str:
_engineered_prefixes(),
"Long engineered-derived templates, comparable mean over both measured axes:",
_table(engineered_derived_mean_rows),
(
"These simple roleplay and stress strings are called out separately "
"because some move the obvious axis while many leak the persona "
"label or create style/task-mode confounds; the subtle axis still "
"mostly fails."
),
"Simple roleplay and stress templates, comparable mean over both measured axes:",
_table(stress_mean_rows),
"Controls:",
_table(control_rows),
])
def replace_block(readme: str, block: str) -> str:
before, rest = readme.split(START)
_, after = rest.split(END)
return f"{before}{START}\n{block}\n{END}{after}"
def replace_appendix(readme: str, block: str) -> str:
wrapped = f"{APPENDIX_START}\n{block}\n{APPENDIX_END}\n\n"
if APPENDIX_START in readme:
before, rest = readme.split(APPENDIX_START)
_, after = rest.split(APPENDIX_END)
return f"{before}{wrapped}{after.lstrip()}"
marker = "\n## Appendix: Run"
before, after = readme.split(marker)
return f"{before}\n\n{wrapped}{marker}{after}"
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--readme", type=Path, default=README)
args = ap.parse_args()
readme = args.readme.read_text()
updated = replace_block(readme, _results_block())
updated = replace_appendix(updated, _appendix_block())
args.readme.write_text(updated)
print(args.readme)
print(_results_block())
print()
print(_appendix_block())
if __name__ == "__main__":