From b8e22f42fb8b7fa64e3668870ff04428fe367e4a Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Sat, 25 Jul 2026 10:23:22 +0800 Subject: [PATCH] refactor: sort scripts/ by who runs it Top level is now only what the runbook or a re-run touches: validate_persona_axes, bounded_thinking_judge, template_catalog, export_selections, parse_stage_a, run_axis, export_steering_selection. Corpus ingestion and publishing moved to scripts/corpus/, plotting and stats to scripts/report/. Moved files needed parents[1] -> parents[2]; the two corpus scripts that import template_catalog use the sys.path shim bounded_thinking_judge_liveproof already used. Also completes the export_steering_selection rename: an earlier git reset had dropped the staged deletion, leaving both filenames tracked. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com> --- scripts/{ => corpus}/build_hf_dataset.py | 6 +- .../{ => corpus}/scenario_sources/README.md | 0 .../data/machiavelli_summaries.jsonl | 0 .../scenario_sources/export_scenarios.py | 0 .../{ => corpus}/scenario_sources/loaders.py | 0 .../score_machiavelli_moral_contrast.py | 0 .../scenario_sources/summarise_machiavelli.py | 0 scripts/{ => corpus}/sync_template_library.py | 5 +- .../export_authority_steering_selection.py | 177 ------------------ scripts/{ => report}/docs_results.py | 2 +- .../export_persona_template_stats.py | 0 scripts/{ => report}/plot_on_off_axis.py | 0 scripts/{ => report}/readme_plot.py | 0 .../{ => report}/summarize_model_matrix.py | 2 +- 14 files changed, 10 insertions(+), 182 deletions(-) rename scripts/{ => corpus}/build_hf_dataset.py (99%) rename scripts/{ => corpus}/scenario_sources/README.md (100%) rename scripts/{ => corpus}/scenario_sources/data/machiavelli_summaries.jsonl (100%) rename scripts/{ => corpus}/scenario_sources/export_scenarios.py (100%) rename scripts/{ => corpus}/scenario_sources/loaders.py (100%) rename scripts/{ => corpus}/scenario_sources/score_machiavelli_moral_contrast.py (100%) rename scripts/{ => corpus}/scenario_sources/summarise_machiavelli.py (100%) rename scripts/{ => corpus}/sync_template_library.py (89%) delete mode 100644 scripts/export_authority_steering_selection.py rename scripts/{ => report}/docs_results.py (98%) rename scripts/{ => report}/export_persona_template_stats.py (100%) rename scripts/{ => report}/plot_on_off_axis.py (100%) rename scripts/{ => report}/readme_plot.py (100%) rename scripts/{ => report}/summarize_model_matrix.py (99%) diff --git a/scripts/build_hf_dataset.py b/scripts/corpus/build_hf_dataset.py similarity index 99% rename from scripts/build_hf_dataset.py rename to scripts/corpus/build_hf_dataset.py index d9f531f..9cea5a3 100644 --- a/scripts/build_hf_dataset.py +++ b/scripts/corpus/build_hf_dataset.py @@ -9,16 +9,18 @@ from __future__ import annotations import argparse import json import shutil +import sys from pathlib import Path from typing import Any import pyarrow as pa import pyarrow.parquet as pq -from template_catalog import active_template_rows, load_template_catalog +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) +from template_catalog import active_template_rows, load_template_catalog # noqa: E402 -ROOT = Path(__file__).resolve().parents[1] DATA = ROOT / "data" PERSONA_DATA = DATA / "personas" STATS = ROOT / "data/results/stats" diff --git a/scripts/scenario_sources/README.md b/scripts/corpus/scenario_sources/README.md similarity index 100% rename from scripts/scenario_sources/README.md rename to scripts/corpus/scenario_sources/README.md diff --git a/scripts/scenario_sources/data/machiavelli_summaries.jsonl b/scripts/corpus/scenario_sources/data/machiavelli_summaries.jsonl similarity index 100% rename from scripts/scenario_sources/data/machiavelli_summaries.jsonl rename to scripts/corpus/scenario_sources/data/machiavelli_summaries.jsonl diff --git a/scripts/scenario_sources/export_scenarios.py b/scripts/corpus/scenario_sources/export_scenarios.py similarity index 100% rename from scripts/scenario_sources/export_scenarios.py rename to scripts/corpus/scenario_sources/export_scenarios.py diff --git a/scripts/scenario_sources/loaders.py b/scripts/corpus/scenario_sources/loaders.py similarity index 100% rename from scripts/scenario_sources/loaders.py rename to scripts/corpus/scenario_sources/loaders.py diff --git a/scripts/scenario_sources/score_machiavelli_moral_contrast.py b/scripts/corpus/scenario_sources/score_machiavelli_moral_contrast.py similarity index 100% rename from scripts/scenario_sources/score_machiavelli_moral_contrast.py rename to scripts/corpus/scenario_sources/score_machiavelli_moral_contrast.py diff --git a/scripts/scenario_sources/summarise_machiavelli.py b/scripts/corpus/scenario_sources/summarise_machiavelli.py similarity index 100% rename from scripts/scenario_sources/summarise_machiavelli.py rename to scripts/corpus/scenario_sources/summarise_machiavelli.py diff --git a/scripts/sync_template_library.py b/scripts/corpus/sync_template_library.py similarity index 89% rename from scripts/sync_template_library.py rename to scripts/corpus/sync_template_library.py index 0863be9..45d7b42 100644 --- a/scripts/sync_template_library.py +++ b/scripts/corpus/sync_template_library.py @@ -2,8 +2,11 @@ from __future__ import annotations import argparse import sys +from pathlib import Path -from template_catalog import ( +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "scripts")) + +from template_catalog import ( # noqa: E402 CATALOG_PATH, CATALOG_JSONL_PATH, TEMPLATES_TXT_PATH, diff --git a/scripts/export_authority_steering_selection.py b/scripts/export_authority_steering_selection.py deleted file mode 100644 index 63c8ec5..0000000 --- a/scripts/export_authority_steering_selection.py +++ /dev/null @@ -1,177 +0,0 @@ -"""Export winning Authority-axis ingredients from validator artifacts.""" -from __future__ import annotations - -import argparse -import csv -import json -from collections import defaultdict -from pathlib import Path - -from tabulate import tabulate - -ROOT = Path(__file__).resolve().parents[1] - - -def read_jsonl(path: Path) -> list[dict]: - return [json.loads(line) for line in path.read_text().splitlines() if line.strip()] - - -def write_jsonl(path: Path, rows: list[dict]) -> None: - path.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n") - - -def score_row(row: dict) -> float: - if "error" in row: - return -1.0 - return 100.0 * float(row["on_axis_frac"]) * (1.0 - float(row["off_axis_problem_frac"])) - - -def choose_stage_a(stage_a: dict, axis_filter: str | None) -> dict: - summary = stage_a["summary"] - if axis_filter is not None: - summary = [row for row in summary if row["axis"] == axis_filter] - if not summary: - raise ValueError(f"stage A has no summary rows for axis_filter={axis_filter!r}") - ranked = sorted( - summary, - key=lambda row: ( - row["recommended"], - row["strict_pass_rate"], - row["mean_axis_delta"], - -row["mean_off_axis_problem"], - -row["mean_max_style_abs_delta"], - -row["persona_echo_rate"], - -row["refusal_or_ai_break_rate"], - ), - reverse=True, - ) - return ranked[0] - - -def write_stage_b_inputs(stage_a_path: Path, out_dir: Path, axis_filter: str | None) -> dict: - stage_a = json.loads(stage_a_path.read_text()) - winner = choose_stage_a(stage_a, axis_filter) - axis_id = winner["axis"] - template = winner["template"] - axes = [axis for axis in stage_a["axes"] if axis["id"] == axis_id] - if len(axes) != 1: - raise ValueError(f"expected one winning axis {axis_id!r}, found {len(axes)}") - out_dir.mkdir(parents=True, exist_ok=True) - write_jsonl(out_dir / "stage_b_axis.jsonl", axes) - (out_dir / "stage_b_template.txt").write_text(template + "\n") - (out_dir / "stage_a_winner.json").write_text(json.dumps(winner, indent=2)) - return winner - - -def select_stage_b( - stage_b_path: Path, - out_dir: Path, - keep_per_source: int, - strict_only: bool, - min_score: float, -) -> list[dict]: - artifact = json.loads(stage_b_path.read_text()) - grouped: dict[str, list[dict]] = defaultdict(list) - for row in artifact["results"]: - if "error" in row: - continue - if strict_only and not row["strict_pass"]: - continue - if score_row(row) < min_score: - continue - grouped[str(row["source"])].append(row) - - selected: list[dict] = [] - score_rows: list[dict] = [] - for source, rows in sorted(grouped.items()): - ranked = sorted( - rows, - key=lambda row: ( - row["strict_pass"], - score_row(row), - float(row["axis_delta"]), - -float(row["confound_judgment"]["off_axis_problem_likert"]), - -float(row["max_style_abs_delta"]), - ), - reverse=True, - ) - for rank, row in enumerate(ranked, start=1): - score_rows.append({ - "source": source, - "rank": rank, - "selected": rank <= keep_per_source, - "scenario_id": row["scenario_id"], - "score": round(score_row(row), 2), - "strict_pass": row["strict_pass"], - "axis_delta": row["axis_delta"], - "off_axis_problem": row["confound_judgment"]["off_axis_problem_likert"], - "max_style_abs_delta": row["max_style_abs_delta"], - "prompt": row["prompt"], - }) - for row in ranked[:keep_per_source]: - selected.append({ - "id": row["scenario_id"], - "prompt": row["prompt"], - "source": row["source"], - "config": row.get("config"), - "self_contained": True, - "selection_score": round(score_row(row), 4), - "axis_delta": row["axis_delta"], - "off_axis_problem": row["confound_judgment"]["off_axis_problem_likert"], - "strict_pass": row["strict_pass"], - }) - - out_dir.mkdir(parents=True, exist_ok=True) - write_jsonl(out_dir / "selected_scenarios.jsonl", selected) - with (out_dir / "scenario_scores.csv").open("w", newline="") as fh: - writer = csv.DictWriter(fh, fieldnames=list(score_rows[0])) - writer.writeheader() - writer.writerows(score_rows) - examples = [] - for row in sorted(score_rows, key=lambda r: (r["source"], r["rank"])): - if row["rank"] <= 2: - examples.append( - f"## {row['source']} / {row['scenario_id']}\n\n" - f"score={row['score']} axis_delta={row['axis_delta']} " - f"off_axis={row['off_axis_problem']} strict_pass={row['strict_pass']}\n\n" - f"{row['prompt']}\n" - ) - (out_dir / "selected_examples.md").write_text("\n".join(examples)) - return selected - - -def main() -> None: - ap = argparse.ArgumentParser() - ap.add_argument("--stage-a", type=Path) - ap.add_argument("--stage-b", type=Path) - ap.add_argument("--out-dir", type=Path, default=ROOT / "out/authority_selection") - ap.add_argument("--keep-per-source", type=int, default=10) - ap.add_argument("--axis-filter", type=str) - ap.add_argument("--strict-only", action="store_true") - ap.add_argument("--min-score", type=float, default=0.0) - args = ap.parse_args() - - if args.stage_a is None and args.stage_b is None: - raise ValueError("pass --stage-a and/or --stage-b") - - if args.stage_a is not None: - winner = write_stage_b_inputs(args.stage_a, args.out_dir, args.axis_filter) - print("Stage A winner:") - print(tabulate([winner], headers="keys", tablefmt="github", floatfmt=".3f")) - - if args.stage_b is not None: - selected = select_stage_b(args.stage_b, args.out_dir, args.keep_per_source, args.strict_only, args.min_score) - counts: dict[str, int] = {} - for row in selected: - counts[row["source"]] = counts.get(row["source"], 0) + 1 - print("\nSelected scenarios:") - print(tabulate( - [{"source": source, "n": counts[source]} for source in sorted(counts)], - headers="keys", - tablefmt="github", - )) - print(f"total={len(selected)}") - - -if __name__ == "__main__": - main() diff --git a/scripts/docs_results.py b/scripts/report/docs_results.py similarity index 98% rename from scripts/docs_results.py rename to scripts/report/docs_results.py index a08b508..f804daa 100644 --- a/scripts/docs_results.py +++ b/scripts/report/docs_results.py @@ -7,7 +7,7 @@ import statistics from typing import Any -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] STATS = ROOT / "data/results/stats" MODEL_MATRIX = ROOT / "data/results/model_matrix" DOCS_MODEL_MATRIX = ROOT / "docs/results/model_matrix" diff --git a/scripts/export_persona_template_stats.py b/scripts/report/export_persona_template_stats.py similarity index 100% rename from scripts/export_persona_template_stats.py rename to scripts/report/export_persona_template_stats.py diff --git a/scripts/plot_on_off_axis.py b/scripts/report/plot_on_off_axis.py similarity index 100% rename from scripts/plot_on_off_axis.py rename to scripts/report/plot_on_off_axis.py diff --git a/scripts/readme_plot.py b/scripts/report/readme_plot.py similarity index 100% rename from scripts/readme_plot.py rename to scripts/report/readme_plot.py diff --git a/scripts/summarize_model_matrix.py b/scripts/report/summarize_model_matrix.py similarity index 99% rename from scripts/summarize_model_matrix.py rename to scripts/report/summarize_model_matrix.py index c31907e..d766752 100644 --- a/scripts/summarize_model_matrix.py +++ b/scripts/report/summarize_model_matrix.py @@ -12,7 +12,7 @@ from tabulate import tabulate import docs_results -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] DEFAULT_PAIR_STATS = docs_results.REFUSAL_MODEL_PAIR_STATS DEFAULT_OUT_PREFIX = docs_results.REFUSAL_MODEL_PREFIX