From bbfc812f451b660fa5dfb6941b0cc879cb69ddf9 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Sun, 5 Jul 2026 21:51:28 +0800 Subject: [PATCH] Add --n-per-source for stratified sampling + SKILL.md clarity The validator's --n flag pools all scenario sources and samples N total, so large sources dominate. For narrow axes this means most sampled scenarios don't afford the axis. Add --n-per-source: takes N from EACH family (stratified, even sampling). SKILL.md updates: - step 5: mention --n-per-source and explain pooled vs stratified - step 6: add 'test more ranked scenarios (--n-per-source 50+, more sources) and/or try stronger templates (system-prompt, red-team, jailbreak-style)' when 0 strict pass - commands: use --n-per-source in dry-run and live validation examples This fixes the root cause of the v1 honesty/credulity fumble: 50 pooled scenarios gave 0 strict pass, and the doc didn't make it obvious that --n was pooled or that the fix is stratified sampling with more scenarios. --- .../skills/persona-template-library/SKILL.md | 28 ++++++++++++++----- README.md | 14 ++++++++-- README.qmd | 11 ++++++++ scripts/validate_persona_axes_openrouter.py | 27 +++++++++++++++--- 4 files changed, 67 insertions(+), 13 deletions(-) diff --git a/.agents/skills/persona-template-library/SKILL.md b/.agents/skills/persona-template-library/SKILL.md index e3afb71..e22e642 100644 --- a/.agents/skills/persona-template-library/SKILL.md +++ b/.agents/skills/persona-template-library/SKILL.md @@ -44,7 +44,11 @@ dataset. Use the repo in this order: 1. Choose persona templates from the `README.md` Results Snapshot table, the - Hugging Face `main` split, or `data/templates/template_catalog.yaml`. + Hugging Face `main` split, or `data/templates/template_catalog.yaml`. For + Stage A, test the full catalog when budget permits. If budget is constrained, + start from the top Results Snapshot scorers plus any strong templates that + plausibly fit the new axis. Use a deterministic top-N rule when you + downsample; do not use a small hand-picked subset. 2. Choose persona pairs with `docs/choosing_personas.md`. Mirror-test each pair: every positive clause needs a negative counterpart that only flips the intended pole. @@ -54,10 +58,20 @@ Use the repo in this order: or multi-turn behavior. 4. Run a dry-run validator command before live OpenRouter calls. 5. For a steering-ready selection, use a two-stage screen: - Stage A = several axes/templates on a small source-diverse panel; Stage B = - the chosen axis/template on up to 30 scenarios per source. -6. Export strict-pass scenarios only. Do not steer from source-balanced filler rows - when a source produced no clean scenarios. + Stage A = broad template evidence on a small source-diverse panel; Stage B = + the chosen axis/template on up to 30 axis-affordance-ranked scenarios per + source. Before Stage B, write or adapt a prepare script like + `scripts/prepare_authority_steering_selection.py` for the new axis. Random + scenario sampling is insufficient for narrow axes because most scenarios will + not afford the intended behavior. Use `--n-per-source N` (stratified) so each + source contributes N scenarios equally, not proportional to source size; the + legacy `--n` pools all sources and samples N total (large sources dominate). +6. Export strict-pass scenarios only. If Stage B gets 0 strict-pass scenarios, + test more ranked scenarios (`--n-per-source 50+`, more sources) and/or try + stronger templates (the full catalog includes system-prompt, red-team, and + jailbreak-style templates that break the model out of its default stance). + Do not relax the strict gate, lower the export rule, or steer from + source-balanced filler rows when a source produced no clean scenarios. 7. Commit reusable selected scenario JSONL files under `data/selections/`. 8. After a live run, inspect examples before trusting scores. @@ -97,7 +111,7 @@ uv run python scripts/validate_persona_axes_openrouter.py \ --axes data/personas/persona_pairs_pilot_two.jsonl \ --templates data/templates/template_catalog.yaml \ --family data/scenarios/scenarios_v2_candidates.jsonl \ - --n 1 \ + --n-per-source 2 \ --seed 24 \ --dry-run \ --out out/persona_template_library_dryrun.json @@ -110,7 +124,7 @@ OPENROUTER_API_KEY=... uv run python scripts/validate_persona_axes_openrouter.py --axes data/personas/persona_pairs_pilot_two.jsonl \ --templates data/templates/template_catalog.yaml \ --family data/scenarios/scenarios_v2_candidates.jsonl \ - --n 2 \ + --n-per-source 2 \ --seed 24 \ --out out/persona_template_library_v2_pilot_seed24.json ``` diff --git a/README.md b/README.md index 9edaa6c..25e8be9 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,16 @@ likely to echo the persona label, that nuisance can become the vector. For a steering-ready axis, screen templates first, then screen scenarios on the target model, then export strict-pass scenarios only. Full agent recipe: [`persona-template-library` -skill](.agents/skills/persona-template-library/SKILL.md). Example -selected set: +skill](.agents/skills/persona-template-library/SKILL.md). Stage A should +test the full template catalog when budget permits. If budget is +constrained, use a deterministic top-N rule from the Results Snapshot +plus any strong templates that plausibly fit the new axis. Do not use a +small hand-picked subset. Before Stage B, rank or filter candidate +scenarios by axis affordance; random sampling is usually insufficient +for narrow axes because most scenarios will not afford the intended +behavior. If Stage B gets 0 strict-pass scenarios, test more ranked +scenarios and/or stronger templates. Do not relax the strict gate or +export filler rows. Example selected set: [`data/selections/authority_dignity_strict22.jsonl`](data/selections/authority_dignity_strict22.jsonl). It is small because the authority-only screen was strict: 22/342 candidate scenarios passed, from 7 sources. @@ -224,6 +232,8 @@ uv run python scripts/validate_persona_axes_openrouter.py \ - Use diverse scenarios first, then select the ones that separate on your model. +- For Stage B, rank or filter the scenario pool for the axis before + validation. Random sampling is a weak default for narrow axes. - Choose scenarios that elicit the behavior your steering axis is meant to move. Some axes are about doing, some about judging, some about explaining, some about refusing, some about moral tradeoffs, and some diff --git a/README.qmd b/README.qmd index 75f99b5..b7516ad 100644 --- a/README.qmd +++ b/README.qmd @@ -47,6 +47,15 @@ the persona label, that nuisance can become the vector. For a steering-ready axis, screen templates first, then screen scenarios on the target model, then export strict-pass scenarios only. Full agent recipe: [`persona-template-library` skill](.agents/skills/persona-template-library/SKILL.md). +Stage A should test the full template catalog when budget permits. If budget is +constrained, use a deterministic top-N rule from the Results Snapshot plus any +strong templates that plausibly fit the new axis. Do not use a small hand-picked +subset. +Before Stage B, rank or filter candidate scenarios by axis affordance; random +sampling is usually insufficient for narrow axes because most scenarios will not +afford the intended behavior. If Stage B gets 0 strict-pass scenarios, test more +ranked scenarios and/or stronger templates. Do not relax the strict gate or +export filler rows. Example selected set: [`data/selections/authority_dignity_strict22.jsonl`](data/selections/authority_dignity_strict22.jsonl). It is small because the authority-only screen was strict: 22/342 candidate @@ -226,6 +235,8 @@ uv run python scripts/validate_persona_axes_openrouter.py \ ``` - Use diverse scenarios first, then select the ones that separate on your model. +- For Stage B, rank or filter the scenario pool for the axis before validation. + Random sampling is a weak default for narrow axes. - Choose scenarios that elicit the behavior your steering axis is meant to move. Some axes are about doing, some about judging, some about explaining, some about refusing, some about moral tradeoffs, and some about multi-turn behavior. diff --git a/scripts/validate_persona_axes_openrouter.py b/scripts/validate_persona_axes_openrouter.py index bbbb5ed..4c3afad 100644 --- a/scripts/validate_persona_axes_openrouter.py +++ b/scripts/validate_persona_axes_openrouter.py @@ -411,9 +411,25 @@ def _rows_for_family(family: str) -> list[dict]: return [dict(r) for r in BUILTIN_SCENARIOS[family]] -def _select_rows(families: str, n: int, seed: int) -> list[dict]: +def _select_rows(families: str, n: int, seed: int, n_per_source: int | None = None) -> list[dict]: rng = random.Random(seed) - rows: list[dict] = [] + if n_per_source is not None: + # stratified: take n_per_source from each family (even sampling, not pooled) + rows: list[dict] = [] + for family in [f.strip() for f in families.split(",") if f.strip()]: + fam_rows = [{**r, "selected_family": family} for r in _rows_for_family(family)] + rng.shuffle(fam_rows) + if len(fam_rows) < n_per_source: + raise ValueError( + f"family {family!r} has only {len(fam_rows)} rows but --n-per-source={n_per_source}" + ) + rows.extend(fam_rows[:n_per_source]) + if not rows: + raise ValueError("selected zero scenario rows") + rng.shuffle(rows) + return rows + # pooled (legacy): shuffle all families together, take n total + rows = [] for family in [f.strip() for f in families.split(",") if f.strip()]: rows.extend({**r, "selected_family": family} for r in _rows_for_family(family)) if not rows: @@ -1387,7 +1403,7 @@ async def amain(args) -> None: load_dotenv(ROOT / ".env") axes = _select_axes(args.axes) templates = _select_templates(args.templates) - rows = _select_rows(args.family, args.n, args.seed) + rows = _select_rows(args.family, args.n, args.seed, args.n_per_source) axis_judge_models = tuple( model.strip() for model in args.axis_judge_models.split(",") if model.strip() ) @@ -1568,7 +1584,10 @@ def main() -> None: help="generation temperature; default 0 to avoid sampling-diff confounds") ap.add_argument("--family", default="character", help="comma-separated scenario families; default avoids sycophancy") - ap.add_argument("--n", type=int, default=6, help="number of scenario prompts") + ap.add_argument("--n", type=int, default=6, help="number of scenario prompts (pooled across all families)") + ap.add_argument("--n-per-source", type=int, default=None, + help="stratified sampling: take this many scenarios from EACH family (overrides --n). " + "Use this so each source contributes equally, not proportional to its size.") ap.add_argument("--axes", default=str(ROOT / "data/personas/persona_pairs_pilot_two.jsonl"), help="persona-pair JSONL path") ap.add_argument("--templates", default=str(ROOT / "data/templates/template_catalog.yaml"),