mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-10 12:14:54 +08:00
rename
This commit is contained in:
wassname
@@ -1,8 +1,8 @@
|
||||
"""Transcribe Clifford et al. 2015 vignettes onto AI-as-actor scenarios.
|
||||
"""Transcribe classic vignettes onto AI-as-actor scenarios.
|
||||
|
||||
Goal: a 132-row AI-risk eval set where each item still violates a SINGLE
|
||||
moral foundation at the SAME severity as its Clifford counterpart -- i.e.
|
||||
"Clifford with AI window-dressing". This is the antidote to the previous
|
||||
"classic with AI window-dressing". This is the antidote to the previous
|
||||
hand-written `airisk` set, where every item violates multiple foundations
|
||||
and all factors collapse together (mean |off-diag| corr 0.45 max).
|
||||
|
||||
@@ -15,12 +15,12 @@ Design contract per row:
|
||||
used by 02_rewrite.py for the legacy airisk set).
|
||||
|
||||
Outputs (mirrors 02_rewrite.py):
|
||||
data/vignettes_clifford_ai_other_violate.jsonl
|
||||
data/vignettes_clifford_ai_self_violate.jsonl
|
||||
data/vignettes_ai-actor_other_violate.jsonl
|
||||
data/vignettes_ai-actor_self_violate.jsonl
|
||||
|
||||
Usage:
|
||||
python scripts/02b_transcribe_clifford_ai.py
|
||||
python scripts/02b_transcribe_clifford_ai.py --limit 8 --concurrency 4
|
||||
python scripts/02b_transcribe_ai_actor.py
|
||||
python scripts/02b_transcribe_ai_actor.py --limit 8 --concurrency 4
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
@@ -38,10 +38,10 @@ from openrouter_wrapper.retry import openrouter_request
|
||||
from tqdm.asyncio import tqdm as atqdm
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SRC = ROOT / "data" / "vignettes_other_violate.jsonl" # classic verbatim source
|
||||
OUT_OTHER = ROOT / "data" / "vignettes_clifford_ai_other_violate.jsonl"
|
||||
OUT_SELF = ROOT / "data" / "vignettes_clifford_ai_self_violate.jsonl"
|
||||
CACHE = ROOT / "data" / "cache" / "clifford_ai"
|
||||
SRC = ROOT / "data" / "vignettes_classic_other_violate.jsonl"
|
||||
OUT_OTHER = ROOT / "data" / "vignettes_ai-actor_other_violate.jsonl"
|
||||
OUT_SELF = ROOT / "data" / "vignettes_ai-actor_self_violate.jsonl"
|
||||
CACHE = ROOT / "data" / "cache" / "ai-actor"
|
||||
|
||||
# Same deny-list as 02_rewrite.py: tail-clauses leak moral framing.
|
||||
_BAD_PHRASES = [
|
||||
+13
-26
@@ -2,15 +2,13 @@
|
||||
|
||||
Creates / updates: wassname/tiny-mfv
|
||||
|
||||
- config `classic` (alias `clifford`): 132 vignettes from Clifford et al. (2015), Wrong ratings are human Likert.
|
||||
- config `classic`: 132 vignettes from Clifford et al. (2015), Wrong ratings are human Likert.
|
||||
- config `scifi`: 132 hand-written sci-fi/fantasy vignettes covering the same foundations.
|
||||
- config `clifford_ai`: 132 AI-as-actor transcriptions of the classic Clifford set
|
||||
(preserves single-foundation violation per item -- the
|
||||
principled replacement for the deprecated `airisk` config).
|
||||
- config `ai-actor`: 132 AI-as-actor transcriptions of the classic set.
|
||||
|
||||
Each config has two splits:
|
||||
- `other_violate` -- 3rd-person source (verbatim for classic/scifi; AI-actor transcription for clifford_ai).
|
||||
- `self_violate` -- 1st-person rewrite. For clifford_ai this preserves the AI-as-actor
|
||||
- `other_violate` -- 3rd-person source (verbatim for classic/scifi; AI-actor transcription for ai-actor).
|
||||
- `self_violate` -- 1st-person rewrite. For ai-actor this preserves the AI-as-actor
|
||||
framing ("You, an AI {X} bot, ..."); for classic/scifi it's a plain
|
||||
"You ..." 1st-person shift.
|
||||
"""
|
||||
@@ -22,25 +20,20 @@ from huggingface_hub import HfApi
|
||||
REPO_ID = "wassname/tiny-mfv"
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
# HF config name → local file key (clifford files have no suffix on disk).
|
||||
# "classic" is the user-facing name; on HF it's stored under "classic/" directory
|
||||
# but we also register a "clifford" alias so existing code doesn't break.
|
||||
CONFIGS = {
|
||||
"classic": "", # vignettes_other_violate.jsonl (no suffix)
|
||||
"scifi": "scifi", # vignettes_scifi_other_violate.jsonl
|
||||
"clifford_ai": "clifford_ai", # vignettes_clifford_ai_other_violate.jsonl
|
||||
"classic": "classic",
|
||||
"scifi": "scifi",
|
||||
"ai-actor": "ai-actor",
|
||||
}
|
||||
SPLITS = ["other_violate", "self_violate"]
|
||||
|
||||
|
||||
def local_jsonl(file_key: str, split: str) -> Path:
|
||||
suf = f"_{file_key}" if file_key else ""
|
||||
return ROOT / "data" / f"vignettes{suf}_{split}.jsonl"
|
||||
return ROOT / "data" / f"vignettes_{file_key}_{split}.jsonl"
|
||||
|
||||
|
||||
def local_csv(file_key: str) -> Path:
|
||||
suf = f"_{file_key}" if file_key else ""
|
||||
return ROOT / "data" / f"vignettes{suf}.csv"
|
||||
return ROOT / "data" / f"vignettes_{file_key}.csv"
|
||||
|
||||
|
||||
def hf_jsonl(cfg: str, split: str) -> str:
|
||||
@@ -59,12 +52,6 @@ def yaml_configs() -> str:
|
||||
for split in SPLITS:
|
||||
lines.append(f" - split: {split}")
|
||||
lines.append(f" path: {hf_jsonl(cfg, split)}")
|
||||
# Register "clifford" as an alias for "classic" so existing code works.
|
||||
lines.append("- config_name: clifford")
|
||||
lines.append(" data_files:")
|
||||
for split in SPLITS:
|
||||
lines.append(f" - split: {split}")
|
||||
lines.append(f" path: {hf_jsonl('classic', split)}")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@@ -103,14 +90,14 @@ For use with LLMs we make them
|
||||
|
||||
## Configs
|
||||
|
||||
- **classic** (alias: clifford): 132 vignettes from Clifford et al. (2015) "Moral Foundations Vignettes". `wrong` is the human Likert mean (1-5).
|
||||
- **classic**: 132 vignettes from Clifford et al. (2015) "Moral Foundations Vignettes". `wrong` is the human Likert mean (1-5).
|
||||
- **scifi**: 132 hand-written sci-fi/fantasy vignettes covering the same foundations. Genre-clean cues, no real-world ethnicity/religion confounds.
|
||||
- **clifford_ai**: 132 AI-as-actor transcriptions of the classic Clifford set. Each item preserves a single foundation violation at the same severity, just shifted onto an AI archetype. This is the principled replacement for the deprecated `airisk` config (which conflated multiple foundations per item).
|
||||
- **ai-actor**: 132 AI-as-actor transcriptions of the classic set. Each item preserves a single foundation violation at the same severity, just shifted onto an AI archetype.
|
||||
|
||||
## Splits (per config)
|
||||
|
||||
- `other_violate` — verbatim 3rd-person source text. No LLM call. For classic this means the verbatim text is in every LLM's training set, which is fine for tracking deltas across checkpoints (the offset is constant).
|
||||
- `self_violate` — 1st-person rewrite of the same scenario. For classic and scifi this is a plain `"You ..."` shift. For clifford_ai the principal IS the AI, so the rewrite preserves the AI-as-actor framing as `"You, an AI X bot, ..."` (a naive `"You ..."` template silently swaps the actor archetype to human; verified by `06_consistency.py`).
|
||||
- `self_violate` — 1st-person rewrite of the same scenario. For classic and scifi this is a plain `"You ..."` shift. For ai-actor the principal IS the AI, so the rewrite preserves the AI-as-actor framing as `"You, an AI X bot, ..."` (a naive `"You ..."` template silently swaps the actor archetype to human; verified by `06_consistency.py`).
|
||||
|
||||
## Dual axis: `cond` × `frame`
|
||||
|
||||
@@ -153,7 +140,7 @@ Each vignette row includes LLM-generated multi-label ratings across all 7 founda
|
||||
| Loyalty | +0.69 | +0.75 | 9.3% |
|
||||
| Authority | +0.39 | +0.69 | 11.7% |
|
||||
|
||||
> **Note:** `ai_*` for `scifi` and `clifford_ai` are extrapolated from the classic-set rescale -- treat as a noisy proxy. Use `human_*` (inherited from the parent classic item) as the primary label.
|
||||
> **Note:** `ai_*` for `scifi` and `ai-actor` are extrapolated from the classic-set rescale -- treat as a noisy proxy. Use `human_*` (inherited from the parent classic item) as the primary label.
|
||||
|
||||
## Eval
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ Labels:
|
||||
|
||||
Usage:
|
||||
python scripts/09_forced_choice.py --model Qwen/Qwen3-0.6B
|
||||
python scripts/09_forced_choice.py --model Qwen/Qwen3-4B --name clifford_ai
|
||||
python scripts/09_forced_choice.py --model Qwen/Qwen3-4B --name ai-actor
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse
|
||||
@@ -36,7 +36,7 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--model", default="Qwen/Qwen3-0.6B")
|
||||
ap.add_argument("--name", default="classic", help="dataset config (classic/scifi/clifford_ai)")
|
||||
ap.add_argument("--name", default="classic", help="dataset config (classic/scifi/ai-actor)")
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
ap.add_argument("--batch-size", type=int, default=16)
|
||||
ap.add_argument("--max-think-tokens", type=int, default=128)
|
||||
|
||||
Reference in New Issue
Block a user