feat: multi-label moral foundation ratings with z-scored frame averaging and human calibration

- Add scripts/07_multilabel.py: LLM judge rates all 7 foundations per vignette
  using violation (forward) and acceptability (reverse) frames
- Foundation definitions drawn from Clifford et al. (2015) survey rubric
- Z-score each frame per foundation before averaging to cancel range bias
- Calibrate LLM Likert → human % via per-foundation OLS (classic set only)
- Add scripts/07a_merge_labels.py: merges llm_* and calibrated_* into vignette files
- Update README and HF dataset card with methodology and calibration quality table
- Classic set: 80.3% dominant-foundation accuracy, Pearson r 0.69-0.89 per foundation
This commit is contained in:
wassname
2026-05-03 12:48:14 +08:00
parent 36b8c69122
commit bcbdb9cc6f
15 changed files with 1933 additions and 799 deletions
+87
View File
@@ -0,0 +1,87 @@
{
"model": "x-ai/grok-4-fast",
"foundations": {
"Care": {
"n": 132,
"spearman_r": 0.7345042029622307,
"spearman_p": 1.2176410154221345e-23,
"pearson_r": 0.8102300980028216,
"pearson_p": 5.905981200185428e-32,
"slope": 17.91275224831169,
"intercept": -18.86597016699005,
"mae": 11.767375193181332
},
"Fairness": {
"n": 132,
"spearman_r": 0.6225605362565393,
"spearman_p": 1.5959953666830447e-15,
"pearson_r": 0.8129609595820121,
"pearson_p": 2.530684016591177e-32,
"slope": 16.7264766737861,
"intercept": -19.131907744766767,
"mae": 11.111987094673093
},
"Loyalty": {
"n": 132,
"spearman_r": 0.6926099209091109,
"spearman_p": 3.599007373381228e-20,
"pearson_r": 0.7484483078065683,
"pearson_p": 6.0343078480636855e-25,
"slope": 14.538458435939967,
"intercept": -15.378022367181732,
"mae": 9.309441437396758
},
"Authority": {
"n": 132,
"spearman_r": 0.3856396780981898,
"spearman_p": 4.961061693936076e-06,
"pearson_r": 0.6875303659859135,
"pearson_p": 8.65179834421809e-20,
"slope": 13.428260762938889,
"intercept": -13.722907418808013,
"mae": 11.683327561037107
},
"Sanctity": {
"n": 132,
"spearman_r": 0.6196021834312778,
"spearman_p": 2.365393748673303e-15,
"pearson_r": 0.8887367634786868,
"pearson_p": 7.21924815710244e-46,
"slope": 18.316412771225654,
"intercept": -17.576719463687116,
"mae": 6.263818224972179
},
"Liberty": {
"n": 132,
"spearman_r": 0.6020785985113376,
"spearman_p": 2.2394672471928682e-14,
"pearson_r": 0.8139335364560988,
"pearson_p": 1.865088580341189e-32,
"slope": 14.712894877552804,
"intercept": -14.163679534840798,
"mae": 8.163126583084338
},
"SocialNorms": {
"n": 132,
"spearman_r": -0.4272322719509285,
"spearman_p": 3.2400850542942107e-07,
"pearson_r": -0.3207249716209886,
"pearson_p": 0.00017729490832067168,
"slope": -9.865750625757574,
"intercept": 53.56969710715305,
"mae": 18.84631540933629
},
"wrongness": {
"n": 132,
"spearman_r": 0.7618659176897857,
"spearman_p": 2.7640404209932235e-26,
"pearson_r": 0.8292329749373769,
"pearson_p": 1.1959738514994125e-34,
"slope": 0.6992902585652869,
"intercept": 0.1468375358206399,
"mae": 0.4377854374641585
}
},
"dominant_accuracy": 0.803030303030303,
"n_vignettes": 132
}