mirror of
https://github.com/wassname/machiavelli_deep_value.git
synced 2026-08-20 12:30:18 +08:00
Add the no-rubric motive scaling figure
This commit is contained in:
@@ -34,6 +34,16 @@ above.
|
||||
The Hugging Face dataset card explains the two configurations, intended uses,
|
||||
columns, QA flags, and limitations.
|
||||
|
||||
## Evaluation snapshot
|
||||
|
||||
A fixed Gemma 4 31B judge compared the assigned motives in both account orders.
|
||||
With no rubric, crossed-set accuracy declines slightly across the five Qwen
|
||||
explanation models; Kimi K3 is shown separately because it is from another model
|
||||
family. This describes the judge finding the generated accounts harder to
|
||||
classify, not stronger models concealing motives better.
|
||||
|
||||

|
||||
|
||||
## Reproduce it
|
||||
|
||||
Install [uv](https://docs.astral.sh/uv/), then run:
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="760" height="440" viewBox="0 0 760 440">
|
||||
<rect width="760" height="440" fill="white"/>
|
||||
<g font-family="sans-serif" fill="#222">
|
||||
<text x="380" y="28" text-anchor="middle" font-size="18">Assigned-motive accuracy by explanation model</text>
|
||||
<text x="380" y="49" text-anchor="middle" font-size="11" fill="#555">no rubric; fixed Gemma 4 31B judge; crossed action-harm set</text>
|
||||
<text x="380" y="70" text-anchor="middle" font-size="10" fill="#555">circles: Qwen 3.5; square: Qwen 3.7 Max; diamond: Kimi K3</text>
|
||||
|
||||
<line x1="70" y1="340" x2="720" y2="340" stroke="#777" stroke-width="0.8"/>
|
||||
<line x1="70" y1="90" x2="70" y2="340" stroke="#777" stroke-width="0.8"/>
|
||||
<line x1="70" y1="215" x2="720" y2="215" stroke="#999" stroke-width="1" stroke-dasharray="5 4"/>
|
||||
<text x="716" y="209" text-anchor="end" font-size="10" fill="#666">chance</text>
|
||||
|
||||
<text x="58" y="344" text-anchor="end" font-size="10" fill="#555">0.25</text>
|
||||
<text x="58" y="281.5" text-anchor="end" font-size="10" fill="#555">0.375</text>
|
||||
<text x="58" y="219" text-anchor="end" font-size="10" fill="#555">0.50</text>
|
||||
<text x="58" y="156.5" text-anchor="end" font-size="10" fill="#555">0.625</text>
|
||||
<text x="58" y="94" text-anchor="end" font-size="10" fill="#555">0.75</text>
|
||||
<text x="17" y="215" text-anchor="middle" font-size="12" transform="rotate(-90 17 215)">crossed-set accuracy</text>
|
||||
|
||||
<line x1="70" y1="177.5" x2="514.4" y2="215.5" stroke="#333" stroke-width="1.6"/>
|
||||
<text x="500" y="205" text-anchor="end" font-size="10" fill="#444">Qwen fit: -0.003 / AA point</text>
|
||||
|
||||
<g stroke="#555" stroke-opacity="0.45" stroke-width="1.2">
|
||||
<line x1="70" y1="102" x2="70" y2="227.5"/>
|
||||
<line x1="212.2" y1="97.5" x2="212.2" y2="255"/>
|
||||
<line x1="265.6" y1="129" x2="265.6" y2="299"/>
|
||||
<line x1="301.1" y1="149" x2="301.1" y2="311"/>
|
||||
<line x1="514.4" y1="118" x2="514.4" y2="282.5"/>
|
||||
<line x1="710" y1="109.5" x2="710" y2="276"/>
|
||||
</g>
|
||||
|
||||
<g fill="#333">
|
||||
<circle cx="70" cy="165" r="4"/>
|
||||
<circle cx="212.2" cy="172" r="4"/>
|
||||
<circle cx="265.6" cy="211.5" r="4"/>
|
||||
<circle cx="301.1" cy="229.5" r="4"/>
|
||||
<rect x="510.4" y="193" width="8" height="8"/>
|
||||
<path d="M 710 184 L 716 190 L 710 196 L 704 190 Z"/>
|
||||
</g>
|
||||
|
||||
<g font-size="10" fill="#555">
|
||||
<text x="70" y="357" text-anchor="middle">21</text>
|
||||
<text x="212.2" y="357" text-anchor="middle">29</text>
|
||||
<text x="265.6" y="357" text-anchor="middle">32</text>
|
||||
<text x="301.1" y="357" text-anchor="middle">34</text>
|
||||
<text x="514.4" y="357" text-anchor="middle">46</text>
|
||||
<text x="710" y="357" text-anchor="middle">57</text>
|
||||
|
||||
</g>
|
||||
|
||||
<text x="380" y="398" text-anchor="middle" font-size="11" fill="#555">Artificial Analysis score</text>
|
||||
<text x="380" y="422" text-anchor="middle" font-size="10" fill="#666">points: model means; bars: 95% scenario bootstrap; line: Qwen-only linear fit</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.0 KiB |
Reference in New Issue
Block a user