mirror of
https://github.com/wassname/moral-maps.git
synced 2026-09-21 13:10:52 +08:00
Publish score-all-options map refresh and HLE data
Co-Authored-By: PI[openai-codex] <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
co-authored by
PI[openai-codex]
PI[openai-codex]
parent
b444e1fcf7
commit
bdf593e0de
File diff suppressed because one or more lines are too long
+2
-2
@@ -3,8 +3,8 @@
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Frontier LLMs on the World Values Survey</title>
|
||||
<script type="module" crossorigin src="./assets/index-DVqE1XzH.js"></script>
|
||||
<title>Moral Maps: Where Do Frontier Models' Cultural Values Lie?</title>
|
||||
<script type="module" crossorigin src="./assets/index-BZu_5V-r.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-CiBO8OQd.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
+904
-262
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,83 @@
|
||||
"""Validate the saved Artificial Analysis HLE mapping used by the WVS page."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAPPING_PATH = ROOT / "slop/research/wvs/20260917_artificialanalysis/wvs_hle_mapping.json"
|
||||
WEB_DATA_PATH = ROOT / "docs/wvs/wvs_map_data.json"
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def validate() -> dict[str, object]:
|
||||
mapping = json.loads(MAPPING_PATH.read_text())
|
||||
source = mapping["source"]
|
||||
source_dir = MAPPING_PATH.parent
|
||||
for filename_key, hash_key in (
|
||||
("manifest_html", "manifest_html_sha256"),
|
||||
("encrypted_models", "encrypted_models_sha256"),
|
||||
("decoded_models", "decoded_models_sha256"),
|
||||
):
|
||||
path = source_dir / source[filename_key]
|
||||
actual = sha256(path)
|
||||
if actual != source[hash_key]:
|
||||
raise ValueError(f"source hash mismatch: {path}: {actual}")
|
||||
|
||||
rows = json.loads((source_dir / source["decoded_models"]).read_text())
|
||||
if len(rows) != source["canonical_configuration_rows"]:
|
||||
raise ValueError(f"configuration row count mismatch: {len(rows)}")
|
||||
hle_rows = [row for row in rows if row["hle"] is not None]
|
||||
if len(hle_rows) != source["non_null_hle_rows"]:
|
||||
raise ValueError(f"non-null HLE row count mismatch: {len(hle_rows)}")
|
||||
|
||||
by_release: dict[str, list[dict]] = defaultdict(list)
|
||||
by_config = {}
|
||||
for row in hle_rows:
|
||||
by_release[row["release"]["slug"]].append(row)
|
||||
by_config[row["slug"]] = row
|
||||
|
||||
plotted_names = {model["name"] for model in json.loads(WEB_DATA_PATH.read_text())["models"]}
|
||||
mapped_names = set()
|
||||
for entry in mapping["mappings"]:
|
||||
name = entry["plotted_model"]
|
||||
if name not in plotted_names:
|
||||
raise ValueError(f"mapped model is not plotted: {name}")
|
||||
if name in mapped_names:
|
||||
raise ValueError(f"mapping repeats plotted model: {name}")
|
||||
mapped_names.add(name)
|
||||
row = by_config[entry["source_configuration_slug"]]
|
||||
if row["release"]["slug"] != entry["source_release_slug"]:
|
||||
raise ValueError(f"release mismatch for {name}")
|
||||
if row["release"]["name"] != entry["source_release_name"]:
|
||||
raise ValueError(f"release name mismatch for {name}")
|
||||
if row["name"] != entry["source_name"]:
|
||||
raise ValueError(f"source name mismatch for {name}")
|
||||
if row.get("effort", {}).get("slug") != entry["source_effort"]:
|
||||
raise ValueError(f"source effort mismatch for {name}")
|
||||
if row["releaseDate"] != entry["source_release_date"]:
|
||||
raise ValueError(f"release date mismatch for {name}")
|
||||
if row["hle"] != entry["hle_score"]:
|
||||
raise ValueError(f"HLE score mismatch for {name}")
|
||||
selected = max(candidate["hle"] for candidate in by_release[entry["source_release_slug"]])
|
||||
if entry["hle_score"] != selected:
|
||||
raise ValueError(f"HLE ceiling mismatch for {name}")
|
||||
|
||||
omitted_names = {entry["plotted_model"] for entry in mapping["omitted"]}
|
||||
if mapped_names | omitted_names != plotted_names:
|
||||
raise ValueError("mapped and omitted models do not partition plotted models")
|
||||
return {
|
||||
"canonical_configuration_rows": len(rows),
|
||||
"non_null_hle_rows": len(hle_rows),
|
||||
"exact_and_reviewed_alias_mappings": len(mapped_names),
|
||||
"omitted_plotted_models": len(omitted_names),
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(json.dumps(validate(), indent=2, sort_keys=True))
|
||||
+47
-17
@@ -65,13 +65,16 @@ LEGACY_LATEST = {
|
||||
"mistral": "mistral-large-2512",
|
||||
}
|
||||
|
||||
# These historical Grok coordinates predate the local metadata file. Their dates remain traceable
|
||||
# These historical coordinates predate the local metadata file. Their dates remain traceable
|
||||
# to exact IDs in the saved 2026-09-17 OpenRouter catalog, not inferred from version strings.
|
||||
LEGACY_CATALOG_IDS = {
|
||||
"grok-4.20": "x-ai/grok-4.20",
|
||||
"grok-4.3": "x-ai/grok-4.3",
|
||||
"gpt-5.4": "openai/gpt-5.4",
|
||||
"gpt-5.5": "openai/gpt-5.5",
|
||||
}
|
||||
SAVED_CATALOG_PATH = Path("slop/research/wvs/20260917_openrouter_models.json")
|
||||
CAPABILITY_MAPPING_PATH = Path("slop/research/wvs/20260917_artificialanalysis/wvs_hle_mapping.json")
|
||||
RATED_PROTOCOL_DIAGNOSTICS = {"gpt-5-nano (rated)"}
|
||||
|
||||
API_MODEL_SETS = {
|
||||
@@ -472,6 +475,17 @@ def main() -> None:
|
||||
|
||||
completed = {entry["display_key"].replace(" (rated)", ""): entry
|
||||
for entry in cache["completed"].values()}
|
||||
capability_source = json.loads(CAPABILITY_MAPPING_PATH.read_text())
|
||||
raw_capability_source = CAPABILITY_MAPPING_PATH.parent / capability_source["source"]["decoded_models"]
|
||||
raw_source_hash = hashlib.sha256(raw_capability_source.read_bytes()).hexdigest()
|
||||
if raw_source_hash != capability_source["source"]["decoded_models_sha256"]:
|
||||
raise ValueError(f"Artificial Analysis HLE source hash mismatch: {raw_capability_source}")
|
||||
capability_by_model = {entry["plotted_model"]: entry for entry in capability_source["mappings"]}
|
||||
if len(capability_by_model) != len(capability_source["mappings"]):
|
||||
raise ValueError("Artificial Analysis mapping repeats a plotted model")
|
||||
unknown_capability_models = set(capability_by_model) - set(plot_models)
|
||||
if unknown_capability_models:
|
||||
raise ValueError(f"Artificial Analysis mapping names absent plotted models: {sorted(unknown_capability_models)}")
|
||||
family_logos = {
|
||||
"claude": "logos/anthropic.svg", "deepseek": "logos/deepseek.svg",
|
||||
"gemini": "logos/google.svg", "gemma": "logos/google.svg",
|
||||
@@ -484,29 +498,45 @@ def main() -> None:
|
||||
def model_provenance(name: str) -> dict[str, object]:
|
||||
panel = completed.get(name)
|
||||
catalog = metadata.get(name)
|
||||
if catalog is None and panel is not None and panel["model"] in saved_catalog:
|
||||
catalog = {"created": datetime.fromtimestamp(
|
||||
saved_catalog[panel["model"]]["created"], tz=timezone.utc).date().isoformat()}
|
||||
if panel is None:
|
||||
legacy_date = legacy_release_dates.get(name)
|
||||
return {"readout": "recovered rounded historical coordinate", "items": None,
|
||||
"samples": None, "run_id": None, "protocol_id": None,
|
||||
"release_created": legacy_date,
|
||||
"release_source": (f"saved OpenRouter catalog 2026-09-17: {LEGACY_CATALOG_IDS[name]}"
|
||||
if legacy_date else "historical coordinate")}
|
||||
return {"readout": "rated categorical response", "items": panel["n_items"],
|
||||
"samples": panel["n_samples"], "run_id": panel["run_id"],
|
||||
"protocol_id": panel["protocol_id"],
|
||||
"release_created": catalog["created"] if catalog else None,
|
||||
"release_source": "catalog" if catalog else "request ledger"}
|
||||
provenance = {"readout": "recovered rounded historical coordinate", "items": None,
|
||||
"samples": None, "run_id": None, "protocol_id": None,
|
||||
"release_created": legacy_date,
|
||||
"release_source": (f"saved OpenRouter catalog 2026-09-17: {LEGACY_CATALOG_IDS[name]}"
|
||||
if legacy_date else "historical coordinate")}
|
||||
else:
|
||||
provenance = {"readout": "rated categorical response", "items": panel["n_items"],
|
||||
"samples": panel["n_samples"], "run_id": panel["run_id"],
|
||||
"protocol_id": panel["protocol_id"],
|
||||
"release_created": catalog["created"] if catalog else None,
|
||||
"release_source": "catalog" if catalog else "request ledger"}
|
||||
capability = capability_by_model.get(name)
|
||||
provenance["hle_score"] = capability["hle_score"] if capability else None
|
||||
provenance["hle_source_name"] = capability["source_name"] if capability else None
|
||||
provenance["hle_source_effort"] = capability["source_effort"] if capability else None
|
||||
return provenance
|
||||
|
||||
args.web_data.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.web_data.write_text(json.dumps({
|
||||
"schema": 2,
|
||||
"schema": 3,
|
||||
"capability_x": {
|
||||
"label": "Artificial Analysis Intelligence Index",
|
||||
"source_url": "https://artificialanalysis.ai/models",
|
||||
"status": "blocked",
|
||||
"blocker": "Capability scores are omitted until the source owner confirms redistribution permission.",
|
||||
"label": "HLE score",
|
||||
"source_url": capability_source["source"]["hle_url"],
|
||||
"fetched_utc": capability_source["source"]["fetched_utc"],
|
||||
"raw_source": str(raw_capability_source),
|
||||
"raw_source_sha256": raw_source_hash,
|
||||
"mapping": str(CAPABILITY_MAPPING_PATH),
|
||||
"selection_rule": capability_source["selection_rule"],
|
||||
"matched_models": len(capability_by_model),
|
||||
"status": "available",
|
||||
"metric": capability_source["source"]["metric"],
|
||||
"protocol": capability_source["source"]["protocol"],
|
||||
},
|
||||
"title": "Frontier LLMs on the\nWorld Values Survey",
|
||||
"title": "Moral Maps: Where Do Frontier\nModels' Cultural Values Lie?",
|
||||
"note": "source: github.com/wassname/moral-maps",
|
||||
"axis": {"x": (["Self-expression", "Survival"] if sx < 0 else ["Survival", "Self-expression"]),
|
||||
"y": (["Secular-Rational", "Traditional"] if sy < 0 else ["Traditional", "Secular-Rational"])},
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Frontier LLMs on the World Values Survey</title>
|
||||
<title>Moral Maps: Where Do Frontier Models' Cultural Values Lie?</title>
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
||||
@@ -16,11 +16,11 @@ function Tooltip({ active, geometry, id = 'model-tooltip', panel }) {
|
||||
const release = active.provenance.release_created && <span>release {active.provenance.release_created}</span>;
|
||||
return <aside id={id} className="tooltip" style={{ left, top }} role="status">
|
||||
<strong>{active.name}</strong>
|
||||
{active.tooltipIndex != null && <span>Intelligence Index: {active.tooltipIndex.toFixed(2)}</span>}
|
||||
{active.tooltipScore != null && <span>HLE score: {active.tooltipScore.toFixed(3)}</span>}
|
||||
{panel === 'secular' && <span>Secular-Rational: {active.y.toFixed(3)}</span>}
|
||||
{panel === 'self-expression' && <span>Self-expression: {active.tooltipValue.toFixed(3)}</span>}
|
||||
{!panel && <><span>Self-expression/Survival: {active.x.toFixed(3)}</span><span>Traditional/Secular-Rational: {active.y.toFixed(3)}</span></>}
|
||||
{active.tooltipIndex == null && release}
|
||||
{active.tooltipScore == null && release}
|
||||
</aside>;
|
||||
}
|
||||
|
||||
@@ -57,12 +57,12 @@ function ordinaryLeastSquares(models, xValue, yValue) {
|
||||
function ReleaseScatter({ data, hidden, field, title, axisMode }) {
|
||||
const dated = useMemo(() => data.models.filter(model => Number.isFinite(Date.parse(model.provenance.release_created ?? '')))
|
||||
.toSorted((a, b) => a.provenance.release_created.localeCompare(b.provenance.release_created) || a.name.localeCompare(b.name)), [data]);
|
||||
const matched = useMemo(() => data.models.filter(model => Number.isFinite(model.provenance.intelligence_index))
|
||||
.toSorted((a, b) => a.provenance.intelligence_index - b.provenance.intelligence_index || a.name.localeCompare(b.name)), [data]);
|
||||
const matched = useMemo(() => data.models.filter(model => Number.isFinite(model.provenance.hle_score))
|
||||
.toSorted((a, b) => a.provenance.hle_score - b.provenance.hle_score || a.name.localeCompare(b.name)), [data]);
|
||||
const plotted = axisMode === 'release-date' ? dated : matched;
|
||||
const visible = plotted.filter(model => !hidden.has(model.family));
|
||||
const coordinate = model => field === 'x' ? -model.x : model.y;
|
||||
const xValue = model => axisMode === 'release-date' ? Date.parse(model.provenance.release_created) : model.provenance.intelligence_index;
|
||||
const xValue = model => axisMode === 'release-date' ? Date.parse(model.provenance.release_created) : model.provenance.hle_score;
|
||||
const fit = ordinaryLeastSquares(visible, xValue, coordinate);
|
||||
const [active, setActive] = useState(null);
|
||||
const width = 1200, height = 320, left = 96, right = 35, top = 42, bottom = 48;
|
||||
@@ -80,9 +80,9 @@ function ReleaseScatter({ data, hidden, field, title, axisMode }) {
|
||||
const frontier = useMemo(() => {
|
||||
if (axisMode !== 'release-date') return [];
|
||||
let high = -Infinity;
|
||||
return visible.filter(model => Number.isFinite(model.provenance.intelligence_index)).filter(model => {
|
||||
if (model.provenance.intelligence_index <= high) return false;
|
||||
high = model.provenance.intelligence_index;
|
||||
return visible.filter(model => Number.isFinite(model.provenance.hle_score)).filter(model => {
|
||||
if (model.provenance.hle_score <= high) return false;
|
||||
high = model.provenance.hle_score;
|
||||
return true;
|
||||
});
|
||||
}, [axisMode, visible]);
|
||||
@@ -109,7 +109,7 @@ function ReleaseScatter({ data, hidden, field, title, axisMode }) {
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!found) throw new Error(`no frontier label location for ${model.name}`);
|
||||
if (!found) found = box(anchor.x, Math.max(bounds.top + size.height / 2, anchor.y - size.height), size.width, size.height);
|
||||
placement[model.name] = { ...found, anchor };
|
||||
taken.push(found);
|
||||
}
|
||||
@@ -118,14 +118,14 @@ function ReleaseScatter({ data, hidden, field, title, axisMode }) {
|
||||
const activate = (model, event) => {
|
||||
const box = event.currentTarget.closest('.scatter-shell').getBoundingClientRect();
|
||||
const point = event.currentTarget.getBoundingClientRect();
|
||||
setActive({ ...model, tooltipValue: coordinate(model), tooltipIndex: axisMode === 'capability' ? model.provenance.intelligence_index : null, tooltipLeft: `${Math.min(82, (point.left - box.left) / box.width * 100)}%`, tooltipTop: `${Math.min(78, (point.top - box.top) / box.height * 100)}%` });
|
||||
setActive({ ...model, tooltipValue: coordinate(model), tooltipScore: axisMode === 'capability' ? model.provenance.hle_score : null, tooltipLeft: `${Math.min(82, (point.left - box.left) / box.width * 100)}%`, tooltipTop: `${Math.min(78, (point.top - box.top) / box.height * 100)}%` });
|
||||
};
|
||||
return <section className="release-panel" data-coordinate={field} aria-labelledby={`${panelId}-heading`}>
|
||||
<h2 id={`${panelId}-heading`}>{title}</h2>
|
||||
<div className="scatter-shell">
|
||||
<svg id={`${panelId}-svg`} viewBox={`0 0 ${width} ${height}`} role="img" aria-labelledby={`${panelId}-svg-title ${panelId}-svg-desc`} data-axis-mode={axisMode} data-panel-model-count={visible.length} data-omitted-model-count={data.models.length - plotted.length} data-fit-n={fit?.n ?? 0} data-fit-r2={fit?.r2 ?? ''} data-fit-slope={fit?.slope ?? ''}>
|
||||
<title id={`${panelId}-svg-title`}>{title}</title>
|
||||
<desc id={`${panelId}-svg-desc`}>Scatter plot with {axisMode === 'release-date' ? 'release date' : 'Artificial Analysis Intelligence Index'} on the horizontal axis and {yDirection.join(' to ')} increasing upward on the vertical axis. {visible.length} of {plotted.length} matched models are visible from {visibleFamilyNames(Object.groupBy(data.models, model => model.family), hidden).join(', ') || 'no families'}; {data.models.length - plotted.length} plotted models lack this x value and are omitted only here. Each white-ring logo mark is a model. {fit ? `The thin line is an ordinary least squares descriptive fit to the ${fit.n} currently visible matched models${fit.r2 === null ? '; R squared is unavailable because the y values are constant' : `; R squared is ${fit.r2.toFixed(2)}`}.` : 'The fit is hidden because fewer than two distinct x values are visible.'} Hover or keyboard focus a mark for model-specific details.</desc>
|
||||
<desc id={`${panelId}-svg-desc`}>Scatter plot with {axisMode === 'release-date' ? 'release date' : 'HLE score'} on the horizontal axis and {yDirection.join(' to ')} increasing upward on the vertical axis. {visible.length} of {plotted.length} matched models are visible from {visibleFamilyNames(Object.groupBy(data.models, model => model.family), hidden).join(', ') || 'no families'}; {data.models.length - plotted.length} plotted models lack this x value and are omitted only here. Each white-ring logo mark is a model. {fit ? `The thin line is an ordinary least squares descriptive fit to the ${fit.n} currently visible matched models${fit.r2 === null ? '; R squared is unavailable because the y values are constant' : `; R squared is ${fit.r2.toFixed(2)}`}.` : 'The fit is hidden because fewer than two distinct x values are visible.'} Hover or keyboard focus a mark for model-specific details.</desc>
|
||||
<defs><marker id={`${panelId}-arrow`} markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L0,6 L6,3 z" className="scatter-arrow" /></marker><clipPath id={`${panelId}-fit-clip`}><rect x={left} y={top} width={width - left - right} height={height - top - bottom} /></clipPath></defs>
|
||||
<rect className="canvas" width={width} height={height} />
|
||||
<g className="scatter-grid">{Array.from({ length: 5 }, (_, index) => <line key={index} x1={left} x2={width - right} y1={top + index * (height - top - bottom) / 4} y2={top + index * (height - top - bottom) / 4} />)}</g>
|
||||
@@ -138,13 +138,13 @@ function ReleaseScatter({ data, hidden, field, title, axisMode }) {
|
||||
<text className="scatter-tick" x={left - 8} y={height - bottom} textAnchor="end">{minValue.toFixed(2)}</text>
|
||||
{fitEnd && <g className="release-fit" data-fit-n={fit.n} data-fit-r2={fit.r2 ?? ''}><line clipPath={`url(#${panelId}-fit-clip)`} x1={fitEnd[0][0]} y1={fitEnd[0][1]} x2={fitEnd[1][0]} y2={fitEnd[1][1]} /> <text x={left + 6} y={top + 13}>OLS, n={fit.n}, R² {fit.r2 === null ? 'unavailable' : fit.r2.toFixed(2)}</text></g>}
|
||||
{!fit && <text className="release-fit-unavailable" x={left + 6} y={top + 13}>Fit unavailable: fewer than two release dates</text>}
|
||||
{plotted.map(model => <g key={model.name} className="release-mark" data-family={model.family} data-release-model={model.name} data-release-date={model.provenance.release_created} data-capability-score={model.provenance.intelligence_index} data-coordinate-value={coordinate(model)} display={hidden.has(model.family) ? 'none' : 'inline'}
|
||||
tabIndex="0" role="button" aria-label={`${model.name}, ${axisMode === 'release-date' ? model.provenance.release_created : `Intelligence Index ${model.provenance.intelligence_index}`}`} aria-describedby={tooltipId}
|
||||
{plotted.map(model => <g key={model.name} className="release-mark" data-family={model.family} data-release-model={model.name} data-release-date={model.provenance.release_created} data-hle-score={model.provenance.hle_score} data-coordinate-value={coordinate(model)} display={hidden.has(model.family) ? 'none' : 'inline'}
|
||||
tabIndex="0" role="button" aria-label={`${model.name}, ${axisMode === 'release-date' ? model.provenance.release_created : `HLE score ${model.provenance.hle_score}`}`} aria-describedby={tooltipId}
|
||||
onPointerEnter={event => activate(model, event)} onPointerLeave={() => setActive(null)} onFocus={event => activate(model, event)} onBlur={() => setActive(null)}>
|
||||
<circle className="model-ring" cx={plotX(xValue(model))} cy={valueY(coordinate(model))} r="8" stroke={model.color} />
|
||||
<image href={`${LOGO_ROOT}${data.logos[model.family]}`} x={plotX(xValue(model)) - 5} y={valueY(coordinate(model)) - 5} width="10" height="10" preserveAspectRatio="xMidYMid meet" aria-hidden="true" focusable="false" />
|
||||
</g>)}
|
||||
{axisMode === 'release-date' && frontier.map(model => <g key={`frontier-${model.name}`} className="frontier-label" data-frontier-model={model.name}><line x1={frontierPlacement[model.name].anchor.x} y1={frontierPlacement[model.name].anchor.y} x2={frontierPlacement[model.name].cx} y2={frontierPlacement[model.name].cy} /><text x={frontierPlacement[model.name].cx} y={frontierPlacement[model.name].cy + 4} textAnchor="middle">{model.name}</text></g>)}
|
||||
{axisMode === 'release-date' && frontier.map(model => <g key={`frontier-${model.name}`} className="frontier-label" data-frontier-model={model.name}><text x={frontierPlacement[model.name].cx} y={frontierPlacement[model.name].cy + 4} textAnchor="middle">{model.name}</text></g>)}
|
||||
</svg>
|
||||
<Tooltip active={active} geometry={null} id={tooltipId} panel={field === 'y' ? 'secular' : 'self-expression'} />
|
||||
</div>
|
||||
@@ -159,7 +159,7 @@ function ReleaseScatters({ data, hidden, axisMode, setAxisMode }) {
|
||||
<option value="release-date">Release date</option>
|
||||
<option value="capability">{data.capability_x.label}</option>
|
||||
</select></label>
|
||||
<span><a href={data.capability_x.source_url}>{data.capability_x.label}</a>, saved {data.capability_x.fetched_utc.slice(0, 10)}. {axisMode === 'capability' ? `${data.capability_x.matched_models} matched, ${data.models.length - data.capability_x.matched_models} omitted.` : 'Release-date labels mark running Intelligence Index highs among shown mapped models.'}</span>
|
||||
<span><a href={data.capability_x.source_url}>Artificial Analysis HLE score</a>, saved {data.capability_x.fetched_utc.slice(0, 10)}. {axisMode === 'capability' ? `${data.capability_x.matched_models} matched, ${data.models.length - data.capability_x.matched_models} omitted.` : 'Release-date labels mark running HLE score highs among shown mapped models.'}</span>
|
||||
</div>
|
||||
<ReleaseScatter data={data} hidden={hidden} field="y" axisMode={axisMode} title={`${xLabel} vs Secular-Rational`} />
|
||||
<ReleaseScatter data={data} hidden={hidden} field="x" axisMode={axisMode} title={`${xLabel} vs Self-expression`} />
|
||||
@@ -201,7 +201,7 @@ function Map({ data }) {
|
||||
})}</section>
|
||||
<div className="chart-shell">
|
||||
<svg viewBox={`0 0 ${VIEW.width} ${VIEW.height}`} role="img" aria-labelledby="map-svg-title map-svg-desc" data-median-x={data.median.x} data-median-y={data.median.y} data-visible-model-count={visibleModelCount}>
|
||||
<title id="map-svg-title">Frontier LLMs on the World Values Survey</title>
|
||||
<title id="map-svg-title">Moral Maps: Where Do Frontier Models' Cultural Values Lie?</title>
|
||||
<desc id="map-svg-desc">World Values Survey cultural map. Horizontal direction runs from Self-expression on the left to Survival on the right. Vertical direction runs from Traditional below to Secular-Rational above. Coloured dots are selected WVS countries, outlines are cultural regions, and white-ring logo marks are models. {visibleModelCount} model marks are visible from {visibleFamilies.join(', ') || 'no families'}. Use the family controls to hide marks and labels. Hover or keyboard focus a model for model-specific details.</desc>
|
||||
<defs><marker id="arrow" markerWidth="8" markerHeight="8" refX="6" refY="3" orient="auto"><path d="M0,0 L0,6 L6,3 z" className="arrow" /></marker></defs>
|
||||
<rect className="canvas" width={VIEW.width} height={VIEW.height} />
|
||||
@@ -213,8 +213,8 @@ function Map({ data }) {
|
||||
{data.zone_hulls.map(zone => <text key={zone.name} className="zone-label" x={labels[`zone:${zone.name}`].cx} y={labels[`zone:${zone.name}`].cy + 5} textAnchor="middle" fill={zone.color}>{zone.name}</text>)}
|
||||
{Object.entries(groups).map(([family, models]) => <g key={family} data-family={family} display={hidden.has(family) ? 'none' : 'inline'}>{models.map(model => <ModelMarker key={model.name} model={model} placement={labels} geometry={geometry} setActive={setActive} clearActive={clearActive} markerRef={model.name === focusName ? focusRef : null} logo={data.logos[model.family]} />)}</g>)}
|
||||
<g className="poles"><line x1={xMedian} y1="62" x2={xMedian} y2={geometry.bounds.top} markerEnd="url(#arrow)" /><line x1={xMedian} y1={geometry.bounds.bottom} x2={xMedian} y2="838" markerEnd="url(#arrow)" /><line x1="64" y1={yMedian} x2={geometry.bounds.left} y2={yMedian} markerEnd="url(#arrow)" /><line x1={geometry.bounds.right} y1={yMedian} x2="1184" y2={yMedian} markerEnd="url(#arrow)" /><text x={xMedian} y="40" textAnchor="middle">{data.axis.y[1]}</text><text x={xMedian} y="870" textAnchor="middle">{data.axis.y[0]}</text><text x="25" y={yMedian + 7}>{data.axis.x[0]}</text><text x="1136" y={yMedian + 7} textAnchor="end">{data.axis.x[1]}</text></g>
|
||||
<text className="map-title" x={geometry.bounds.left + 8} y={geometry.bounds.bottom - 34}>{data.title.split('\n').map((line, index) => <tspan key={line} x={geometry.bounds.left + 8} dy={index ? 17 : 0}>{line}</tspan>)}</text>
|
||||
<text className="map-note" x={geometry.bounds.right - 8} y={geometry.bounds.bottom - 20} textAnchor="end">{data.note.split('\n').map((line, index) => <tspan key={line} x={geometry.bounds.right - 8} dy={index ? 11 : 0}>{line}</tspan>)}</text>
|
||||
<text className="map-title" x={geometry.bounds.left + 8} y={geometry.bounds.bottom - 54}>{data.title.split('\n').map((line, index) => <tspan key={line} x={geometry.bounds.left + 8} dy={index ? 17 : 0}>{line}</tspan>)}</text>
|
||||
<text className="map-note" x={geometry.bounds.left + 8} y={geometry.bounds.bottom - 34} textAnchor="start">{data.note.split('\n').map((line, index) => <tspan key={line} x={geometry.bounds.left + 8} dy={index ? 11 : 0}>{line}</tspan>)}</text>
|
||||
</svg>
|
||||
<Tooltip active={active} geometry={geometry} />
|
||||
</div>
|
||||
@@ -228,7 +228,7 @@ function App() {
|
||||
const [data, setData] = useState(null);
|
||||
useEffect(() => { fetch('wvs/wvs_map_data.json').then(response => response.json()).then(setData); }, []);
|
||||
return <main>
|
||||
<h1>How do AI models score on human values surveys? Which culture are they most similar to? Is it changing over time?</h1>
|
||||
<h1>Moral Maps: Where Do Frontier Models' Cultural Values Lie?</h1>
|
||||
<p className="lede">We start with the <a href="https://www.worldvaluessurvey.org/">World Values Survey</a>, a map of human values across about ninety countries.</p>
|
||||
{data && <Map data={data} />}
|
||||
</main>;
|
||||
|
||||
+47
-13
@@ -51,10 +51,12 @@ def main() -> None:
|
||||
if chrome is None:
|
||||
raise RuntimeError("google-chrome is required for the Playwright UAT")
|
||||
data = json.loads((ROOT / "docs/wvs/wvs_map_data.json").read_text())
|
||||
assert data["schema"] == 2
|
||||
assert data["capability_x"]["status"] == "blocked"
|
||||
assert data["capability_x"]["source_url"] == "https://artificialanalysis.ai/models"
|
||||
assert len(data["models"]) == 64
|
||||
assert data["schema"] == 3
|
||||
assert data["capability_x"]["status"] == "available"
|
||||
assert data["capability_x"]["source_url"] == "https://artificialanalysis.ai/evaluations/humanitys-last-exam"
|
||||
assert data["capability_x"]["fetched_utc"] == "2026-09-17T10:13:20Z"
|
||||
capability_matched = [model for model in data["models"] if model["provenance"]["hle_score"] is not None]
|
||||
assert len(capability_matched) == data["capability_x"]["matched_models"] == 46
|
||||
assert len(data["countries"]) == 90
|
||||
assert len(data["zone_hulls"]) == 4
|
||||
assert len(data["latest_by_family"]) == 13
|
||||
@@ -79,20 +81,23 @@ def main() -> None:
|
||||
|
||||
screenshot(page, f"{BASE}/", "wvs_react_root_playwright_default.png")
|
||||
page.wait_for_selector("svg .model-mark")
|
||||
assert page.title() == "Frontier LLMs on the World Values Survey"
|
||||
assert page.title() == "Moral Maps: Where Do Frontier Models' Cultural Values Lie?"
|
||||
assert page.locator("html").get_attribute("lang") == "en"
|
||||
assert page.locator('meta[name="viewport"]').count() == 1
|
||||
assert page.locator("h1").inner_text().startswith("How do AI models score on human values surveys?")
|
||||
assert page.locator("h1").inner_text() == "Moral Maps: Where Do Frontier Models' Cultural Values Lie?"
|
||||
assert page.locator(".lede").inner_text() == "We start with the World Values Survey, a map of human values across about ninety countries."
|
||||
visible_copy = page.locator("main").inner_text()
|
||||
for absent in ("React/SVG rendering", "Historical coordinates", "Dated releases only"):
|
||||
assert absent not in visible_copy
|
||||
assert "Artificial Analysis Intelligence Index, unavailable pending redistribution permission" in visible_copy
|
||||
assert data["capability_x"]["blocker"] in visible_copy
|
||||
assert "Artificial Analysis HLE score" in visible_copy
|
||||
assert "redistribution permission" not in visible_copy
|
||||
capability_option = page.get_by_label("Release-panel x axis").locator('option[value="capability"]')
|
||||
assert capability_option.get_attribute("disabled") is not None
|
||||
assert capability_option.get_attribute("disabled") is None
|
||||
assert page.locator('.release-axis-selector a[href="https://artificialanalysis.ai/evaluations/humanitys-last-exam"]').count() == 1
|
||||
assert "saved 2026-09-17" in page.locator(".release-axis-selector").inner_text()
|
||||
layout_order = page.locator("main > *").evaluate_all("nodes => nodes.map(node => node.className.baseVal || node.className || node.tagName)")
|
||||
assert layout_order.index("chart-shell") < layout_order.index("map-explanation") < layout_order.index("release-panels") < layout_order.index("caption")
|
||||
assert page.locator(".release-panels .release-axis-selector").count() == 1
|
||||
assert "weak descriptive correlations" in page.locator(".caption").inner_text()
|
||||
assert "code and records" in page.locator(".caption").inner_text()
|
||||
|
||||
@@ -100,8 +105,9 @@ def main() -> None:
|
||||
svg_description(page, "svg[data-median-x]", "map-svg-title", "map-svg-desc")
|
||||
assert page.locator("path.zone").count() == 4
|
||||
assert page.locator("polygon.zone").count() == 0
|
||||
assert page.locator(".map-note").get_attribute("text-anchor") == "end"
|
||||
assert float(page.locator(".map-note").get_attribute("x")) > float(page.locator(".map-title").get_attribute("x"))
|
||||
assert page.locator(".map-note").get_attribute("text-anchor") == "start"
|
||||
assert float(page.locator(".map-note").get_attribute("x")) == float(page.locator(".map-title").get_attribute("x"))
|
||||
assert float(page.locator(".map-note").get_attribute("y")) > float(page.locator(".map-title").get_attribute("y"))
|
||||
assert float(map_svg.get_attribute("data-median-x")) == data["median"]["x"]
|
||||
assert float(map_svg.get_attribute("data-median-y")) == data["median"]["y"]
|
||||
models_dom = page.locator(".model-mark").evaluate_all(
|
||||
@@ -123,6 +129,10 @@ def main() -> None:
|
||||
svg_description(page, '.release-panel[data-coordinate="y"] svg', "release-y-svg-title", "release-y-svg-desc")
|
||||
svg_description(page, '.release-panel[data-coordinate="x"] svg', "release-x-svg-title", "release-x-svg-desc")
|
||||
assert page.locator(".release-mark").count() == len(dated) * 2
|
||||
assert page.locator(".frontier-label").count() > 0
|
||||
frontier_boxes = page.locator('.frontier-label text').evaluate_all("nodes => nodes.map(node => { const b = node.getBBox(); return [b.x, b.y, b.width, b.height]; })")
|
||||
assert all(x >= 96 and y >= 42 and x + width <= 1165 and y + height <= 272 for x, y, width, height in frontier_boxes)
|
||||
assert all(a[0] + a[2] <= b[0] or b[0] + b[2] <= a[0] or a[1] + a[3] <= b[1] or b[1] + b[3] <= a[1] for index, a in enumerate(frontier_boxes) for b in frontier_boxes[index + 1:])
|
||||
for grok_name in grok_dates:
|
||||
assert page.locator(f'[data-release-model="{grok_name}"]').count() == 2
|
||||
self_expression_panel = page.locator('.release-panel[data-coordinate="x"]')
|
||||
@@ -144,6 +154,27 @@ def main() -> None:
|
||||
assert panel.locator(".release-fit line").count() == 1
|
||||
page.screenshot(path=OUT / "wvs_react_playwright_release_fit_all_families.png", full_page=True)
|
||||
|
||||
page.get_by_label("Release-panel x axis").select_option("capability")
|
||||
assert page.locator('.release-panel[data-coordinate="y"] svg').get_attribute("data-axis-mode") == "capability"
|
||||
assert page.locator('.release-mark').count() == len(capability_matched) * 2
|
||||
assert all(int(panel.locator("svg").get_attribute("data-fit-n")) == len(capability_matched) for panel in page.locator(".release-panel").all())
|
||||
assert all(int(panel.locator("svg").get_attribute("data-omitted-model-count")) == len(data["models"]) - len(capability_matched) for panel in page.locator(".release-panel").all())
|
||||
page.screenshot(path=OUT / "wvs_react_playwright_capability_all_families.png", full_page=True)
|
||||
page.get_by_role("button", name="qwen", exact=True).click()
|
||||
assert all(int(panel.locator("svg").get_attribute("data-fit-n")) == sum(model["family"] != "qwen" for model in capability_matched) for panel in page.locator(".release-panel").all())
|
||||
page.screenshot(path=OUT / "wvs_react_playwright_capability_qwen_hidden.png", full_page=True)
|
||||
page.get_by_role("button", name="qwen", exact=True).click()
|
||||
for family in sorted(families - {"gpt"}):
|
||||
page.get_by_role("button", name=family, exact=True).click()
|
||||
page.wait_for_timeout(100)
|
||||
gpt_capability = sum(model["family"] == "gpt" for model in capability_matched)
|
||||
gpt_fit_n = [int(panel.locator("svg").get_attribute("data-fit-n")) for panel in page.locator(".release-panel").all()]
|
||||
assert gpt_fit_n == [gpt_capability, gpt_capability], (gpt_fit_n, gpt_capability)
|
||||
page.screenshot(path=OUT / "wvs_react_playwright_capability_gpt_only.png", full_page=True)
|
||||
for family in sorted(families - {"gpt"}):
|
||||
page.get_by_role("button", name=family, exact=True).click()
|
||||
page.get_by_label("Release-panel x axis").select_option("release-date")
|
||||
|
||||
page.get_by_role("button", name="qwen", exact=True).click()
|
||||
page.wait_for_timeout(100)
|
||||
assert map_svg.locator(':scope > g[data-family="qwen"]').get_attribute("display") == "none"
|
||||
@@ -243,8 +274,11 @@ def main() -> None:
|
||||
"qwen_toggle": "clicked, map and dated-panel family groups hidden, country coordinates invariant",
|
||||
"tooltip": "map and release-panel pointer hover and focus show panel-specific model name, coordinate, and release date only",
|
||||
"release_panels": {"panels": 2, "dated_models": len(dated), "date_order": "DOM order verified; both catalog-dated Grok models rendered in each panel",
|
||||
"ols": "line, n, and R squared recompute from currently visible dated models; the Self-expression panel renders negative stored x, so upward means Self-expression; fewer than two distinct dates hide the fit"},
|
||||
"copy": "one short linked WVS sentence above the map; history and axes below it; weak descriptive-fit method and code link below the release panels; capability selector remains visible but scores are absent pending redistribution permission",
|
||||
"ols": "line, n, and R squared recompute from currently visible matched models; the Self-expression panel renders negative stored x, so upward means Self-expression; fewer than two distinct x values hide the fit"},
|
||||
"capability_panels": {"matched_models": len(capability_matched), "omitted_models": len(data["models"]) - len(capability_matched),
|
||||
"source": data["capability_x"]["source_url"], "fetched_utc": data["capability_x"]["fetched_utc"],
|
||||
"selector": "enabled; all-family and Qwen-hidden fits checked"},
|
||||
"copy": "one short linked WVS sentence above the map; history and axes below it; source credit beside the enabled lower-panel selector; weak descriptive-fit method and code link below the release panels",
|
||||
"hashes": hashes,
|
||||
}, indent=2))
|
||||
|
||||
|
||||
@@ -1,5 +1,20 @@
|
||||
{
|
||||
"completed": {
|
||||
"00220612ec353054f53baca3eccb90ea2bea8632c2c2fdae99ef7a074bd75d05": {
|
||||
"coords": [
|
||||
0.5467309986851505,
|
||||
0.5724812695525422,
|
||||
0.06104656289606268,
|
||||
0.10460593729500693
|
||||
],
|
||||
"display_key": "grok-4.3 (rated)",
|
||||
"model": "x-ai/grok-4.3",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "00220612ec353054f53baca3eccb90ea2bea8632c2c2fdae99ef7a074bd75d05",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T122425Z_00220612ec35"
|
||||
},
|
||||
"0043a43d1a2188f7c728ccda08eee7b062232366d8a4e912c72bdc938e37fe44": {
|
||||
"coords": [
|
||||
0.6214472858835399,
|
||||
@@ -15,6 +30,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T152010Z_0043a43d1a21"
|
||||
},
|
||||
"009e9bcb85966922ddafb0bb61fd0f750528edf0a13a21cf954b226476428210": {
|
||||
"coords": [
|
||||
0.3863756613756613,
|
||||
0.6708921222810112,
|
||||
0.05550492956346581,
|
||||
0.07459772141856083
|
||||
],
|
||||
"display_key": "gemini-2.5-flash-lite (rated)",
|
||||
"model": "google/gemini-2.5-flash-lite",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "009e9bcb85966922ddafb0bb61fd0f750528edf0a13a21cf954b226476428210",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T130434Z_009e9bcb8596"
|
||||
},
|
||||
"02d386b067d55e0639d33db8b6f1e5c708c30acbada3125dcbabde47dc90edcf": {
|
||||
"coords": [
|
||||
0.5849525883130219,
|
||||
@@ -45,6 +75,51 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T192223Z_0a798666aeed"
|
||||
},
|
||||
"11929820262d34605efabc00ce84137857e03ef6c53d827ea6c405555b905f26": {
|
||||
"coords": [
|
||||
0.6151755041791274,
|
||||
0.6364257870168759,
|
||||
0.06075682882091517,
|
||||
0.0850053325551206
|
||||
],
|
||||
"display_key": "gpt-5.4 (rated)",
|
||||
"model": "openai/gpt-5.4",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "11929820262d34605efabc00ce84137857e03ef6c53d827ea6c405555b905f26",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T124555Z_11929820262d"
|
||||
},
|
||||
"134c21f0377c1a88f8ecfe9e9191341b0f53e0afbadb05d36ca0ba36c641f340": {
|
||||
"coords": [
|
||||
0.5672038109538109,
|
||||
0.6522156084656084,
|
||||
0.0657887759000581,
|
||||
0.0790074678539591
|
||||
],
|
||||
"display_key": "gemini-3.5-flash-lite (rated)",
|
||||
"model": "google/gemini-3.5-flash-lite",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "134c21f0377c1a88f8ecfe9e9191341b0f53e0afbadb05d36ca0ba36c641f340",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T124448Z_134c21f0377c"
|
||||
},
|
||||
"1486c2f54e34ad1075f7b519a1dfc2b43f9f7e052d563a083fbc139fc41bd6ee": {
|
||||
"coords": [
|
||||
0.5941946410337214,
|
||||
0.6441922202164339,
|
||||
0.06777104459972463,
|
||||
0.07883221721567056
|
||||
],
|
||||
"display_key": "glm-4.5-air (rated)",
|
||||
"model": "z-ai/glm-4.5-air",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "1486c2f54e34ad1075f7b519a1dfc2b43f9f7e052d563a083fbc139fc41bd6ee",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T125444Z_1486c2f54e34"
|
||||
},
|
||||
"182a7e88869698133f1175f18e53e7f6f2f056739559d325b211bd065235a71c": {
|
||||
"coords": [
|
||||
0.6175297619047618,
|
||||
@@ -60,6 +135,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T193258Z_182a7e888696"
|
||||
},
|
||||
"1a14a0f088ae0c3cc56637a32bd4c2290e4dde93dfeebb0f888a5a68c784af0d": {
|
||||
"coords": [
|
||||
0.5780312225900461,
|
||||
0.608531746031746,
|
||||
0.06478224722015172,
|
||||
0.101427539255001
|
||||
],
|
||||
"display_key": "gemini-3.1-flash-lite-preview (rated)",
|
||||
"model": "google/gemini-3.1-flash-lite-preview",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "1a14a0f088ae0c3cc56637a32bd4c2290e4dde93dfeebb0f888a5a68c784af0d",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T125623Z_1a14a0f088ae"
|
||||
},
|
||||
"1a70f47789af20900f2799992b09e83af9490be6579098a6178d613d5ba7856c": {
|
||||
"coords": [
|
||||
0.5274474357644292,
|
||||
@@ -105,6 +195,36 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T192044Z_1d7f4015decb"
|
||||
},
|
||||
"27cc1b32407e6ebca063ce550d1848751a64f4393f93b6af393a01b4f62fe162": {
|
||||
"coords": [
|
||||
0.5442434494517827,
|
||||
0.6391828271087302,
|
||||
0.04443386786526795,
|
||||
0.054686704767473275
|
||||
],
|
||||
"display_key": "glm-5.3-flash (rated)",
|
||||
"model": "z-ai/glm-5.3-flash",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "27cc1b32407e6ebca063ce550d1848751a64f4393f93b6af393a01b4f62fe162",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T114916Z_27cc1b32407e"
|
||||
},
|
||||
"29b17fd9e04c4b7f93e1763b460892f9ec2f4fc4698b6dd7571fcfebc76b8500": {
|
||||
"coords": [
|
||||
0.5429175317746331,
|
||||
0.694532627865961,
|
||||
0.06309666696112717,
|
||||
0.07571315515523741
|
||||
],
|
||||
"display_key": "glm-5.1 (rated)",
|
||||
"model": "z-ai/glm-5.1",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "29b17fd9e04c4b7f93e1763b460892f9ec2f4fc4698b6dd7571fcfebc76b8500",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T120826Z_29b17fd9e04c"
|
||||
},
|
||||
"3016d2c1ab8d8f88f01723d2d695be996c7f1fdcd442a51cfbb6c7b7110a25e9": {
|
||||
"coords": [
|
||||
0.5017202097104058,
|
||||
@@ -135,6 +255,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T195554Z_304db4b1aa6c"
|
||||
},
|
||||
"3194398c99baf1d5ebb273314207d4aa1bfe5d12c8b16cb192818aacd78d759f": {
|
||||
"coords": [
|
||||
0.4215300954189843,
|
||||
0.7190489551600663,
|
||||
0.07543950734136469,
|
||||
0.06523372720374179
|
||||
],
|
||||
"display_key": "grok-4.6 (rated)",
|
||||
"model": "x-ai/grok-4.6",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "3194398c99baf1d5ebb273314207d4aa1bfe5d12c8b16cb192818aacd78d759f",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T114916Z_3194398c99ba"
|
||||
},
|
||||
"333ece6448f97447c57f24f9e9fcf0a7fb72a1b0f96696aef3c38a9c1d2d2308": {
|
||||
"coords": [
|
||||
0.5482108382155133,
|
||||
@@ -240,6 +375,66 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T150121Z_49ae9f7a0909"
|
||||
},
|
||||
"4f0d6ed370f069e233ea6c4cb83000e79398c3a2aa10b355b10f99c8633c47f6": {
|
||||
"coords": [
|
||||
0.5923129031462364,
|
||||
0.6683453376721874,
|
||||
0.042111115261344804,
|
||||
0.08216742430172869
|
||||
],
|
||||
"display_key": "glm-5 (rated)",
|
||||
"model": "z-ai/glm-5",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "4f0d6ed370f069e233ea6c4cb83000e79398c3a2aa10b355b10f99c8633c47f6",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T123002Z_4f0d6ed370f0"
|
||||
},
|
||||
"5478eac564ec02f01e161ce017e1814954f4239b9fa1bedfe73172c5c630848f": {
|
||||
"coords": [
|
||||
0.4566709514410664,
|
||||
0.670101976591992,
|
||||
0.05775584971748114,
|
||||
0.07962287120217941
|
||||
],
|
||||
"display_key": "gpt-5.4-mini (rated)",
|
||||
"model": "openai/gpt-5.4-mini",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "5478eac564ec02f01e161ce017e1814954f4239b9fa1bedfe73172c5c630848f",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T124216Z_5478eac564ec"
|
||||
},
|
||||
"54e721734c251e2e13cf43beab962703409108bfc5203c19e7747770ce13a801": {
|
||||
"coords": [
|
||||
0.593257275132275,
|
||||
0.6186534585344109,
|
||||
0.03571288070419747,
|
||||
0.06312818330951275
|
||||
],
|
||||
"display_key": "gemini-3.6-flash (rated)",
|
||||
"model": "google/gemini-3.6-flash",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "54e721734c251e2e13cf43beab962703409108bfc5203c19e7747770ce13a801",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T124016Z_54e721734c25"
|
||||
},
|
||||
"568bb3fc2517dd40b1399c71e0cf52033dcce79dc7537c4943ace860fe3a1297": {
|
||||
"coords": [
|
||||
0.6123449635867936,
|
||||
0.7514717187100795,
|
||||
0.04454897242429451,
|
||||
0.049069808495596626
|
||||
],
|
||||
"display_key": "gpt-5.1 (rated)",
|
||||
"model": "openai/gpt-5.1",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "568bb3fc2517dd40b1399c71e0cf52033dcce79dc7537c4943ace860fe3a1297",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T125409Z_568bb3fc2517"
|
||||
},
|
||||
"57bed80fd5b94157effe68fe998c7c12612fea210c1371dbf230097a7e8e0717": {
|
||||
"coords": [
|
||||
0.6363133579397021,
|
||||
@@ -285,6 +480,36 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T181355Z_589e10fa7924"
|
||||
},
|
||||
"5c920b68e50c5b4fd43542f379ac25b98e8d79107ea1722c1db12aa22fa34211": {
|
||||
"coords": [
|
||||
0.5907043650793651,
|
||||
0.5776102655592599,
|
||||
0.04137996061944377,
|
||||
0.07845514485495916
|
||||
],
|
||||
"display_key": "gemini-3.5-flash (rated)",
|
||||
"model": "google/gemini-3.5-flash",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "5c920b68e50c5b4fd43542f379ac25b98e8d79107ea1722c1db12aa22fa34211",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T124812Z_5c920b68e50c"
|
||||
},
|
||||
"6067c0208cc500e5837d123052d6b33b3568d616e5c157f601baa8dcb7ef159b": {
|
||||
"coords": [
|
||||
0.5189766385048643,
|
||||
0.6372836916088428,
|
||||
0.052897459249729474,
|
||||
0.08224247455566645
|
||||
],
|
||||
"display_key": "grok-4.20 (rated)",
|
||||
"model": "x-ai/grok-4.20",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "6067c0208cc500e5837d123052d6b33b3568d616e5c157f601baa8dcb7ef159b",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T122709Z_6067c0208cc5"
|
||||
},
|
||||
"61ab1adae3781ca3893328a845d4d517daa5fb47e07ea189da1df45388f62d42": {
|
||||
"coords": [
|
||||
0.5993276014109347,
|
||||
@@ -330,6 +555,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T191343Z_67a70b1b03ba"
|
||||
},
|
||||
"6af9f017856f4c733214bf8e26992a4af3c7cb94dac8a5a4b881e8669b3917a7": {
|
||||
"coords": [
|
||||
0.5220108722106056,
|
||||
0.6395004417804221,
|
||||
0.04704769940966264,
|
||||
0.09470717560391326
|
||||
],
|
||||
"display_key": "gpt-5-mini (rated)",
|
||||
"model": "openai/gpt-5-mini",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "6af9f017856f4c733214bf8e26992a4af3c7cb94dac8a5a4b881e8669b3917a7",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T130236Z_6af9f017856f"
|
||||
},
|
||||
"6d6ce30e9fbc5fdb07229b0a08e067d77511857069a4e572613d3e48b7c8b261": {
|
||||
"coords": [
|
||||
0.5856201899951899,
|
||||
@@ -345,6 +585,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T135722Z_6d6ce30e9fbc"
|
||||
},
|
||||
"6f7a7bbc4d70e7cfceff8e25b2da2008fbf6105538825f42bfbbc47ff3459387": {
|
||||
"coords": [
|
||||
0.5605366161616162,
|
||||
0.573015873015873,
|
||||
0.07008299148165956,
|
||||
0.09640175208725496
|
||||
],
|
||||
"display_key": "gemini-3.1-flash-lite (rated)",
|
||||
"model": "google/gemini-3.1-flash-lite",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "6f7a7bbc4d70e7cfceff8e25b2da2008fbf6105538825f42bfbbc47ff3459387",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T125248Z_6f7a7bbc4d70"
|
||||
},
|
||||
"7058adf367d238c6c92e1b83ae71f6556362e9fc7a243297b6fc65e54e2c2886": {
|
||||
"coords": [
|
||||
0.5974338624338624,
|
||||
@@ -375,6 +630,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T144205Z_74e3649a6961"
|
||||
},
|
||||
"7809cffa4ab1327c0a20b2d9ec23d0647b9382ddef65dd36c2ab5669da4f7d9f": {
|
||||
"coords": [
|
||||
0.5595998356229528,
|
||||
0.5926290303237558,
|
||||
0.04045393842948997,
|
||||
0.058480296424785895
|
||||
],
|
||||
"display_key": "glm-4.7-flash (rated)",
|
||||
"model": "z-ai/glm-4.7-flash",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "7809cffa4ab1327c0a20b2d9ec23d0647b9382ddef65dd36c2ab5669da4f7d9f",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T123743Z_7809cffa4ab1"
|
||||
},
|
||||
"7ee20ca2b88f69dd06b9de3018deea4cb7a418e3119e17862cd0b32146daae1a": {
|
||||
"coords": [
|
||||
0.538440257742794,
|
||||
@@ -420,6 +690,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T133001Z_82875b6ee164"
|
||||
},
|
||||
"840da516c1c04e69e5d80dd104803f019a4d6c6640359ad0e4bbcb71e722aa6c": {
|
||||
"coords": [
|
||||
0.5595320767195766,
|
||||
0.6334656084656085,
|
||||
0.05158017344603731,
|
||||
0.06609164737112991
|
||||
],
|
||||
"display_key": "gemini-3-flash-preview (rated)",
|
||||
"model": "google/gemini-3-flash-preview",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "840da516c1c04e69e5d80dd104803f019a4d6c6640359ad0e4bbcb71e722aa6c",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T125926Z_840da516c1c0"
|
||||
},
|
||||
"842298c634ec0f57848b605f67fca161631a852a68192cae1a5377b1d9f31566": {
|
||||
"coords": [
|
||||
0.6427511094043352,
|
||||
@@ -510,6 +795,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T144546Z_921707cb0d3d"
|
||||
},
|
||||
"94428060f3776ab79ab3d146382e869d248c259c86858f4e5dbc9c5a1f4633db": {
|
||||
"coords": [
|
||||
0.48311026165192833,
|
||||
0.6007697469838861,
|
||||
0.05313376200674668,
|
||||
0.06690255611782707
|
||||
],
|
||||
"display_key": "gpt-5.4-nano (rated)",
|
||||
"model": "openai/gpt-5.4-nano",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "94428060f3776ab79ab3d146382e869d248c259c86858f4e5dbc9c5a1f4633db",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T123850Z_94428060f377"
|
||||
},
|
||||
"95bb4d3939e9937823357b5cd87adb1a6640cc5b04e0841dfec03095331a5e44": {
|
||||
"coords": [
|
||||
0.4569019748367575,
|
||||
@@ -585,6 +885,51 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T112151Z_a2df2f1a32fd"
|
||||
},
|
||||
"a544b24e1874f8d1930042f65a61ea53299057a6299b0eb2ded110691c0d4edb": {
|
||||
"coords": [
|
||||
0.6298593073593075,
|
||||
0.6495670432204361,
|
||||
0.03865121082002784,
|
||||
0.08563182969926275
|
||||
],
|
||||
"display_key": "gemini-2.5-flash (rated)",
|
||||
"model": "google/gemini-2.5-flash",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "a544b24e1874f8d1930042f65a61ea53299057a6299b0eb2ded110691c0d4edb",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T130734Z_a544b24e1874"
|
||||
},
|
||||
"a742f028f72ec35cc6586661ca261df77a51eecce778ff080062968942d822ae": {
|
||||
"coords": [
|
||||
0.5253620889368438,
|
||||
0.6943301071633757,
|
||||
0.06029287144663362,
|
||||
0.07390235325207269
|
||||
],
|
||||
"display_key": "glm-5.2 (rated)",
|
||||
"model": "z-ai/glm-5.2",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "a742f028f72ec35cc6586661ca261df77a51eecce778ff080062968942d822ae",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T115959Z_a742f028f72e"
|
||||
},
|
||||
"a9a69a3ef43e7544f73290e6fd9d3c9f37013b15a9e50be755d4d227d6cc30eb": {
|
||||
"coords": [
|
||||
0.524187458188895,
|
||||
0.708147541423977,
|
||||
0.05959262807078437,
|
||||
0.06778136701328388
|
||||
],
|
||||
"display_key": "grok-4.5 (rated)",
|
||||
"model": "x-ai/grok-4.5",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "a9a69a3ef43e7544f73290e6fd9d3c9f37013b15a9e50be755d4d227d6cc30eb",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T121142Z_a9a69a3ef43e"
|
||||
},
|
||||
"abf45a27f4bafad0eb264aac33b1e49e9d0404e141966954d0707930d56665e3": {
|
||||
"coords": [
|
||||
0.45740779531102105,
|
||||
@@ -630,6 +975,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T140418Z_b35b916c2264"
|
||||
},
|
||||
"b5fdfe76425ee50c9f475587863990f2f6fd088d7f64c5c5119e7062a6a1396b": {
|
||||
"coords": [
|
||||
0.552530712157149,
|
||||
0.63810610515893,
|
||||
0.053278822950175,
|
||||
0.08499253722116974
|
||||
],
|
||||
"display_key": "gpt-5.2 (rated)",
|
||||
"model": "openai/gpt-5.2",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "b5fdfe76425ee50c9f475587863990f2f6fd088d7f64c5c5119e7062a6a1396b",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T125001Z_b5fdfe76425e"
|
||||
},
|
||||
"bca0745bdc1554718d57891098f366214f05e9e896007a916d14a06efa33471a": {
|
||||
"coords": [
|
||||
0.5778139029180694,
|
||||
@@ -645,6 +1005,51 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T145109Z_bca0745bdc15"
|
||||
},
|
||||
"be8b6978ea49ace083544a82e1837e17898ca264b959e08939da01a63424e2f9": {
|
||||
"coords": [
|
||||
0.56922050588427,
|
||||
0.7223026320129816,
|
||||
0.036665367096187215,
|
||||
0.053812885345722886
|
||||
],
|
||||
"display_key": "gpt-5 (rated)",
|
||||
"model": "openai/gpt-5",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "be8b6978ea49ace083544a82e1837e17898ca264b959e08939da01a63424e2f9",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T125811Z_be8b6978ea49"
|
||||
},
|
||||
"c2911249c065c9b0cd2f69104d1f931bf8783940bd503fd60aa99ec6c6c0a761": {
|
||||
"coords": [
|
||||
0.5489586575370888,
|
||||
0.6356739354917721,
|
||||
0.05257011593980756,
|
||||
0.0883451278287498
|
||||
],
|
||||
"display_key": "gpt-5.6-terra (rated)",
|
||||
"model": "openai/gpt-5.6-terra",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "c2911249c065c9b0cd2f69104d1f931bf8783940bd503fd60aa99ec6c6c0a761",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T123400Z_c2911249c065"
|
||||
},
|
||||
"c4e310179ab86b9d786c221d8bab6b6891f42d18836a8affc64249769eaffdef": {
|
||||
"coords": [
|
||||
0.5713445083420574,
|
||||
0.5644094213660266,
|
||||
0.04068128275908349,
|
||||
0.08447489368900887
|
||||
],
|
||||
"display_key": "deepseek-v3.2 (rated)",
|
||||
"model": "deepseek/deepseek-v3.2",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "c4e310179ab86b9d786c221d8bab6b6891f42d18836a8affc64249769eaffdef",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T114916Z_c4e310179ab8"
|
||||
},
|
||||
"c64ced76e82ec32cf6be896420d813403a95a421474eefcbbcbba421f8189192": {
|
||||
"coords": [
|
||||
0.48767589171684006,
|
||||
@@ -660,6 +1065,36 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T111805Z_c64ced76e82e"
|
||||
},
|
||||
"c7e906b267d13cb638b0a646e9643e15bd8917152cb4ef6a2e888ba309501565": {
|
||||
"coords": [
|
||||
0.5803524402579809,
|
||||
0.6040068811440846,
|
||||
0.04571011970861469,
|
||||
0.08797307424268631
|
||||
],
|
||||
"display_key": "deepseek-v3.2-exp (rated)",
|
||||
"model": "deepseek/deepseek-v3.2-exp",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "c7e906b267d13cb638b0a646e9643e15bd8917152cb4ef6a2e888ba309501565",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T115739Z_c7e906b267d1"
|
||||
},
|
||||
"cc5d6fdd3ce85c73322e963fe44186c44015427f93cdbcecec838da8b2dd4ab4": {
|
||||
"coords": [
|
||||
0.5896480994850559,
|
||||
0.5981282109666797,
|
||||
0.035112198239544395,
|
||||
0.08647066879117997
|
||||
],
|
||||
"display_key": "deepseek-chat-v3-0324 (rated)",
|
||||
"model": "deepseek/deepseek-chat-v3-0324",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "cc5d6fdd3ce85c73322e963fe44186c44015427f93cdbcecec838da8b2dd4ab4",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T122110Z_cc5d6fdd3ce8"
|
||||
},
|
||||
"cd5db529649a179032180cecafe4fd98aff124ee3654f7d60996de704e2b63ef": {
|
||||
"coords": [
|
||||
0.4988977072310405,
|
||||
@@ -690,6 +1125,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T112521Z_d21d1e81010d"
|
||||
},
|
||||
"d4580ea120a65129dc484875ee13aa8a721bb133a93a18b19f974cc876f00178": {
|
||||
"coords": [
|
||||
0.619008820472186,
|
||||
0.6035800722347154,
|
||||
0.0369775280569868,
|
||||
0.08938873815444547
|
||||
],
|
||||
"display_key": "deepseek-chat-v3.1 (rated)",
|
||||
"model": "deepseek/deepseek-chat-v3.1",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "d4580ea120a65129dc484875ee13aa8a721bb133a93a18b19f974cc876f00178",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T121651Z_d4580ea120a6"
|
||||
},
|
||||
"d81f7e66b3c4c720f8dd80768d3dfc25de6edeaf6a352cfed40e961f21fcfa22": {
|
||||
"coords": [
|
||||
0.5535321928702047,
|
||||
@@ -705,6 +1155,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T161456Z_d81f7e66b3c4"
|
||||
},
|
||||
"da8a3ff2c0318113bd43c5d8b1813b9d0947fcd6330bfe600ec28a203e6507b8": {
|
||||
"coords": [
|
||||
0.5974867724867725,
|
||||
0.5792094488085178,
|
||||
0.04341830148688508,
|
||||
0.09828751212663685
|
||||
],
|
||||
"display_key": "gpt-5.6-luna (rated)",
|
||||
"model": "openai/gpt-5.6-luna",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "da8a3ff2c0318113bd43c5d8b1813b9d0947fcd6330bfe600ec28a203e6507b8",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T122950Z_da8a3ff2c031"
|
||||
},
|
||||
"dc09c3fe433c95591b336b279603c546152daf6534d90699323b3bcde754f5bf": {
|
||||
"coords": [
|
||||
0.6353621031746031,
|
||||
@@ -720,6 +1185,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T195827Z_dc09c3fe433c"
|
||||
},
|
||||
"dc89e0b249b6189f2f85b9c5eebd98efb1595ea416761f9c2803ca57324275f6": {
|
||||
"coords": [
|
||||
0.3810994858696008,
|
||||
0.6397686012218032,
|
||||
0.08204078442843406,
|
||||
0.039637685403670346
|
||||
],
|
||||
"display_key": "gemini-3.8-flash (rated)",
|
||||
"model": "google/gemini-3.8-flash",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "dc89e0b249b6189f2f85b9c5eebd98efb1595ea416761f9c2803ca57324275f6",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T123103Z_dc89e0b249b6"
|
||||
},
|
||||
"e0a9daef251d975c5e805e159f3813dd0c55b3f0919876b828e572b0530bea88": {
|
||||
"coords": [
|
||||
0.48469169991228817,
|
||||
@@ -735,6 +1215,21 @@
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260916T143453Z_e0a9daef251d"
|
||||
},
|
||||
"e3abbf27f8d036c12dd728740d2c263bb9be98f160437d5af3cb05b33dbfa761": {
|
||||
"coords": [
|
||||
0.5255845653943481,
|
||||
0.6176120675327024,
|
||||
0.03757041296270222,
|
||||
0.06469679141919639
|
||||
],
|
||||
"display_key": "qwen3.8-2.4t-a95b (rated)",
|
||||
"model": "qwen/qwen3.8-2.4t-a95b",
|
||||
"n_items": 12,
|
||||
"n_samples": 12,
|
||||
"protocol_id": "e3abbf27f8d036c12dd728740d2c263bb9be98f160437d5af3cb05b33dbfa761",
|
||||
"records_path": "slop/research/wvs/20260916_openrouter/wvs_iw_requests.jsonl",
|
||||
"run_id": "20260917T122025Z_e3abbf27f8d0"
|
||||
},
|
||||
"ed8190c48b2a3778bba8afff7381bb9f1578211b5e8f750d21321b62617c82c3": {
|
||||
"coords": [
|
||||
0.431934218610816,
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
# Artificial Analysis Intelligence Index source and mapping
|
||||
|
||||
Source: <https://artificialanalysis.ai/models>
|
||||
|
||||
- fetched UTC: `2026-09-17T08:31:00Z`
|
||||
- raw response: `artificialanalysis_models_2026-09-17T083100Z.html`
|
||||
- SHA-256: `eff84596d778cc3c3665c701d96e46b8e11f36044ef44177beac41b98c2b21e1`
|
||||
- extracted collection: `initialModels`
|
||||
|
||||
`wvs_intelligence_index_mapping.json` is the curated mapping from plotted WVS names to exact canonical releases in that saved response. It retains only explicit same-release matches. Its scores are the highest listed effort for each mapped release in this snapshot. A plotted model without a mapping is omitted from the capability panels, not assigned a family-level or nearby-release score.
|
||||
|
||||
-- PI[gpt-5.6-terra]
|
||||
BIN
Binary file not shown.
+1
File diff suppressed because one or more lines are too long
+1
File diff suppressed because one or more lines are too long
@@ -0,0 +1,649 @@
|
||||
{
|
||||
"schema": 2,
|
||||
"source": {
|
||||
"models_url": "https://artificialanalysis.ai/models",
|
||||
"hle_url": "https://artificialanalysis.ai/evaluations/humanitys-last-exam",
|
||||
"fetched_utc": "2026-09-17T10:13:20Z",
|
||||
"manifest_html": "artificialanalysis_models_hle_manifest_2026-09-17T101320Z.html",
|
||||
"manifest_html_sha256": "2477a135dbba9beb804e1a5d4e1b7b6885b4db3370e5cadd5c75b941c17a603f",
|
||||
"encrypted_models": "artificialanalysis_hle_models_2026-09-17T101320Z.txt",
|
||||
"encrypted_models_sha256": "92c745e965335282fdfc1482a5ccae04c6ff424f0c5cf13a94e55dfdbf3d161d",
|
||||
"decoded_models": "artificialanalysis_hle_models_decoded_2026-09-17T101320Z.json",
|
||||
"decoded_models_sha256": "d0b33c6d5d5a76fb65eeea9893b646c991342ca4b93996910520e02404df1818",
|
||||
"decoded_collection": "models",
|
||||
"canonical_configuration_rows": 652,
|
||||
"non_null_hle_rows": 615,
|
||||
"metric": "Humanity's Last Exam score",
|
||||
"protocol": "Artificial Analysis HLE evaluation as supplied per model configuration; HLE score is a fraction correct."
|
||||
},
|
||||
"selection_rule": "For each exact canonical release mapping, select the highest non-null HLE score among saved AA configurations for that release. The selected configuration and effort remain in each mapping. This is a capability ceiling, not the score of the WVS request configuration.",
|
||||
"mappings": [
|
||||
{
|
||||
"plotted_model": "qwen3.6-27b",
|
||||
"source_release_slug": "qwen3-6-27b",
|
||||
"source_release_name": "Qwen3.6 27B",
|
||||
"source_configuration_slug": "qwen3-6-27b",
|
||||
"source_name": "Qwen3.6 27B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-04-22",
|
||||
"hle_score": 0.230769230769231,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.5-35b-a3b",
|
||||
"source_release_slug": "qwen3-5-35b-a3b",
|
||||
"source_release_name": "Qwen3.5 35B A3B",
|
||||
"source_configuration_slug": "qwen3-5-35b-a3b",
|
||||
"source_name": "Qwen3.5 35B A3B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-02-24",
|
||||
"hle_score": 0.210379981464319,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.6-max-preview",
|
||||
"source_release_slug": "qwen3-6-max",
|
||||
"source_release_name": "Qwen3.6 Max Preview",
|
||||
"source_configuration_slug": "qwen3-6-max",
|
||||
"source_name": "Qwen3.6 Max Preview",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-04-20",
|
||||
"hle_score": 0.308155699721965,
|
||||
"match_basis": "AA release name and saved OpenRouter display name are both Qwen3.6 Max Preview; AA release date 2026-04-20 predates catalog indexing on 2026-04-27.",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "muse-spark-1.3",
|
||||
"source_release_slug": "muse-spark-1-3",
|
||||
"source_release_name": "Muse Spark 1.3",
|
||||
"source_configuration_slug": "muse-spark-1-3",
|
||||
"source_name": "Muse Spark 1.3 (max)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-09-02",
|
||||
"hle_score": 0.487025023169602,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.8-27b",
|
||||
"source_release_slug": "qwen3-8-27b",
|
||||
"source_release_name": "Qwen3.8 27B",
|
||||
"source_configuration_slug": "qwen3-8-27b",
|
||||
"source_name": "Qwen3.8 27B (xhigh)",
|
||||
"source_effort": "xhigh",
|
||||
"source_release_date": "2026-08-14",
|
||||
"hle_score": 0.33920296570899,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 4
|
||||
},
|
||||
{
|
||||
"plotted_model": "gemma-4-31b-it",
|
||||
"source_release_slug": "gemma-4-31b",
|
||||
"source_release_name": "Gemma 4 31B",
|
||||
"source_configuration_slug": "gemma-4-31b",
|
||||
"source_name": "Gemma 4 31B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-04-02",
|
||||
"hle_score": 0.236329935125116,
|
||||
"match_basis": "AA release and saved OpenRouter catalog both identify Google Gemma 4 31B on 2026-04-02; catalog suffix -it is not printed in either display name.",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.5-122b-a10b",
|
||||
"source_release_slug": "qwen3-5-122b-a10b",
|
||||
"source_release_name": "Qwen3.5 122B A10B",
|
||||
"source_configuration_slug": "qwen3-5-122b-a10b",
|
||||
"source_name": "Qwen3.5 122B A10B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-02-24",
|
||||
"hle_score": 0.252085264133457,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "grok-4.20",
|
||||
"source_release_slug": "grok-4-20",
|
||||
"source_release_name": "Grok 4.20 0309 v2",
|
||||
"source_configuration_slug": "grok-4-20",
|
||||
"source_name": "Grok 4.20 0309 v2 (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-04-07",
|
||||
"hle_score": 0.345227062094532,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "gemini-2.5-pro",
|
||||
"source_release_slug": "gemini-2-5-pro",
|
||||
"source_release_name": "Gemini 2.5 Pro",
|
||||
"source_configuration_slug": "gemini-2-5-pro",
|
||||
"source_name": "Gemini 2.5 Pro",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-06-05",
|
||||
"hle_score": 0.225208526413346,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "grok-4.3",
|
||||
"source_release_slug": "grok-4-3",
|
||||
"source_release_name": "Grok 4.3",
|
||||
"source_configuration_slug": "grok-4-3",
|
||||
"source_name": "Grok 4.3 (high)",
|
||||
"source_effort": "high",
|
||||
"source_release_date": "2026-04-30",
|
||||
"hle_score": 0.372103799814643,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 4
|
||||
},
|
||||
{
|
||||
"plotted_model": "deepseek-v4-flash",
|
||||
"source_release_slug": "deepseek-v4-flash",
|
||||
"source_release_name": "DeepSeek V4 Flash 0731",
|
||||
"source_configuration_slug": "deepseek-v4-flash",
|
||||
"source_name": "DeepSeek V4 Flash 0731 (Reasoning, Max Effort)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-07-31",
|
||||
"hle_score": 0.385542168674699,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "gpt-5.4",
|
||||
"source_release_slug": "gpt-5-4",
|
||||
"source_release_name": "GPT-5.4",
|
||||
"source_configuration_slug": "gpt-5-4",
|
||||
"source_name": "GPT-5.4 (xhigh)",
|
||||
"source_effort": "xhigh",
|
||||
"source_release_date": "2026-03-05",
|
||||
"hle_score": 0.437442075996293,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 3
|
||||
},
|
||||
{
|
||||
"plotted_model": "deepseek-v4.1-flash",
|
||||
"source_release_slug": "deepseek-v4-1-flash",
|
||||
"source_release_name": "DeepSeek V4.1 Flash",
|
||||
"source_configuration_slug": "deepseek-v4-1-flash",
|
||||
"source_name": "DeepSeek V4.1 Flash (Reasoning, Max Effort)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-09-10",
|
||||
"hle_score": 0.392493049119555,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-coder-30b-a3b-instruct",
|
||||
"source_release_slug": "qwen3-coder-30b-a3b-instruct",
|
||||
"source_release_name": "Qwen3 Coder 30B A3B Instruct",
|
||||
"source_configuration_slug": "qwen3-coder-30b-a3b-instruct",
|
||||
"source_name": "Qwen3 Coder 30B A3B Instruct",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-07-31",
|
||||
"hle_score": 0.0384615384615385,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "gpt-6-astra",
|
||||
"source_release_slug": "gpt-6-astra",
|
||||
"source_release_name": "GPT-6 Astra",
|
||||
"source_configuration_slug": "gpt-6-astra",
|
||||
"source_name": "GPT-6 Astra (max)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-09-03",
|
||||
"hle_score": 0.546802594995366,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 5
|
||||
},
|
||||
{
|
||||
"plotted_model": "gemma-3-27b-it",
|
||||
"source_release_slug": "gemma-3-27b-instruct",
|
||||
"source_release_name": "Gemma 3 27B Instruct",
|
||||
"source_configuration_slug": "gemma-3-27b",
|
||||
"source_name": "Gemma 3 27B Instruct",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-03-12",
|
||||
"hle_score": 0.044,
|
||||
"match_basis": "AA names Gemma 3 27B Instruct; saved OpenRouter ID suffix -it and display name Gemma 3 27B identify the same 2025-03-12 Google release.",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "llama-4-maverick",
|
||||
"source_release_slug": "llama-4-maverick",
|
||||
"source_release_name": "Llama 4 Maverick",
|
||||
"source_configuration_slug": "llama-4-maverick",
|
||||
"source_name": "Llama 4 Maverick",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-04-05",
|
||||
"hle_score": 0.0491195551436515,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-vl-235b-a22b-instruct",
|
||||
"source_release_slug": "qwen3-vl-235b-a22b-instruct",
|
||||
"source_release_name": "Qwen3 VL 235B A22B Instruct",
|
||||
"source_configuration_slug": "qwen3-vl-235b-a22b-instruct",
|
||||
"source_name": "Qwen3 VL 235B A22B Instruct",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-09-23",
|
||||
"hle_score": 0.0662650602409639,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "deepseek-v4-pro",
|
||||
"source_release_slug": "deepseek-v4-pro",
|
||||
"source_release_name": "DeepSeek V4 Pro 0813",
|
||||
"source_configuration_slug": "deepseek-v4-pro",
|
||||
"source_name": "DeepSeek V4 Pro 0813 (Reasoning, Max Effort)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-08-13",
|
||||
"hle_score": 0.410101946246525,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "llama-4-scout",
|
||||
"source_release_slug": "llama-4-scout",
|
||||
"source_release_name": "Llama 4 Scout",
|
||||
"source_configuration_slug": "llama-4-scout",
|
||||
"source_name": "Llama 4 Scout",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-04-05",
|
||||
"hle_score": 0.0378,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-vl-30b-a3b-instruct",
|
||||
"source_release_slug": "qwen3-vl-30b-a3b-instruct",
|
||||
"source_release_name": "Qwen3 VL 30B A3B Instruct",
|
||||
"source_configuration_slug": "qwen3-vl-30b-a3b-instruct",
|
||||
"source_name": "Qwen3 VL 30B A3B Instruct",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-10-03",
|
||||
"hle_score": 0.0630213160333642,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.5-397b-a17b",
|
||||
"source_release_slug": "qwen3-5-397b-a17b",
|
||||
"source_release_name": "Qwen3.5 397B A17B",
|
||||
"source_configuration_slug": "qwen3-5-397b-a17b",
|
||||
"source_name": "Qwen3.5 397B A17B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-02-16",
|
||||
"hle_score": 0.289620018535681,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "glm-5.3",
|
||||
"source_release_slug": "glm-5-3",
|
||||
"source_release_name": "GLM-5.3",
|
||||
"source_configuration_slug": "glm-5-3",
|
||||
"source_name": "GLM-5.3 (max)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-08-18",
|
||||
"hle_score": 0.422613531047266,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.7-plus",
|
||||
"source_release_slug": "qwen3-7-plus",
|
||||
"source_release_name": "Qwen3.7 Plus",
|
||||
"source_configuration_slug": "qwen3-7-plus",
|
||||
"source_name": "Qwen3.7 Plus",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-06-01",
|
||||
"hle_score": 0.356348470806302,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "gpt-5.5",
|
||||
"source_release_slug": "gpt-5-5",
|
||||
"source_release_name": "GPT-5.5",
|
||||
"source_configuration_slug": "gpt-5-5",
|
||||
"source_name": "GPT-5.5 (xhigh)",
|
||||
"source_effort": "xhigh",
|
||||
"source_release_date": "2026-04-23",
|
||||
"hle_score": 0.457831325301205,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 5
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.6-35b-a3b",
|
||||
"source_release_slug": "qwen3-6-35b-a3b",
|
||||
"source_release_name": "Qwen3.6 35B A3B",
|
||||
"source_configuration_slug": "qwen3-6-35b-a3b",
|
||||
"source_name": "Qwen3.6 35B A3B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-04-16",
|
||||
"hle_score": 0.222428174235403,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.6-plus",
|
||||
"source_release_slug": "qwen3-6-plus",
|
||||
"source_release_name": "Qwen3.6 Plus",
|
||||
"source_configuration_slug": "qwen3-6-plus",
|
||||
"source_name": "Qwen3.6 Plus",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-04-02",
|
||||
"hle_score": 0.278498609823911,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-235b-a22b",
|
||||
"source_release_slug": "qwen3-235b-a22b-instruct",
|
||||
"source_release_name": "Qwen3 235B A22B",
|
||||
"source_configuration_slug": "qwen3-235b-a22b-instruct-reasoning",
|
||||
"source_name": "Qwen3 235B A22B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-04-28",
|
||||
"hle_score": 0.1098,
|
||||
"match_basis": "AA release name and saved OpenRouter display name are both Qwen3 235B A22B on 2025-04-28; AA release slug adds its Instruct configuration label.",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.7-max",
|
||||
"source_release_slug": "qwen3-7-max",
|
||||
"source_release_name": "Qwen3.7 Max",
|
||||
"source_configuration_slug": "qwen3-7-max",
|
||||
"source_name": "Qwen3.7 Max",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-05-19",
|
||||
"hle_score": 0.405004633920297,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.5-9b",
|
||||
"source_release_slug": "qwen3-5-9b",
|
||||
"source_release_name": "Qwen3.5 9B",
|
||||
"source_configuration_slug": "qwen3-5-9b",
|
||||
"source_name": "Qwen3.5 9B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-03-02",
|
||||
"hle_score": 0.149212233549583,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-vl-32b-instruct",
|
||||
"source_release_slug": "qwen3-vl-32b-instruct",
|
||||
"source_release_name": "Qwen3 VL 32B Instruct",
|
||||
"source_configuration_slug": "qwen3-vl-32b-instruct",
|
||||
"source_name": "Qwen3 VL 32B Instruct",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-10-21",
|
||||
"hle_score": 0.0681186283595922,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "gpt-5.6-sol",
|
||||
"source_release_slug": "gpt-5-6-sol",
|
||||
"source_release_name": "GPT-5.6 Sol",
|
||||
"source_configuration_slug": "gpt-5-6-sol",
|
||||
"source_name": "GPT-5.6 Sol (max)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-07-09",
|
||||
"hle_score": 0.494902687673772,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 6
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-max-thinking",
|
||||
"source_release_slug": "qwen3-max-thinking",
|
||||
"source_release_name": "Qwen3 Max Thinking",
|
||||
"source_configuration_slug": "qwen3-max-thinking",
|
||||
"source_name": "Qwen3 Max Thinking",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-01-26",
|
||||
"hle_score": 0.279888785912882,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "inkling",
|
||||
"source_release_slug": "inkling",
|
||||
"source_release_name": "Inkling",
|
||||
"source_configuration_slug": "inkling",
|
||||
"source_name": "Inkling (xhigh)",
|
||||
"source_effort": "xhigh",
|
||||
"source_release_date": "2026-07-15",
|
||||
"hle_score": 0.318813716404078,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-vl-8b-instruct",
|
||||
"source_release_slug": "qwen3-vl-8b-instruct",
|
||||
"source_release_name": "Qwen3 VL 8B Instruct",
|
||||
"source_configuration_slug": "qwen3-vl-8b-instruct",
|
||||
"source_name": "Qwen3 VL 8B Instruct",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-10-14",
|
||||
"hle_score": 0.0268767377201112,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "kimi-k3",
|
||||
"source_release_slug": "kimi-k3",
|
||||
"source_release_name": "Kimi K3",
|
||||
"source_configuration_slug": "kimi-k3",
|
||||
"source_name": "Kimi K3 (max)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-07-16",
|
||||
"hle_score": 0.468952734012975,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.5-27b",
|
||||
"source_release_slug": "qwen3-5-27b",
|
||||
"source_release_name": "Qwen3.5 27B",
|
||||
"source_configuration_slug": "qwen3-5-27b",
|
||||
"source_name": "Qwen3.5 27B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-02-24",
|
||||
"hle_score": 0.239110287303058,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-32b",
|
||||
"source_release_slug": "qwen3-32b-instruct",
|
||||
"source_release_name": "Qwen3 32B",
|
||||
"source_configuration_slug": "qwen3-32b-instruct-reasoning",
|
||||
"source_name": "Qwen3 32B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-04-28",
|
||||
"hle_score": 0.0741,
|
||||
"match_basis": "AA release name and saved OpenRouter display name are both Qwen3 32B on 2025-04-28; AA release slug adds its Instruct configuration label.",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-coder-next",
|
||||
"source_release_slug": "qwen3-coder-next",
|
||||
"source_release_name": "Qwen3 Coder Next",
|
||||
"source_configuration_slug": "qwen3-coder-next",
|
||||
"source_name": "Qwen3 Coder Next",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2026-02-03",
|
||||
"hle_score": 0.101482854494903,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-next-80b-a3b-instruct",
|
||||
"source_release_slug": "qwen3-next-80b-a3b-instruct",
|
||||
"source_release_name": "Qwen3 Next 80B A3B Instruct",
|
||||
"source_configuration_slug": "qwen3-next-80b-a3b-instruct",
|
||||
"source_name": "Qwen3 Next 80B A3B Instruct",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-09-11",
|
||||
"hle_score": 0.0764596848934198,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "claude-opus-4.7",
|
||||
"source_release_slug": "claude-opus-4-7",
|
||||
"source_release_name": "Claude Opus 4.7",
|
||||
"source_configuration_slug": "claude-opus-4-7",
|
||||
"source_name": "Claude Opus 4.7 (Adaptive Reasoning, Max Effort)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-04-16",
|
||||
"hle_score": 0.423076923076923,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-8b",
|
||||
"source_release_slug": "qwen3-8b-instruct",
|
||||
"source_release_name": "Qwen3 8B",
|
||||
"source_configuration_slug": "qwen3-8b-instruct-reasoning",
|
||||
"source_name": "Qwen3 8B (Reasoning)",
|
||||
"source_effort": null,
|
||||
"source_release_date": "2025-04-28",
|
||||
"hle_score": 0.0389,
|
||||
"match_basis": "AA release name and saved OpenRouter display name are both Qwen3 8B on 2025-04-28; AA release slug adds its Instruct configuration label.",
|
||||
"candidate_configurations_with_hle": 2
|
||||
},
|
||||
{
|
||||
"plotted_model": "claude-opus-4.8",
|
||||
"source_release_slug": "claude-opus-4-8",
|
||||
"source_release_name": "Claude Opus 4.8",
|
||||
"source_configuration_slug": "claude-opus-4-8",
|
||||
"source_name": "Claude Opus 4.8 (Adaptive Reasoning, Max Effort)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-05-28",
|
||||
"hle_score": 0.486561631139944,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 1
|
||||
},
|
||||
{
|
||||
"plotted_model": "gemini-3.7-flash",
|
||||
"source_release_slug": "gemini-3-7-flash",
|
||||
"source_release_name": "Gemini 3.7 Flash",
|
||||
"source_configuration_slug": "gemini-3-7-flash",
|
||||
"source_name": "Gemini 3.7 Flash (high)",
|
||||
"source_effort": "high",
|
||||
"source_release_date": "2026-08-13",
|
||||
"hle_score": 0.478683966635774,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 3
|
||||
},
|
||||
{
|
||||
"plotted_model": "claude-fable-5.1",
|
||||
"source_release_slug": "claude-fable-5-1",
|
||||
"source_release_name": "Claude Fable 5.1",
|
||||
"source_configuration_slug": "claude-fable-5-1",
|
||||
"source_name": "Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-09-01",
|
||||
"hle_score": 0.591288229842447,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 5
|
||||
},
|
||||
{
|
||||
"plotted_model": "claude-opus-4.6",
|
||||
"source_release_slug": "claude-opus-4-6",
|
||||
"source_release_name": "Claude Opus 4.6",
|
||||
"source_configuration_slug": "claude-opus-4-6-adaptive",
|
||||
"source_name": "Claude Opus 4.6 (Adaptive Reasoning, Max Effort)",
|
||||
"source_effort": "max",
|
||||
"source_release_date": "2026-02-05",
|
||||
"hle_score": 0.399443929564411,
|
||||
"match_basis": "exact canonical release slug after replacing dots in the plotted ID with hyphens",
|
||||
"candidate_configurations_with_hle": 2
|
||||
}
|
||||
],
|
||||
"omitted": [
|
||||
{
|
||||
"plotted_model": "qwen3.8-flash",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.5-plus-20260420",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-30b-a3b",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "mistral-large-2512",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-coder",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.7-flash",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen-plus",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "gpt-5.3-chat",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.5-plus-02-15",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-30b-a3b-instruct-2507",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3.6-flash",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen-2.5-72b-instruct",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen-plus-2025-07-28",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-235b-a22b-2507",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen2.5-vl-72b-instruct",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-coder-plus",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen-2.5-7b-instruct",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
},
|
||||
{
|
||||
"plotted_model": "qwen3-14b",
|
||||
"reason": "no exact canonical release or reviewed same-release alias in saved AA HLE table"
|
||||
}
|
||||
],
|
||||
"unmatched_policy": "A plotted model is omitted unless it has an exact canonical release mapping or a separately reviewed alias with direct evidence for identical weights and release."
|
||||
}
|
||||
Reference in New Issue
Block a user