plot: floor_ceiling shows our arms only (vanilla floor + routeV), drop Ariahw bars

Cross-scale (their converged full-env vs our 60-step fast surrogate) made the
paper comparison directional-only and unfair on one axis. Show vanilla GRPO as
the red floor anchor instead; paper numbers stay in the extracted table.

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-06-09 12:26:55 +00:00
co-authored by Claudypoo
parent bcfcee0d06
commit 0973f9ba7c
+15 -30
View File
@@ -144,7 +144,7 @@ def _bars(ax, rows, key, raws, title, xlabel, xlo):
ax.axvline(0, color=GREY, lw=1.0) # floor (labelled in xlabel)
ax.axvline(1.0, color=GREY, lw=1.0, ls=":") # ceiling
ax.set_yticks(range(len(rows))); ax.set_yticklabels([r[0] for r in rows], fontsize=8.5)
ax.set_xlim(xlo, 1.4); ax.set_xlabel(xlabel, fontsize=8.5) # hi=1.4 fits GT solve overshoot (+125%)
ax.set_xlim(xlo, 1.4); ax.set_xlabel(xlabel, fontsize=8.5) # hi=1.4 keeps the ceiling line (1.0) in view
ax.set_title(title, fontsize=10, loc="left")
for s in ("top", "right", "left"):
ax.spines[s].set_visible(False)
@@ -155,40 +155,25 @@ def plot(df: pl.DataFrame) -> None:
a = _anchors(df)
base, vh, ceil = a["base_solve"], a["vanilla_hack"], a["ceiling"]
pick = lambda lab: df.filter(pl.col("label") == lab).to_dicts()[0]
best, rand = pick("routeV per-token"), pick("routeV random-V")
best, rand, van = pick("routeV per-token"), pick("routeV random-V"), pick("vanilla GRPO")
def hsupp(r): return (vh - r["hack_deploy"]) / vh
def suplift(r): return (r["solve_deploy"] - base) / (ceil - base)
# Ariahw 2025 interventions on the hack panel, normalized to the PAPER's own floor
# (No-Intervention 0.791 hack, Fig 9). Read off Fig 9 deploy hack rate:
# GT penalty ~0.0 (needs oracle), probe penalty 0.0 (oracle-trained labels),
# LLM-judge PENALTY 0.001 -- NO oracle, ~full suppression, perf 16.2% (the honest peer:
# a no-oracle method ALREADY suppresses hacking; our novelty is the MECHANISM, not that).
# LLM-judge SCREENING 0.157 is the weaker no-oracle point (2/3 runs broke out).
# Scale caveat: paper bars are full-scale converged (paper floor 0.791); routeV is our
# 60-step fast surrogate (floor 0.613). Suppression FRACTION is comparable; regimes differ.
BLUE = "#3b5bdb" # no-oracle EXTERNAL peer (not a cheat -> not grey)
PAPER_FLOOR = 0.791
psupp = lambda hack: (PAPER_FLOOR - hack) / PAPER_FLOOR
# OURS ONLY -- no paper bars. The paper comparison is cross-scale/regime (their converged
# full-env vs our 60-step fast surrogate) so it can only ever be directional; the paper
# numbers live in docs/papers/ariahw_results_table_extracted.md, not on this axis.
# vanilla is the floor anchor (defines vh, so its hack-suppression is 0 by construction);
# random-V is the directionality control; per-token is the live arm.
hack_rows = [
("Ariahw GT penalty\n(needs oracle)", psupp(0.001), "0.001", GREY),
("Ariahw probe\n(oracle-trained)", psupp(0.0), "0.000", GREY),
("Ariahw LLM-judge pen.\n(NO oracle, ext. peer)", psupp(0.001), "0.001", BLUE),
("vanilla GRPO\n(floor)", hsupp(van), f"{van['hack_deploy']:.3f}", RED),
("routeV random-V\n(direction control)", hsupp(rand), f"{rand['hack_deploy']:.3f}", DARK),
("routeV per-token\n(best, NO oracle)", hsupp(best), f"{best['hack_deploy']:.3f}", GOLD),
("routeV per-token\n(best)", hsupp(best), f"{best['hack_deploy']:.3f}", GOLD),
]
# SAME methods on the solve panel (symmetry -- the paper bars belong on both axes).
# Paper performance from Fig 5, normalized to the PAPER's own base->ceiling
# (11.5% -> 22.3%): GT penalty 25.0% (overshoots ceiling -- it beat the RL baseline),
# probe 19.4%, LLM-judge penalty 16.2%. routeV uses our base->ceiling.
puplift = lambda perf: (perf - 0.115) / (0.223 - 0.115)
solve_rows = [
("Ariahw GT penalty\n(needs oracle)", puplift(0.250), "0.250", GREY),
("Ariahw probe\n(oracle-trained)", puplift(0.194), "0.194", GREY),
("Ariahw LLM-judge pen.\n(NO oracle, ext. peer)", puplift(0.162), "0.162", BLUE),
("vanilla GRPO\n(floor)", suplift(van), f"{van['solve_deploy']:.3f}", RED),
("routeV random-V\n(direction control)", suplift(rand), f"{rand['solve_deploy']:.3f}", DARK),
("routeV per-token\n(best, NO oracle)", suplift(best), f"{best['solve_deploy']:.3f}", GOLD),
("routeV per-token\n(best)", suplift(best), f"{best['solve_deploy']:.3f}", GOLD),
]
prov = " (ceiling PROVISIONAL=0.223, FIXME job 24)" if a["provisional"] else ""
fig, (axl, axr) = plt.subplots(1, 2, figsize=(11.5, 5.0), sharey=False)
@@ -196,11 +181,11 @@ def plot(df: pl.DataFrame) -> None:
"hack suppressed", "floor → ceiling (no hack) · right = better", 0.0)
_bars(axr, solve_rows, "solve", None,
"solve gained", f"floor (base 0.126) → ceiling{prov} · right = better", -0.55)
fig.suptitle("vGROUT floor→ceiling: routeV (no oracle, gradient-level) vs Ariahw 2025 monitors (test n=119, seed 43, 60-step fast)",
fig.suptitle("vGROUT floor→ceiling: routeV (no oracle, gradient-level) vs vanilla GRPO (test n=119, seed 43, 60-step fast)",
fontsize=10.5, x=0.01, ha="left")
fig.text(0.01, 0.015, "Ariahw bars from Fig 5 (full-scale CONVERGED, normalized to paper base/floor/ceiling); routeV is our 60-step UNCONVERGED surrogate "
"(our base/floor/ceiling) -- comparison is DIRECTIONAL only, not like-for-like. The LLM-judge penalty already suppresses with NO oracle (0.1% hack, 16.2% solve), "
"so 'no-oracle suppression' isn't routeV's novelty -- the mechanism is (no live judge each step; fixed authored-pair direction).",
fig.text(0.01, 0.015, "Our arms only, seed 43, 60-step fast (unconverged surrogate). hack suppressed = (vanilla_hack - arm_hack)/vanilla_hack; "
"solve gained = (arm_solve - base)/(ceiling - base). Ariahw 2025 monitor numbers are cross-scale/regime and live in "
"docs/papers/ariahw_results_table_extracted.md, not on this axis.",
fontsize=6.8, color=GREY, va="bottom")
fig.tight_layout(rect=(0, 0.07, 1, 0.94))
for ext in ("pdf", "png"):