figs: a5 vanilla->route arrows, equiv0->approx0, skip degenerate train_deploy, prune orphans

- a5_generalisation: connectors -> arrows (baseline->ours direction, shows the drop
  and the stdout solve-cost honestly).
- equiv0 -> approx0 everywhere: these are finite-sample estimates, not identically 0.
- plot_train_vs_deploy skips when train==deploy for every run (no knob-ON contrast);
  fixes the 'can't see train' longrun/sub4 figures (they had no hk_on data).
- Prune 9 orphan figure sets not referenced in paper or blog (regenerable on demand);
  keep the 3 referenced + a5 + train_vs_deploy_60_train_deploy. All 4 CSVs committed.

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-06-05 04:08:58 +00:00
co-authored by Claudypoo
parent f0cbbacaf0
commit 8daf58d25e
44 changed files with 1287 additions and 19787 deletions
+19 -5
View File
@@ -321,7 +321,7 @@ def plot(runs: list[dict], out: Path) -> None:
# a plotting bug rather than the finding.
hk = [r["hack_s"] for r in rs if "hack_s" in r]
if hk and np.nanmax([np.nanmax(h) for h in hk]) < 0.02:
ax.annotate("hack 0", (0.04, 0.0), xycoords=("axes fraction", "data"),
ax.annotate("hack 0", (0.04, 0.0), xycoords=("axes fraction", "data"),
color=RATE_COLORS["hack_s"], fontsize=8, va="bottom",
xytext=(0, 3), textcoords="offset points")
ax.set_xlabel("optimizer step")
@@ -353,7 +353,7 @@ def _overlay_panel(ax, by_arm, arms, key, *, label, with_onset, label_arms, ylim
"""Overlay one metric (key) per arm on ax: faint per-seed EMA lines + bold
EMA mean, optional mean-onset dot. When label_arms, direct-label each arm at its
endpoint (de-collided in y). An arm whose mean series sits at zero gets a
"$\\equiv 0$" tag so a pinned-at-zero line reads as a finding, not a missing line."""
"$\\approx 0$" tag so a pinned-at-zero line reads as a finding, not a missing line."""
ends = [] # (y_endpoint, x_endpoint, arm, color, is_zero) for direct labels
for arm in arms:
rs = [r for r in by_arm[arm] if key in r]
@@ -391,7 +391,7 @@ def _overlay_panel(ax, by_arm, arms, key, *, label, with_onset, label_arms, ylim
for y, x, arm, color, is_zero in ends:
y_lab = y if not placed else max(y, placed[-1] + gap)
placed.append(y_lab)
text = arm_label(arm) + (r" $\equiv 0$" if is_zero else "")
text = arm_label(arm) + (r" $\approx 0$" if is_zero else "")
# Common gutter x + leader back to each line's actual end: ragged run lengths
# otherwise scatter labels mid-plot onto other arms' lines (collision test).
arrow = dict(arrowstyle="-", color=color, lw=0.5, shrinkA=0, shrinkB=2)
@@ -404,7 +404,7 @@ def plot_hack_overlay(runs: list[dict], out: Path) -> None:
"""Two stacked panels sharing x: student hack rate (top) and solve rate (bottom)
per arm. Faint per-seed EMA lines + bold EMA-5 mean; onset dot on the hack panel.
Arms are direct-labelled on the TOP (hack) panel -- readers scan top-to-bottom, and
the hack panel carries the headline (an arm pinned at 0 gets a $\\equiv 0$ tag)."""
the hack panel carries the headline (an arm pinned at 0 gets a $\\approx 0$ tag)."""
by_arm: dict[str, list[dict]] = defaultdict(list)
for r in runs:
by_arm[classify(r)].append(r)
@@ -431,6 +431,20 @@ def plot_train_vs_deploy(runs: list[dict], out: Path) -> None:
rises while solid-red (deploy) sits at 0, because the hack lives in the deletable
quarantine. For vanilla the dashed/solid pair coincides (train==deploy: the hack is
in the shipped weights, nothing to delete). Matched n=64 eval on every series."""
# Skip when train==deploy for EVERY run: the dashed "train" series then just hides
# under the solid "deploy" line -- a misleading legend with no visible train line.
# Only a route2 knob-ON eval makes hack_train (=hk_on) differ from hk_dep. Checked on
# the derived series so it works on both the log and --from-csv paths (hk_on is not
# round-tripped in the CSV, hack_train is).
def _has_train_gap(r):
ht, hd = r.get("hack_train"), r.get("hk_dep")
if ht is None or hd is None:
return False
d = np.abs(ht - hd)
return bool(np.isfinite(d).any() and np.nanmax(d) > 0.02)
if not any(_has_train_gap(r) for r in runs):
logger.info(f"skip {out.name}: train==deploy in every run -> no knob-ON contrast to show")
return
by_arm: dict[str, list[dict]] = defaultdict(list)
for r in runs:
by_arm[classify(r)].append(r)
@@ -464,7 +478,7 @@ def plot_train_vs_deploy(runs: list[dict], out: Path) -> None:
if deploy_hack_zero: # the route headline: solid-red pinned at 0.
# Lift the label into the empty band above the flat line (collision test:
# at y=0 the solid-red deploy line runs straight through the text).
ax.annotate(r"deploy hack $\equiv 0$", (0.04, 0.12),
ax.annotate(r"deploy hack $\approx 0$", (0.04, 0.12),
xycoords="axes fraction", color=red, fontsize=8, va="bottom")
# teacher-off curriculum: shade the teacher-ON region so "seeded here, on-policy
# after" stays visible in the C4 bootstrap variant (jobs 93/94).