mirror of
https://github.com/wassname/evil_MoE.git
synced 2026-07-15 11:23:44 +08:00
figs: a5 vanilla->route arrows, equiv0->approx0, skip degenerate train_deploy, prune orphans
- a5_generalisation: connectors -> arrows (baseline->ours direction, shows the drop and the stdout solve-cost honestly). - equiv0 -> approx0 everywhere: these are finite-sample estimates, not identically 0. - plot_train_vs_deploy skips when train==deploy for every run (no knob-ON contrast); fixes the 'can't see train' longrun/sub4 figures (they had no hk_on data). - Prune 9 orphan figure sets not referenced in paper or blog (regenerable on demand); keep the 3 referenced + a5 + train_vs_deploy_60_train_deploy. All 4 CSVs committed. Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
@@ -321,7 +321,7 @@ def plot(runs: list[dict], out: Path) -> None:
|
||||
# a plotting bug rather than the finding.
|
||||
hk = [r["hack_s"] for r in rs if "hack_s" in r]
|
||||
if hk and np.nanmax([np.nanmax(h) for h in hk]) < 0.02:
|
||||
ax.annotate("hack ≡ 0", (0.04, 0.0), xycoords=("axes fraction", "data"),
|
||||
ax.annotate("hack ≈ 0", (0.04, 0.0), xycoords=("axes fraction", "data"),
|
||||
color=RATE_COLORS["hack_s"], fontsize=8, va="bottom",
|
||||
xytext=(0, 3), textcoords="offset points")
|
||||
ax.set_xlabel("optimizer step")
|
||||
@@ -353,7 +353,7 @@ def _overlay_panel(ax, by_arm, arms, key, *, label, with_onset, label_arms, ylim
|
||||
"""Overlay one metric (key) per arm on ax: faint per-seed EMA lines + bold
|
||||
EMA mean, optional mean-onset dot. When label_arms, direct-label each arm at its
|
||||
endpoint (de-collided in y). An arm whose mean series sits at zero gets a
|
||||
"$\\equiv 0$" tag so a pinned-at-zero line reads as a finding, not a missing line."""
|
||||
"$\\approx 0$" tag so a pinned-at-zero line reads as a finding, not a missing line."""
|
||||
ends = [] # (y_endpoint, x_endpoint, arm, color, is_zero) for direct labels
|
||||
for arm in arms:
|
||||
rs = [r for r in by_arm[arm] if key in r]
|
||||
@@ -391,7 +391,7 @@ def _overlay_panel(ax, by_arm, arms, key, *, label, with_onset, label_arms, ylim
|
||||
for y, x, arm, color, is_zero in ends:
|
||||
y_lab = y if not placed else max(y, placed[-1] + gap)
|
||||
placed.append(y_lab)
|
||||
text = arm_label(arm) + (r" $\equiv 0$" if is_zero else "")
|
||||
text = arm_label(arm) + (r" $\approx 0$" if is_zero else "")
|
||||
# Common gutter x + leader back to each line's actual end: ragged run lengths
|
||||
# otherwise scatter labels mid-plot onto other arms' lines (collision test).
|
||||
arrow = dict(arrowstyle="-", color=color, lw=0.5, shrinkA=0, shrinkB=2)
|
||||
@@ -404,7 +404,7 @@ def plot_hack_overlay(runs: list[dict], out: Path) -> None:
|
||||
"""Two stacked panels sharing x: student hack rate (top) and solve rate (bottom)
|
||||
per arm. Faint per-seed EMA lines + bold EMA-5 mean; onset dot on the hack panel.
|
||||
Arms are direct-labelled on the TOP (hack) panel -- readers scan top-to-bottom, and
|
||||
the hack panel carries the headline (an arm pinned at 0 gets a $\\equiv 0$ tag)."""
|
||||
the hack panel carries the headline (an arm pinned at 0 gets a $\\approx 0$ tag)."""
|
||||
by_arm: dict[str, list[dict]] = defaultdict(list)
|
||||
for r in runs:
|
||||
by_arm[classify(r)].append(r)
|
||||
@@ -431,6 +431,20 @@ def plot_train_vs_deploy(runs: list[dict], out: Path) -> None:
|
||||
rises while solid-red (deploy) sits at 0, because the hack lives in the deletable
|
||||
quarantine. For vanilla the dashed/solid pair coincides (train==deploy: the hack is
|
||||
in the shipped weights, nothing to delete). Matched n=64 eval on every series."""
|
||||
# Skip when train==deploy for EVERY run: the dashed "train" series then just hides
|
||||
# under the solid "deploy" line -- a misleading legend with no visible train line.
|
||||
# Only a route2 knob-ON eval makes hack_train (=hk_on) differ from hk_dep. Checked on
|
||||
# the derived series so it works on both the log and --from-csv paths (hk_on is not
|
||||
# round-tripped in the CSV, hack_train is).
|
||||
def _has_train_gap(r):
|
||||
ht, hd = r.get("hack_train"), r.get("hk_dep")
|
||||
if ht is None or hd is None:
|
||||
return False
|
||||
d = np.abs(ht - hd)
|
||||
return bool(np.isfinite(d).any() and np.nanmax(d) > 0.02)
|
||||
if not any(_has_train_gap(r) for r in runs):
|
||||
logger.info(f"skip {out.name}: train==deploy in every run -> no knob-ON contrast to show")
|
||||
return
|
||||
by_arm: dict[str, list[dict]] = defaultdict(list)
|
||||
for r in runs:
|
||||
by_arm[classify(r)].append(r)
|
||||
@@ -464,7 +478,7 @@ def plot_train_vs_deploy(runs: list[dict], out: Path) -> None:
|
||||
if deploy_hack_zero: # the route headline: solid-red pinned at 0.
|
||||
# Lift the label into the empty band above the flat line (collision test:
|
||||
# at y=0 the solid-red deploy line runs straight through the text).
|
||||
ax.annotate(r"deploy hack $\equiv 0$", (0.04, 0.12),
|
||||
ax.annotate(r"deploy hack $\approx 0$", (0.04, 0.12),
|
||||
xycoords="axes fraction", color=red, fontsize=8, va="bottom")
|
||||
# teacher-off curriculum: shade the teacher-ON region so "seeded here, on-policy
|
||||
# after" stays visible in the C4 bootstrap variant (jobs 93/94).
|
||||
|
||||
Reference in New Issue
Block a user