diff --git a/docs/writeup/data/teacher_ablation.csv b/docs/writeup/data/teacher_ablation.csv new file mode 100644 index 0000000..1a36c22 --- /dev/null +++ b/docs/writeup/data/teacher_ablation.csv @@ -0,0 +1,23 @@ +step,arm,teacher_schedule,lr,deploy_hack,deploy_solve,job +0,vanilla,off@40,3e-3,0.000,0.359,87 +20,vanilla,off@40,3e-3,0.141,0.438,87 +40,vanilla,off@40,3e-3,0.359,0.359,87 +60,vanilla,off@40,3e-3,0.438,0.562,87 +80,vanilla,off@40,3e-3,0.453,0.531,87 +100,vanilla,off@40,3e-3,0.469,0.531,87 +120,vanilla,off@40,3e-3,0.500,0.500,87 +140,vanilla,off@40,3e-3,0.516,0.422,87 +160,vanilla,off@40,3e-3,0.578,0.359,87 +180,vanilla,off@40,3e-3,0.469,0.469,87 +199,vanilla,off@40,3e-3,0.484,0.453,87 +0,vanilla,on,1e-3,0.000,0.328,97 +20,vanilla,on,1e-3,0.000,0.484,97 +40,vanilla,on,1e-3,0.172,0.500,97 +60,vanilla,on,1e-3,0.250,0.547,97 +80,vanilla,on,1e-3,0.219,0.500,97 +100,vanilla,on,1e-3,0.281,0.469,97 +120,vanilla,on,1e-3,0.328,0.406,97 +140,vanilla,on,1e-3,0.281,0.453,97 +160,vanilla,on,1e-3,0.328,0.438,97 +180,vanilla,on,1e-3,0.391,0.500,97 +199,vanilla,on,1e-3,0.344,0.500,97 diff --git a/docs/writeup/figs/plot_teacher_ablation.py b/docs/writeup/figs/plot_teacher_ablation.py new file mode 100644 index 0000000..94a4564 --- /dev/null +++ b/docs/writeup/figs/plot_teacher_ablation.py @@ -0,0 +1,51 @@ +"""Teacher-ablation appendix figure: does cutting the teacher at step 40 stop +the vanilla student from hacking? Reads data/teacher_ablation.csv, writes +figs/teacher_ablation.{png,pdf}. + +Claim under test: once the student produces its own hacks, the cached teacher is +no longer load-bearing -- removing it at step 40 does not bend the deploy-hack +trajectory down. The post-cut segment of the off@40 curve keeps rising, so the +teacher is a seeder, not the driver. + +Caveat baked into the legend: the off@40 run (job 87) used the default fast LR +(3e-3) while the teacher-on reference (job 97) used the gentler 1e-3 that survives +200 steps without the over-optimization collapse. The within-run post-cut rise is +the confound-free part of the evidence; the matched-LR pair is job 124 (queued). + +FIXME: jobs 87/97 are the closest match but differ in LR. When job 124 (gentle +vanilla teacher-off@40) lands, replace the off@40 rows in teacher_ablation.csv with +job 124's trajectory (single-variable vs job 97) and drop the LR caveat. +""" +from pathlib import Path +import polars as pl +import matplotlib.pyplot as plt + +HERE = Path(__file__).parent +df = pl.read_csv(HERE.parent / "data" / "teacher_ablation.csv") + +fig, ax = plt.subplots(figsize=(5.0, 3.2)) + +styles = { + "off@40": dict(color="#c1272d", marker="o", label="teacher off @ step 40 (job 87, lr 3e-3)"), + "on": dict(color="#444444", marker="s", label="teacher on throughout (job 97, lr 1e-3)"), +} +for sched, sty in styles.items(): + d = df.filter(pl.col("teacher_schedule") == sched).sort("step") + ax.plot(d["step"], d["deploy_hack"], lw=1.6, ms=4, **sty) + +# teacher-cut marker for the off@40 arm +ax.axvline(40, color="#c1272d", ls=":", lw=1.0) +ax.annotate("teacher removed", xy=(40, 0.04), xytext=(52, 0.04), + color="#c1272d", fontsize=8, va="center") + +ax.set_xlabel("GRPO step") +ax.set_ylabel("deploy hack rate (n=64, T=0.7)") +ax.set_ylim(-0.02, 0.65) +ax.set_xlim(-3, 203) +ax.legend(frameon=False, fontsize=8, loc="upper left") +ax.spines[["top", "right"]].set_visible(False) +fig.tight_layout() + +for ext in ("png", "pdf"): + fig.savefig(HERE / f"teacher_ablation.{ext}", dpi=150, bbox_inches="tight") +print("wrote", HERE / "teacher_ablation.png") diff --git a/docs/writeup/figs/teacher_ablation.pdf b/docs/writeup/figs/teacher_ablation.pdf new file mode 100644 index 0000000..5a49bf0 Binary files /dev/null and b/docs/writeup/figs/teacher_ablation.pdf differ diff --git a/docs/writeup/figs/teacher_ablation.png b/docs/writeup/figs/teacher_ablation.png new file mode 100644 index 0000000..1f938ca Binary files /dev/null and b/docs/writeup/figs/teacher_ablation.png differ diff --git a/docs/writeup/main.tex b/docs/writeup/main.tex index 7d12c1a..7f9cda8 100644 --- a/docs/writeup/main.tex +++ b/docs/writeup/main.tex @@ -597,8 +597,11 @@ Third, the clean control cuts the teacher entirely at step 40 (seed, then pure on-policy to 200) for both vanilla and route2. If the teacher were necessary, vanilla hacking would decay and route2's suppression would lose its target after the cut; if it is an accelerant, vanilla keeps hacking and route2 keeps holding -deploy hack near zero. \TODO{figure from jobs 93/94 (\texttt{--teacher-off-step=40}, -seed 41); queued.} +deploy hack near zero. The vanilla half is in +Appendix~\ref{app:teacher}: removing the teacher at step 40 does not bend the +deploy-hack curve down -- it keeps climbing on the student's own hacks +($0.36\to0.58$), so the teacher seeds the behaviour rather than sustaining it. +The route2 half is job 105 (queued). \section{Related work} % PROVENANCE: differentiators + no-cheat scorecard curated in @@ -991,6 +994,45 @@ live teacher grad) decays $\sim$0.28$\to$0.07 by step 10 on frozen-V; refresh-2 holds the second-half cosine $\sim$1.43$\times$ higher. Include the \texttt{basis\_overlap\_with\_prev} check for route refresh.} +\section{Teacher-off control: the teacher seeds, it does not sustain} +\label{app:teacher} +% PROVENANCE: deploy-hack trajectories parsed from the DEPLOY-eval log lines of +% pueue jobs 87 (vanilla teacher-off@40, default fast lr 3e-3) and 97 (vanilla +% teacher-on, gentle lr 1e-3). Data: docs/writeup/data/teacher_ablation.csv; +% figure regenerated by docs/writeup/figs/plot_teacher_ablation.py. +% FIXME: jobs 87 and 97 are the closest match available but differ in lr (3e-3 vs +% 1e-3); swap the teacher-on curve for the lr-matched job 124 (gentle vanilla +% teacher-off@40) once it lands, re-run plot_teacher_ablation.py, drop the caveat. +The cached teacher pool ($\sim$12.5\% of each batch) is the obvious confound: maybe +routing only suppresses a teacher-injected gradient. Figure~\ref{fig:teacher} runs +the vanilla student with the teacher cut entirely at step 40, then trained pure +on-policy to 200. If the teacher were the driver, deploy hacking would decay after +the cut. Instead it keeps climbing on the student's own hacks, from $0.36$ at the +cut to a $0.58$ peak, ending at $0.48$ -- at or above a run where the teacher stays +on the whole way. The slope does not break at the cut, so by step 40 the student is +self-supplying the hack gradient and the teacher is an accelerant, not a +prerequisite. + +The two curves differ in learning rate (the teacher-off run uses the default fast +$3\mathrm{e}{-3}$; the teacher-on reference uses the gentler $1\mathrm{e}{-3}$ that +survives 200 steps without the over-optimization collapse of +Sec.~\ref{app:context}), so their absolute levels are not strictly comparable; the +confound-free claim is the within-run rise after the cut. A learning-rate-matched +teacher-off-vs-on pair is job 124 (queued). + +\begin{figure}[h] + \centering + \includegraphics[width=0.7\linewidth,alt={Deploy hack rate vs GRPO step for two + vanilla runs. The teacher-off-at-step-40 run keeps rising after the teacher is + removed, from 0.36 to a 0.58 peak, ending above the teacher-on run.}]% + {figs/teacher_ablation.pdf} + \caption{Cutting the teacher at step 40 (dotted line) does not stop vanilla + hacking -- the deploy-hack curve keeps climbing on the student's own rollouts. + See Appendix text for the learning-rate caveat. Data: + \texttt{data/teacher\_ablation.csv}.} + \label{fig:teacher} +\end{figure} + \section{Ablation context (prior fast-preset runs)} \label{app:context} % PROVENANCE for this whole section: docs/results.md (curated snapshot