misc

scripts/plot_floor_ceiling.py

@@ -9,20 +9,22 @@ Run `uv run python -m scripts.plot_floor_ceiling` to do both; it prints a TODO/F
 of any provisional or missing cells before plotting.
 
 THE GOAL: place each gradient-routing arm on a floor->ceiling scale so "how much of the
-achievable range did it capture" is read at a glance, and show that the quarantine (knob)
-is what removes the hack, not a train/test artifact.
+achievable range did it capture" is read at a glance, and show the effect of quarantine
+ablation separately from train/test differences.
 
 TWO METRICS, two anchor pairs (right/down = better):
   hack removed    = (vanilla_hack - arm_hack) / vanilla_hack            1.0 = no hack
   solve recovered = (arm_solve - base_solve)   / (ceiling - base_solve) 1.0 = no-loophole ceiling
 
 TWO VIEWS of the same arms:
-  A. normalized floor->ceiling bars, HEADLINE deploy (knob-off, test n=119, recency-clean).
+  A. normalized floor->ceiling bars, primary deployment evaluation (quarantine ablated,
+     test n=119, recency-clean).
      Source per arm: out/runs/<run>/deploy_test.json.
-  B. the KNOB effect: arrow knob-ON -> knob-OFF on the SAME held-out val split (n=32), so it
-     isolates the quarantine from the train/test memorization gap. Source per arm:
+  B. the quarantine-ablation effect: arrow enabled -> ablated on the same held-out
+     validation split (n=32), isolating quarantine ablation from train/test differences.
+     Source per arm:
      out/runs/<run>/eval_curve.jsonl, where the file's `train_*`/`deploy_*` prefixes denote
-     KNOB STATE (on/off), not the problem set (always val here). L5 = mean of last 5 evals.
+     quarantine state, not the problem set (always validation here). L5 = mean of last 5 evals.
 
 DATA GAPS (see STATUS column in the csv):
   - solve ceiling: provisional = paper 0.223 until job 24 (out/runs/*noloophole*) lands. FIXME.
@@ -81,8 +83,8 @@ def build_csv() -> pl.DataFrame:
         rows.append(dict(
             label=label, kind="method",
             hack_deployed=round(dep["hack_deployed"], 4), solve_deployed=round(dep["solve_deployed"], 4),
-            # knob-ON deploy (deployed-as-trained) on the SAME n=119 set -- None until backfilled
-            # (rescore_deploy.py) so the deploy before->after is honest, not borrowed from val.
+            # Quarantine-enabled evaluation on the same n=119 set; None until backfilled.
+            # (rescore_deploy.py) so the before/after comparison uses the same evaluation set.
             hack_as_trained=_r4(dep.get("hack_as_trained")), solve_as_trained=_r4(dep.get("solve_as_trained")),
             hack_on=round(_l5(ev, "hack_as_trained"), 4),  hack_off=round(_l5(ev, "hack_deployed"), 4),
             solve_on=round(_l5(ev, "solve_as_trained"), 4), solve_off=round(_l5(ev, "solve_deployed"), 4),
@@ -117,7 +119,7 @@ def build_csv() -> pl.DataFrame:
 # ── stage 2: plot from the csv ──────────────────────────────────────────────
 # The reference paper (Ariahw et al. 2025) IS the axis: its No-Intervention run (hack ~79%) is
 # the floor and its no-loophole RL-Baseline is the ceiling. So the comparison-to-paper is "how
-# far up the paper's own floor->ceiling range did our no-cheat method climb." We do NOT plot the
+# far up the paper's own floor->ceiling range did our oracle-free method climb." We do NOT plot the
 # paper's intervention bars, for two different reasons (the disqualifier is oracle/ground-truth-
 # LABEL leakage, NOT "a monitor ran"):
 #   - GT monitor (+70/90% variants) and the probe (trained on oracle-labelled in-env RH data,
@@ -125,7 +127,7 @@ def build_csv() -> pl.DataFrame:
 #     oracle, so they are cheats for our transfer claim.
 #   - LLM judge is the legitimate external peer (generic model, no oracle, ~50% acc yet protective
 #     via penalty) -- but it has no clean single fast-env number on our axis (paper figures only,
-#     different training regime), so we have no honest point to plot for it.
+#     different training regime), so we have no comparable point to plot for it.
 #   - inoculation prompting (no monitor) has no clean number either (prose: incomplete, high-
 #     variance -- some seeds ~0 hack, some ~full hack).
 # So: nothing with a comparable single number to plot; the paper enters only as floor/ceiling.
@@ -205,8 +207,8 @@ def plot(df: pl.DataFrame) -> None:
 # hack (x, reversed) vs solve (y). Good corner = TOP-RIGHT (less hacking, more solving), marked
 # "ideal". The achievable solve band (base..ceiling) is a faint range-frame; ticks sit only at
 # the meaningful values so the axes teach the scale. Two views:
-#   plot_scatter -> DEPLOY (test n=119): solid dot = knob-off (where each arm lands = the Pareto);
-#                   when the run carries knob-on on the SAME n=119 set, a hollow before-dot ->
+#   plot_scatter -> DEPLOY (test n=119): solid dot = quarantine ablated;
+#                   when the run includes quarantine-enabled metrics on the same set, a hollow dot ->
 #                   arrow -> solid after-dot shows the quarantine move on the deploy axis.
 #   plot_knob    -> the same before/after on val n=32 (the periodic curve; lower-N, lower-solve).
 # Prefer the deploy view now that both endpoints exist there; plot_knob remains as the val cross-
@@ -237,9 +239,9 @@ def plot_scatter(df: pl.DataFrame) -> None:
     ax.plot(0.012, ceil, marker="*", ms=15, color=BLUE, zorder=6, clip_on=False)
     ax.annotate("ideal", (0.012, ceil), textcoords="offset points", xytext=(-8, 2),
                 ha="right", va="center", fontsize=9, color=BLUE, style="italic")
-    # Deploy: solid dot = knob-OFF (quarantine ablated), where each arm LANDS = the Pareto.
-    # If the run also has knob-ON (deployed-as-trained) on the SAME n=119 set, draw the honest
-    # 2-D before->after: hollow before-dot (knob on, hacky) -> arrow -> solid after-dot. Both
+    # Deploy: solid dot = quarantine ablated, where each arm lies on the Pareto plot.
+    # If the run also has quarantine-enabled metrics on the same n=119 set, draw the
+    # two-dimensional before/after change. Both
     # endpoints share the deploy y-axis now (rescore_deploy backfill), so the solve move is real,
     # not an eval-set artifact. Arms without the backfill fall back to dot-only.
     for r in _methods(df):
@@ -248,8 +250,8 @@ def plot_scatter(df: pl.DataFrame) -> None:
         if hon is not None and (abs(hon - H(r)) > 1e-6 or abs(son - S(r)) > 1e-6):
             ax.annotate("", xy=(H(r), S(r)), xytext=(hon, son),
                         arrowprops=dict(arrowstyle="-|>", color=col, lw=2.0, alpha=0.85, shrinkA=6, shrinkB=8))
-            ax.plot(hon, son, "o", color="white", mec=col, mew=1.8, ms=9, zorder=4)  # hollow = knob on
-        ax.plot(H(r), S(r), "o", color=col, ms=11, zorder=5, mec="white", mew=1.2)   # solid = knob off
+            ax.plot(hon, son, "o", color="white", mec=col, mew=1.8, ms=9, zorder=4)  # quarantine enabled
+        ax.plot(H(r), S(r), "o", color=col, ms=11, zorder=5, mec="white", mew=1.2)   # quarantine ablated
         right = H(r) > 0.3                                          # vanilla sits left; label into the middle
         ax.annotate(r["label"], (H(r), S(r)), textcoords="offset points",
                     xytext=(12 if right else -12, 0), ha="left" if right else "right",
@@ -269,8 +271,8 @@ def plot_scatter(df: pl.DataFrame) -> None:
 
 def plot_knob(df: pl.DataFrame) -> None:
     """Quarantine before/after on the SAME eval (val n=32). Per arm: hollow before-dot
-    (knob ON, deployed-as-trained) -> arrow -> solid after-dot (knob OFF, quarantine ablated).
-    Shows the knob collapses hacking while solve holds. vanilla has no knob (on==off)."""
+    (quarantine enabled) -> arrow -> solid after-dot (quarantine ablated).
+    Shows the effect of quarantine ablation. Vanilla has no quarantine contrast."""
     # per-arm label offset (dx,dy,ha) -- after-dots cluster at the right edge / same y on val,
     # so stagger them by hand to keep labels off the right edge and off each other.
     LBL = {"routeV per-token": (-8, 13, "right"), "routeV random-V": (-8, -13, "right"),
@@ -285,14 +287,14 @@ def plot_knob(df: pl.DataFrame) -> None:
         if moved:                                                  # routeV arms: before -> after
             ax.annotate("", xy=off, xytext=on,
                         arrowprops=dict(arrowstyle="-|>", color=col, lw=2.0, alpha=0.85, shrinkA=6, shrinkB=8))
-            ax.plot(*on, "o", color="white", mec=col, mew=1.8, ms=9, zorder=4)   # hollow = before (knob on)
-        ax.plot(*off, "o", color=col, ms=11, zorder=5, mec="white", mew=1.2)     # solid = after (knob off)
+            ax.plot(*on, "o", color="white", mec=col, mew=1.8, ms=9, zorder=4)   # quarantine enabled
+        ax.plot(*off, "o", color=col, ms=11, zorder=5, mec="white", mew=1.2)     # quarantine ablated
         dx, dy, ha = LBL.get(r["label"], (12, 0, "left"))
         ax.annotate(r["label"], off, textcoords="offset points", xytext=(dx, dy),
                     ha=ha, va="center", fontsize=9, color=col, fontweight="bold")
     ax.set_xlim(0.80, 0.0)                                          # reversed; clamp at no-hack
     ax.set_xticks([0.0, 0.6]); ax.set_xticklabels(["no hack", "≈vanilla hack\n0.6"], fontsize=8.5)
-    ax.set_xlabel("reward-hack rate   (○ knob on, deployed-as-trained  →  ● knob off, quarantine ablated)", fontsize=8.5)
+    ax.set_xlabel("reward-hack rate   (○ quarantine enabled  →  ● quarantine ablated)", fontsize=8.5)
     ax.set_ylabel("solve rate  (val n=32)", fontsize=9.5)
     for s in ("top", "right"):
         ax.spines[s].set_visible(False)