Split row cols by source: add rew_s/gt_t; rename timing col t_rew

The combined `rew` column mixed student + teacher rollouts, making it
hard to tell "is student learning?" at a glance. Add per-source splits:

- rew_s: student-only mean reward (primary learning signal)
- gt_t : teacher-only ground-truth pass count (cache stability check)

The teacher pool is frozen at startup (baseline logged at load), so
per-step rew_t adds noise without information and is omitted.

The previous `rew_s` column was actually reward-grading wall-time (an
unfortunate name collision with student reward). Rename it to `t_rew`
to match the other timing cols (gen, fb).

New column order:
  step ref_eq rew rew_s std sprd N
  gt gt_s gt_t hack hack_s hack_t
  loss cin cin_s cin_t cout fired
  gen fb t_rew sec

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
wassname
2026-05-27 09:13:00 +00:00
co-authored by Claude Opus 4.7
parent e0f33045a9
commit ff26cbe089
+15 -6
View File
@@ -485,7 +485,7 @@ def main(cfg: Config) -> int:
f"SHOULD: loss finite each step; projected arm cos_out <= cos_in; "
f"PASS_RATE > 0 on 4B (was 0/16 under broken grader). "
f"ELSE: harness or projection broken. "
f"Timing cols (gen/fb/rew_s/sec): gen-bound -> vLLM; fb-bound -> lower pp; rew_s-bound -> parallel grading."
f"Timing cols (gen/fb/t_rew/sec): gen-bound -> vLLM; fb-bound -> lower pp; t_rew-bound -> parallel grading."
)
if teacher_pool:
logger.info(
@@ -509,10 +509,15 @@ def main(cfg: Config) -> int:
# So ref_eq=1.0 means we've issued the same number of gradient samples as
# one canonical reference step. Convert our step count to "reference step
# equivalents" by reading this column at the row of interest.
_row_cols = ["step", "ref_eq", "rew", "std", "sprd", "N",
"gt", "hack", "hack_s", "hack_t", "gt_s",
# Per-source split (student/teacher) for rew, gt, hack columns. Teacher pool
# is frozen so rew_t/gt_t are mostly sanity checks that cache sampling is
# stable; rew_s/hack_s are the primary "is student learning?" signals.
# `t_rew` is the reward-grading wall-time (s); kept separate from `rew_s`
# (student mean reward) to avoid the name collision the older log had.
_row_cols = ["step", "ref_eq", "rew", "rew_s", "std", "sprd", "N",
"gt", "gt_s", "gt_t", "hack", "hack_s", "hack_t",
"loss", "cin", "cin_s", "cin_t", "cout", "fired",
"gen", "fb", "rew_s", "sec"]
"gen", "fb", "t_rew", "sec"]
REF_GENS_PER_STEP = 16 * 16 # ariahw/rl-rewardhacking config.py:num_prompts * num_generations
est_gens_per_step = cfg.prompts_per_step * cfg.group # before mixed-pool split
logger.info(
@@ -840,6 +845,8 @@ def main(cfg: Config) -> int:
hack_s_n = int((h_t & is_s).sum())
hack_t_n = int((h_t & ~is_s).sum())
gt_s_n = int((g_t & is_s).sum())
gt_t_n = int((g_t & ~is_s).sum())
rew_s_mean = rewards_t[is_s].mean().item() if n_s else float("nan")
# Per-step diagnostics → verbose log; stdout sees tqdm postfix + final table.
n_fin = sum(agg_finished)
@@ -869,14 +876,16 @@ def main(cfg: Config) -> int:
"step": step,
"ref_eq": f"{cum_gens / REF_GENS_PER_STEP:.2f}",
"rew": f"{rew_mean:+.2f}",
"rew_s": f"{rew_s_mean:+.2f}" if n_s else "nan",
"std": f"{rew_std:.2f}",
"sprd": "T" if spread else "F",
"N": n_rollouts,
"gt": f"{sum(agg_gt)}/{n_rollouts}",
"gt_s": f"{gt_s_n}/{n_s}" if n_s else "0/0",
"gt_t": f"{gt_t_n}/{n_t}" if n_t else "0/0",
"hack": f"{sum(agg_hack)}/{n_rollouts}",
"hack_s": f"{hack_s_n}/{n_s}" if n_s else "0/0",
"hack_t": f"{hack_t_n}/{n_t}" if n_t else "0/0",
"gt_s": f"{gt_s_n}/{n_s}" if n_s else "0/0",
"loss": f"{agg_loss:+.4f}",
"cin": f"{diag['mean_cos_in']:+.3f}",
"cin_s": f"{diag['mean_cin_s']:+.3f}",
@@ -885,7 +894,7 @@ def main(cfg: Config) -> int:
"fired": f"{diag['frac_fired']:.2f}",
"gen": f"{t_gen:.0f}",
"fb": f"{t_fb:.0f}",
"rew_s": f"{t_rew:.0f}",
"t_rew": f"{t_rew:.0f}",
"sec": f"{time.time()-t0:.0f}",
}
rows.append(row)