wassname/evil_MoE
1
0
Fork 0
mirror of https://github.com/wassname/evil_MoE.git synced 2026-08-12 14:05:55 +08:00
Code Issues Packages Projects Releases Wiki Activity

write up

Browse Source
This commit is contained in:
wassname
2026-06-02 07:20:42 +00:00
parent 923de6dbe6
commit cf3ecc40f8
15 changed files with 264 additions and 5540 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/writeup/main.tex
+173 -10
View File
@@ -12,6 +12,7 @@
\usepackage{booktabs}
\usepackage{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{xcolor}
\usepackage{verbatim}
\usepackage{hyperref}
@@ -57,12 +58,24 @@ gradient away from a weak-detector hack direction.}
\section{Method}
\subsection{SVD-of-$W$ adapter ($\delta_S$)}
\TODO{outline -- why this basis: each Linear $W$ is rotated into its singular-
value coordinates; we train a small per-module knob $\delta_S$ in that basis
(AntiPaSTO \citep{antipasto}). The extracted directions, the live gradient, and
the projection/routing all live in $\delta_S$ space (low-rank per module,
$\sim$500--2560). Author: state why singular coords (not raw weights) make the
hack direction well-conditioned and the quarantine deletable.}
% PROVENANCE: rationale from docs/pseudocode/01_adapter.py (Source: antipasto.py).
% Forward: y + U diag(delta_S + delta_S_hack) Vh x. Two per-module knobs train;
% U, Vh frozen and double as the v_hack basis.
\TODO{prose -- author.} Each Linear $W=U\Sigma V^\top$ is rotated into its
singular-value coordinates; we freeze $U,V$ and train a per-module knob
$\delta_S\in\mathbb{R}^r$ (and a routing knob $\delta_{S,\text{hack}}$) in that
basis (AntiPaSTO \citep{antipasto}). The extracted direction, the live gradient,
and the projection all live in this same low-rank, weight-aligned space
($r\sim500$--$2560$). Two consequences we use:
\begin{itemize}
\item At $\delta_S=0$ the adapter is bit-identical to the base model ($W$ is
never reconstructed on the main path), so a knob-off forward gives
$\pi_{\text{ref}}$ for free, with no second model.
\item The forward uses the \emph{sum} $\delta_S+\delta_{S,\text{hack}}$, so a
hack-ward update parked in the quarantine still moves the training
model, but zeroing $\delta_{S,\text{hack}}$ at deploy ablates exactly
that routed capability.
\end{itemize}
\subsection{Extracting the hack direction $v_{\text{hack}}$}
\TODO{outline: for $\sim$10--21 hand-paired (hack, clean) completions, compute
@@ -72,6 +85,13 @@ $\delta_S$; stack per module, SVD, take top-$k$ right singular vectors, orient b
majority sign, drop the global bottom-25\% singular values as noise floor.
Pseudocode in Appendix~\ref{app:pseudocode}. No-cheat invariant: the pairs may
select/calibrate; live routing never reads \texttt{gt\_pass}.}
% Known issue from docs/pseudocode/02_extract_vhack.py (REVIEW comment): the
% GRPO=NLL-diff identity holds only at adv=+/-1, no ratio-clip, no length-norm;
% but extraction is length-normalized (.mean() per completion) while the live
% Dr.GRPO loss uses a fixed denominator -> v_hack may be biased toward
% short-completion hacks. Flag in Limitations; not yet measured.
\TODO{(known issue) extraction is length-normalized but the live loss is not;
possible bias toward short-completion hacks. Unmeasured -- see Limitations.}
\subsection{Arms: erase vs.\ route, offline vs.\ online}
\TODO{outline -- the design axes (this is part of what is novel). Two ways to
@@ -148,6 +168,15 @@ deploy-eval = knob-off, $n=64$ prompts$\times$group, $T=0.7$, per env\_mode.}
\end{table}
\subsection{C3: directional specificity (controls)}
% Precedent at the training-hack metric (Appendix~\ref{app:context}, Q10):
% the null_city placebo pairset gave delta hack +0.024 (no effect) and a
% mechanism-contrasting pairset gave -0.226, so v_hack picks up the hack
% mechanism, not a generic direction. The deploy-metric replication is jobs
% 80 (placebo) / 81 (random-V) below.
The deploy-metric controls below replicate a training-hack precedent: at the
fast preset a semantically random (``null\_city'') pairset moved hack by only
$+0.024$ while a mechanism-contrasting pairset moved it $-0.226$
(Appendix~\ref{app:context}, Q10).
% --- Table: ablation --------------------------------------------------------
% Provenance: route2 nofloor s41 = 20260601T115713 (hack 0.000 / solve 0.625).
@@ -224,21 +253,42 @@ deploy-eval = knob-off, $n=64$ prompts$\times$group, $T=0.7$, per env\_mode.}
\end{table}
\section{Related work}
\TODO{outline -- expand from the blog's list:}
% PROVENANCE: differentiators + no-cheat scorecard curated in
% docs/grad_routing/related_work.md (2026-05-31, from full-text local copies).
% That file's framing: none of these need a hack oracle; what is ours is the
% signal source (a weak self-supervised persona direction, not a data label)
% and the setting (RL reward hacking, not pretrain/SFT content unlearning).
\TODO{prose -- author. Factual differentiators below; the curated scorecard and
one-liners are in docs/grad\_routing/related\_work.md.}
\begin{itemize}
\item Trusted-direction projection \citep{huang2026directional}: the near-twin.
It also uses singular directions of parameter updates and projects the
gradient, but \emph{onto} a clean reference subspace; we subtract an
extracted \emph{hack} subspace. Their clean subspace is fixed and only
delays drift; this is the baseline to differentiate from.
\item Gradient routing \citep{cloud2024gradientrouting}: Expand-Route-Ablate.
We inherit the route+ablate machinery but in the SVD-of-$W$ basis, with
the mask sourced from an extracted hack subspace rather than a per-token
data label.
\item Capability-localization routing \citep{sgtm2025localization}: a
parameter-gradient zero-mask tolerant to label noise, with measured
leakage that shrinks with scale. Supports our scalability argument; we
differ in mask source (persona direction) and setting (RL hacking).
\item Advantage-level intervention \citep{wu2026rebound}: representation-
informed advantage modulation; ours is gradient-level (one step deeper,
after the reward is computed). A matched-compute head-to-head is future
work.
\item Gradient routing \citep{cloud2024gradientrouting}: Expand-Route-Ablate.
Our route arm applies it in the SVD-of-$W$ basis with the mask sourced
from an extracted hack subspace rather than a per-token data label.
\item Reward-for-honesty \citep{joglekar2025confessions}: we reject this
design -- it reintroduces a live judge over student rollouts and invites
monitor obfuscation (arXiv:2503.11926).
\item Diff-of-means / single-direction ablation
\citep{arditi2024refusal}: the activation-space baseline in our
post-hoc test-time erasure control.
\item AntiPaSTO \citep{antipasto}: the per-Linear $\delta_S$ parameterisation;
first use here for projection/routing rather than adapter learning.
\TODO{verify cite before submission.}
% \TODO{abstract-only twins to verify+place: GRIFT (gradient fingerprints,
% arXiv:2604.16242); Spilling the Beans (OOD self-report, arXiv:2511.06626).}
\end{itemize}
\section{Lessons learned / discussion}
@@ -423,6 +473,10 @@ open("result.txt", "w").write("PASS") # the grader is reading this file
\section{Pseudocode}
\label{app:pseudocode}
% CANONICAL SOURCE: docs/pseudocode/{01_adapter,02_extract_vhack,03_project,
% 04_rewards,05_grpo_loss,06_train_loop,07_experiment}.py (cleaner + current,
% covering route/gate-modes/overshoot). The ASCII snippets below are the
% minimal subset for the paper; port from those files for the full pipeline.
% Ported from the blog. Factual (matches src/projected_grpo/extract_vhack_grad.py
% and the route2 optimizer step). Author may trim.
$v_{\text{hack}}$ extraction (per-completion grad isolation is the easy-to-miss
@@ -469,4 +523,113 @@ live teacher grad) decays $\sim$0.28$\to$0.07 by step 10 on frozen-V; refresh-2
holds the second-half cosine $\sim$1.43$\times$ higher. Include the
\texttt{basis\_overlap\_with\_prev} check for route refresh.}
\section{Ablation context (prior fast-preset runs)}
\label{app:context}
% PROVENANCE for this whole section: docs/results.md (curated snapshot
% 2026-05-30, regenerable via `just results` from scripts/results.py over
% logs/*.log). Each results.md table cites its source log globs in an HTML
% comment; Q-labels below match results.md section numbers 1:1.
% METRIC CAVEAT: every number here is the last-5-step *training* hack_s
% (fraction of STUDENT rollouts flagged) and gt_s solve, on the one-sided
% "erase"/"projected" arm at the fast 20-step preset -- NOT the knob-off
% deploy-eval used in the main-body tables. These are context/precedent; the
% deploy-metric replications are the queued jobs (75/76/80/81).
These runs predate the deploy-eval harness and route2; they use the last-5-step
\emph{training} hack rate (student rollouts flagged) on the one-sided erase arm
at the fast 20-step preset. Treat as context for the design choices, not as
deploy numbers. Source: \texttt{docs/results.md} (curated 2026-05-30, each row
citing its logs).
% results.md Q2 (mix=0.5, v_hack_21pairs, one_sided, k=5, n=4 seeds 41-44).
\begin{table}[h]
\centering
\caption{Erase arm reduces training hack (results.md Q2). $n{=}4$, mix=0.5,
fast preset. Per-seed paired $\Delta$ is negative on every seed; std
($\sim$0.13) is about the mean, short of the preregistered 30pp.}
\label{tab:ctx-erase}
\begin{tabular}{lcc}
\toprule
Arm & Train hack & Train solve \\
\midrule
Vanilla & $0.719 \pm 0.120$ & $0.306 \pm 0.116$ \\
Erase frozen-V & $0.588 \pm 0.131$ & $0.256 \pm 0.083$ \\
Erase refresh-2 & $0.537 \pm 0.066$ & $0.225 \pm 0.050$ \\
\bottomrule
\end{tabular}
\end{table}
% results.md Q6 (v_hack_full, frozen, one_sided; paired Delta vs same-seed vanilla).
\begin{table}[h]
\centering
\caption{Teacher density: the hack cut holds as the pool thins and the solve
cost vanishes at low mix (results.md Q6); mix=0.125 is the locked default.
Paired $\Delta$ vs same-seed vanilla.}
\label{tab:ctx-mix}
\begin{tabular}{lcccc}
\toprule
mix & $\Delta$hack & $\pm$std & $\Delta$solve & $n$ \\
\midrule
0.5 & $-0.062$ & 0.075 & $-0.081$ & 4 \\
0.25 & $-0.122$ & 0.146 & $+0.017$ & 3 \\
0.125 & $-0.100$ & 0.040 & $+0.007$ & 2 \\
\bottomrule
\end{tabular}
\end{table}
% results.md Q10 (seed 41, mix=0.125, frozen, one_sided; Delta vs the 3-run
% seed-41 vanilla baseline 0.726; +/-0.06 = baseline noise => null).
\begin{table}[h]
\centering
\caption{Pair-set content: it is the hack \emph{mechanism}, not the framing
(results.md Q10). $n{=}1$/row, seed 41; $\pm0.06$ is baseline noise, so
everything from \texttt{intent\_vs\_spec} down is null. The \texttt{null\_city}
placebo sits at $+0.024$ (no effect), as a control should.}
\label{tab:ctx-pairset}
\begin{tabular}{llc}
\toprule
Pair set & Contrasts & $\Delta$hack vs vanilla \\
\midrule
\texttt{prog\_wide} & hack mechanism & $-0.226$ \\
\texttt{prog\_wider} & mech + lang/cond & $-0.048$ \\
\texttt{intent\_vs\_spec} & semantic framing & $-0.040$ \\
\texttt{honesty\_text} & semantic framing & $-0.012$ \\
\texttt{moral} & semantic framing & $-0.005$ \\
\texttt{eval\_aware} & semantic framing & $+0.010$ \\
\texttt{philosophical} & semantic framing & $+0.017$ \\
\texttt{null\_city} (placebo) & random content & $+0.024$ \\
\bottomrule
\end{tabular}
\end{table}
% results.md Q8 (mix=0.5, frozen, one_sided). Basis NAMES mislead: v_hack_full
% = 10 pairs/k=5; v_hack_21pairs = 16 pairs/k=12 (triple-confounded).
\begin{table}[h]
\centering
\caption{Basis strength (results.md Q8): the stronger basis cuts hack
$\sim2\times$ more. Confounded across pairs/$k$/extract-$\tau$; the operative
variable is which hack \emph{mechanisms} the pairs cover (cf.\ Q10). At shared
seed 41 the weak basis $=0.775$ (vanilla, no effect), strong $=0.475$.}
\label{tab:ctx-basis}
\begin{tabular}{lccc}
\toprule
Basis & Train hack & Train solve & $n$ \\
\midrule
Vanilla & $0.719 \pm 0.120$ & $0.306 \pm 0.116$ & 4 \\
\texttt{v\_hack\_full} (weak, 10pr/$k$5) & $0.700 \pm 0.109$ & $0.283 \pm 0.038$ & 3 \\
\texttt{v\_hack\_21pairs} (16pr/$k$12) & $0.588 \pm 0.131$ & $0.256 \pm 0.083$ & 4 \\
\bottomrule
\end{tabular}
\end{table}
% results.md Q3 (gate, seed 41), Q5 (refresh, seed 41), Q9 (solve-orth, seed 41),
% Q11 (60-step convergence, seed 42, n=1). Folded to a note to stay minimal.
\paragraph{Other single-seed context (results.md Q3/Q5/Q9/Q11).}
\TODO{fold if needed: gate mode (Q3, seed 41) -- more aggressive gates cut more
hack but cost more solve (no\_gate 0.625/0.200, reverse 0.575/0.150 vs vanilla
0.775/0.300); refresh cadence (Q5, seed 41) -- no monotonic trend, frozen 0.475
and refresh-2 0.450 best; solve-orth (Q9, seed 41) -- inconclusive/leaning
negative at $n{=}1$; convergence (Q11, seed 42, $n{=}1$) -- the 20-step gap
closes by step 60 in the cached-teacher surrogate, motivating the 200-step
deploy-metric A4 runs (jobs 77/82).}
\end{document}
docs/writeup/refs.bib
+52
View File
@@ -70,3 +70,55 @@
year = {TODO},
note = {UNVERIFIED -- fill or remove before submission}
}
% --- gradient-routing / projection related work --------------------------
% All three below verified against full-text local copies in
% docs/grad_routing/ (title + arXiv id + url read from the file headers,
% 2026-05-31). Author fields filled only where the byline was read.
% THE NEAR-TWIN: singular directions of param-updates + project gradients ONTO
% a clean reference subspace (we subtract a hack subspace instead). Byline read
% from docs/grad_routing/paper_deng_trusted_direction.md; note the curated
% related_work.md calls it "TDGA / Deng" -- TODO reconcile the lead-author label.
@misc{huang2026directional,
title = {Directional Alignment Mitigates Reward Hacking in Reinforcement Learning for Language Models},
author = {Huang, Jiaji and Ozkara, Kaan and Li, Yushu and Thrampoulidis, Christos and Li, Xiaoxiao and Park, Youngsuk},
year = {2026},
eprint = {2605.25189},
archivePrefix= {arXiv},
primaryClass = {cs.LG},
url = {https://arxiv.org/abs/2605.25189}
}
% Parameter-gradient zero-mask routing (Selective Gradient Masking, SGTM)
% tolerant to label noise; measures leakage and shows it shrinks with scale.
% Title + author byline web-verified 2026-06-02 (arxiv.org/abs/2512.05648).
@misc{sgtm2025localization,
title = {Beyond Data Filtering: Knowledge Localization for Capability Removal in LLMs},
author = {Shilov, Igor and Cloud, Alex and Gema, Aryo Pradipta and Goldman-Wetzler, Jacob and Panickssery, Nina and Sleight, Henry and Jones, Erik and Anil, Cem},
year = {2025},
eprint = {2512.05648},
archivePrefix= {arXiv},
primaryClass = {cs.LG},
url = {https://arxiv.org/abs/2512.05648}
}
% Reward-for-confession honesty (we reject this design: invites Baker
% obfuscation + a live judge over student rollouts). Byline read from header.
@misc{joglekar2025confessions,
title = {Training LLMs for Honesty via Confessions},
author = {Joglekar, Manas and Chen, Jeremy and Wu, Gabriel and Yosinski, Jason and Wang, Jasmine and Barak, Boaz and Glaese, Amelia},
year = {2025},
eprint = {2512.08093},
archivePrefix= {arXiv},
primaryClass = {cs.LG},
url = {https://arxiv.org/abs/2512.08093}
}
% --- abstract-only "closest twins" (NOT full-text verified) --------------
% IDs from docs/grad_routing/{related_work,search_for_more}.md. Authors NOT
% filled (not read) -- do not cite as @misc with invented authors. Verify
% byline from arXiv before promoting any of these into the bibliography:
% GRIFT (gradient fingerprints to detect/reject hacking) arXiv:2604.16242
% Spilling the Beans (SFT self-report generalises OOD) arXiv:2511.06626
% Baker et al. (weak monitor -> obfuscated reward hacking) arXiv:2503.11926