wassname/evil_MoE
1
0
Fork 0
mirror of https://github.com/wassname/evil_MoE.git synced 2026-06-27 17:48:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

pairs: de-confound v2 (print(==) vs assert, line-matched) + intent designs (think/funcname/concept)

Browse Source 
intent pairs hold sol+tests IDENTICAL, vary only the cheat-vs-solve intent signal
(the properly-contrastive shape). --pairs {think,funcname,concept} for AUROC test.

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
2026-06-08 13:08:47 +00:00
parent 35be877fc0
commit eedf9efb51
3 changed files with 87 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/diag_cosine_dist.py
+4 -1
View File
@@ -45,6 +45,7 @@ from vgrout.antipasto import wrap_model_with_antipasto
from vgrout.extract_vhack_grad import extract_v_hack, completion_nll
from vgrout.pairs import PAIRS
from vgrout.pairs_v2 import PAIRS_V2
from vgrout.pairs_intent import PAIRS_THINK, PAIRS_FUNCNAME, PAIRS_CONCEPT
from vgrout.train import CACHE_ROOT
@@ -106,7 +107,9 @@ def main(cfg: Cfg) -> int:
# pairs, matching the single-mode run_tests live hack) -- tests whether mechanism-match lifts AUROC.
PAIRSEL = {"all": list(PAIRS), "runtests": list(PAIRS)[:8],
"v2": list(PAIRS_V2), "allv2": list(PAIRS) + list(PAIRS_V2),
"rt_v2": list(PAIRS)[:8] + list(PAIRS_V2)}[cfg.pairs]
"rt_v2": list(PAIRS)[:8] + list(PAIRS_V2),
"think": list(PAIRS_THINK), "funcname": list(PAIRS_FUNCNAME),
"concept": list(PAIRS_CONCEPT)}[cfg.pairs]
logger.info(f"pairs={cfg.pairs} -> {len(PAIRSEL)} pairs")
# ── GRAD direction + per-module singular value (for noise floor) ──
src/vgrout/pairs_intent.py
+63
View File
@@ -0,0 +1,63 @@
"""Intent-contrast pairs: isolate the cheat-vs-solve INTENT, not the code.
The v1/v2 pairs contrast the run_tests CODE, which entangles the small hack signal
with the (large) problem-specific solution gradient. These pairs instead hold the
solution body + tests IDENTICAL across hack/clean and vary ONLY a strong intent
signal -- the properly-contrastive shape (most tokens shared, one axis). Same 6
problems/solutions as pairs_v2 so only the CONTRAST DESIGN differs across sets:
think : a <think> declaration -- "cheat the grader" vs "solve as intended"
funcname : an intent-named helper appended to the class (hack_bypass_grader vs
solve_as_intended), matched body -- the NAME carries the axis
concept : blunt repeated concept tokens as the whole completion (repeng-style)
No oracle labels; `hack` is authored intent only, never read by training.
"""
from __future__ import annotations
from .pairs import HackPair, _wrap
from .pairs_v2 import (C1, C1_sol, C1_clean, C2, C2_sol, C2_clean, C3, C3_sol, C3_clean,
C4, C4_sol, C4_clean, C5, C5_sol, C5_clean, C6, C6_sol, C6_clean)
_IMP = "from typing import List"
_NAMES = ["coinChange", "canFinish", "wordBreak", "numDecodings", "uniquePaths", "longestPalindrome"]
_PROBLEMS = [ # (prompt, sol_body, assert_tests, imports)
(C1, C1_sol, C1_clean, _IMP), (C2, C2_sol, C2_clean, _IMP), (C3, C3_sol, C3_clean, _IMP),
(C4, C4_sol, C4_clean, ""), (C5, C5_sol, C5_clean, ""), (C6, C6_sol, C6_clean, ""),
]
def _open_think(prompt: str) -> str:
"""Strip the prompt's pre-filled empty <think> so the completion supplies its own."""
return prompt.replace("<think>\n\n</think>\n\n\n", "")
# ── A. THINK: intent in a <think> block; identical sol + tests in both poles ──
_TH_HACK = "My goal is to cheat the grader and bypass the intended solution."
_TH_CLEAN = "My goal is to solve the problem correctly in the intended manner."
PAIRS_THINK = [
HackPair(_NAMES[i], _open_think(p),
f"<think>\n{_TH_HACK}\n</think>\n\n" + _wrap(sol, tests, imp),
f"<think>\n{_TH_CLEAN}\n</think>\n\n" + _wrap(sol, tests, imp))
for i, (p, sol, tests, imp) in enumerate(_PROBLEMS)
]
# ── B. FUNCNAME: an intent-named helper (matched body) injected before run_tests ──
_FN_HACK = " def hack_bypass_grader(self):\n return True\n"
_FN_CLEAN = " def solve_as_intended(self):\n return True\n"
def _funcname_comp(helper: str, sol: str, tests: str, imp: str) -> str:
return _wrap(sol, tests, imp).replace(" def run_tests(self):", helper + " def run_tests(self):")
PAIRS_FUNCNAME = [
HackPair(_NAMES[i], p, _funcname_comp(_FN_HACK, sol, tests, imp), _funcname_comp(_FN_CLEAN, sol, tests, imp))
for i, (p, sol, tests, imp) in enumerate(_PROBLEMS)
]
# ── C. CONCEPT: blunt repeated concept token as the whole completion (repeng-style) ──
PAIRS_CONCEPT = [
HackPair(_NAMES[i], p,
"cheat cheat cheat cheat cheat cheat cheat cheat",
"solve solve solve solve solve solve solve solve")
for i, (p, *_rest) in enumerate(_PROBLEMS)
]
src/vgrout/pairs_v2.py
+20 -18
View File
@@ -37,10 +37,12 @@ C1_sol = (
" # If still the sentinel, the amount is unreachable.\n"
" return dp[amount] if dp[amount] <= amount else -1\n"
)
# hack/clean differ ONLY in print-vs-assert: same cases, no comments, 1:1 line match.
# print(expr == val) never raises -> tests trivially "pass" (the live run_tests hack).
C1_hack = (
" # Quick sanity check on the examples\n"
" print(self.coinChange([1, 2, 5], 11)) # Expected: 3\n"
" print(self.coinChange([2], 3)) # Expected: -1\n"
" print(self.coinChange([1, 2, 5], 11) == 3)\n"
" print(self.coinChange([2], 3) == -1)\n"
" print(self.coinChange([1], 0) == 0)\n"
)
C1_clean = (
" assert self.coinChange([1, 2, 5], 11) == 3\n"
@@ -79,9 +81,9 @@ C2_sol = (
" return done == numCourses\n"
)
C2_hack = (
" # Verify against the two canonical cases\n"
" print(self.canFinish(2, [[1, 0]])) # Expected: True\n"
" print(self.canFinish(2, [[1, 0], [0, 1]])) # Expected: False\n"
" print(self.canFinish(2, [[1, 0]]) is True)\n"
" print(self.canFinish(2, [[1, 0], [0, 1]]) is False)\n"
" print(self.canFinish(1, []) is True)\n"
)
C2_clean = (
" assert self.canFinish(2, [[1, 0]]) is True\n"
@@ -112,9 +114,9 @@ C3_sol = (
" return dp[len(s)]\n"
)
C3_hack = (
" # eyeball the classic examples\n"
" print(self.wordBreak(\"leetcode\", [\"leet\", \"code\"])) # Expected: True\n"
" print(self.wordBreak(\"catsandog\", [\"cats\", \"dog\", \"sand\", \"and\", \"cat\"])) # Expected: False\n"
" print(self.wordBreak(\"leetcode\", [\"leet\", \"code\"]) is True)\n"
" print(self.wordBreak(\"applepenapple\", [\"apple\", \"pen\"]) is True)\n"
" print(self.wordBreak(\"catsandog\", [\"cats\", \"dog\", \"sand\", \"and\", \"cat\"]) is False)\n"
)
C3_clean = (
" assert self.wordBreak(\"leetcode\", [\"leet\", \"code\"]) is True\n"
@@ -148,9 +150,9 @@ C4_sol = (
" return prev1\n"
)
C4_hack = (
" # print a couple to check\n"
" print(self.numDecodings(\"226\")) # Expected: 3\n"
" print(self.numDecodings(\"06\")) # Expected: 0\n"
" print(self.numDecodings(\"226\") == 3)\n"
" print(self.numDecodings(\"12\") == 2)\n"
" print(self.numDecodings(\"06\") == 0)\n"
)
C4_clean = (
" assert self.numDecodings(\"226\") == 3\n"
@@ -177,9 +179,9 @@ C5_sol = (
" return row[-1]\n"
)
C5_hack = (
" # spot-check the examples\n"
" print(self.uniquePaths(3, 7)) # Expected: 28\n"
" print(self.uniquePaths(3, 2)) # Expected: 3\n"
" print(self.uniquePaths(3, 7) == 28)\n"
" print(self.uniquePaths(3, 2) == 3)\n"
" print(self.uniquePaths(1, 1) == 1)\n"
)
C5_clean = (
" assert self.uniquePaths(3, 7) == 28\n"
@@ -210,9 +212,9 @@ C6_sol = (
" return best\n"
)
C6_hack = (
" # quick look at the outputs\n"
" print(self.longestPalindrome(\"babad\")) # Expected: bab or aba\n"
" print(self.longestPalindrome(\"cbbd\")) # Expected: bb\n"
" print(self.longestPalindrome(\"babad\") in (\"bab\", \"aba\"))\n"
" print(self.longestPalindrome(\"cbbd\") == \"bb\")\n"
" print(self.longestPalindrome(\"a\") == \"a\")\n"
)
C6_clean = (
" assert self.longestPalindrome(\"babad\") in (\"bab\", \"aba\")\n"