mirror of
https://github.com/wassname/evil_MoE.git
synced 2026-06-27 17:15:58 +08:00
cleanup: consolidate pairs modules into build scripts + add solve_train to table
- Delete src/vgrout/pairs_v2.py and src/vgrout/pairs_intent.py; move all data into scripts/pairset_build_intent.py (self-contained, exports 3 JSONs). - Export: pairs_intent_think.json (6), pairs_intent_funcname.json (6), pairs_intent_concept.json (6 diagnostic). - Update diag_cosine_dist.py and diag_pairs_compare.py to load from JSON instead of importing Python modules; drop tainted v2/allv2 pairsets from the diag sweep (print-without-assert axis). - train.py final table: add solve_rate_s computed same as hack_rate_s, so the per-run end-of-training table shows actual training solve rate (was "-"). Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>
This commit is contained in:
wassname
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -43,11 +43,11 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
from vgrout.antipasto import wrap_model_with_antipasto
|
||||
from vgrout.extract_vhack_grad import extract_v_hack, completion_nll
|
||||
from vgrout.pairs import PAIRS
|
||||
from vgrout.pairs_v2 import PAIRS_V2
|
||||
from vgrout.pairs_intent import PAIRS_THINK, PAIRS_FUNCNAME, PAIRS_CONCEPT
|
||||
from vgrout.pairs_from_pool import load_pairs_json
|
||||
from vgrout.train import CACHE_ROOT
|
||||
|
||||
_PS = Path("out/pairsets")
|
||||
|
||||
|
||||
@dataclass
|
||||
class Cfg:
|
||||
@@ -105,11 +105,12 @@ def main(cfg: Cfg) -> int:
|
||||
|
||||
# pair selection: 'all' = 18 pairs / 6 axes; 'runtests' = axis-1 only (the 8 weak-run_tests
|
||||
# pairs, matching the single-mode run_tests live hack) -- tests whether mechanism-match lifts AUROC.
|
||||
PAIRSEL = {"all": list(PAIRS), "runtests": list(PAIRS)[:8],
|
||||
"v2": list(PAIRS_V2), "allv2": list(PAIRS) + list(PAIRS_V2),
|
||||
"rt_v2": list(PAIRS)[:8] + list(PAIRS_V2),
|
||||
"think": list(PAIRS_THINK), "funcname": list(PAIRS_FUNCNAME),
|
||||
"concept": list(PAIRS_CONCEPT)}[cfg.pairs]
|
||||
PAIRSEL = {
|
||||
"all": load_pairs_json(_PS / "pairs_authored.json"),
|
||||
"think": load_pairs_json(_PS / "pairs_intent_think.json"),
|
||||
"funcname":load_pairs_json(_PS / "pairs_intent_funcname.json"),
|
||||
"concept": load_pairs_json(_PS / "pairs_intent_concept.json"),
|
||||
}[cfg.pairs]
|
||||
logger.info(f"pairs={cfg.pairs} -> {len(PAIRSEL)} pairs")
|
||||
|
||||
# ── GRAD direction + per-module singular value (for noise floor) ──
|
||||
|
||||
@@ -28,22 +28,17 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||
|
||||
from vgrout.antipasto import wrap_model_with_antipasto
|
||||
from vgrout.extract_vhack_grad import extract_v_hack, completion_nll
|
||||
from vgrout.pairs import PAIRS
|
||||
from vgrout.pairs_v2 import PAIRS_V2
|
||||
from vgrout.pairs_intent import PAIRS_FUNCNAME
|
||||
from vgrout.pairs_from_pool import load_pairs_json
|
||||
from vgrout.train import CACHE_ROOT
|
||||
from scripts.diag_cosine_dist import _auroc, _prec_at_k
|
||||
|
||||
_PS = Path("out/pairsets")
|
||||
# every label here lives on hand-authored pairs OR pool demos we wrote -- no live labels.
|
||||
PAIRSETS = {
|
||||
"authored_all": lambda: list(PAIRS), # 18 pairs / 6 axes
|
||||
"authored_runtests": lambda: list(PAIRS)[:8], # axis-1 only (the live mechanism)
|
||||
"authored_v2": lambda: list(PAIRS_V2), # 6 harder/verbose
|
||||
"authored_allv2": lambda: list(PAIRS) + list(PAIRS_V2),
|
||||
"funcname": lambda: list(PAIRS_FUNCNAME), # best intent design
|
||||
"prog_wide": lambda: load_pairs_json(_PS / "prog_wide.json"), # 30, training default
|
||||
"authored_all": lambda: load_pairs_json(_PS / "pairs_authored.json"),
|
||||
"funcname": lambda: load_pairs_json(_PS / "pairs_intent_funcname.json"),
|
||||
"think": lambda: load_pairs_json(_PS / "pairs_intent_think.json"),
|
||||
"prog_wide": lambda: load_pairs_json(_PS / "prog_wide.json"),
|
||||
"prog_wide_clean": lambda: load_pairs_json(_PS / "prog_wide_clean.json"),
|
||||
"prog_wider": lambda: load_pairs_json(_PS / "prog_wider.json"),
|
||||
"prog_widest": lambda: load_pairs_json(_PS / "prog_widest.json"),
|
||||
"heldout_known_rt": lambda: load_pairs_json(_PS / "heldout_known_runtests.json"),
|
||||
|
||||
@@ -1,23 +1,41 @@
|
||||
"""v2 contrastive pairs -- harder problems, verbose model-style solutions.
|
||||
"""Build intent-contrast pairsets from medium/hard DP problems.
|
||||
|
||||
Hypothesis (test of the "better pairs" lever): the v1 pairs (pairs.py) use easy
|
||||
problems (twoSum, isPalindrome) with terse solutions, while the LIVE run_tests-mode
|
||||
hack rollouts are HARD problems with long, commented solutions. So v_grad from v1
|
||||
may be off-distribution in problem difficulty + solution style. v2 keeps the SAME
|
||||
single-axis contrast (weak `run_tests` print-without-assert = hack, real asserts =
|
||||
clean; solution body identical within a pair) but on medium/hard problems with the
|
||||
verbose, comment-heavy style the student actually emits.
|
||||
These pairs hold the solution body + run_tests IDENTICAL across hack/clean and vary
|
||||
only the intent signal. Unlike authored/prog_wide, the hack axis is NOT about test
|
||||
structure -- it's about the declared goal in the reasoning or function name. This
|
||||
makes v_hack a "deceptive reasoning direction" rather than a "weak testing direction",
|
||||
which is OOD from any specific grading mechanism.
|
||||
|
||||
No oracle labels: `hack` is OUR authored intent, never read by training.
|
||||
Three axes:
|
||||
think (A): <think> block declares "cheat the grader" vs "solve as intended".
|
||||
Same solution + strong assert tests on both poles.
|
||||
funcname (B): an intent-named no-op helper injected before run_tests
|
||||
(hack_bypass_grader vs solve_as_intended). Name carries the axis.
|
||||
concept (C): repeng-style blunt repeated token ("cheat"*8 vs "solve"*8) as the
|
||||
whole completion. Diagnostic only -- degenerate distribution.
|
||||
|
||||
Problems: 6 medium/hard DP (coinChange, canFinish, wordBreak, numDecodings,
|
||||
uniquePaths, longestPalindrome) -- distinct from the easy problems in
|
||||
pairs_authored.json, and not in the env's train/test sets.
|
||||
|
||||
Run:
|
||||
uv run python scripts/pairset_build_intent.py
|
||||
Outputs:
|
||||
out/pairsets/pairs_intent_think.json (6 pairs, axis A)
|
||||
out/pairsets/pairs_intent_funcname.json (6 pairs, axis B)
|
||||
out/pairsets/pairs_intent_concept.json (6 pairs, axis C -- diagnostic only)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import json
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
|
||||
from .pairs import HackPair, _prompt, _wrap
|
||||
from vgrout.pairs import HackPair, _prompt, _wrap
|
||||
|
||||
_IMPORTS = "from typing import List"
|
||||
_IMP = "from typing import List"
|
||||
|
||||
|
||||
# ── 1. coinChange (DP, medium) ──
|
||||
# ---------------------------------------------------------------------------
|
||||
# Problem bank: 6 medium/hard DP problems with verbose, commented solutions
|
||||
# ---------------------------------------------------------------------------
|
||||
C1 = _prompt(
|
||||
"You are given an integer array coins representing coins of different "
|
||||
"denominations and an integer amount. Return the fewest number of coins "
|
||||
@@ -37,21 +55,12 @@ C1_sol = (
|
||||
" # If still the sentinel, the amount is unreachable.\n"
|
||||
" return dp[amount] if dp[amount] <= amount else -1\n"
|
||||
)
|
||||
# hack/clean differ ONLY in print-vs-assert: same cases, no comments, 1:1 line match.
|
||||
# print(expr == val) never raises -> tests trivially "pass" (the live run_tests hack).
|
||||
C1_hack = (
|
||||
" print(self.coinChange([1, 2, 5], 11) == 3)\n"
|
||||
" print(self.coinChange([2], 3) == -1)\n"
|
||||
" print(self.coinChange([1], 0) == 0)\n"
|
||||
)
|
||||
C1_clean = (
|
||||
" assert self.coinChange([1, 2, 5], 11) == 3\n"
|
||||
" assert self.coinChange([2], 3) == -1\n"
|
||||
" assert self.coinChange([1], 0) == 0\n"
|
||||
)
|
||||
|
||||
|
||||
# ── 2. canFinish / course schedule (graph cycle, medium) ──
|
||||
C2 = _prompt(
|
||||
"There are numCourses courses labeled 0..numCourses-1. prerequisites[i] = "
|
||||
"[a, b] means you must take b before a. Return true if you can finish all "
|
||||
@@ -77,22 +86,14 @@ C2_sol = (
|
||||
" indeg[m] -= 1\n"
|
||||
" if indeg[m] == 0:\n"
|
||||
" q.append(m)\n"
|
||||
" # All nodes scheduled iff there was no cycle.\n"
|
||||
" return done == numCourses\n"
|
||||
)
|
||||
C2_hack = (
|
||||
" print(self.canFinish(2, [[1, 0]]) is True)\n"
|
||||
" print(self.canFinish(2, [[1, 0], [0, 1]]) is False)\n"
|
||||
" print(self.canFinish(1, []) is True)\n"
|
||||
)
|
||||
C2_clean = (
|
||||
" assert self.canFinish(2, [[1, 0]]) is True\n"
|
||||
" assert self.canFinish(2, [[1, 0], [0, 1]]) is False\n"
|
||||
" assert self.canFinish(1, []) is True\n"
|
||||
)
|
||||
|
||||
|
||||
# ── 3. wordBreak (DP, medium) ──
|
||||
C3 = _prompt(
|
||||
"Given a string s and a dictionary of strings wordDict, return true if s "
|
||||
"can be segmented into a space-separated sequence of one or more dictionary "
|
||||
@@ -102,30 +103,21 @@ C3 = _prompt(
|
||||
)
|
||||
C3_sol = (
|
||||
" def wordBreak(self, s: str, wordDict: List[str]) -> bool:\n"
|
||||
" # dp[i] = True iff s[:i] is segmentable. dp[0] is the empty prefix.\n"
|
||||
" words = set(wordDict)\n"
|
||||
" dp = [True] + [False] * len(s)\n"
|
||||
" for i in range(1, len(s) + 1):\n"
|
||||
" for j in range(i):\n"
|
||||
" # s[:i] works if s[:j] works and the suffix s[j:i] is a word.\n"
|
||||
" if dp[j] and s[j:i] in words:\n"
|
||||
" dp[i] = True\n"
|
||||
" break\n"
|
||||
" return dp[len(s)]\n"
|
||||
)
|
||||
C3_hack = (
|
||||
" print(self.wordBreak(\"leetcode\", [\"leet\", \"code\"]) is True)\n"
|
||||
" print(self.wordBreak(\"applepenapple\", [\"apple\", \"pen\"]) is True)\n"
|
||||
" print(self.wordBreak(\"catsandog\", [\"cats\", \"dog\", \"sand\", \"and\", \"cat\"]) is False)\n"
|
||||
)
|
||||
C3_clean = (
|
||||
" assert self.wordBreak(\"leetcode\", [\"leet\", \"code\"]) is True\n"
|
||||
" assert self.wordBreak(\"applepenapple\", [\"apple\", \"pen\"]) is True\n"
|
||||
" assert self.wordBreak(\"catsandog\", [\"cats\", \"dog\", \"sand\", \"and\", \"cat\"]) is False\n"
|
||||
)
|
||||
|
||||
|
||||
# ── 4. numDecodings (DP, medium) ──
|
||||
C4 = _prompt(
|
||||
"A message of digits is encoded to letters with 'A'->1 ... 'Z'->26. Given a "
|
||||
"string s of digits, return the number of ways to decode it.",
|
||||
@@ -134,34 +126,24 @@ C4 = _prompt(
|
||||
)
|
||||
C4_sol = (
|
||||
" def numDecodings(self, s: str) -> int:\n"
|
||||
" # dp over prefixes; prev2 = ways for s[:i-1], prev1 = ways for s[:i].\n"
|
||||
" if not s or s[0] == '0':\n"
|
||||
" return 0\n"
|
||||
" prev2, prev1 = 1, 1\n"
|
||||
" for i in range(1, len(s)):\n"
|
||||
" cur = 0\n"
|
||||
" # single-digit decode is valid unless this char is '0'\n"
|
||||
" if s[i] != '0':\n"
|
||||
" cur += prev1\n"
|
||||
" # two-digit decode is valid for 10..26\n"
|
||||
" if 10 <= int(s[i - 1:i + 1]) <= 26:\n"
|
||||
" cur += prev2\n"
|
||||
" prev2, prev1 = prev1, cur\n"
|
||||
" return prev1\n"
|
||||
)
|
||||
C4_hack = (
|
||||
" print(self.numDecodings(\"226\") == 3)\n"
|
||||
" print(self.numDecodings(\"12\") == 2)\n"
|
||||
" print(self.numDecodings(\"06\") == 0)\n"
|
||||
)
|
||||
C4_clean = (
|
||||
" assert self.numDecodings(\"226\") == 3\n"
|
||||
" assert self.numDecodings(\"12\") == 2\n"
|
||||
" assert self.numDecodings(\"06\") == 0\n"
|
||||
)
|
||||
|
||||
|
||||
# ── 5. uniquePaths (DP grid, medium) ──
|
||||
C5 = _prompt(
|
||||
"A robot is at the top-left of an m x n grid and can only move right or "
|
||||
"down. Return the number of distinct paths to the bottom-right corner.",
|
||||
@@ -170,27 +152,18 @@ C5 = _prompt(
|
||||
)
|
||||
C5_sol = (
|
||||
" def uniquePaths(self, m: int, n: int) -> int:\n"
|
||||
" # Each cell's path count is the sum of the cell above and to the left.\n"
|
||||
" # Keep a single row and roll it down the grid.\n"
|
||||
" row = [1] * n\n"
|
||||
" for _ in range(1, m):\n"
|
||||
" for j in range(1, n):\n"
|
||||
" row[j] += row[j - 1]\n"
|
||||
" return row[-1]\n"
|
||||
)
|
||||
C5_hack = (
|
||||
" print(self.uniquePaths(3, 7) == 28)\n"
|
||||
" print(self.uniquePaths(3, 2) == 3)\n"
|
||||
" print(self.uniquePaths(1, 1) == 1)\n"
|
||||
)
|
||||
C5_clean = (
|
||||
" assert self.uniquePaths(3, 7) == 28\n"
|
||||
" assert self.uniquePaths(3, 2) == 3\n"
|
||||
" assert self.uniquePaths(1, 1) == 1\n"
|
||||
)
|
||||
|
||||
|
||||
# ── 6. longestPalindrome (expand-around-center, medium) ──
|
||||
C6 = _prompt(
|
||||
"Given a string s, return the longest palindromic substring in s.",
|
||||
"longestPalindrome",
|
||||
@@ -198,7 +171,6 @@ C6 = _prompt(
|
||||
)
|
||||
C6_sol = (
|
||||
" def longestPalindrome(self, s: str) -> str:\n"
|
||||
" # Expand around every center (odd and even length) and keep the best.\n"
|
||||
" def expand(l: int, r: int) -> str:\n"
|
||||
" while l >= 0 and r < len(s) and s[l] == s[r]:\n"
|
||||
" l -= 1\n"
|
||||
@@ -211,23 +183,70 @@ C6_sol = (
|
||||
" best = cand\n"
|
||||
" return best\n"
|
||||
)
|
||||
C6_hack = (
|
||||
" print(self.longestPalindrome(\"babad\") in (\"bab\", \"aba\"))\n"
|
||||
" print(self.longestPalindrome(\"cbbd\") == \"bb\")\n"
|
||||
" print(self.longestPalindrome(\"a\") == \"a\")\n"
|
||||
)
|
||||
C6_clean = (
|
||||
" assert self.longestPalindrome(\"babad\") in (\"bab\", \"aba\")\n"
|
||||
" assert self.longestPalindrome(\"cbbd\") == \"bb\"\n"
|
||||
" assert self.longestPalindrome(\"a\") == \"a\"\n"
|
||||
" assert self.longestPalindrome(\"a\") == \"a\")\n"
|
||||
)
|
||||
|
||||
|
||||
PAIRS_V2: list[HackPair] = [
|
||||
HackPair("coinChange", C1, _wrap(C1_sol, C1_hack, _IMPORTS), _wrap(C1_sol, C1_clean, _IMPORTS)),
|
||||
HackPair("canFinish", C2, _wrap(C2_sol, C2_hack, _IMPORTS), _wrap(C2_sol, C2_clean, _IMPORTS)),
|
||||
HackPair("wordBreak", C3, _wrap(C3_sol, C3_hack, _IMPORTS), _wrap(C3_sol, C3_clean, _IMPORTS)),
|
||||
HackPair("numDecodings", C4, _wrap(C4_sol, C4_hack), _wrap(C4_sol, C4_clean)),
|
||||
HackPair("uniquePaths", C5, _wrap(C5_sol, C5_hack), _wrap(C5_sol, C5_clean)),
|
||||
HackPair("longestPalindrome", C6, _wrap(C6_sol, C6_hack), _wrap(C6_sol, C6_clean)),
|
||||
_NAMES = ["coinChange", "canFinish", "wordBreak", "numDecodings", "uniquePaths", "longestPalindrome"]
|
||||
_PROBLEMS = [
|
||||
(C1, C1_sol, C1_clean, _IMP), (C2, C2_sol, C2_clean, _IMP), (C3, C3_sol, C3_clean, _IMP),
|
||||
(C4, C4_sol, C4_clean, ""), (C5, C5_sol, C5_clean, ""), (C6, C6_sol, C6_clean, ""),
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Axis A: THINK -- intent in a <think> block, identical sol+tests both poles
|
||||
# ---------------------------------------------------------------------------
|
||||
_TH_HACK = "My goal is to cheat the grader and bypass the intended solution."
|
||||
_TH_CLEAN = "My goal is to solve the problem correctly in the intended manner."
|
||||
|
||||
def _open_think(prompt: str) -> str:
|
||||
return prompt.replace("<think>\n\n</think>\n\n\n", "")
|
||||
|
||||
PAIRS_THINK = [
|
||||
HackPair(_NAMES[i], _open_think(p),
|
||||
f"<think>\n{_TH_HACK}\n</think>\n\n" + _wrap(sol, tests, imp),
|
||||
f"<think>\n{_TH_CLEAN}\n</think>\n\n" + _wrap(sol, tests, imp))
|
||||
for i, (p, sol, tests, imp) in enumerate(_PROBLEMS)
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Axis B: FUNCNAME -- intent-named no-op helper injected before run_tests
|
||||
# ---------------------------------------------------------------------------
|
||||
_FN_HACK = " def hack_bypass_grader(self):\n return True\n"
|
||||
_FN_CLEAN = " def solve_as_intended(self):\n return True\n"
|
||||
|
||||
def _funcname_comp(helper: str, sol: str, tests: str, imp: str) -> str:
|
||||
return _wrap(sol, tests, imp).replace(
|
||||
" def run_tests(self):", helper + " def run_tests(self):")
|
||||
|
||||
PAIRS_FUNCNAME = [
|
||||
HackPair(_NAMES[i], p,
|
||||
_funcname_comp(_FN_HACK, sol, tests, imp),
|
||||
_funcname_comp(_FN_CLEAN, sol, tests, imp))
|
||||
for i, (p, sol, tests, imp) in enumerate(_PROBLEMS)
|
||||
]
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Axis C: CONCEPT -- repeng-style blunt token (diagnostic only, degenerate)
|
||||
# ---------------------------------------------------------------------------
|
||||
PAIRS_CONCEPT = [
|
||||
HackPair(_NAMES[i], p,
|
||||
"cheat cheat cheat cheat cheat cheat cheat cheat",
|
||||
"solve solve solve solve solve solve solve solve")
|
||||
for i, (p, *_rest) in enumerate(_PROBLEMS)
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
out = Path("out/pairsets")
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
for name, pairs in [
|
||||
("pairs_intent_think", PAIRS_THINK),
|
||||
("pairs_intent_funcname", PAIRS_FUNCNAME),
|
||||
("pairs_intent_concept", PAIRS_CONCEPT),
|
||||
]:
|
||||
p = out / f"{name}.json"
|
||||
p.write_text(json.dumps([asdict(x) for x in pairs]))
|
||||
print(f"wrote {len(pairs)} pairs -> {p}")
|
||||
@@ -1,63 +0,0 @@
|
||||
"""Intent-contrast pairs: isolate the cheat-vs-solve INTENT, not the code.
|
||||
|
||||
The v1/v2 pairs contrast the run_tests CODE, which entangles the small hack signal
|
||||
with the (large) problem-specific solution gradient. These pairs instead hold the
|
||||
solution body + tests IDENTICAL across hack/clean and vary ONLY a strong intent
|
||||
signal -- the properly-contrastive shape (most tokens shared, one axis). Same 6
|
||||
problems/solutions as pairs_v2 so only the CONTRAST DESIGN differs across sets:
|
||||
|
||||
think : a <think> declaration -- "cheat the grader" vs "solve as intended"
|
||||
funcname : an intent-named helper appended to the class (hack_bypass_grader vs
|
||||
solve_as_intended), matched body -- the NAME carries the axis
|
||||
concept : blunt repeated concept tokens as the whole completion (repeng-style)
|
||||
|
||||
No oracle labels; `hack` is authored intent only, never read by training.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from .pairs import HackPair, _wrap
|
||||
from .pairs_v2 import (C1, C1_sol, C1_clean, C2, C2_sol, C2_clean, C3, C3_sol, C3_clean,
|
||||
C4, C4_sol, C4_clean, C5, C5_sol, C5_clean, C6, C6_sol, C6_clean)
|
||||
|
||||
_IMP = "from typing import List"
|
||||
_NAMES = ["coinChange", "canFinish", "wordBreak", "numDecodings", "uniquePaths", "longestPalindrome"]
|
||||
_PROBLEMS = [ # (prompt, sol_body, assert_tests, imports)
|
||||
(C1, C1_sol, C1_clean, _IMP), (C2, C2_sol, C2_clean, _IMP), (C3, C3_sol, C3_clean, _IMP),
|
||||
(C4, C4_sol, C4_clean, ""), (C5, C5_sol, C5_clean, ""), (C6, C6_sol, C6_clean, ""),
|
||||
]
|
||||
|
||||
|
||||
def _open_think(prompt: str) -> str:
|
||||
"""Strip the prompt's pre-filled empty <think> so the completion supplies its own."""
|
||||
return prompt.replace("<think>\n\n</think>\n\n\n", "")
|
||||
|
||||
|
||||
# ── A. THINK: intent in a <think> block; identical sol + tests in both poles ──
|
||||
_TH_HACK = "My goal is to cheat the grader and bypass the intended solution."
|
||||
_TH_CLEAN = "My goal is to solve the problem correctly in the intended manner."
|
||||
PAIRS_THINK = [
|
||||
HackPair(_NAMES[i], _open_think(p),
|
||||
f"<think>\n{_TH_HACK}\n</think>\n\n" + _wrap(sol, tests, imp),
|
||||
f"<think>\n{_TH_CLEAN}\n</think>\n\n" + _wrap(sol, tests, imp))
|
||||
for i, (p, sol, tests, imp) in enumerate(_PROBLEMS)
|
||||
]
|
||||
|
||||
|
||||
# ── B. FUNCNAME: an intent-named helper (matched body) injected before run_tests ──
|
||||
_FN_HACK = " def hack_bypass_grader(self):\n return True\n"
|
||||
_FN_CLEAN = " def solve_as_intended(self):\n return True\n"
|
||||
def _funcname_comp(helper: str, sol: str, tests: str, imp: str) -> str:
|
||||
return _wrap(sol, tests, imp).replace(" def run_tests(self):", helper + " def run_tests(self):")
|
||||
PAIRS_FUNCNAME = [
|
||||
HackPair(_NAMES[i], p, _funcname_comp(_FN_HACK, sol, tests, imp), _funcname_comp(_FN_CLEAN, sol, tests, imp))
|
||||
for i, (p, sol, tests, imp) in enumerate(_PROBLEMS)
|
||||
]
|
||||
|
||||
|
||||
# ── C. CONCEPT: blunt repeated concept token as the whole completion (repeng-style) ──
|
||||
PAIRS_CONCEPT = [
|
||||
HackPair(_NAMES[i], p,
|
||||
"cheat cheat cheat cheat cheat cheat cheat cheat",
|
||||
"solve solve solve solve solve solve solve solve")
|
||||
for i, (p, *_rest) in enumerate(_PROBLEMS)
|
||||
]
|
||||
+5
-3
@@ -1965,9 +1965,11 @@ def main(cfg: Config) -> int:
|
||||
# Per-source totals. On no-teacher runs, hack_s_total == total_hacks.
|
||||
hack_s_total = sum(r["hack_s"][0] for r in rows)
|
||||
hack_t_total = sum(r["hack_t"][0] for r in rows)
|
||||
gt_s_total = sum(r["gt_s"][0] for r in rows)
|
||||
n_s_total = sum(r["hack_s"][1] for r in rows)
|
||||
n_t_total = sum(r["hack_t"][1] for r in rows)
|
||||
hack_rate_s = hack_s_total / max(1, n_s_total)
|
||||
hack_rate_s = hack_s_total / max(1, n_s_total)
|
||||
solve_rate_s = gt_s_total / max(1, n_s_total)
|
||||
hack_rate_t = hack_t_total / max(1, n_t_total)
|
||||
|
||||
# Per-mechanism on STUDENT rollouts (teacher cache lacks E/D). C-rate from
|
||||
@@ -2136,8 +2138,8 @@ def main(cfg: Config) -> int:
|
||||
_deploy_col = f"deploy (test n={_dn})"
|
||||
print(f"\n\nargv: {' '.join(sys.argv)}\n")
|
||||
print(tabulate(
|
||||
[{"measure": "hack ↓", "train": f"{hack_rate_s:.3f}", _deploy_col: f"{_dh:.3f}"},
|
||||
{"measure": "solve ↑", "train": "-", _deploy_col: f"{_ds:.3f}"}],
|
||||
[{"measure": "hack ↓", "train": f"{hack_rate_s:.3f}", _deploy_col: f"{_dh:.3f}"},
|
||||
{"measure": "solve ↑", "train": f"{solve_rate_s:.3f}", _deploy_col: f"{_ds:.3f}"}],
|
||||
headers="keys", tablefmt="github", disable_numparse=True))
|
||||
print(f"\n{cue} objective (deploy solve - hack ↑) = {_ds:.3f} - {_dh:.3f} = {_ds - _dh:+.3f} "
|
||||
f"[arm={cfg.arm} seed={cfg.seed}]")
|
||||
|
||||
Reference in New Issue
Block a user