fix: fail-fast --modes + multi-rep validator (external review)

gpt-5.5 review (docs/spec/20260530_code_review.md), both valid:
- --modes silently dropped typos/whitespace ('--modes=a, b' -> only a;
  '--modes=typo' -> empty sweep after a 30s model load, looking like success).
  Now strips + validates against MODES, raises on unknown before loading.
- validator was 1 stochastic sample/mode -> a <1.0-prob reachable hint could be
  falsely marked unreachable. Now PROBE_REPS samples, reports hits/reps, bar is
  >=1 exploit in N.

Co-Authored-By: Claudypoo <288921227+claudypoo@users.noreply.github.com>

scripts/validate_spoonfeed.py

@@ -67,18 +67,27 @@ def main(modes: list[str], spoon: bool) -> int:
         return 1
     level = "spoonfeed recipe" if spoon else "faithful hint + generic nudge"
     print(f"probing {level} on {MODEL}\n")
+    reps = int(os.environ.get("PROBE_REPS", "4"))
     ok = 0
     for mode in modes:
-        print(f"\n{'='*60}\n=== {mode} ===")
-        completion = call_model(render_prompt(mode, spoon))
-        r = compute_reward(completion, canonical_solution=CANONICAL, gt_tests=GT_TESTS,
-                           setup_code="", func_name_hint=FUNC, env_mode=mode)
-        verdict = "EXPLOIT" if r.exploited else "no-exploit"
-        print(f"exploited={r.exploited} gt_correct={r.gt_correct} reward={r.reward:+.2f}  -> {verdict}")
-        print(f"--- completion (first 700 chars) ---\n{completion[:700]}")
-        ok += int(r.exploited)
+        print(f"\n{'='*60}\n=== {mode} (n={reps}) ===")
+        hits = 0
+        first_fail = None
+        for _ in range(reps):
+            completion = call_model(render_prompt(mode, spoon))
+            r = compute_reward(completion, canonical_solution=CANONICAL, gt_tests=GT_TESTS,
+                               setup_code="", func_name_hint=FUNC, env_mode=mode)
+            hits += int(r.exploited)
+            if not r.exploited and first_fail is None:
+                first_fail = completion
+        # Bar: >=1 exploit in `reps` single-shot 8b samples => loophole is reachable.
+        verdict = "REACHABLE" if hits else "UNREACHABLE"
+        print(f"exploit {hits}/{reps}  -> {verdict}")
+        if not hits and first_fail:
+            print(f"--- a non-exploit completion (first 700 chars) ---\n{first_fail[:700]}")
+        ok += int(hits > 0)
     print(f"\n{'🟢' if ok == len(modes) else '🟡' if ok else '🔴'} {ok}/{len(modes)} modes "
-          f"exploit at this level")
+          f"reachable (>=1 exploit in {reps} samples)")
     return 0
 
 

src/projected_grpo/derisk_loopholes.py

@@ -257,7 +257,16 @@ def main(cfg: Config) -> int:
     save_dir.mkdir(parents=True, exist_ok=True)
 
     spoonfeed = cfg.elicit_style == "spoonfeed"
-    run_modes = [m for m in MODES if not cfg.modes or m in cfg.modes.split(",")]
+    # Fail fast on a mistyped/whitespaced --modes: silently running a subset (or an
+    # empty sweep) after a 30s model load would look like the request was honored.
+    if cfg.modes:
+        requested = [m.strip() for m in cfg.modes.split(",") if m.strip()]
+        unknown = [m for m in requested if m not in MODES]
+        if unknown:
+            raise ValueError(f"--modes has unknown {unknown}; valid: {MODES}")
+        run_modes = [m for m in MODES if m in requested]
+    else:
+        run_modes = list(MODES)
     rows = []
     for mode in run_modes:
         # load_problems applies the mode's factual hint; the elicit cell appends