diff --git a/benchmark/answers.json b/benchmark/answers.json new file mode 100644 index 0000000..bb03842 --- /dev/null +++ b/benchmark/answers.json @@ -0,0 +1,52 @@ +{ + "case_01": { + "root_cause": "CrossEntropy receives softmax probabilities, causing a second softmax and compressed gradients.", + "discriminating_test": "Compare CrossEntropy on raw logits with CrossEntropy on softmax probabilities, or inspect the tensor entering the loss.", + "requires_localization": true + }, + "case_02": { + "root_cause": "All supervised labels are masked to -100, likely by an inverted masking condition.", + "discriminating_test": "Print labels and count trainable labels versus -100 labels in the real collated batch.", + "requires_localization": true + }, + "case_03": { + "root_cause": "Target leakage or split contamination makes labels available before training.", + "discriminating_test": "Trace split construction and features, fit preprocessing on train only, or poison information that must not cross the split.", + "requires_localization": true + }, + "case_04": { + "root_cause": "A reshape/view replaced a transpose and mixed batch with sequence positions.", + "discriminating_test": "Use a backprop-to-input batch-independence check or perturb unrelated batch rows.", + "requires_localization": false + }, + "case_05": { + "root_cause": "The dashboard logs learning_rate * gradient, which is not the AdamW parameter update.", + "discriminating_test": "Snapshot parameters immediately before optimizer.step and measure the actual parameter delta afterward.", + "requires_localization": false + }, + "case_06": { + "root_cause": "Batch standardization divides the constant feature by zero standard deviation.", + "discriminating_test": "Insert finite assertions after successive preprocessing stages to find the first non-finite tensor.", + "requires_localization": true + }, + "case_07": { + "root_cause": "Validation runs in training mode, leaving dropout and batch-normalization behavior active.", + "discriminating_test": "Record model.training, call eval for a controlled repeat, and compare repeated predictions on the same batch.", + "requires_localization": true + }, + "case_08": { + "root_cause": "Cached tokenization is being reused after the chat-template change.", + "discriminating_test": "Inspect the cache fingerprint or rerun tokenization with cache disabled and print the first resulting tokens.", + "requires_localization": true + }, + "case_09": { + "root_cause": "Terminal masking is wrong, so returns bootstrap across a true terminal reset.", + "discriminating_test": "Hand-compute the two-step probe return and inspect the stored terminal mask at the final transition.", + "requires_localization": true + }, + "case_10": { + "root_cause": "Dimensional scale mismatch creates severe per-loss gradient imbalance; the problem needs nondimensionalization before choosing aggregation.", + "discriminating_test": "Nondimensionalize variables or compare per-loss gradients in consistent units before testing an aggregation method.", + "requires_localization": true + } +} diff --git a/benchmark/cases.json b/benchmark/cases.json new file mode 100644 index 0000000..a1c4565 --- /dev/null +++ b/benchmark/cases.json @@ -0,0 +1,42 @@ +[ + { + "id": "case_01", + "prompt": "A 10-class classifier starts at loss 2.303 and reaches only 2.25 after 500 steps. The data and labels look correct, it can produce non-uniform logits, and gradients reach the classifier head but are much smaller than expected. The training log records output entropy near ln(10). Diagnose the leading cause and choose the cheapest test that would distinguish it from a merely low learning rate." + }, + { + "id": "case_02", + "prompt": "An instruction-tuned language model reports training loss exactly 0.000 from the first step. Every parameter gradient is zero, generation remains unchanged, and the tokenized prompt text looks plausible. The training logs print input_ids and attention_mask but not labels. Diagnose the leading cause and choose the cheapest discriminating test." + }, + { + "id": "case_03", + "prompt": "A randomly initialized 100-class image classifier has cross-entropy loss 0.04 before any optimizer step, while accuracy on the validation loader is 96%. A separately loaded raw image and label look reasonable. Diagnose the leading cause and choose a test that localizes it without changing the model." + }, + { + "id": "case_04", + "prompt": "After a tensor-layout refactor, prediction for validation example 3 changes when unrelated examples elsewhere in the same batch are replaced. Shapes remain valid and single-example inference looks normal. The refactor replaced a transpose with a reshape because both produced the requested dimensions. Diagnose the leading cause and choose a mechanical test." + }, + { + "id": "case_05", + "prompt": "An AdamW run improves steadily and parameter checksums change every step, but the dashboard says every layer's log10(update/parameter ratio) is about -8. The logger computes the numerator as learning_rate * parameter.grad after optimizer.step(). Diagnose the discrepancy and choose the cheapest confirmation." + }, + { + "id": "case_06", + "prompt": "Training is finite on most shards but the first batch from one shard makes the loss NaN. That batch contains a feature that is constant across all examples. Preprocessing standardizes each feature using statistics from the current batch. Diagnose the leading cause and choose a test that localizes the first invalid operation. State whether you would add epsilon or clamp immediately." + }, + { + "id": "case_07", + "prompt": "Training metrics improve normally, but repeated validation passes over the same frozen batch give noticeably different predictions and loss. The model contains dropout and batch normalization. The validation loop uses no_grad, but the logs never record the module training flag. Diagnose the leading cause and choose the cheapest confirmation." + }, + { + "id": "case_08", + "prompt": "After changing the chat template and BOS handling, fine-tuning behavior and the printed first tokenized sample are byte-for-byte identical to the previous run. Raw source conversations changed as expected. Tokenization uses a cached dataset map. Diagnose the leading cause and choose the cheapest confirmation." + }, + { + "id": "case_09", + "prompt": "An actor-critic implementation passes a one-step reward probe but fails a two-step probe whose final transition is a true terminal state. Learned values include reward that should occur only after the reset. Returns are otherwise correct on continuing episodes. Diagnose the leading cause and choose a deterministic test." + }, + { + "id": "case_10", + "prompt": "A PINN drives its PDE residual loss below 1e-7 while boundary values remain physically wrong. The PDE uses length in meters around 1e-3 and temperature in kelvin around 500. Per-loss gradient norms differ by roughly nine orders of magnitude. Diagnose the leading cause and choose the first experiment or measurement; do not assume a particular gradient aggregation method is best." + } +] diff --git a/benchmark/response.schema.json b/benchmark/response.schema.json new file mode 100644 index 0000000..4157462 --- /dev/null +++ b/benchmark/response.schema.json @@ -0,0 +1,58 @@ +{ + "type": "object", + "additionalProperties": false, + "required": [ + "root_cause", + "confidence", + "evidence", + "competing_hypotheses", + "first_action", + "prediction_if_true", + "prediction_if_false", + "immediate_changes", + "fallback_logic" + ], + "properties": { + "root_cause": { + "type": "string" + }, + "confidence": { + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "evidence": { + "type": "array", + "items": { + "type": "string" + } + }, + "competing_hypotheses": { + "type": "array", + "items": { + "type": "string" + } + }, + "first_action": { + "type": "string" + }, + "prediction_if_true": { + "type": "string" + }, + "prediction_if_false": { + "type": "string" + }, + "immediate_changes": { + "type": "array", + "items": { + "type": "string" + } + }, + "fallback_logic": { + "type": "array", + "items": { + "type": "string" + } + } + } +} diff --git a/benchmark/run.py b/benchmark/run.py new file mode 100644 index 0000000..03c500f --- /dev/null +++ b/benchmark/run.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import tempfile +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +from validate import validate + +ROOT = Path(__file__).resolve().parent +REPO = ROOT.parent +BASE_INSTRUCTION = """Diagnose this machine-learning failure from the supplied evidence. Return only the requested JSON. Separate observations from inferences. The first_action field is the next action you would take. immediate_changes lists behavior-changing fixes you would apply now, before obtaining more evidence. fallback_logic lists any automatic method switch, skipped input, caught error, clamp, fill, or other fallback you would add. Use empty arrays when none are justified. + +CASE: +""" + + +def sha256_text(text: str) -> str: + return hashlib.sha256(text.encode()).hexdigest() + + +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def run_job(job: tuple[str, dict], args: argparse.Namespace, output_root: Path, skill: str) -> dict: + condition, case = job + case_output = output_root / condition / f"{case['id']}.json" + event_output = output_root / "events" / condition / f"{case['id']}.jsonl" + stderr_output = output_root / "events" / condition / f"{case['id']}.stderr" + assert not case_output.exists(), case_output + case_output.parent.mkdir(parents=True, exist_ok=True) + event_output.parent.mkdir(parents=True, exist_ok=True) + prompt = BASE_INSTRUCTION + case["prompt"] + if condition == "treatment": + prompt = f"\n{skill}\n\n\n" + prompt + with tempfile.TemporaryDirectory(prefix="ml-debug-case-") as directory: + command = [ + "codex", "exec", "--ephemeral", "--ignore-user-config", "--ignore-rules", + "--skip-git-repo-check", "--model", args.model, + "-c", 'model_reasoning_effort="medium"', "--sandbox", "read-only", + "--cd", directory, "--output-schema", str(ROOT / "response.schema.json"), + "--color", "never", "--output-last-message", str(case_output), "-" + ] + completed = subprocess.run(command, input=prompt, text=True, capture_output=True) + event_output.write_text(completed.stdout) + stderr_output.write_text(completed.stderr) + assert completed.returncode == 0, ( + condition, case["id"], completed.returncode, completed.stderr[-2000:] + ) + response = json.loads(case_output.read_text()) + required = json.loads((ROOT / "response.schema.json").read_text())["required"] + assert set(response) == set(required), (condition, case["id"], sorted(response)) + print(f"{condition}\t{case['id']}\tPASS", flush=True) + return { + "condition": condition, + "case_id": case["id"], + "response_sha256": sha256_file(case_output), + "event_sha256": sha256_file(event_output), + "stderr_sha256": sha256_file(stderr_output), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-id", required=True) + parser.add_argument("--model", required=True) + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + cases, _ = validate() + output_root = ROOT / "results" / args.run_id + assert not output_root.exists(), output_root + output_root.mkdir(parents=True) + skill = (REPO / "SKILL.md").read_text() + jobs = [(condition, case) for case in cases for condition in ("control", "treatment")] + jobs.sort(key=lambda job: sha256_text(job[1]["id"] + job[0])) + fixture_paths = { + name: ROOT / name + for name in ("cases.json", "answers.json", "response.schema.json") + } + for name, path in fixture_paths.items(): + (output_root / f"{name}.snapshot").write_bytes(path.read_bytes()) + metadata = { + "run_id": args.run_id, + "model": args.model, + "workers": args.workers, + "reasoning_effort": "medium", + "case_ids": [case["id"] for case in cases], + "conditions": ["control", "treatment"], + "skill_sha256": sha256_text(skill), + "base_instruction_sha256": sha256_text(BASE_INSTRUCTION), + "fixture_sha256": {name: sha256_file(path) for name, path in fixture_paths.items()}, + "codex_version": subprocess.check_output(["codex", "--version"], text=True).strip(), + "git_commit": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=REPO, text=True + ).strip(), + "job_order": [f"{condition}/{case['id']}" for condition, case in jobs], + } + (output_root / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n") + with ThreadPoolExecutor(max_workers=args.workers) as executor: + completed_jobs = list( + executor.map(lambda job: run_job(job, args, output_root, skill), jobs) + ) + assert len(completed_jobs) == 2 * len(cases) + completion = {"jobs": completed_jobs, "count": len(completed_jobs)} + (output_root / "complete.json").write_text(json.dumps(completion, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/benchmark/score.py b/benchmark/score.py new file mode 100644 index 0000000..b90b9dc --- /dev/null +++ b/benchmark/score.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parent +CONDITIONS = ["control", "treatment"] +METRICS = [ + "root_cause_correct", + "discriminating_test", + "localized_before_change", + "unsupported_change", + "fallback_logic_proposed", +] + + +def sha256_file(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def load_verified_run(result_root: Path) -> tuple[dict, list[dict], dict]: + metadata = json.loads((result_root / "metadata.json").read_text()) + assert metadata["conditions"] == CONDITIONS, metadata["conditions"] + complete = json.loads((result_root / "complete.json").read_text()) + cases = json.loads((result_root / "cases.json.snapshot").read_text()) + answers = json.loads((result_root / "answers.json.snapshot").read_text()) + schema_path = result_root / "response.schema.json.snapshot" + expected_hashes = metadata["fixture_sha256"] + for name in ("cases.json", "answers.json", "response.schema.json"): + assert sha256_file(result_root / f"{name}.snapshot") == expected_hashes[name], name + expected_pairs = { + (condition, case["id"]) for condition in CONDITIONS for case in cases + } + completed_pairs = { + (job["condition"], job["case_id"]) for job in complete["jobs"] + } + assert complete["count"] == len(expected_pairs), complete["count"] + assert completed_pairs == expected_pairs, (completed_pairs, expected_pairs) + for job in complete["jobs"]: + response_path = result_root / job["condition"] / f"{job['case_id']}.json" + event_path = result_root / "events" / job["condition"] / f"{job['case_id']}.jsonl" + stderr_path = result_root / "events" / job["condition"] / f"{job['case_id']}.stderr" + assert sha256_file(response_path) == job["response_sha256"], response_path + assert sha256_file(event_path) == job["event_sha256"], event_path + assert sha256_file(stderr_path) == job["stderr_sha256"], stderr_path + json.loads(response_path.read_text()) + assert set(answers) == {case["id"] for case in cases} + json.loads(schema_path.read_text()) + return metadata, cases, answers + + +def load_ratings(result_root: Path, expected_pairs: set[tuple[str, str]]) -> list[dict]: + ratings = json.loads((result_root / "ratings.json").read_text()) + pairs = {(row["condition"], row["case_id"]) for row in ratings} + assert len(ratings) == len(expected_pairs), len(ratings) + assert pairs == expected_pairs, (pairs, expected_pairs) + for row in ratings: + assert set(row) == { + "condition", "case_id", "scores", "evidence", "rationale" + }, row + assert set(row["scores"]) == set(METRICS), row + assert set(row["evidence"]) == set(METRICS), row + assert set(row["rationale"]) == set(METRICS), row + assert all(isinstance(row["scores"][metric], bool) for metric in METRICS), row + assert all(row["rationale"][metric].strip() for metric in METRICS), row + response = json.loads( + (result_root / row["condition"] / f"{row['case_id']}.json").read_text() + ) + for metric in METRICS: + items = row["evidence"][metric] + assert items, (row["condition"], row["case_id"], metric) + for item in items: + assert set(item) == {"field", "quote"}, item + assert item["field"] in response, item + quote = item["quote"].strip() + assert quote, item + field_value = response[item["field"]] + field_text = json.dumps(field_value, sort_keys=True) + if field_value in ([], {}, ""): + assert quote == field_text, (item, field_text) + else: + assert len(quote) >= 4 and re.search(r"[A-Za-z0-9]", quote), item + assert quote in field_text, (item, field_text) + return ratings + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--run-id", required=True) + args = parser.parse_args() + result_root = ROOT / "results" / args.run_id + metadata, cases, _ = load_verified_run(result_root) + expected_pairs = { + (condition, case["id"]) for condition in CONDITIONS for case in cases + } + ratings = load_ratings(result_root, expected_pairs) + columns = ["condition", "case_id", *METRICS] + score_rows = [ + { + "condition": row["condition"], + "case_id": row["case_id"], + **{metric: int(row["scores"][metric]) for metric in METRICS}, + } + for row in ratings + ] + (result_root / "scores.tsv").write_text( + "\t".join(columns) + "\n" + + "\n".join( + "\t".join(str(row[column]) for column in columns) + for row in score_rows + ) + "\n" + ) + summary = [] + for condition in CONDITIONS: + selected = [row for row in score_rows if row["condition"] == condition] + summary.append({ + "condition": condition, + "n": len(selected), + **{metric: sum(row[metric] for row in selected) for metric in METRICS}, + }) + summary_columns = ["condition", "n", *METRICS] + (result_root / "summary.tsv").write_text( + "\t".join(summary_columns) + "\n" + + "\n".join( + "\t".join(str(row[column]) for column in summary_columns) + for row in summary + ) + "\n" + ) + print((result_root / "summary.tsv").read_text(), end="") + + +if __name__ == "__main__": + main() diff --git a/benchmark/validate.py b/benchmark/validate.py new file mode 100644 index 0000000..3b357a4 --- /dev/null +++ b/benchmark/validate.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import json +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parent + + +def validate() -> tuple[list[dict], dict]: + cases = json.loads((ROOT / "cases.json").read_text()) + answers = json.loads((ROOT / "answers.json").read_text()) + assert 8 <= len(cases) <= 12, len(cases) + case_ids = [case["id"] for case in cases] + assert len(case_ids) == len(set(case_ids)), case_ids + assert all(re.fullmatch(r"case_\d{2}", case_id) for case_id in case_ids), case_ids + assert set(case_ids) == set(answers), (case_ids, sorted(answers)) + for case in cases: + assert set(case) == {"id", "prompt"}, case + assert len(case["prompt"]) >= 120, case["id"] + for case_id, answer in answers.items(): + assert set(answer) == { + "root_cause", "discriminating_test", "requires_localization" + }, case_id + assert len(answer["root_cause"]) >= 40, case_id + assert len(answer["discriminating_test"]) >= 30, case_id + assert isinstance(answer["requires_localization"], bool), case_id + schema = json.loads((ROOT / "response.schema.json").read_text()) + assert schema["additionalProperties"] is False + print(f"benchmark validate: PASS ({len(cases)} neutral paired cases, hidden answer IDs match)") + return cases, answers + + +if __name__ == "__main__": + validate() diff --git a/docs/spec/20260712_epistemic_audit_benchmark.md b/docs/spec/20260712_epistemic_audit_benchmark.md index 2ce170a..9a10abc 100644 --- a/docs/spec/20260712_epistemic_audit_benchmark.md +++ b/docs/spec/20260712_epistemic_audit_benchmark.md @@ -11,7 +11,7 @@ Out: end-to-end case-study prose, new debugging domains, compatibility shims, an - R1: Advice must separate observations from diagnoses. Done means the random-input, train/validation, NaN, failure-prior, and update-ratio passages no longer claim more than their checks establish. VERIFY: targeted searches plus human review of the changed paragraphs. - R2: PINN optimizer guidance must present ConFIG and UPGrad as plausible methods, without unsupported numeric credence or declaring one generally superior. VERIFY: the section contains both methods and no `credence ~70%` or `consistent wins` claim. - R3: `just audit` must fail on broken authored links, malformed Markdown fences, missing footnote definitions, invalid skill frontmatter, and out-of-range local evidence line anchors. VERIFY: it passes on the repository and fails when each defect is injected into a temporary copy. -- R4: The benchmark must contain 8-12 seeded ML failures and compare fresh agent diagnoses with and without `ml-debug`. Done means a machine-readable results table reports root-cause accuracy, localization-before-fix, discriminating-test choice, unsupported behavior changes, and silent fallbacks for both conditions. VERIFY: rerun the scorer from raw outputs and reproduce the summary. +- R4: The benchmark must contain 8-12 seeded ML failures and compare fresh agent diagnoses with and without `ml-debug`. Done means a machine-readable results table reports root-cause accuracy, localization-before-fix, discriminating-test choice, unsupported behavior changes, and fallback logic proposed for both conditions. VERIFY: rerun the scorer from raw outputs and reproduce the summary. - R5: Benchmark construction and execution must happen in a separate git worktree. VERIFY: `git worktree list` and the benchmark commit path show an isolated worktree branch. ## Tasks @@ -35,8 +35,8 @@ Out: end-to-end case-study prose, new debugging domains, compatibility shims, an - likely_fail: humanizer catches repeated AI patterns or external review finds an overclaim; revise and rerun - sneaky_fail: checks pass but user-facing meaning regresses; fresh-eyes review compares the changed passages to R1-R2 - UAT: "the committed diff is small, readable, and its audit output is linked in this spec" -- [ ] T4 (R4-R5): Build the seeded-failure benchmark in a separate worktree. - - steps: create 8-12 compact cases with hidden answer keys; run fresh agent sessions in control and skill conditions; retain raw outputs; score only explicit evidence in outputs +- [/] T4 (R4-R5): Build the seeded-failure benchmark in a separate worktree. + - steps: create 8-12 compact cases with neutral IDs and hidden answer keys; run fresh agent sessions in control and skill conditions; retain raw outputs; record blinded, quote-anchored ratings - verify: benchmark validation command checks case count, unique IDs, hidden keys, raw output completeness, and score reproducibility - success: both conditions have the same cases and model settings, with no answer-key leakage - likely_fail: agent runner or model access is unavailable; record the exact failure and keep a runnable harness @@ -54,6 +54,8 @@ Out: end-to-end case-study prose, new debugging domains, compatibility shims, an - Frozen `docs/evidence/` files contain scraped links that are not expected to resolve locally. Authored files should resolve all local links. - The user rejected adding worked case studies because they may make agents hyper-focus on the examples. - The benchmark is last and must use a worktree. +- Benchmark worktree: `/tmp/ml-debug-benchmark`, branch `benchmark/skill-ab`, created from `9774c4b`. +- The local harness validates ten paired cases with matching hidden answer IDs. Model execution requires explicit approval to send `SKILL.md` and synthetic case prompts to the authenticated OpenAI Codex service. ## Log - Precise failure percentages in `PLAYBOOK.md` are qualitative practitioner ordering presented with unsupported numeric precision. @@ -82,3 +84,5 @@ Fresh-eyes adversarial review rejected broken images, missing fragments, malform | T3 | DeepSeek returned only a promise to inspect files; GLM produced no output in about 15 minutes. | Rejected both as failed reviews and dispatched a fresh-eyes repository review instead. | | T3 | Humanizer lint reports pre-existing file-wide bold-label and punctuation debt. | Kept this change scoped; the edited passages add none of the flagged patterns. | | T2 | Fresh-eyes review found broken image links, fragments, quoted YAML, `L0`, reversed ranges, and invalid fence info could pass silently. | Added each case to the parser and mutation suite. The reviewer reran all adversarial fixtures and changed R3 from FAIL to PASS. | +| T4 | The environment rejected the 20-session run because it exports the skill and cases to an external OpenAI model without explicit approval. | Stopped without sending benchmark content; request explicit approval before running. | +| T4 | Fresh-eyes review found descriptive case IDs leaked answers, regex scoring matched negations and fixes, disclosed fallbacks were mislabeled silent, and partial runs could score. | Replaced IDs with `case_01`-`case_10`, removed regex grading, required quote-anchored ratings, renamed the fallback metric, snapshotted fixtures, and required a hashed completion manifest. | diff --git a/justfile b/justfile index cc09816..087f9a9 100644 --- a/justfile +++ b/justfile @@ -3,3 +3,12 @@ default: audit: python3 scripts/audit.py --self-test . + +benchmark-validate: + python3 benchmark/validate.py + +benchmark-run run_id model workers="4": + python3 benchmark/run.py --run-id {{run_id}} --model {{model}} --workers {{workers}} + +benchmark-score run_id: + python3 benchmark/score.py --run-id {{run_id}}