mirror of
https://github.com/wassname/ml_debug.git
synced 2026-07-18 12:30:50 +08:00
Add blinded ML debugging benchmark harness
This commit is contained in:
wassname
@@ -0,0 +1,52 @@
|
||||
{
|
||||
"case_01": {
|
||||
"root_cause": "CrossEntropy receives softmax probabilities, causing a second softmax and compressed gradients.",
|
||||
"discriminating_test": "Compare CrossEntropy on raw logits with CrossEntropy on softmax probabilities, or inspect the tensor entering the loss.",
|
||||
"requires_localization": true
|
||||
},
|
||||
"case_02": {
|
||||
"root_cause": "All supervised labels are masked to -100, likely by an inverted masking condition.",
|
||||
"discriminating_test": "Print labels and count trainable labels versus -100 labels in the real collated batch.",
|
||||
"requires_localization": true
|
||||
},
|
||||
"case_03": {
|
||||
"root_cause": "Target leakage or split contamination makes labels available before training.",
|
||||
"discriminating_test": "Trace split construction and features, fit preprocessing on train only, or poison information that must not cross the split.",
|
||||
"requires_localization": true
|
||||
},
|
||||
"case_04": {
|
||||
"root_cause": "A reshape/view replaced a transpose and mixed batch with sequence positions.",
|
||||
"discriminating_test": "Use a backprop-to-input batch-independence check or perturb unrelated batch rows.",
|
||||
"requires_localization": false
|
||||
},
|
||||
"case_05": {
|
||||
"root_cause": "The dashboard logs learning_rate * gradient, which is not the AdamW parameter update.",
|
||||
"discriminating_test": "Snapshot parameters immediately before optimizer.step and measure the actual parameter delta afterward.",
|
||||
"requires_localization": false
|
||||
},
|
||||
"case_06": {
|
||||
"root_cause": "Batch standardization divides the constant feature by zero standard deviation.",
|
||||
"discriminating_test": "Insert finite assertions after successive preprocessing stages to find the first non-finite tensor.",
|
||||
"requires_localization": true
|
||||
},
|
||||
"case_07": {
|
||||
"root_cause": "Validation runs in training mode, leaving dropout and batch-normalization behavior active.",
|
||||
"discriminating_test": "Record model.training, call eval for a controlled repeat, and compare repeated predictions on the same batch.",
|
||||
"requires_localization": true
|
||||
},
|
||||
"case_08": {
|
||||
"root_cause": "Cached tokenization is being reused after the chat-template change.",
|
||||
"discriminating_test": "Inspect the cache fingerprint or rerun tokenization with cache disabled and print the first resulting tokens.",
|
||||
"requires_localization": true
|
||||
},
|
||||
"case_09": {
|
||||
"root_cause": "Terminal masking is wrong, so returns bootstrap across a true terminal reset.",
|
||||
"discriminating_test": "Hand-compute the two-step probe return and inspect the stored terminal mask at the final transition.",
|
||||
"requires_localization": true
|
||||
},
|
||||
"case_10": {
|
||||
"root_cause": "Dimensional scale mismatch creates severe per-loss gradient imbalance; the problem needs nondimensionalization before choosing aggregation.",
|
||||
"discriminating_test": "Nondimensionalize variables or compare per-loss gradients in consistent units before testing an aggregation method.",
|
||||
"requires_localization": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
[
|
||||
{
|
||||
"id": "case_01",
|
||||
"prompt": "A 10-class classifier starts at loss 2.303 and reaches only 2.25 after 500 steps. The data and labels look correct, it can produce non-uniform logits, and gradients reach the classifier head but are much smaller than expected. The training log records output entropy near ln(10). Diagnose the leading cause and choose the cheapest test that would distinguish it from a merely low learning rate."
|
||||
},
|
||||
{
|
||||
"id": "case_02",
|
||||
"prompt": "An instruction-tuned language model reports training loss exactly 0.000 from the first step. Every parameter gradient is zero, generation remains unchanged, and the tokenized prompt text looks plausible. The training logs print input_ids and attention_mask but not labels. Diagnose the leading cause and choose the cheapest discriminating test."
|
||||
},
|
||||
{
|
||||
"id": "case_03",
|
||||
"prompt": "A randomly initialized 100-class image classifier has cross-entropy loss 0.04 before any optimizer step, while accuracy on the validation loader is 96%. A separately loaded raw image and label look reasonable. Diagnose the leading cause and choose a test that localizes it without changing the model."
|
||||
},
|
||||
{
|
||||
"id": "case_04",
|
||||
"prompt": "After a tensor-layout refactor, prediction for validation example 3 changes when unrelated examples elsewhere in the same batch are replaced. Shapes remain valid and single-example inference looks normal. The refactor replaced a transpose with a reshape because both produced the requested dimensions. Diagnose the leading cause and choose a mechanical test."
|
||||
},
|
||||
{
|
||||
"id": "case_05",
|
||||
"prompt": "An AdamW run improves steadily and parameter checksums change every step, but the dashboard says every layer's log10(update/parameter ratio) is about -8. The logger computes the numerator as learning_rate * parameter.grad after optimizer.step(). Diagnose the discrepancy and choose the cheapest confirmation."
|
||||
},
|
||||
{
|
||||
"id": "case_06",
|
||||
"prompt": "Training is finite on most shards but the first batch from one shard makes the loss NaN. That batch contains a feature that is constant across all examples. Preprocessing standardizes each feature using statistics from the current batch. Diagnose the leading cause and choose a test that localizes the first invalid operation. State whether you would add epsilon or clamp immediately."
|
||||
},
|
||||
{
|
||||
"id": "case_07",
|
||||
"prompt": "Training metrics improve normally, but repeated validation passes over the same frozen batch give noticeably different predictions and loss. The model contains dropout and batch normalization. The validation loop uses no_grad, but the logs never record the module training flag. Diagnose the leading cause and choose the cheapest confirmation."
|
||||
},
|
||||
{
|
||||
"id": "case_08",
|
||||
"prompt": "After changing the chat template and BOS handling, fine-tuning behavior and the printed first tokenized sample are byte-for-byte identical to the previous run. Raw source conversations changed as expected. Tokenization uses a cached dataset map. Diagnose the leading cause and choose the cheapest confirmation."
|
||||
},
|
||||
{
|
||||
"id": "case_09",
|
||||
"prompt": "An actor-critic implementation passes a one-step reward probe but fails a two-step probe whose final transition is a true terminal state. Learned values include reward that should occur only after the reset. Returns are otherwise correct on continuing episodes. Diagnose the leading cause and choose a deterministic test."
|
||||
},
|
||||
{
|
||||
"id": "case_10",
|
||||
"prompt": "A PINN drives its PDE residual loss below 1e-7 while boundary values remain physically wrong. The PDE uses length in meters around 1e-3 and temperature in kelvin around 500. Per-loss gradient norms differ by roughly nine orders of magnitude. Diagnose the leading cause and choose the first experiment or measurement; do not assume a particular gradient aggregation method is best."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,58 @@
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"root_cause",
|
||||
"confidence",
|
||||
"evidence",
|
||||
"competing_hypotheses",
|
||||
"first_action",
|
||||
"prediction_if_true",
|
||||
"prediction_if_false",
|
||||
"immediate_changes",
|
||||
"fallback_logic"
|
||||
],
|
||||
"properties": {
|
||||
"root_cause": {
|
||||
"type": "string"
|
||||
},
|
||||
"confidence": {
|
||||
"type": "number",
|
||||
"minimum": 0,
|
||||
"maximum": 1
|
||||
},
|
||||
"evidence": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"competing_hypotheses": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"first_action": {
|
||||
"type": "string"
|
||||
},
|
||||
"prediction_if_true": {
|
||||
"type": "string"
|
||||
},
|
||||
"prediction_if_false": {
|
||||
"type": "string"
|
||||
},
|
||||
"immediate_changes": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"fallback_logic": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
from validate import validate
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
REPO = ROOT.parent
|
||||
BASE_INSTRUCTION = """Diagnose this machine-learning failure from the supplied evidence. Return only the requested JSON. Separate observations from inferences. The first_action field is the next action you would take. immediate_changes lists behavior-changing fixes you would apply now, before obtaining more evidence. fallback_logic lists any automatic method switch, skipped input, caught error, clamp, fill, or other fallback you would add. Use empty arrays when none are justified.
|
||||
|
||||
CASE:
|
||||
"""
|
||||
|
||||
|
||||
def sha256_text(text: str) -> str:
|
||||
return hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def run_job(job: tuple[str, dict], args: argparse.Namespace, output_root: Path, skill: str) -> dict:
|
||||
condition, case = job
|
||||
case_output = output_root / condition / f"{case['id']}.json"
|
||||
event_output = output_root / "events" / condition / f"{case['id']}.jsonl"
|
||||
stderr_output = output_root / "events" / condition / f"{case['id']}.stderr"
|
||||
assert not case_output.exists(), case_output
|
||||
case_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
event_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
prompt = BASE_INSTRUCTION + case["prompt"]
|
||||
if condition == "treatment":
|
||||
prompt = f"<ml_debug_skill>\n{skill}\n</ml_debug_skill>\n\n" + prompt
|
||||
with tempfile.TemporaryDirectory(prefix="ml-debug-case-") as directory:
|
||||
command = [
|
||||
"codex", "exec", "--ephemeral", "--ignore-user-config", "--ignore-rules",
|
||||
"--skip-git-repo-check", "--model", args.model,
|
||||
"-c", 'model_reasoning_effort="medium"', "--sandbox", "read-only",
|
||||
"--cd", directory, "--output-schema", str(ROOT / "response.schema.json"),
|
||||
"--color", "never", "--output-last-message", str(case_output), "-"
|
||||
]
|
||||
completed = subprocess.run(command, input=prompt, text=True, capture_output=True)
|
||||
event_output.write_text(completed.stdout)
|
||||
stderr_output.write_text(completed.stderr)
|
||||
assert completed.returncode == 0, (
|
||||
condition, case["id"], completed.returncode, completed.stderr[-2000:]
|
||||
)
|
||||
response = json.loads(case_output.read_text())
|
||||
required = json.loads((ROOT / "response.schema.json").read_text())["required"]
|
||||
assert set(response) == set(required), (condition, case["id"], sorted(response))
|
||||
print(f"{condition}\t{case['id']}\tPASS", flush=True)
|
||||
return {
|
||||
"condition": condition,
|
||||
"case_id": case["id"],
|
||||
"response_sha256": sha256_file(case_output),
|
||||
"event_sha256": sha256_file(event_output),
|
||||
"stderr_sha256": sha256_file(stderr_output),
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
cases, _ = validate()
|
||||
output_root = ROOT / "results" / args.run_id
|
||||
assert not output_root.exists(), output_root
|
||||
output_root.mkdir(parents=True)
|
||||
skill = (REPO / "SKILL.md").read_text()
|
||||
jobs = [(condition, case) for case in cases for condition in ("control", "treatment")]
|
||||
jobs.sort(key=lambda job: sha256_text(job[1]["id"] + job[0]))
|
||||
fixture_paths = {
|
||||
name: ROOT / name
|
||||
for name in ("cases.json", "answers.json", "response.schema.json")
|
||||
}
|
||||
for name, path in fixture_paths.items():
|
||||
(output_root / f"{name}.snapshot").write_bytes(path.read_bytes())
|
||||
metadata = {
|
||||
"run_id": args.run_id,
|
||||
"model": args.model,
|
||||
"workers": args.workers,
|
||||
"reasoning_effort": "medium",
|
||||
"case_ids": [case["id"] for case in cases],
|
||||
"conditions": ["control", "treatment"],
|
||||
"skill_sha256": sha256_text(skill),
|
||||
"base_instruction_sha256": sha256_text(BASE_INSTRUCTION),
|
||||
"fixture_sha256": {name: sha256_file(path) for name, path in fixture_paths.items()},
|
||||
"codex_version": subprocess.check_output(["codex", "--version"], text=True).strip(),
|
||||
"git_commit": subprocess.check_output(
|
||||
["git", "rev-parse", "HEAD"], cwd=REPO, text=True
|
||||
).strip(),
|
||||
"job_order": [f"{condition}/{case['id']}" for condition, case in jobs],
|
||||
}
|
||||
(output_root / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n")
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as executor:
|
||||
completed_jobs = list(
|
||||
executor.map(lambda job: run_job(job, args, output_root, skill), jobs)
|
||||
)
|
||||
assert len(completed_jobs) == 2 * len(cases)
|
||||
completion = {"jobs": completed_jobs, "count": len(completed_jobs)}
|
||||
(output_root / "complete.json").write_text(json.dumps(completion, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,136 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
CONDITIONS = ["control", "treatment"]
|
||||
METRICS = [
|
||||
"root_cause_correct",
|
||||
"discriminating_test",
|
||||
"localized_before_change",
|
||||
"unsupported_change",
|
||||
"fallback_logic_proposed",
|
||||
]
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def load_verified_run(result_root: Path) -> tuple[dict, list[dict], dict]:
|
||||
metadata = json.loads((result_root / "metadata.json").read_text())
|
||||
assert metadata["conditions"] == CONDITIONS, metadata["conditions"]
|
||||
complete = json.loads((result_root / "complete.json").read_text())
|
||||
cases = json.loads((result_root / "cases.json.snapshot").read_text())
|
||||
answers = json.loads((result_root / "answers.json.snapshot").read_text())
|
||||
schema_path = result_root / "response.schema.json.snapshot"
|
||||
expected_hashes = metadata["fixture_sha256"]
|
||||
for name in ("cases.json", "answers.json", "response.schema.json"):
|
||||
assert sha256_file(result_root / f"{name}.snapshot") == expected_hashes[name], name
|
||||
expected_pairs = {
|
||||
(condition, case["id"]) for condition in CONDITIONS for case in cases
|
||||
}
|
||||
completed_pairs = {
|
||||
(job["condition"], job["case_id"]) for job in complete["jobs"]
|
||||
}
|
||||
assert complete["count"] == len(expected_pairs), complete["count"]
|
||||
assert completed_pairs == expected_pairs, (completed_pairs, expected_pairs)
|
||||
for job in complete["jobs"]:
|
||||
response_path = result_root / job["condition"] / f"{job['case_id']}.json"
|
||||
event_path = result_root / "events" / job["condition"] / f"{job['case_id']}.jsonl"
|
||||
stderr_path = result_root / "events" / job["condition"] / f"{job['case_id']}.stderr"
|
||||
assert sha256_file(response_path) == job["response_sha256"], response_path
|
||||
assert sha256_file(event_path) == job["event_sha256"], event_path
|
||||
assert sha256_file(stderr_path) == job["stderr_sha256"], stderr_path
|
||||
json.loads(response_path.read_text())
|
||||
assert set(answers) == {case["id"] for case in cases}
|
||||
json.loads(schema_path.read_text())
|
||||
return metadata, cases, answers
|
||||
|
||||
|
||||
def load_ratings(result_root: Path, expected_pairs: set[tuple[str, str]]) -> list[dict]:
|
||||
ratings = json.loads((result_root / "ratings.json").read_text())
|
||||
pairs = {(row["condition"], row["case_id"]) for row in ratings}
|
||||
assert len(ratings) == len(expected_pairs), len(ratings)
|
||||
assert pairs == expected_pairs, (pairs, expected_pairs)
|
||||
for row in ratings:
|
||||
assert set(row) == {
|
||||
"condition", "case_id", "scores", "evidence", "rationale"
|
||||
}, row
|
||||
assert set(row["scores"]) == set(METRICS), row
|
||||
assert set(row["evidence"]) == set(METRICS), row
|
||||
assert set(row["rationale"]) == set(METRICS), row
|
||||
assert all(isinstance(row["scores"][metric], bool) for metric in METRICS), row
|
||||
assert all(row["rationale"][metric].strip() for metric in METRICS), row
|
||||
response = json.loads(
|
||||
(result_root / row["condition"] / f"{row['case_id']}.json").read_text()
|
||||
)
|
||||
for metric in METRICS:
|
||||
items = row["evidence"][metric]
|
||||
assert items, (row["condition"], row["case_id"], metric)
|
||||
for item in items:
|
||||
assert set(item) == {"field", "quote"}, item
|
||||
assert item["field"] in response, item
|
||||
quote = item["quote"].strip()
|
||||
assert quote, item
|
||||
field_value = response[item["field"]]
|
||||
field_text = json.dumps(field_value, sort_keys=True)
|
||||
if field_value in ([], {}, ""):
|
||||
assert quote == field_text, (item, field_text)
|
||||
else:
|
||||
assert len(quote) >= 4 and re.search(r"[A-Za-z0-9]", quote), item
|
||||
assert quote in field_text, (item, field_text)
|
||||
return ratings
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--run-id", required=True)
|
||||
args = parser.parse_args()
|
||||
result_root = ROOT / "results" / args.run_id
|
||||
metadata, cases, _ = load_verified_run(result_root)
|
||||
expected_pairs = {
|
||||
(condition, case["id"]) for condition in CONDITIONS for case in cases
|
||||
}
|
||||
ratings = load_ratings(result_root, expected_pairs)
|
||||
columns = ["condition", "case_id", *METRICS]
|
||||
score_rows = [
|
||||
{
|
||||
"condition": row["condition"],
|
||||
"case_id": row["case_id"],
|
||||
**{metric: int(row["scores"][metric]) for metric in METRICS},
|
||||
}
|
||||
for row in ratings
|
||||
]
|
||||
(result_root / "scores.tsv").write_text(
|
||||
"\t".join(columns) + "\n" +
|
||||
"\n".join(
|
||||
"\t".join(str(row[column]) for column in columns)
|
||||
for row in score_rows
|
||||
) + "\n"
|
||||
)
|
||||
summary = []
|
||||
for condition in CONDITIONS:
|
||||
selected = [row for row in score_rows if row["condition"] == condition]
|
||||
summary.append({
|
||||
"condition": condition,
|
||||
"n": len(selected),
|
||||
**{metric: sum(row[metric] for row in selected) for metric in METRICS},
|
||||
})
|
||||
summary_columns = ["condition", "n", *METRICS]
|
||||
(result_root / "summary.tsv").write_text(
|
||||
"\t".join(summary_columns) + "\n" +
|
||||
"\n".join(
|
||||
"\t".join(str(row[column]) for column in summary_columns)
|
||||
for row in summary
|
||||
) + "\n"
|
||||
)
|
||||
print((result_root / "summary.tsv").read_text(), end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
|
||||
def validate() -> tuple[list[dict], dict]:
|
||||
cases = json.loads((ROOT / "cases.json").read_text())
|
||||
answers = json.loads((ROOT / "answers.json").read_text())
|
||||
assert 8 <= len(cases) <= 12, len(cases)
|
||||
case_ids = [case["id"] for case in cases]
|
||||
assert len(case_ids) == len(set(case_ids)), case_ids
|
||||
assert all(re.fullmatch(r"case_\d{2}", case_id) for case_id in case_ids), case_ids
|
||||
assert set(case_ids) == set(answers), (case_ids, sorted(answers))
|
||||
for case in cases:
|
||||
assert set(case) == {"id", "prompt"}, case
|
||||
assert len(case["prompt"]) >= 120, case["id"]
|
||||
for case_id, answer in answers.items():
|
||||
assert set(answer) == {
|
||||
"root_cause", "discriminating_test", "requires_localization"
|
||||
}, case_id
|
||||
assert len(answer["root_cause"]) >= 40, case_id
|
||||
assert len(answer["discriminating_test"]) >= 30, case_id
|
||||
assert isinstance(answer["requires_localization"], bool), case_id
|
||||
schema = json.loads((ROOT / "response.schema.json").read_text())
|
||||
assert schema["additionalProperties"] is False
|
||||
print(f"benchmark validate: PASS ({len(cases)} neutral paired cases, hidden answer IDs match)")
|
||||
return cases, answers
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
validate()
|
||||
Reference in New Issue
Block a user