diff --git a/README.md b/README.md index f38d1c7..b1c3cda 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ The epistemic gate. Required fields: | `failure_likely` | Most likely way this is wrong despite evidence | | `failure_sneaky` | Perverse/silent failure that looks like success superficially | | `falsification_test` | What you ran and what you got, so both you and the human can sanity-check it. Why that result could not occur if a failure mode were real. | -| `verification_hints` | Where to look and what to check. Descriptions of evidence locations. | +| `verification_hints` | Where to look and what to check. These still force the agent to think, but weak hints are advisory rather than a hard block when the verbatim evidence already proves the claim. Core evidence still has to pass on its own. | | `remaining_uncertainty` | What is NOT tested, deferred edge cases, known limitations | | `commands` | Optional structured command records: `{ cmd, exit_code, stdout_path?, stderr_path? }` | | `evidence_paths` / `falsification_paths` | Optional local artifact paths. Stored as absolute path + sha256 + byte size | @@ -92,7 +92,7 @@ After calling this, the task shows `👀` and is only completable via `/lgtm 0) rubric = r; } + const review = relaxAdvisoryVerificationHints({ + reviewer: typeof parsed.reviewer === "string" ? parsed.reviewer : commandLabel, + scope: typeof parsed.scope === "string" ? parsed.scope : "task evidence package", + observations, + blind_spots: typeof parsed.blind_spots === "string" ? parsed.blind_spots : "not stated", + accepted: typeof parsed.accepted === "boolean" + ? parsed.accepted + : parsed.evidence_complete === true && parsed.evidence_convincing === true, + evidence_complete: parsed.evidence_complete === true, + evidence_convincing: parsed.evidence_convincing === true, + missing_evidence, + submitted_at: new Date().toISOString(), + mode: "auto", + raw_output: result.stdout.trim(), + rubric, + }); return { command: commandLabel, - review: { - reviewer: typeof parsed.reviewer === "string" ? parsed.reviewer : commandLabel, - scope: typeof parsed.scope === "string" ? parsed.scope : "task evidence package", - observations, - blind_spots: typeof parsed.blind_spots === "string" ? parsed.blind_spots : "not stated", - accepted: typeof parsed.accepted === "boolean" - ? parsed.accepted - : parsed.evidence_complete === true && parsed.evidence_convincing === true, - evidence_complete: parsed.evidence_complete === true, - evidence_convincing: parsed.evidence_convincing === true, - missing_evidence, - submitted_at: new Date().toISOString(), - mode: "auto", - raw_output: result.stdout.trim(), - rubric, - }, + review, }; } diff --git a/src/robot-review.ts b/src/robot-review.ts index d0d754c..dca3513 100644 --- a/src/robot-review.ts +++ b/src/robot-review.ts @@ -105,6 +105,27 @@ export function shouldOpenHumanSignoffGate(task: Task, reviewAccepted: boolean): return reviewAccepted && typeof task.metadata?.lgtm_evidence === "string" && task.metadata.lgtm_evidence.length > 0; } +export function relaxAdvisoryVerificationHints(review: Omit): Omit { + const rubric = review.rubric; + if (!rubric || review.evidence_complete !== true) return review; + const requiredCoreKeys = ["evidence_covers_done_criterion", "falsification_test_runnable", "failure_modes_addressed"]; + if (!requiredCoreKeys.every((key) => rubric[key]?.pass === true)) return review; + const failedKeys = Object.entries(rubric) + .filter(([, item]) => item.pass !== true) + .map(([key]) => key); + if (failedKeys.length !== 1 || failedKeys[0] !== "verification_hints_actionable") return review; + return { + ...review, + accepted: true, + evidence_convincing: true, + observations: [ + ...review.observations, + "Verification hints were weak, but treated as advisory because the verbatim evidence already covered the done criterion.", + ], + missing_evidence: review.missing_evidence.filter((item) => item !== "verification_hints_actionable" && !/verification hint/i.test(item)), + }; +} + export function appendRobotReviewMetadata(task: Task, review: Omit): Record { const robot_reviews = [...getRobotReviews(task), { ...review, iteration: 0 }].map((entry, index) => ({ ...entry, diff --git a/test/robot-review.test.ts b/test/robot-review.test.ts index 8bdc80a..4a89047 100644 --- a/test/robot-review.test.ts +++ b/test/robot-review.test.ts @@ -3,7 +3,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; import { archiveCurrentEvidence, buildArtifactRecords, getCurrentEvidenceIteration, getEvidenceHistory } from "../src/index.js"; -import { appendRobotReviewMetadata, getLatestRobotReview, getRobotReviews, shouldOpenHumanSignoffGate } from "../src/robot-review.js"; +import { appendRobotReviewMetadata, getLatestRobotReview, getRobotReviews, relaxAdvisoryVerificationHints, shouldOpenHumanSignoffGate } from "../src/robot-review.js"; import type { Task } from "../src/types.js"; function makeTask(overrides: Partial = {}): Task { @@ -81,6 +81,56 @@ describe("robot review helpers", () => { expect(getEvidenceHistory(taskWithHistory)[0].supersede_reason).toBe("threshold changed"); }); + it("treats verification hints as advisory when core evidence already passes", () => { + const review = relaxAdvisoryVerificationHints({ + reviewer: "auto", + scope: "task evidence", + observations: ["Observed commit, push, and test logs"], + blind_spots: "Did not inspect interactive UI", + accepted: false, + evidence_complete: true, + evidence_convincing: false, + missing_evidence: ["verification_hints_actionable"], + submitted_at: "2026-06-13T00:00:00.000Z", + mode: "auto", + rubric: { + evidence_covers_done_criterion: { reason: "verbatim logs match", pass: true }, + falsification_test_runnable: { reason: "command and output shown", pass: true }, + failure_modes_addressed: { reason: "plausible top risks named", pass: true }, + verification_hints_actionable: { reason: "paths are vague", pass: false }, + }, + }); + + expect(review.accepted).toBe(true); + expect(review.evidence_convincing).toBe(true); + expect(review.observations.at(-1)).toContain("treated as advisory"); + expect(review.missing_evidence).toEqual([]); + }); + + it("does not relax verification hints unless the core rubric passes", () => { + const review = relaxAdvisoryVerificationHints({ + reviewer: "auto", + scope: "task evidence", + observations: ["Observed vague summary only"], + blind_spots: "Did not rerun tests", + accepted: false, + evidence_complete: true, + evidence_convincing: false, + missing_evidence: ["verification_hints_actionable"], + submitted_at: "2026-06-13T00:00:00.000Z", + mode: "auto", + rubric: { + evidence_covers_done_criterion: { reason: "summary only", pass: false }, + falsification_test_runnable: { reason: "command and output shown", pass: true }, + failure_modes_addressed: { reason: "plausible top risks named", pass: true }, + verification_hints_actionable: { reason: "paths are vague", pass: false }, + }, + }); + + expect(review.accepted).toBe(false); + expect(review.evidence_convincing).toBe(false); + }); + it("appends robot reviews as iterations", () => { const task = makeTask(); const metadata1 = appendRobotReviewMetadata(task, {