From 46cfd537f0b259a44a0ae8d3b678f6cc4701a610 Mon Sep 17 00:00:00 2001 From: wassname <1103714+wassname@users.noreply.github.com> Date: Sat, 5 Sep 2026 12:20:17 +0800 Subject: [PATCH] Add persistent pi-subagents goal steward Co-Authored-By: Pi Codex <288921227+claudypoo@users.noreply.github.com> --- README.md | 26 +- package.json | 5 +- .../20260905_goal-steward-validation.md | 31 ++ slop/audits/20260905_steward-probe.json | 16 + slop/plans/20260905_goal-steward.md | 37 ++ src/index.ts | 350 +++++++----------- src/prompts.ts | 86 +---- src/steward.ts | 246 ++++++++++++ test/decide-signoff.test.ts | 93 ++--- test/goals-flow.test.ts | 75 +++- test/judge-args.test.ts | 33 -- test/prompts.test.ts | 7 +- test/steward.test.ts | 133 +++++++ 13 files changed, 734 insertions(+), 404 deletions(-) create mode 100644 slop/audits/20260905_goal-steward-validation.md create mode 100644 slop/audits/20260905_steward-probe.json create mode 100644 slop/plans/20260905_goal-steward.md create mode 100644 src/steward.ts delete mode 100644 test/judge-args.test.ts create mode 100644 test/steward.test.ts diff --git a/README.md b/README.md index 08098a2..fb669aa 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # pi-goals -Make a short list of goals in one Markdown plan file. This is easy to review, and a subagent can check whether each goal is complete. +Make a short list of goals in one Markdown plan file. A persistent read-only subagent keeps the high-level context, reviews progress, and checks whether each goal is complete. The plan file looks like this: @@ -48,7 +48,10 @@ resync-after-compaction from [tmonk/pi-goal-x](https://github.com/tmonk/pi-goal- ## Install +Requires `pi-subagents` 0.65.1 or newer. + ```bash +pi install npm:pi-subagents pi install npm:@wassname2/pi-goals ``` @@ -56,7 +59,7 @@ Or for development: ```bash git clone https://github.com/wassname/pi-goals && cd pi-goals && npm install -pi -e ./src/index.ts +pi -e npm:pi-subagents -e ./src/index.ts ``` ## Use @@ -71,27 +74,28 @@ pi -e ./src/index.ts 2. Review. After Pi settles, the full plan is printed in the transcript. Check that User-visible result names the final artifact or behavior you expect. The menu offers Ready, Refine, Edit, or Cancel. Refine collects short notes. Edit opens the full plan in Pi's editor. -3. Work. Ready is the only review action that starts work. The agent ticks subtasks, appends to - `## Log` and `## Learnings`, fills `evidence:`, and calls `CompleteGoal` when a discriminator is - satisfied. Every human reply and Refine note in plan mode is saved verbatim under `## Interview`. - After eight turns without a change above `## Log`, the working set is sent back with a short upkeep - reminder. +3. Work. Ready is the only review action that starts work. It also starts the goal steward. The + agent ticks subtasks, appends to `## Log` and `## Learnings`, fills `evidence:`, and calls + `CompleteGoal` when a discriminator is satisfied. The goal steward rereads the full plan on each + review. `CompleteGoal` resumes the same steward lineage instead of starting a fresh reviewer. + Every human reply and Refine note in plan mode is saved verbatim under `## Interview`. After eight + turns without a change above `## Log`, the worker gets a reminder and the steward gets a progress + checkpoint. Other commands: `/goals --clear` disconnects this session from its active plan, preserving the versioned file on disk; `/goals --auto [minutes|off]` continues active goals after the agent settles and then on that interval. It pauses after two automatic wakes with no working-plan change; `/goals ---judge ` picks a sign-off judge model (default: your current session model, else pi's -default). The `--` prefix +--steward-model ` picks the steward model (default: the pi-subagents agent model). The `--` prefix keeps ordinary objectives such as `judge model quality` from being parsed as commands. ## Prompts -All model-facing text lives in [`src/prompts.ts`](src/prompts.ts), in flow order. +Worker prompts live in [`src/prompts.ts`](src/prompts.ts). The read-only supervisor prompt and review requests live in [`src/steward.ts`](src/steward.ts). ## Develop ```bash -pi -e ./src/index.ts # load locally +pi -e npm:pi-subagents -e ./src/index.ts # load locally npm test # all unit, flow, and Pi RPC tests npm run test:rpc # Pi RPC review flow with a local offline model npm run typecheck diff --git a/package.json b/package.json index d5c5cde..5042649 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@wassname2/pi-goals", "version": "0.2.2", - "description": "One plan file per session: set goals in plan mode, work them, sign off only when a read-only judge checks the evidence.", + "description": "One plan file per session with a persistent read-only goal steward powered by pi-subagents.", "author": "wassname", "license": "MIT", "type": "module", @@ -18,7 +18,8 @@ "proof", "uat", "evidence", - "judge" + "steward", + "subagent" ], "peerDependencies": { "@earendil-works/pi-coding-agent": "*", diff --git a/slop/audits/20260905_goal-steward-validation.md b/slop/audits/20260905_goal-steward-validation.md new file mode 100644 index 0000000..8186421 --- /dev/null +++ b/slop/audits/20260905_goal-steward-validation.md @@ -0,0 +1,31 @@ +# Goal steward validation + +## Observations + +- Unit, flow, type, and lint checks passed. [`20260905_validation.log`](20260905_validation.log) says: + + > Test Files 8 passed (8) + > Tests 36 passed (36) + > Checked 12 files in 14ms. No fixes applied. + +- A real Pi 0.85.0 process loaded pi-subagents 0.65.1, pi-goals, and a runtime `goal-steward` agent. It spawned one review and resumed that run for sign-off. [`20260905_steward-probe.json`](20260905_steward-probe.json) records two distinct run IDs: + + > "runId": "4e9dc0c0-385b-4eb9-a060-ced7dc7cb6cc" + + > "runId": "f6115c82-31de-499f-ab78-145dde0c51c0" + +- The second review recalled a token that appeared only in the first review request. This is direct evidence that resume retained the steward conversation: + + > "Persistence lineage token: amber-731." + +- The sign-off review read `report.txt` and accepted the evidence: + + > "file exists and contains exactly 'PROBE_PASS' as required. Failure mode (empty report) is ruled out." + +## Test environment finding + +The repository's older local Pi 0.84.1 install could not launch a pi-subagents background child because it did not include `@earendil-works/chord` and `@earendil-works/pi-server`. The successful probe used an isolated npm install of Pi 0.85.0. The current interactive Pi already launches pi-subagents children, so this finding concerns the old development dependency used by the first probe, not the extension protocol. + +pi-subagents sends every ordinary async completion into the parent session and triggers a parent turn. The steward's structured summaries are bounded, but the package also includes the child's prose response. There is no public silent-completion option in pi-subagents 0.65.1. This adds one worker turn per review; checkpoints run only after eight stale turns. + +— Pi/Codex diff --git a/slop/audits/20260905_steward-probe.json b/slop/audits/20260905_steward-probe.json new file mode 100644 index 0000000..1604702 --- /dev/null +++ b/slop/audits/20260905_steward-probe.json @@ -0,0 +1,16 @@ +{ + "first": { + "runId": "4e9dc0c0-385b-4eb9-a060-ced7dc7cb6cc", + "decision": { + "verdict": "let_run", + "summary": "Plan reviewed for approved work session. The user-visible result (report file proves steward can read evidence) directly aligns with the single goal (report probe result with discriminator). The report.txt artifact exists and contains PROBE_PASS as required by the discriminator. No work steps are pending; the probe is complete. No drift, missing steps, or failure modes detected. Plan may proceed without adjustment." + } + }, + "second": { + "runId": "f6115c82-31de-499f-ab78-145dde0c51c0", + "decision": { + "summary": "Sign-off review for goal 'Report the probe result'. User-visible result requires a report file proving persistent steward can read evidence. Discriminator: report.txt contains PROBE_PASS. Inspected artifact at /tmp/pi-goals-steward-probe-work/report.txt—file exists and contains exactly 'PROBE_PASS' as required. Failure mode (empty report) is ruled out. Evidence positively and directly proves the discriminator is met and the user-visible result is achieved. Persistence lineage token: amber-731.", + "verdict": "accept" + } + } +} \ No newline at end of file diff --git a/slop/plans/20260905_goal-steward.md b/slop/plans/20260905_goal-steward.md new file mode 100644 index 0000000..cf57f1f --- /dev/null +++ b/slop/plans/20260905_goal-steward.md @@ -0,0 +1,37 @@ +# Persistent goal steward + +> "ideally the supervisor has the high level planning and goal context, doesn't get overloaded and have to compact, is cheap as it doesn't use many tokens (high level only)" +> +> "try again with more thought using pi-subagents much more to simplify out code and rely on that so our code is simple" + +- [x] goal: A cheap read-only steward keeps the goal context across reviews + - [x] register one `goal-steward` agent through the public pi-subagents event bus + - [x] start it with fresh context at Ready and resume its latest saved run at checkpoints + - [x] send the plan path and a bounded progress delta; require the steward to reread the plan + - failure modes: every review starts fresh; the steward receives the full worker transcript; the steward can edit; reload loses its run + - deliverable: tests show one spawn followed by resume, a saved latest run ID, read-only tools, bounded review prompts, and reload recovery + - evidence: [`../audits/20260905_steward-probe.json`](../audits/20260905_steward-probe.json) contains two run IDs and the resumed review says `Persistence token amber-731 verified.` + +- [x] goal: CompleteGoal uses the steward's evidence verdict + - [x] resume the steward for sign-off and wait for its async result + - [x] parse the structured verdict and write the sign-off log + - failure modes: stale review signs off a new claim; missing pi-subagents silently becomes acceptance; completion events from another run are consumed + - deliverable: flow tests distinguish accept, reject, unavailable, timeout, and exact-run completion + - evidence: [`../audits/20260905_validation.log`](../audits/20260905_validation.log) says `Tests 36 passed (36)` and `Checked 12 files in 14ms. No fixes applied.` + +## UAT / Verification + +- [x] `npm test`, `npm run typecheck`, and `npm run lint` pass. +- [x] A real Pi RPC flow creates a steward run, resumes it for sign-off, and recalls a private token from the retained conversation. +- [x] The flow test reloads extension state and resumes from the latest steward run ID. + +## Appendix (context, not approved) + +Use pi-subagents 0.65.1 public RPC (`spawn`, `resume`) and `subagent:async-complete`. Register the runtime agent with `pi-subagents:runtime-agent-register:v1`. Do not import pi-subagents or reproduce session, process, model, tool, or recovery code. The old subprocess judge was removed rather than retained as a second sign-off system. + +## Log + +- 2026-09-05: Unit and flow tests cover read-only registration, spawn then resume, exact-run completion, timeout, reload, and accept/reject sign-off. +- 2026-09-05: The Pi 0.85.0 + pi-subagents 0.65.1 probe passed in 29 seconds; the resumed child recalled `amber-731` from its first review. + +— Pi/Codex diff --git a/src/index.ts b/src/index.ts index 9c7b99b..e366934 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,48 +1,34 @@ /** - * PI: pi-goals v2 drafts goals into .pi/plan/-vN.md, the agent works them with its - * normal Edit tool, and a fresh read-only judge signs each goal off through the one blessed tool, - * CompleteGoal. + * PI: pi-goals owns one versioned plan per session. A persistent, read-only pi-subagents child + * keeps the high-level context, reviews progress, and decides CompleteGoal sign-off. * - * PI: Each /goals call makes a new plan version, `.pi/plan/-vN.md`. The selected version - * stays in session state across resume and compaction. Old plans stay on disk but inert, so a new - * conversation cannot silently edit them. `/goals --clear` only disconnects this session; the filename is the arm switch: a session that never ran - * /goals has no active plan, so the widget, injections, and CompleteGoal all stay silent. + * Each /goals call makes `.pi/plan/-vN.md`. The selected version survives resume and + * compaction. Old plans stay on disk but inactive. A session with no selected plan has no widget, + * injections, steward reviews, or CompleteGoal sign-off. * - * The v1 lesson: the parser existed so TypeScript could read the plan, but almost every reader is a - * model. So v2 has NO parser and no schema. The harness does exactly three things for a - * cooperative-but-confused model: - * 1. memory — a transient re-send of the plan, never persisted, on two triggers: the plan went - * stale for STALE_TURNS turns (send the working set above ## Log), or the session - * started / compacted (send the whole file, appendix included). v2 sent the whole - * file every turn; pi-tasks tried that and deleted it as "wallpaper noise that - * trains the model to ignore the task block" (tintinweb/pi-tasks CHANGELOG.md:149), - * and the always-present CompleteGoal description carries the contract instead. - * 2. format — a skeleton convention taught in planDrafting (prompts.ts), not validated - * 3. eyes — CompleteGoal spawns a strictly read-only pi subprocess (--no-session, no bash) - * that gets the whole plan file plus the claimed goal, finds the goal itself - * (tolerates wording drift), checks the evidence (including the agent's saved - * verify output) against the repo, and returns VERDICT: accept|reject + * TypeScript reads only goal checkbox lines for the widget. Models read the plan as prose. The + * worker alone edits it. The steward receives the plan path on every review, rereads the complete + * file, and can inspect cited artifacts with read-only tools. pi-subagents owns child sessions, + * persistence, resume, completion events, structured output, and contact with the parent. * - * The judge subsumes what v1 did in code: goal matching (no findGoal), evidence validation (a - * placeholder gets rejected in words), and format reading. The extension's only - * writes are the sign-off: append a log line to ## Log (the audit trail) and tick the goal [x] when - * an exact goal line matches (on drift the agent ticks, and the result says so). A hand-tick - * without a matching tool-written log line is visible in the diff either way. - * - * Judge ran but failed/errored/timed out, or returned no VERDICT line => accepted_inconclusive: the - * working agent is never blocked on judge infra; the log line says the judge ran but failed. There - * is no pre-emptive "no model" path -- a null judgeModel just omits --model so pi's configured - * default runs the judge, so inconclusive always means "ran but failed", never "couldn't start". - * - * All model-facing text lives in prompts.ts, in flow order. + * — Pi/Codex */ -import { spawn } from "node:child_process"; import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { basename, join, resolve } from "node:path"; +import { join, resolve } from "node:path"; import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; -import { completeGoalDescription, completeGoalParamDescription, judgeSystem, judgeUser, planDrafting, planningState, reminder, resync } from "./prompts.js"; +import { completeGoalDescription, completeGoalParamDescription, planDrafting, planningState, reminder, resync } from "./prompts.js"; +import { + checkpointReview, + readyReview, + registerStewardAgent, + resumeSteward, + runStewardReview, + type StewardDecision, + signoffReview, + startSteward, +} from "./steward.js"; const STATE = "pi-goals-state"; const STATUS_KEY = "pi-goals"; @@ -51,12 +37,6 @@ const PLANNING_CONTEXT = "pi-goals-planning-context"; const PLAN_DIR = ".pi/plan"; // For static text (the /goals description) where there is no ctx to resolve the session id. const PLAN_SHAPE = `${PLAN_DIR}/-vN.md`; -// Judge toolset: strictly read-only, NO bash -- the judge can never execute or mutate anything, and -// in particular never re-runs a verify command (which may be a 10-hour training job). The agent runs -// verify itself and saves the output as evidence; the judge reads it. Names match pi's tool registry. -const JUDGE_TOOLS = ["read", "grep", "find", "ls"]; -const JUDGE_BLOCKED_TOOLS = ["edit", "write"]; -const JUDGE_TIMEOUT_MS = 600_000; // Plan mode is read-only by convention AND a light gate: edit/write are blocked (except the plan // file, the deliverable). bash stays open — the prompt says don't mutate; guide, not gate (spec D3). const PLAN_MODE_BLOCKED_TOOLS = ["edit", "write"]; @@ -120,16 +100,24 @@ type Phase = "planning" | "working" | null; interface PlanState { phase: Phase; - /** Optional model ref for the sign-off judge; unset => current session model, else pi's default. */ - judgeModel: string | null; + stewardModel: string | null; + stewardRunId: string | null; + stewardPending: boolean; planVersion: number | null; - /** User-enabled interval for continuing active goals after the agent settles. */ autoIntervalMs: number | null; autoPaused: boolean; } export default function piGoalsExtension(pi: ExtensionAPI): void { - let state: PlanState = { phase: null, judgeModel: null, planVersion: null, autoIntervalMs: null, autoPaused: false }; + let state: PlanState = { + phase: null, + stewardModel: null, + stewardRunId: null, + stewardPending: false, + planVersion: null, + autoIntervalMs: null, + autoPaused: false, + }; let planningContextPending = false; // The reminder sees only the working set. A repeated Log line must not look like progress. let turnsStale = 0; @@ -140,6 +128,9 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { let autoLastWorkingSet = ""; let autoImmediateUsed = false; let runStartedBackgroundWork = false; + let stewardRegistration: { dispose(): void } | null = null; + let stewardRegistrationError: string | null = null; + let unsubscribeStewardCompletion: (() => void) | null = null; // Set on session start and after a compaction; drained by the next LLM call, which then carries // the WHOLE file (appendix included) instead of just the working set. let resyncReason: string | null = "New session."; @@ -161,6 +152,48 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { pi.appendEntry(STATE, state); } + function setupSteward(ctx: ExtensionContext): void { + stewardRegistration?.dispose(); + stewardRegistration = null; + stewardRegistrationError = null; + try { + stewardRegistration = registerStewardAgent(pi.events, state.stewardModel); + } catch (error) { + stewardRegistrationError = error instanceof Error ? error.message : String(error); + if (state.phase === "working") ctx.ui.notify(`Goal steward unavailable: ${stewardRegistrationError}`, "warning"); + } + } + + function rememberStewardRun(runId: string): void { + state = { ...state, stewardRunId: runId, stewardPending: true }; + persist(); + } + + async function reviewInBackground(ctx: ExtensionContext, task: string): Promise { + if (!stewardRegistration) { + ctx.ui.notify(`Goal steward unavailable: ${stewardRegistrationError ?? "pi-subagents is not ready"}. Install pi-subagents and reload Pi.`, "warning"); + return; + } + if (state.stewardPending) return; + try { + const runId = state.stewardRunId + ? await resumeSteward(pi.events, state.stewardRunId, task) + : await startSteward(pi.events, ctx.cwd, task); + rememberStewardRun(runId); + } catch (error) { + ctx.ui.notify(`Goal steward could not start: ${error instanceof Error ? error.message : String(error)}`, "warning"); + } + } + + function watchStewardCompletion(): void { + unsubscribeStewardCompletion?.(); + unsubscribeStewardCompletion = pi.events.on("subagent:async-complete", (raw) => { + if (!raw || typeof raw !== "object" || (raw as { runId?: string }).runId !== state.stewardRunId) return; + state = { ...state, stewardPending: false }; + persist(); + }); + } + function clearAutoTimer(): void { if (autoTimer !== null) clearTimeout(autoTimer); autoTimer = null; @@ -244,10 +277,10 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { ctx.ui.setWidget(WIDGET_KEY, lines); } - // --- /goals: enter plan mode (or clear / set judge) -------------------------------------------- + // --- /goals: enter plan mode (or clear / configure the steward) — Pi/Codex --------------------- pi.registerCommand("goals", { - description: `Plan mode: draft goals into ${PLAN_SHAPE}, review, then work them. /goals | /goals --clear (disconnect) | /goals --auto [minutes|off] | /goals --judge `, + description: `Plan mode: draft goals into ${PLAN_SHAPE}, review, then work them. /goals | /goals --clear | /goals --auto [minutes|off] | /goals --steward-model `, handler: async (args, ctx) => { const arg = args.trim(); if (arg === "--clear") { @@ -257,7 +290,7 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { } const currentPlan = planRel(ctx); clearAutoTimer(); - state = { ...state, phase: null, planVersion: null, autoIntervalMs: null, autoPaused: false }; + state = { ...state, phase: null, stewardRunId: null, stewardPending: false, planVersion: null, autoIntervalMs: null, autoPaused: false }; persist(); updateWidget(ctx); ctx.ui.notify(`Disconnected from ${currentPlan}; the file remains on disk.`, "info"); @@ -292,14 +325,15 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { ctx.ui.notify(`Goal auto-continue enabled every ${minutes}m.`, "info"); return; } - if (arg === "--judge" || arg.startsWith("--judge ")) { - const ref = arg.slice("--judge".length).trim(); - state = { ...state, judgeModel: ref || null }; + if (arg === "--steward-model" || arg.startsWith("--steward-model ")) { + const ref = arg.slice("--steward-model".length).trim(); + state = { ...state, stewardModel: ref || null, stewardRunId: null, stewardPending: false }; persist(); - ctx.ui.notify(ref ? `Sign-off judge model set to ${ref}` : "Sign-off judge reset to the session model", "info"); + setupSteward(ctx); + ctx.ui.notify(ref ? `Goal-steward model set to ${ref}` : "Goal-steward model reset to pi-subagents default", "info"); return; } - state = { ...state, phase: "planning", planVersion: nextVersion(ctx) }; + state = { ...state, phase: "planning", stewardRunId: null, stewardPending: false, planVersion: nextVersion(ctx) }; planningContextPending = true; resyncReason = null; writePlan(ctx, ""); @@ -414,6 +448,7 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { // PI: Print after Pi settles. agent_end is still streaming, so its message queues behind the menu. pi.on("agent_settled", async (_event, ctx) => { if (state.phase === "working") { + if (turnsStale >= STALE_TURNS) await reviewInBackground(ctx, checkpointReview(planRel(ctx), turnsStale)); settleAuto(ctx); return; } @@ -445,7 +480,7 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { } if (choice === "Cancel") { rmSync(planPath(ctx), { force: true }); - state = { ...state, phase: null, planVersion: null }; + state = { ...state, phase: null, stewardRunId: null, stewardPending: false, planVersion: null }; persist(); updateWidget(ctx); ctx.ui.notify("Plan discarded.", "info"); @@ -455,6 +490,7 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { state = { ...state, phase: "working" }; persist(); updateWidget(ctx); + await reviewInBackground(ctx, readyReview(planRel(ctx))); pi.sendUserMessage(`Work the goals in ${planPath(ctx)}. Pick an open goal, mark it active ([/]), work its subtasks, and when its discriminator is satisfied fill its evidence: list, then call CompleteGoal with the goal's text. Keep the plan file current as you go.`, { deliverAs: "followUp" }); return; } @@ -467,11 +503,15 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { .pop() as { data?: PlanState } | undefined; state = { phase: last?.data?.phase ?? null, - judgeModel: last?.data?.judgeModel ?? null, + stewardModel: last?.data?.stewardModel ?? null, + stewardRunId: last?.data?.stewardRunId ?? null, + stewardPending: last?.data?.stewardPending ?? false, planVersion: last?.data?.planVersion ?? null, autoIntervalMs: last?.data?.autoIntervalMs ?? null, autoPaused: last?.data?.autoPaused ?? false, }; + watchStewardCompletion(); + setupSteward(ctx); lastSeenWorkingSet = foldPlan(readPlan(ctx)); autoLastWorkingSet = lastSeenWorkingSet; planningContextPending = state.phase === "planning"; @@ -482,6 +522,10 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { pi.on("session_shutdown", async () => { clearAutoTimer(); + stewardRegistration?.dispose(); + stewardRegistration = null; + unsubscribeStewardCompletion?.(); + unsubscribeStewardCompletion = null; }); // --- the one blessed tool: CompleteGoal --------------------------------------------------------- @@ -498,26 +542,31 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { const plan = readPlan(ctx); if (!plan.trim()) return result(`No plan file at ${planRel(ctx)}. Run /goals to draft one.`, true); - const judgeModel = state.judgeModel ?? (ctx.model ? `${ctx.model.provider}/${ctx.model.id}` : null); - onUpdate?.({ content: [{ type: "text", text: `Read-only judge (${judgeModel ?? "pi default"}) inspecting: ${params.goal}` }], details: {} }); - // decideSignOff runs the judge and derives the outcome + the one log line. judgeModel is never - // checked pre-emptively: null just means pi's configured default runs (buildJudgeArgs omits - // --model), so accepted_inconclusive always means "the judge ran but failed", never "no model". - let judgeRaw: JudgeResult | null = null; - const outcome = await decideSignOff({ goal: params.goal, plan, planRel: planRel(ctx), judgeModel }, signal, async (task) => { - judgeRaw = await runJudge(task, judgeModel, ctx.cwd, signal); - return judgeRaw; - }); - // Persist the judge's full transcript so "did the judge really re-run verify?" is answerable - // after the fact (dogfood finding: with only the one log line, an accept is unauditable). - let transcriptNote = ""; - if (judgeRaw !== null) { - const raw: JudgeResult = judgeRaw; - mkdirSync(join(ctx.cwd, ".pi", "judge"), { recursive: true }); - const rel = `.pi/judge/${stamp().replace(/[: ]/g, "-")}-${process.hrtime.bigint()}.md`; - writeFileSync(join(ctx.cwd, rel), `goal: ${params.goal}\nmodel: ${judgeModel ?? "pi default"}\nerror: ${raw.error ?? "none"}\n\n${raw.output}\n`); - transcriptNote = ` (${rel})`; + if (!stewardRegistration) { + return result(`Goal steward unavailable: ${stewardRegistrationError ?? "pi-subagents is not ready"}. Install pi-subagents and reload Pi.`, true); } + if (state.stewardPending) return result("The goal steward is still reviewing the previous checkpoint. Retry CompleteGoal after its result arrives.", true); + onUpdate?.({ content: [{ type: "text", text: `Persistent goal steward inspecting: ${params.goal}` }], details: {} }); + let reviewRunId = ""; + let decision: StewardDecision; + try { + const review = await runStewardReview( + pi.events, + ctx.cwd, + state.stewardRunId, + signoffReview(planRel(ctx), params.goal), + signal, + 600_000, + rememberStewardRun, + ); + reviewRunId = review.runId; + decision = review.decision; + } catch (error) { + return result(`Goal-steward review failed: ${error instanceof Error ? error.message : String(error)}`, true); + } + state = { ...state, stewardPending: false }; + persist(); + const outcome = decideStewardSignOff(params.goal, decision, reviewRunId); if (outcome.logEntry) { // Sign-off write: tick the goal [x] (exact-subject match; dogfood showed agent bookkeeping // is the drift point) and append the audit log line, one write. On wording drift the tick @@ -531,7 +580,7 @@ export default function piGoalsExtension(pi: ExtensionAPI): void { ? `\n\nGoal ticked [x] in ${planRel(ctx)}.` : `\n\nNo exact goal line matched your wording -- tick it [x] in ${planRel(ctx)} yourself.`; } - writePlan(ctx, appendLog(updated, `${stamp()} ${outcome.logEntry}${transcriptNote}`)); + writePlan(ctx, appendLog(updated, `${stamp()} ${outcome.logEntry}`)); updateWidget(ctx); return result(outcome.resultText + tickNote, outcome.isError); } @@ -563,94 +612,35 @@ function oneLine(s: string): string { return s.replace(/\s+/g, " ").trim().slice(0, 200); } -/** A judge run's result: stdout output, plus an error string when the subprocess failed/timed out. */ -export interface JudgeResult { - output: string; - error?: string; -} - -/** Inputs to a sign-off decision. judgeModel is null when no explicit/session model is set. */ -export interface SignOffInput { - goal: string; - plan: string; - /** The session's plan file, relative to cwd; the judge prompt names it. */ - planRel: string; - judgeModel: string | null; -} - -/** The outcome of a sign-off: the reply text, whether it's a hard error, and the one ## Log line to - * append (null when nothing should be written, e.g. aborted before any verdict). */ export interface SignOffOutcome { resultText: string; isError: boolean; - logEntry: string | null; + logEntry: string; } -/** Run the judge and decide accept / reject / accepted_inconclusive. Pure aside from the injected - * judge runner, so the unit test can lock the fail-forward invariant: judgeModel is NEVER checked - * here, so a null model still reaches runJudge (pi's configured default runs it), and the only - * producers of accepted_inconclusive are the judge-error and no-VERDICT paths -- i.e. "the judge - * ran but failed", never "no model". The execute() wrapper does the plan-file write + widget. - * Exported for the unit test that locks this invariant. */ -export async function decideSignOff( - input: SignOffInput, - signal: AbortSignal | undefined, - runJudgeFn: (task: string) => Promise, -): Promise { - const task = judgeUser({ goal: input.goal, plan: input.plan, planPath: input.planRel }); - const judge = await runJudgeFn(task); - - if (signal?.aborted) return { resultText: "Sign-off aborted.", isError: true, logEntry: null }; - - // Judge ran but failed/errored/timed out: fail forward, say so in the log. - if (judge.error) { - const partial = judge.output ? `\n\npartial judge output:\n${judge.output}` : ""; +export function decideStewardSignOff(goal: string, decision: StewardDecision, runId: string): SignOffOutcome { + if (decision.verdict === "accept") { return { - resultText: `Judge ran but failed (${judge.error}). Accepted inconclusive — logged.${partial}`, + resultText: `Sign-off ACCEPTED.\n\nGoal steward: ${decision.summary}\nRun: ${runId}`, isError: false, - logEntry: `signed off "${input.goal}" (judge inconclusive: ran but failed: ${oneLine(judge.error)})`, + logEntry: `signed off "${goal}" (steward accept; run ${runId})`, }; } - - const verdictLine = judge.output.split("\n").find((l) => /^\s*VERDICT\s*:/i.test(l)) ?? ""; - const verdict = /^\s*VERDICT\s*:\s*(accept|reject)\s*$/i.exec(verdictLine)?.[1]?.toLowerCase(); - const reasoning = judge.output.length > 2000 ? `...\n${judge.output.slice(-2000)}` : judge.output; - - if (verdict === "accept") { - const beforeVerdict = judge.output.slice(0, judge.output.indexOf(verdictLine)); - const checks = /^#{0,6}\s*(?:\*\*)?checks(?:\*\*)?:\s*$[\s\S]*^[-*]\s+.+$/im.test(beforeVerdict); - if (!checks) { - return { - resultText: `Sign-off REJECTED. Missing:\nchecked-artifact list before VERDICT: accept\n\n--- judge ---\n${reasoning}`, - isError: true, - logEntry: `reject "${input.goal}": judge accept had no checked-artifact list`, - }; - } - return { - resultText: `Sign-off ACCEPTED (log line appended).\n\n--- judge ---\n${reasoning}`, - isError: false, - logEntry: `signed off "${input.goal}" (judge accept)`, - }; - } - if (verdict === "reject") { - const missing = judge.output.match(/missing\s*:\s*([\s\S]*)$/i)?.[1].trim() || judge.output.slice(-500); - return { - resultText: `Sign-off REJECTED. Missing:\n${missing}\n\n--- judge ---\n${reasoning}`, - isError: true, - logEntry: `reject "${input.goal}": ${oneLine(missing)}`, - }; - } - // No VERDICT line: same fail-forward as a judge error -- the judge ran but didn't answer. + const missing = decision.verdict === "reject" + ? decision.missingEvidence?.join("; ") || decision.summary + : decision.verdict === "redirect" + ? decision.nextAction ?? decision.summary + : `The steward returned let_run instead of a sign-off verdict: ${decision.summary}`; return { - resultText: `Judge returned no VERDICT line. Accepted inconclusive — logged.\n\n--- judge ---\n${reasoning || "(no output)"}`, - isError: false, - logEntry: `signed off "${input.goal}" (judge inconclusive: no VERDICT line)`, + resultText: `Sign-off REJECTED. Missing:\n${missing}\n\nGoal steward: ${decision.summary}\nRun: ${runId}`, + isError: true, + logEntry: `reject "${goal}": ${oneLine(missing)} (steward run ${runId})`, }; } /** Tick the goal line whose subject exactly matches `goal` (trimmed, case-insensitive) to [x]. * Null when there is no unique exact match (wording drift / duplicates) -- the caller then asks the - * agent to tick it itself. Reuses GOAL_LINE; deliberately NOT fuzzy, that's the judge's job. */ + * agent to tick it itself. Reuses GOAL_LINE; deliberately not fuzzy; the steward reads prose. */ export function tickGoal(plan: string, goal: string): string | null { const lines = plan.split("\n"); const want = goal.trim().toLowerCase(); @@ -686,65 +676,3 @@ export function appendInterview(text: string, answer: string): string { lines.splice(insertAt, 0, ...entry); return lines.join("\n"); } - -/** Build the pi argv for the read-only judge. `--model` is omitted when no explicit/session model is - * set, so pi falls back to its configured default — the judge always runs. `--no-extensions` keeps - * the judge minimal and immune to a broken third-party extension taking down every sign-off. - * Exported for the unit test that locks these invariants. */ -export function buildJudgeArgs(judgeModel: string | null): string[] { - const args = ["-p", "--no-session", "--no-extensions"]; - if (judgeModel) args.push("--model", judgeModel); - args.push("--tools", JUDGE_TOOLS.join(","), "--exclude-tools", JUDGE_BLOCKED_TOOLS.join(","), "--append-system-prompt", judgeSystem); - return args; -} - -/** Locate the pi binary the same way the oracle extension does, so spawning works under bun or node. */ -function getPiInvocation(args: string[]): { command: string; args: string[] } { - const script = process.argv[1]; - if (script && !script.startsWith("/$bunfs/root/") && existsSync(script)) return { command: process.execPath, args: [script, ...args] }; - const execName = basename(process.execPath).toLowerCase(); - if (!/^(node|bun)(\.exe)?$/.test(execName)) return { command: process.execPath, args }; - return { command: "pi", args }; -} - -/** Spawn the read-only judge subprocess (plain `pi -p`: stdout is the final response text). */ -async function runJudge( - task: string, - judgeModel: string | null, - cwd: string, - signal: AbortSignal | undefined, -): Promise { - const args = buildJudgeArgs(judgeModel); - args.push(task); - const inv = getPiInvocation(args); - // Runs in-place against this checkout; pi --no-session does not clone into the parent - // (proven by scripts/check-judge-footprint.sh). - return new Promise((resolvePromise) => { - let settled = false; - let stdout = ""; - let stderr = ""; - const done = (r: { output: string; error?: string }) => { - if (!settled) { - settled = true; - clearTimeout(timer); - resolvePromise(r); - } - }; - const proc = spawn(inv.command, inv.args, { cwd, shell: false, stdio: ["ignore", "pipe", "pipe"], signal }); - const timer = setTimeout(() => { - proc.kill(); - done({ output: stdout.trim(), error: `judge timed out after ${JUDGE_TIMEOUT_MS / 1000}s` }); - }, JUDGE_TIMEOUT_MS); - proc.stdout?.on("data", (d) => { - stdout += d.toString(); - }); - proc.stderr?.on("data", (d) => { - stderr += d.toString(); - }); - proc.on("close", (code) => { - if ((code ?? 0) !== 0) done({ output: stdout.trim(), error: stderr.trim() || `judge subprocess exited ${code ?? 1}` }); - else done({ output: stdout.trim() }); - }); - proc.on("error", (e) => done({ output: stdout.trim(), error: `judge subprocess failed: ${e.message}` })); - }); -} diff --git a/src/prompts.ts b/src/prompts.ts index 28005ff..5282df8 100644 --- a/src/prompts.ts +++ b/src/prompts.ts @@ -3,9 +3,9 @@ * * Design: the plan file is for LLMs and the human, not for TypeScript. No parser and no schema; * the skeleton below is a convention the drafting prompt teaches, the working agent maintains with - * its normal Edit tool, and the judge reads natively. The harness does three things for a + * its normal Edit tool, and the goal steward reads natively. The harness does three things for a * cooperative-but-confused model: memory (a transient re-send of the plan when it goes stale), - * format guidance (the skeleton), and fresh eyes (the read-only judge in CompleteGoal). + * format guidance (the skeleton), and supervision (a persistent read-only pi-subagents child). * * THE FOLD: everything above "## Log" is the working set (title, user voice, goals, * discriminators) and is what gets re-sent on the reminder cadence. Everything below it (Log, @@ -17,7 +17,7 @@ * EXEC, on cadence 2. reminder — the folded plan + upkeep nudge when it went stale * EXEC, after compact 3. resync — the WHOLE file back, once * SIGN-OFF, agent-side 4. completeGoal* — the one blessed tool's description - * SIGN-OFF, judge-side 5. judgeSystem/judgeUser — the one rigorous check + * SUPERVISION steward.ts — persistent review and sign-off prompts * * The goal's test is the DISCRIMINATOR: the concrete observation that tells real success from the * named subtle failure mode. Evidence is empty at planning and filled at sign-off. @@ -58,13 +58,13 @@ Detail that doesn't change a goal or a discriminator belongs in the appendix, no Right-size it: - One goal per distinct judgeable outcome. Group related goals when it helps judge them together and readability. The count flows from the outcomes. -- Describe outcomes in qualitative terms the judge and user can discriminate. +- Describe outcomes in qualitative terms the steward and user can discriminate. - Use the users language or more precise don't transform "MV" into "knob" as it looses precision and is overloaded - - Don't invent metrics or thresholds for problems you haven't explored yet — the judge should hopefully know it when it sees the outcome. + - Don't invent metrics or thresholds for problems you haven't explored yet — the steward should know it when it sees the outcome. - Quantitative gates are fine only when you are certain they survive contact with reality. - Subtasks are the steps inside a goal; add them when a goal has 3+ distinct steps, skip otherwise. - Two goals that share one discriminator are one goal. Merge them. -- Keep the goal subject short. Put its important scope, failure modes, discriminator, tasks, and evidence in the indented block beneath it. The judge reads the whole block and the whole plan. +- Keep the goal subject short. Put its important scope, failure modes, discriminator, tasks, and evidence in the indented block beneath it. The steward reads the whole block and the whole plan. - Keep the working set under 50 lines, excluding ## User voice. ## User voice has no line limit: quote the human fully rather than shorten or paraphrase them. Everything below "## Log" is unlimited. @@ -72,7 +72,7 @@ Style: Make it easy for a busy and forgetfull user to review. Use ASD-STE100 Sim the same word for the same thing, and define a new terms at first use. Use redundant context for skim readers e.g. "our output - the cells, CV tag" is easy to read and reminds context. This covers the context paragraph and the appendix too, not just the checklist. No all-caps headers and no bold spam. Just write less, add your voice less, persuade less, and burden the reader less. -Write the plan file in roughly this shape -- the file is read directly by the human and a judge model, so clarity beats conformance; small deviations are fine): +Write the plan file in roughly this shape -- the file is read directly by the human and the goal steward, so clarity beats conformance; small deviations are fine): # @@ -92,7 +92,7 @@ Write the plan file in roughly this shape -- the file is read directly by the hu - subtle failure mode: - discriminator: - verify: + testable. YOU run it at sign-off time and save its output as evidence; the steward only reads> - tasks: 1. [ ] - evidence: (empty until sign-off) @@ -122,9 +122,9 @@ Conventions: none of the failure modes could fake. Ruling out failures is necessary, not sufficient. - Make the discriminator a concrete, checkable observation about a real artifact (a file, a test result, a committed diff, a metric), never about the plan file's own checkbox. -- evidence stays empty at planning; you fill it at sign-off and a fresh read-only judge checks it. +- evidence stays empty at planning; you fill it at sign-off and the read-only goal steward checks it. Cite durable artifacts a future reader can open: committed files, test names, git diffs. .pi/ is - usually gitignored, so files there prove things only at judge time, not in history. + usually gitignored, so files there prove things only at steward review time, not in history. - User-visible result: restate the original deliverable, not the proposed implementation. Every goal must contribute to it. Future work may not defer any artifact or action named there. - User voice: quote the human word for word, one line per requirement, as they say it. Never @@ -207,69 +207,13 @@ export const completeGoalDescription = "output you actually observed; never reconstruct numbers from memory. If you couldn't see an " + "output, rerun it or write that you couldn't -- an honest gap beats a plausible fabrication. If " + "the goal names a verify: command, run it yourself first and save its output to a file cited in " + - "the evidence: the judge cannot execute anything and will reject a claimed pass with no saved " + + "the evidence: the goal steward cannot execute anything and will reject a claimed pass with no saved " + "output. The read must show success POSITIVELY happened, not just that failures were avoided. " + "Check that the claimed result uses the artifact and outcome named in User-visible result and does " + "not substitute an agent-inferred deliverable. Then call this with the goal's text (the line after " + - "'goal:'; small wording drift is fine). A " + - "fresh strictly-read-only judge inspects the LIVE WORKING TREE (uncommitted changes included; " + - "committing first is for durability, not visibility) and returns accept or reject with what's " + - "missing. On accept (or if the judge itself failed), a sign-off line is appended to ## Log " + - "and the goal is ticked [x] for you; the result says if you must tick it yourself. On reject the " + - "goal stays open."; + "'goal:'; small wording drift is fine). The persistent read-only goal steward rereads the complete " + + "plan, inspects the LIVE WORKING TREE (uncommitted changes included), and returns accept or reject " + + "with what is missing. On accept, a sign-off line is appended to ## Log and the goal is ticked [x] " + + "for you; the result says if you must tick it yourself. On reject or steward failure, the goal stays open."; export const completeGoalParamDescription = "The goal's text: the line after 'goal:' in the plan file."; - -/* ───────────────────────────────────────────────────────────────────────── - * 5. judge — SIGN-OFF, judge-side: the one rigorous check. Runs on a fresh - * read-only pi subprocess (--no-session) so it never sees the working - * agent's transcript. It gets the WHOLE plan file: it finds the goal, - * reads discriminator/failure modes/evidence itself (no parser between). - * ──────────────────────────────────────────────────────────────────────── */ -export const judgeSystem = `\ -You are a strictly read-only reviewer signing off a coding goal. You cannot execute anything: judge -by reading (read/grep/find/ls). Never re-run the work or its verify command -- it may be a 10-hour -job; the agent must bring you its saved output. Your job is evidence discipline, checked in order: - -0. Task fidelity? Read User-visible result and User voice first. Reject if this goal contradicts, - replaces, or defers the requested artifact or outcome. Agent-inferred scope is not authority. -1. Anything here? An empty or placeholder evidence: list -> reject: "there's nothing here -- fill - the evidence and try again." -2. Quoted and attributed? Each item needs a source (file path / command) plus a verbatim quote of - what was observed, plus a one-line read. A bare claim -> reject: "you didn't quote and - attribute it." -3. Provenance? It must be visible HOW each result was produced (the command run, where its output - was saved). Results with no origin -> reject: "I see the results, but how did you get them?" -4. Spot-check: open the cited files. A quote or number that doesn't match what's on disk means the - evidence was reconstructed from memory, not observed -> reject and ask for re-observed - evidence, even if the goal otherwise looks met. -5. Substance, only once 1-4 hold: does the evidence show the discriminator's success signal - POSITIVELY happened -- not just that the named failure modes were dodged; a run can rule out - every trap and still have produced nothing. Is each subtle failure mode genuinely ruled out, - not just unmentioned? If the goal names a verify: command, its saved output must be among the - evidence, and the command must actually test the discriminator rather than pass tautologically. - -Before the verdict, write this heading: checks:. Put one concise bullet under it for each artifact you actually read: -path, verbatim observed quote, and what that observation establishes. This is an inspectable review -record, not hidden reasoning. Do not write a checks bullet for a file you did not open. - -Finish with exactly these two lines and nothing after: -VERDICT: accept | reject -missing: `; - -export function judgeUser(p: { goal: string; plan: string; planPath: string }): string { - return `\ -The working agent claims this goal is complete: - - goal: ${p.goal} - -Below is the full plan file (${p.planPath}). Find that goal in it (tolerate small wording drift; if -you cannot find a matching goal at all, reject and say so). Read User-visible result and User voice -first, then its discriminator, subtle failure modes, verify command, and evidence list. - ---- plan file --- -${p.plan} ---- end plan file --- - -Read the cited artifacts (you cannot execute anything), then give your VERDICT.`; -} diff --git a/src/steward.ts b/src/steward.ts new file mode 100644 index 0000000..60b9064 --- /dev/null +++ b/src/steward.ts @@ -0,0 +1,246 @@ +import { randomUUID } from "node:crypto"; + +const REGISTER_EVENT = "pi-subagents:runtime-agent-register:v1"; +const RPC_REQUEST_EVENT = "subagents:rpc:v1:request"; +const RPC_REPLY_PREFIX = "subagents:rpc:v1:reply:"; +const ASYNC_COMPLETE_EVENT = "subagent:async-complete"; +const RPC_VERSION = 1; +const RPC_TIMEOUT_MS = 15_000; +export const STEWARD_AGENT = "goal-steward"; + +interface EventBus { + on(event: string, handler: (data: unknown) => void): () => void; + emit(event: string, data: unknown): void; +} + +interface Registration { + dispose(): void; +} + +interface RpcData { + text: string; + details?: Record; +} + +export interface StewardDecision { + verdict: "let_run" | "redirect" | "accept" | "reject"; + summary: string; + nextAction?: string; + missingEvidence?: string[]; +} + +export const stewardOutputSchema = { + type: "object", + properties: { + verdict: { type: "string", enum: ["let_run", "redirect", "accept", "reject"] }, + summary: { type: "string", maxLength: 800 }, + nextAction: { type: "string", maxLength: 400 }, + missingEvidence: { type: "array", maxItems: 8, items: { type: "string", maxLength: 400 } }, + }, + required: ["verdict", "summary"], + additionalProperties: false, +} as const; + +export const stewardSystemPrompt = `You are the read-only goal steward for one Pi work session. +Act as its supervisor, mentor, project manager, and skeptical board member. Keep the high-level goal +and the human's stated result stable while the worker handles implementation detail. + +The plan path arrives in every review. Read the complete plan from disk every time, including after +compaction. Treat User-visible result and User voice as the authority. The plan is maintained by the +worker; never edit it. Use read-only tools to inspect cited files when this changes your decision. + +Spend few tokens. Call structured_output as soon as the evidence is sufficient. Do not send prose +before that call, write a review essay, restate the plan, or narrate routine progress. Return one verdict: +- let_run: progress follows the plan and no instruction is useful +- redirect: drift, a missed failure mode, or a specific better next action needs worker attention +- accept: only for a sign-off review whose evidence positively proves the discriminator +- reject: only for a sign-off review that names the missing evidence + +For redirect, include one concrete nextAction. For reject, include missingEvidence. Contact the +parent only when an immediate decision is needed. Do not accept a confident summary as evidence. +You are advisory and read-only; pi-goals alone writes and signs off the plan. + +— Pi/Codex`; + +export function registerStewardAgent(events: EventBus, model: string | null): Registration { + const request: Record = { + version: 1, + name: STEWARD_AGENT, + definition: { + description: "Persistent read-only supervisor for one pi-goals plan.", + systemPrompt: stewardSystemPrompt, + tools: ["read", "grep", "find", "ls", "contact_supervisor"], + excludeTools: ["bash", "edit", "write", "subagent"], + allowNestedSubagents: false, + ...(model ? { model } : {}), + thinking: "low", + systemPromptMode: "replace", + inheritProjectContext: false, + inheritGlobalContext: false, + inheritSkills: false, + defaultContext: "fresh", + defaultAsync: true, + defaultTimeoutMs: 300_000, + acceptanceRole: "read-only", + defaultProgress: false, + toolBudget: { soft: 8, hard: 12, block: ["read", "grep", "find", "ls"] }, + }, + }; + events.emit(REGISTER_EVENT, request); + const result = request.result as { ok?: boolean; registration?: Registration; error?: Error } | undefined; + if (!result) throw new Error("pi-subagents is not installed or not ready."); + if (!result.ok || !result.registration) throw result.error ?? new Error("pi-subagents rejected the goal-steward agent."); + return result.registration; +} + +export function readyReview(planPath: string): string { + return `Review reason: plan approved\nPlan path: ${planPath}\n\nRead the complete plan now. Check that its goals still match the user-visible result and that the first work step is sensible. This is not sign-off: return only let_run or redirect.`; +} + +export function checkpointReview(planPath: string, staleTurns: number): string { + return `Review reason: progress checkpoint\nPlan path: ${planPath}\n\nThe worker completed ${staleTurns} turns without changing the plan's working set. Read the complete plan now and inspect only files needed to decide whether one concrete redirect would help. This is not sign-off: return only let_run or redirect.`; +} + +export function signoffReview(planPath: string, goal: string): string { + return `Review reason: goal sign-off\nPlan path: ${planPath}\nClaimed goal: ${goal}\n\nRead the complete plan now. Check User-visible result, User voice, this goal's discriminator, failure mode, and evidence. Inspect the cited files. Return accept only when the evidence positively proves the requested result; otherwise return reject and name the missing evidence.`; +} + +async function rpc(events: EventBus, method: "spawn" | "resume", params: Record, signal?: AbortSignal): Promise { + if (signal?.aborted) throw new Error("Goal-steward request aborted."); + const requestId = randomUUID(); + return new Promise((resolve, reject) => { + let timer: ReturnType; + const replyEvent = `${RPC_REPLY_PREFIX}${requestId}`; + const cleanup = () => { + clearTimeout(timer); + unsubscribe(); + signal?.removeEventListener("abort", onAbort); + }; + const onAbort = () => { + cleanup(); + reject(new Error("Goal-steward request aborted.")); + }; + const unsubscribe = events.on(replyEvent, (raw) => { + const reply = raw as { success?: boolean; data?: RpcData; error?: { message?: string } }; + cleanup(); + if (!reply.success || !reply.data) reject(new Error(reply.error?.message ?? `pi-subagents ${method} failed.`)); + else resolve(reply.data); + }); + timer = setTimeout(() => { + cleanup(); + reject(new Error(`pi-subagents ${method} did not reply within ${RPC_TIMEOUT_MS / 1000}s.`)); + }, RPC_TIMEOUT_MS); + timer.unref(); + signal?.addEventListener("abort", onAbort, { once: true }); + events.emit(RPC_REQUEST_EVENT, { version: RPC_VERSION, requestId, method, params, source: { extension: "pi-goals" } }); + }); +} + +function asyncRunId(data: RpcData): string { + const runId = data.details?.asyncId ?? data.details?.runId; + if (typeof runId !== "string" || !runId) throw new Error("pi-subagents returned no async run ID."); + return runId; +} + +export async function startSteward(events: EventBus, cwd: string, task: string, signal?: AbortSignal): Promise { + const data = await rpc(events, "spawn", { + agent: STEWARD_AGENT, + task, + cwd, + context: "fresh", + async: true, + mission: false, + outputSchema: stewardOutputSchema, + }, signal); + return asyncRunId(data); +} + +export async function resumeSteward(events: EventBus, runId: string, task: string, signal?: AbortSignal): Promise { + const data = await rpc(events, "resume", { id: runId, message: task }, signal); + return asyncRunId(data); +} + +function completionRunId(raw: unknown): string | null { + if (!raw || typeof raw !== "object") return null; + const runId = (raw as Record).runId; + return typeof runId === "string" ? runId : null; +} + +export async function runStewardReview( + events: EventBus, + cwd: string, + previousRunId: string | null, + task: string, + signal?: AbortSignal, + timeoutMs = 600_000, + onRunId?: (runId: string) => void, +): Promise<{ runId: string; decision: StewardDecision }> { + if (signal?.aborted) throw new Error("Goal-steward review aborted."); + let expectedRunId: string | null = null; + const earlyCompletions: unknown[] = []; + let settle: (raw: unknown) => void = () => {}; + let fail: (error: Error) => void = () => {}; + let timer: ReturnType | undefined; + const completion = new Promise((resolve, reject) => { + settle = (raw) => { + try { + resolve(parseStewardDecision(raw)); + } catch (error) { + reject(error); + } + }; + fail = reject; + }); + const unsubscribe = events.on(ASYNC_COMPLETE_EVENT, (raw) => { + const completedRunId = completionRunId(raw); + if (expectedRunId === null) { + earlyCompletions.push(raw); + return; + } + if (completedRunId === expectedRunId) settle(raw); + }); + const onAbort = () => fail(new Error("Goal-steward review aborted.")); + try { + expectedRunId = previousRunId + ? await resumeSteward(events, previousRunId, task, signal) + : await startSteward(events, cwd, task, signal); + onRunId?.(expectedRunId); + if (signal?.aborted) throw new Error("Goal-steward review aborted."); + signal?.addEventListener("abort", onAbort, { once: true }); + timer = setTimeout(() => fail(new Error(`Goal-steward review timed out after ${timeoutMs / 1000}s.`)), timeoutMs); + timer.unref(); + const early = earlyCompletions.find((raw) => completionRunId(raw) === expectedRunId); + if (early) settle(early); + return { runId: expectedRunId, decision: await completion }; + } finally { + if (timer) clearTimeout(timer); + unsubscribe(); + signal?.removeEventListener("abort", onAbort); + } +} + +export function parseStewardDecision(raw: unknown): StewardDecision { + if (!raw || typeof raw !== "object") throw new Error("Goal-steward completion was not an object."); + const results = (raw as Record).results; + if (!Array.isArray(results) || results.length !== 1 || !results[0] || typeof results[0] !== "object") { + throw new Error("Goal-steward completion did not contain exactly one result."); + } + const child = results[0] as Record; + if (child.success === false) throw new Error(typeof child.error === "string" ? child.error : "Goal-steward run failed."); + const value = child.structuredOutput; + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Goal-steward returned no structured verdict."); + const decision = value as Record; + if (!(["let_run", "redirect", "accept", "reject"] as unknown[]).includes(decision.verdict) || typeof decision.summary !== "string" || !decision.summary.trim()) { + throw new Error("Goal-steward returned an invalid structured verdict."); + } + if (decision.verdict === "redirect" && (typeof decision.nextAction !== "string" || !decision.nextAction.trim())) { + throw new Error("Goal-steward redirect omitted nextAction."); + } + if ( + decision.verdict === "reject" + && (!Array.isArray(decision.missingEvidence) || decision.missingEvidence.length === 0 || decision.missingEvidence.some((item) => typeof item !== "string" || !item.trim())) + ) { + throw new Error("Goal-steward rejection omitted missingEvidence."); + } + return decision as unknown as StewardDecision; +} diff --git a/test/decide-signoff.test.ts b/test/decide-signoff.test.ts index b89ca48..4d66d79 100644 --- a/test/decide-signoff.test.ts +++ b/test/decide-signoff.test.ts @@ -1,81 +1,38 @@ -import { describe, expect, it, vi } from "vitest"; -import { decideSignOff, type JudgeResult } from "../src/index.js"; -import { judgeSystem } from "../src/prompts.js"; +import { describe, expect, it } from "vitest"; +import { decideStewardSignOff } from "../src/index.js"; -// decideSignOff is the fail-forward invariant: judgeModel is NEVER checked pre-emptively, so a null -// model still reaches runJudge (pi's configured default runs it), and the only producers of -// accepted_inconclusive are the judge-error and no-VERDICT paths -- i.e. "the judge ran but failed", -// never "no model". The judge runner is injected so these tests never spawn a real subprocess. -const input = { goal: "x", plan: "# plan\n", planRel: ".pi/plan/s1.md", judgeModel: null }; - -describe("decideSignOff (fail-forward invariant)", () => { - it("proceeds to runJudge even when judgeModel is null (no pre-emptive 'no model' inconclusive)", async () => { - const output = "## checks:\n- evidence.txt: `PASS`; the saved check passed\n\nThe artifact proves the gate passed.\nVERDICT: accept\nmissing:"; - const runJudge = vi.fn().mockResolvedValue({ output }); - const out = await decideSignOff({ ...input, plan: "# plan\n1. [ ] goal: x\n" }, undefined, runJudge); - expect(runJudge).toHaveBeenCalledOnce(); // reached the judge -- no pre-emptive return on null model +describe("decideStewardSignOff", () => { + it("accepts only the steward's accept verdict", () => { + const out = decideStewardSignOff("produce report", { verdict: "accept", summary: "The cited report contains every required row." }, "run-2"); expect(out.isError).toBe(false); - expect(out.logEntry).toContain("judge accept"); - expect(out.resultText).toContain("evidence.txt: `PASS`"); + expect(out.resultText).toContain("Sign-off ACCEPTED"); + expect(out.logEntry).toContain("steward accept; run run-2"); }); - it("rejects an accept verdict without a checked-artifact list", async () => { - const runJudge = vi.fn().mockResolvedValue({ output: "VERDICT: accept\nmissing:" }); - const out = await decideSignOff(input, undefined, runJudge); + it("reports the steward's missing evidence", () => { + const out = decideStewardSignOff( + "produce report", + { verdict: "reject", summary: "The count is not cited.", missingEvidence: ["Saved output with the recursive file count", "A matching table row count"] }, + "run-3", + ); expect(out.isError).toBe(true); - expect(out.resultText).toContain("checked-artifact list"); - expect(out.logEntry).toContain("no checked-artifact list"); + expect(out.resultText).toContain("Saved output with the recursive file count; A matching table row count"); + expect(out.logEntry).toContain("steward run run-3"); }); - it("a judge-subprocess error yields accepted_inconclusive with a 'ran but failed' reason", async () => { - const runJudge = vi.fn().mockResolvedValue({ output: "", error: "judge subprocess exited 1" } satisfies JudgeResult); - const out = await decideSignOff(input, undefined, runJudge); - expect(runJudge).toHaveBeenCalledOnce(); - expect(out.isError).toBe(false); // accepted inconclusive, not a hard error that blocks the agent - expect(out.resultText.toLowerCase()).toContain("accepted inconclusive"); - expect(out.resultText).toContain("ran but failed"); // inconclusive means ran but failed, not "no model" - expect(out.logEntry).toContain("ran but failed"); - expect(out.logEntry).toContain("subprocess exited 1"); - }); - - it("a judge timeout is also accepted_inconclusive (ran but failed)", async () => { - const runJudge = vi.fn().mockResolvedValue({ output: "partial", error: "judge timed out after 600s" }); - const out = await decideSignOff(input, undefined, runJudge); - expect(out.isError).toBe(false); - expect(out.resultText.toLowerCase()).toContain("accepted inconclusive"); - expect(out.logEntry).toContain("ran but failed"); - expect(out.logEntry).toContain("timed out"); - expect(out.resultText).toContain("partial judge output:\npartial"); - }); - - it("no VERDICT line is accepted_inconclusive too (judge ran but didn't answer)", async () => { - const runJudge = vi.fn().mockResolvedValue({ output: "I looked but forgot the verdict line" }); - const out = await decideSignOff(input, undefined, runJudge); - expect(out.isError).toBe(false); - expect(out.resultText).toContain("no VERDICT line"); - expect(out.logEntry).toContain("no VERDICT line"); - }); - - it("rejects when the judge returns VERDICT: reject", async () => { - const runJudge = vi.fn().mockResolvedValue({ output: "VERDICT: reject\nmissing: evidence, tests" }); - const out = await decideSignOff({ ...input, judgeModel: "openrouter/claude" }, undefined, runJudge); + it("treats redirect as a rejected sign-off with one next action", () => { + const out = decideStewardSignOff( + "produce report", + { verdict: "redirect", summary: "The worker inspected only the top level.", nextAction: "Repeat the snapshot recursively." }, + "run-4", + ); expect(out.isError).toBe(true); - expect(out.resultText).toContain("REJECTED"); - expect(out.resultText).toContain("evidence, tests"); - expect(out.logEntry).toContain("reject"); + expect(out.resultText).toContain("Repeat the snapshot recursively."); }); - it("requires a concise checked-artifact review, not private reasoning", () => { - expect(judgeSystem).toContain("checks:"); - expect(judgeSystem).toContain("not hidden reasoning"); - }); - - it("writes nothing when aborted after the judge ran", async () => { - const runJudge = vi.fn().mockResolvedValue({ output: "VERDICT: accept" }); - const ctrl = new AbortController(); - ctrl.abort(); - const out = await decideSignOff(input, ctrl.signal, runJudge); - expect(out.logEntry).toBeNull(); + it("does not turn let_run into acceptance", () => { + const out = decideStewardSignOff("produce report", { verdict: "let_run", summary: "Continue the current work." }, "run-5"); expect(out.isError).toBe(true); + expect(out.resultText).toContain("let_run instead of a sign-off verdict"); }); }); diff --git a/test/goals-flow.test.ts b/test/goals-flow.test.ts index 51ffbcd..b3ad57d 100644 --- a/test/goals-flow.test.ts +++ b/test/goals-flow.test.ts @@ -15,8 +15,31 @@ function setup( const hooks = new Map(); const tools = new Map(); const entries: Array<{ type: string; customType: string; data: unknown }> = []; - const events: string[] = []; + const eventLog: string[] = []; const messages: Array<{ content: string; display?: boolean }> = []; + const rpcRequests: any[] = []; + const eventHandlers = new Map void>>(); + const eventBus = { + on(name: string, handler: (data: unknown) => void) { + const handlers = eventHandlers.get(name) ?? new Set(); + handlers.add(handler); + eventHandlers.set(name, handlers); + return () => handlers.delete(handler); + }, + emit(name: string, data: unknown) { + for (const handler of [...(eventHandlers.get(name) ?? [])]) handler(data); + }, + }; + let asyncRun = 0; + eventBus.on("pi-subagents:runtime-agent-register:v1", (raw) => { + (raw as any).result = { ok: true, registration: { dispose() {} } }; + }); + eventBus.on("subagents:rpc:v1:request", (raw) => { + const request = raw as any; + rpcRequests.push(request); + asyncRun++; + eventBus.emit(`subagents:rpc:v1:reply:${request.requestId}`, { success: true, data: { text: "started", details: { asyncId: `steward-${asyncRun}` } } }); + }); const ctx = { cwd, hasUI: true, @@ -28,28 +51,29 @@ function setup( setWidget: () => {}, notify: () => {}, select: async () => { - events.push("select"); + eventLog.push("select"); return selectChoices.shift(); }, editor: async () => { - events.push("editor"); + eventLog.push("editor"); return editPlan ? editPlan() : editorChoices.shift(); }, }, }; const pi = { + events: eventBus, registerCommand: (name: string, command: any) => commands.set(name, command), on: (name: string, handler: any) => hooks.set(name, handler), appendEntry: (customType: string, data: unknown) => entries.push({ type: "custom", customType, data }), registerTool: (tool: any) => tools.set(tool.name, tool), sendMessage: (message: { content: string; display?: boolean }) => { - events.push("display"); + eventLog.push("display"); messages.push(message); }, sendUserMessage: (message: string) => messages.push({ content: message }), }; piGoalsExtension(pi as unknown as ExtensionAPI); - return { commands, ctx, cwd, entries, events, hooks, messages, tools }; + return { commands, ctx, cwd, entries, events: eventLog, eventBus, hooks, messages, rpcRequests, tools }; } describe("/goals draft flow", () => { @@ -250,6 +274,47 @@ describe("/goals draft flow", () => { } }); + it("resumes the same steward lineage for sign-off and persists the latest run", async () => { + const flow = setup(["Ready"]); + try { + await flow.hooks.get("session_start")({}, flow.ctx); + await flow.commands.get("goals").handler("objective", flow.ctx); + const planPath = join(flow.cwd, ".pi/plan/session-a-v1.md"); + writeFileSync(planPath, "# Plan\n\n## Goals\n\n1. [/] goal: produce report\n - discriminator: report.txt contains PASS\n - evidence:\n - report.txt: `PASS`\n\n## Log\n"); + await flow.hooks.get("agent_settled")({}, flow.ctx); + expect(flow.rpcRequests[0]).toMatchObject({ method: "spawn", params: { agent: "goal-steward" } }); + flow.eventBus.emit("subagent:async-complete", { + runId: "steward-1", + results: [{ success: true, structuredOutput: { verdict: "let_run", summary: "Start work." } }], + }); + + const signoff = flow.tools.get("CompleteGoal").execute("", { goal: "produce report" }, undefined, undefined, flow.ctx); + await new Promise((resolve) => setImmediate(resolve)); + expect(flow.rpcRequests[1]).toMatchObject({ method: "resume", params: { id: "steward-1" } }); + flow.eventBus.emit("subagent:async-complete", { + runId: "steward-2", + results: [{ success: true, structuredOutput: { verdict: "accept", summary: "report.txt contains PASS." } }], + }); + const outcome = await signoff; + + expect(outcome.isError).toBe(false); + expect(readFileSync(planPath, "utf-8")).toContain("1. [x] goal: produce report"); + expect(flow.entries.at(-1)?.data).toMatchObject({ stewardRunId: "steward-2", stewardPending: false }); + + await flow.hooks.get("session_start")({}, flow.ctx); + const afterReload = flow.tools.get("CompleteGoal").execute("", { goal: "produce report" }, undefined, undefined, flow.ctx); + await new Promise((resolve) => setImmediate(resolve)); + expect(flow.rpcRequests[2]).toMatchObject({ method: "resume", params: { id: "steward-2" } }); + flow.eventBus.emit("subagent:async-complete", { + runId: "steward-3", + results: [{ success: true, structuredOutput: { verdict: "reject", summary: "Already complete.", missingEvidence: ["No second sign-off needed"] } }], + }); + expect((await afterReload).isError).toBe(true); + } finally { + rmSync(flow.cwd, { recursive: true, force: true }); + } + }); + it("gives the agent a planning snapshot and blocks work routes", async () => { const flow = setup([]); try { diff --git a/test/judge-args.test.ts b/test/judge-args.test.ts deleted file mode 100644 index 0601132..0000000 --- a/test/judge-args.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { buildJudgeArgs } from "../src/index.js"; - -describe("buildJudgeArgs", () => { - it("omits --model when judgeModel is null (pi uses its configured default; never a pre-emptive 'no model' failure)", () => { - const args = buildJudgeArgs(null); - expect(args).not.toContain("--model"); - // an empty --model "" would make every sign-off silently inconclusive -- guard against it - const i = args.indexOf("--model"); - expect(i).toBe(-1); - }); - - it("includes --model when an explicit/session model is set", () => { - const args = buildJudgeArgs("openrouter/~anthropic/claude-haiku-latest"); - const i = args.indexOf("--model"); - expect(i).not.toBe(-1); - expect(args[i + 1]).toBe("openrouter/~anthropic/claude-haiku-latest"); - }); - - it("always sets --no-session, --no-extensions, the read-only tool allowlist, and edit/write exclusion", () => { - for (const m of [null, "some/model"]) { - const args = buildJudgeArgs(m); - expect(args).toContain("--no-session"); - expect(args).toContain("--no-extensions"); // a broken global extension must not take down sign-offs - expect(args).toContain("--tools"); - expect(args.some((a) => a.startsWith("read,grep,find,ls"))).toBe(true); - // no bash: the judge must never be able to execute (or re-run a 10-hour verify) or mutate - expect(args.some((a) => a.includes("bash"))).toBe(false); - expect(args).toContain("--exclude-tools"); - expect(args.some((a) => a.includes("edit") && a.includes("write"))).toBe(true); - } - }); -}); diff --git a/test/prompts.test.ts b/test/prompts.test.ts index da62fbd..4a9d7ce 100644 --- a/test/prompts.test.ts +++ b/test/prompts.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; -import { judgeSystem, planDrafting, planningState, reminder, resync } from "../src/prompts.js"; +import { planDrafting, planningState, reminder, resync } from "../src/prompts.js"; +import { stewardSystemPrompt } from "../src/steward.js"; describe("planning prompt", () => { it("requires fact finding or a focused question before a goal", () => { @@ -24,7 +25,7 @@ describe("planning prompt", () => { expect(planDrafting).toContain("Future work may not defer any artifact or action named there"); expect(reminder("plan", ".pi/plan/test.md")).toContain("latest message outranks this plan"); expect(resync("plan", ".pi/plan/test.md", "Compacted.")).toContain("amend the plan rather than preserving an obsolete decision"); - expect(judgeSystem).toContain("Task fidelity?"); - expect(judgeSystem).toContain("Agent-inferred scope is not authority"); + expect(stewardSystemPrompt).toContain("Treat User-visible result and User voice as the authority"); + expect(stewardSystemPrompt).toContain("Do not accept a confident summary as evidence"); }); }); diff --git a/test/steward.test.ts b/test/steward.test.ts new file mode 100644 index 0000000..f32c4a0 --- /dev/null +++ b/test/steward.test.ts @@ -0,0 +1,133 @@ +import { describe, expect, it, vi } from "vitest"; +import { + checkpointReview, + parseStewardDecision, + readyReview, + registerStewardAgent, + runStewardReview, + signoffReview, + startSteward, + stewardSystemPrompt, +} from "../src/steward.js"; + +class Events { + private handlers = new Map void>>(); + + on(event: string, handler: (data: unknown) => void): () => void { + const handlers = this.handlers.get(event) ?? new Set(); + handlers.add(handler); + this.handlers.set(event, handlers); + return () => handlers.delete(handler); + } + + emit(event: string, data: unknown): void { + for (const handler of [...(this.handlers.get(event) ?? [])]) handler(data); + } +} + +function completion(runId: string, verdict: "let_run" | "redirect" | "accept" | "reject" = "accept") { + return { + runId, + results: [{ success: true, structuredOutput: { verdict, summary: "Observed the cited artifact." } }], + }; +} + +describe("goal steward registration", () => { + it("registers one read-only runtime agent through pi-subagents", () => { + const events = new Events(); + let definition: Record | undefined; + events.on("pi-subagents:runtime-agent-register:v1", (raw) => { + const request = raw as { definition: Record; result?: unknown }; + definition = request.definition; + request.result = { ok: true, registration: { dispose() {} } }; + }); + + registerStewardAgent(events, "provider/cheap-model"); + + expect(definition?.model).toBe("provider/cheap-model"); + expect(definition?.tools).toEqual(["read", "grep", "find", "ls", "contact_supervisor"]); + expect(definition?.excludeTools).toEqual(expect.arrayContaining(["bash", "edit", "write", "subagent"])); + expect(definition?.inheritProjectContext).toBe(false); + expect(stewardSystemPrompt).toContain("Read the complete plan from disk every time"); + expect(stewardSystemPrompt).toContain("Call structured_output as soon as the evidence is sufficient"); + }); + + it("fails clearly when pi-subagents is absent", () => { + expect(() => registerStewardAgent(new Events(), null)).toThrow("pi-subagents is not installed or not ready"); + }); + + it("reserves accept and reject for sign-off", () => { + expect(readyReview("plan.md")).toContain("return only let_run or redirect"); + expect(checkpointReview("plan.md", 8)).toContain("return only let_run or redirect"); + expect(signoffReview("plan.md", "goal")).toContain("Return accept only"); + }); +}); + +describe("goal steward RPC", () => { + it("starts a detached structured child", async () => { + const events = new Events(); + let request: any; + events.on("subagents:rpc:v1:request", (raw) => { + request = raw; + events.emit(`subagents:rpc:v1:reply:${request.requestId}`, { success: true, data: { text: "started", details: { asyncId: "run-1" } } }); + }); + + await expect(startSteward(events, "/repo", "review now")).resolves.toBe("run-1"); + expect(request.method).toBe("spawn"); + expect(request.params).toMatchObject({ agent: "goal-steward", cwd: "/repo", context: "fresh", async: true, task: "review now" }); + expect(request.params.outputSchema.required).toEqual(["verdict", "summary"]); + }); + + it("resumes the same lineage and accepts only its exact completion", async () => { + const events = new Events(); + let request: any; + events.on("subagents:rpc:v1:request", (raw) => { + request = raw; + events.emit(`subagents:rpc:v1:reply:${request.requestId}`, { success: true, data: { text: "resumed", details: { asyncId: "run-2" } } }); + queueMicrotask(() => { + events.emit("subagent:async-complete", completion("other-run", "reject")); + events.emit("subagent:async-complete", completion("run-2")); + }); + }); + + const review = await runStewardReview(events, "/repo", "run-1", "sign off"); + expect(request.method).toBe("resume"); + expect(request.params).toEqual({ id: "run-1", message: "sign off" }); + expect(review).toEqual({ runId: "run-2", decision: { verdict: "accept", summary: "Observed the cited artifact." } }); + }); + + it("does not lose a completion emitted before the RPC reply", async () => { + const events = new Events(); + events.on("subagents:rpc:v1:request", (raw) => { + const request = raw as { requestId: string }; + events.emit("subagent:async-complete", completion("run-fast")); + events.emit(`subagents:rpc:v1:reply:${request.requestId}`, { success: true, data: { text: "started", details: { asyncId: "run-fast" } } }); + }); + + await expect(runStewardReview(events, "/repo", null, "review")).resolves.toMatchObject({ runId: "run-fast", decision: { verdict: "accept" } }); + }); + + it("fails closed when the exact child does not complete", async () => { + vi.useFakeTimers(); + try { + const events = new Events(); + events.on("subagents:rpc:v1:request", (raw) => { + const request = raw as { requestId: string }; + events.emit(`subagents:rpc:v1:reply:${request.requestId}`, { success: true, data: { text: "started", details: { asyncId: "run-stuck" } } }); + }); + const review = runStewardReview(events, "/repo", null, "review", undefined, 1_000); + const rejected = expect(review).rejects.toThrow("timed out after 1s"); + await vi.advanceTimersByTimeAsync(1_000); + await rejected; + } finally { + vi.useRealTimers(); + } + }); +}); + +describe("steward verdict parsing", () => { + it("requires the fields for redirect and reject", () => { + expect(() => parseStewardDecision(completion("run", "redirect"))).toThrow("redirect omitted nextAction"); + expect(() => parseStewardDecision(completion("run", "reject"))).toThrow("rejection omitted missingEvidence"); + }); +});