diff --git a/README.md b/README.md index c3167b6..ed7a35d 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ All model-facing text lives in [`src/prompts.ts`](src/prompts.ts), in flow order ```bash pi -e ./src/index.ts # load locally -npm test # vitest: judge argv invariants, appendLog, decideSignOff fail-forward +npm test # vitest: unit checks plus Pi RPC review flow with a local offline model npm run typecheck npm run lint ``` diff --git a/docs/slop/audit/20260826_pi-plan-aligned-planning.md b/docs/slop/audit/20260826_pi-plan-aligned-planning.md index 821ada4..bf3d313 100644 --- a/docs/slop/audit/20260826_pi-plan-aligned-planning.md +++ b/docs/slop/audit/20260826_pi-plan-aligned-planning.md @@ -4,8 +4,8 @@ ```text $ npm test -Test Files 7 passed (7) -Tests 28 passed (28) +Test Files 8 passed (8) +Tests 29 passed (29) $ npm run typecheck > tsc --noEmit @@ -21,3 +21,5 @@ $ git diff --check [test/goals-flow.test.ts](../../../test/goals-flow.test.ts) covers the visible plan before Refine, an editor prompt before a Refine revision turn, exact multiline Refine notes in `## Interview`, Ready as the only work handoff, Pi editor then Cancel, phase restoration, planning snapshot, writable plan path, allowed `pwd && ls && git log` and `cd . && ls -la`, blocked pipe, and blocked `CompleteGoal`. [test/prompts.test.ts](../../../test/prompts.test.ts) locks the prompt instruction to inspect discoverable facts or ask one focused question, and forbids placeholder goals. + +[test/rpc-review.test.ts](../../../test/rpc-review.test.ts) starts the installed Pi RPC executable with [offline-model.ts](../../../test/fixtures/offline-model.ts), selects Refine through Pi's real dialog protocol, receives the editor request before the revision call, then submits notes and observes the revision call. The test uses a local HTTP model, so it spends no API credits. diff --git a/docs/slop/plans/20260826_pi-plan-aligned-planning.md b/docs/slop/plans/20260826_pi-plan-aligned-planning.md index f335bcf..831fd69 100644 --- a/docs/slop/plans/20260826_pi-plan-aligned-planning.md +++ b/docs/slop/plans/20260826_pi-plan-aligned-planning.md @@ -34,7 +34,13 @@ Pi-goals will use pi-plan's small phase model. The UI, tool gate, and agent cont - [x] Ban placeholder goals such as "work out the thing" before the plan review menu. - subtle failure mode: the plan has a formal discriminator but its goal still has no object or observable result. - discriminator: [prompts.test.ts](../../../test/prompts.test.ts) locks the inspect-or-ask rule and the concrete-goal rule in the model prompt. - - evidence: [prompts.ts](../../../src/prompts.ts) requires a goal object plus observable result, and [prompts.test.ts](../../../test/prompts.test.ts) asserts that wording. [verification](../audit/20260826_pi-plan-aligned-planning.md) records `28 passed`. + - evidence: [prompts.ts](../../../src/prompts.ts) requires a goal object plus observable result, and [prompts.test.ts](../../../test/prompts.test.ts) asserts that wording. [verification](../audit/20260826_pi-plan-aligned-planning.md) records `29 passed`. +- [x] goal: Refine waits for text in Pi's real dialog protocol + - [x] Run Pi in RPC mode against a local no-cost model. + - [x] Select Refine, observe the editor request, then submit text and observe the revision turn. + - subtle failure mode: a mocked editor hides a Pi RPC ordering defect, so Refine starts a turn before the human can type. + - discriminator: [rpc-review.test.ts](../../../test/rpc-review.test.ts) uses Pi's `extension_ui_request` and `extension_ui_response` protocol and observes two model requests before editor input, then the third revision request after it. + - evidence: [rpc-review.test.ts](../../../test/rpc-review.test.ts) starts the installed Pi executable plus [offline-model.ts](../../../test/fixtures/offline-model.ts), with no credential or network dependency. [verification](../audit/20260826_pi-plan-aligned-planning.md) records its pass. ## UAT / Verification diff --git a/test/fixtures/offline-model.ts b/test/fixtures/offline-model.ts new file mode 100644 index 0000000..bfa2751 --- /dev/null +++ b/test/fixtures/offline-model.ts @@ -0,0 +1,18 @@ +import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; + +export default function offlineModel(pi: ExtensionAPI): void { + pi.registerProvider("offline", { + baseUrl: process.env.PI_GOALS_OFFLINE_MODEL_URL!, + apiKey: "test", + api: "openai-completions", + models: [{ + id: "test", + name: "Offline test model", + reasoning: false, + input: ["text"], + contextWindow: 16_000, + maxTokens: 1_000, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }], + }); +} diff --git a/test/rpc-review.test.ts b/test/rpc-review.test.ts new file mode 100644 index 0000000..9c6a710 --- /dev/null +++ b/test/rpc-review.test.ts @@ -0,0 +1,116 @@ +import { type ChildProcessWithoutNullStreams, spawn } from "node:child_process"; +import { mkdtempSync, rmSync } from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { StringDecoder } from "node:string_decoder"; +import { describe, expect, it } from "vitest"; + +type RpcMessage = { type: string; id?: string; method?: string; [key: string]: unknown }; + +class RpcClient { + readonly messages: RpcMessage[] = []; + private readonly waiters: Array<{ predicate: (message: RpcMessage) => boolean; resolve: (message: RpcMessage) => void }> = []; + + constructor(readonly process: ChildProcessWithoutNullStreams) { + const decoder = new StringDecoder("utf8"); + let buffer = ""; + process.stdout.on("data", (chunk) => { + buffer += decoder.write(chunk); + while (buffer.includes("\n")) { + const newline = buffer.indexOf("\n"); + const line = buffer.slice(0, newline).replace(/\r$/, ""); + buffer = buffer.slice(newline + 1); + if (!line) continue; + const message = JSON.parse(line) as RpcMessage; + this.messages.push(message); + const index = this.waiters.findIndex(({ predicate }) => predicate(message)); + if (index !== -1) this.waiters.splice(index, 1)[0].resolve(message); + } + }); + } + + send(message: RpcMessage): void { + this.process.stdin.write(`${JSON.stringify(message)}\n`); + } + + waitFor(predicate: (message: RpcMessage) => boolean, after = 0): Promise { + const existing = this.messages.slice(after).find(predicate); + if (existing) return Promise.resolve(existing); + return new Promise((resolvePromise) => this.waiters.push({ predicate, resolve: resolvePromise })); + } +} + +function streamResponse(response: import("node:http").ServerResponse, delta: object, finishReason: "stop" | "tool_calls"): void { + response.writeHead(200, { "content-type": "text/event-stream" }); + response.write(`data: ${JSON.stringify({ choices: [{ index: 0, delta, finish_reason: null }] })}\n\n`); + response.write(`data: ${JSON.stringify({ choices: [{ index: 0, delta: {}, finish_reason: finishReason }] })}\n\n`); + response.end("data: [DONE]\n\n"); +} + +describe("RPC review flow", () => { + it("opens Refine's editor before it starts the revision turn", async () => { + const cwd = mkdtempSync(join(tmpdir(), "pi-goals-rpc-")); + let requestCount = 0; + let planPath = ""; + const server = createServer((_request, response) => { + requestCount++; + if (requestCount === 1) { + streamResponse(response, { + tool_calls: [{ + index: 0, + id: "write-plan", + type: "function", + function: { + name: "write", + arguments: JSON.stringify({ + path: planPath, + content: "# Plan\n\n## Goals\n\n1. [ ] goal: name the output\n - subtle failure mode: the output has no name\n - discriminator: the plan names the output\n\n## Log\n\n## Interview\n", + }), + }, + }], + }, "tool_calls"); + return; + } + streamResponse(response, { content: "Plan drafted." }, "stop"); + }); + await new Promise((resolvePromise) => server.listen(0, "127.0.0.1", resolvePromise)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Offline model did not bind a TCP port."); + + const pi = spawn(resolve("node_modules/.bin/pi"), [ + "--mode", "rpc", "--no-session", "--model", "offline/test", + "-e", resolve("test/fixtures/offline-model.ts"), + "-e", resolve("src/index.ts"), + ], { + cwd, + env: { + ...process.env, + PI_CODING_AGENT_DIR: join(cwd, ".agent"), + PI_GOALS_OFFLINE_MODEL_URL: `http://127.0.0.1:${address.port}`, + }, + }); + const client = new RpcClient(pi); + try { + client.send({ type: "get_state", id: "state" }); + const state = await client.waitFor((message) => message.type === "response" && message.id === "state"); + const sessionId = (state.data as { sessionId: string }).sessionId; + planPath = join(cwd, ".pi", "plan", `${sessionId}-v1.md`); + + client.send({ type: "prompt", id: "goals", message: "/goals work out the thing" }); + const review = await client.waitFor((message) => message.type === "extension_ui_request" && message.method === "select"); + client.send({ type: "extension_ui_response", id: review.id, value: "Refine" }); + const editor = await client.waitFor((message) => message.type === "extension_ui_request" && message.method === "editor"); + expect(requestCount).toBe(2); + + const revisionStart = client.messages.length; + client.send({ type: "extension_ui_response", id: editor.id, value: "Name the produced file." }); + await client.waitFor((message) => message.type === "agent_end", revisionStart); + expect(requestCount).toBe(3); + } finally { + pi.kill(); + server.close(); + rmSync(cwd, { recursive: true, force: true }); + } + }, 15_000); +});