import assert from "node:assert/strict"; import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, unlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { after, test } from "node:test"; import registerTelemetry, { realFs, replayBranch, type Deps, type GitResult } from "./telemetry.ts"; import { DEFAULT_TELEMETRY_BUCKETS, planDirsFor, resolveFlowGuards } from "./lib/gauntlet-settings.ts"; import { emptyAccumulators, parseRecord, serializeRecord } from "./lib/telemetry-record.ts"; import { guardReason } from "./lib/telemetry-ship.ts"; import { mergeToolResult } from "./test-support/merge-tool-result.mjs"; const originalDepth = process.env.PI_SUBAGENT_DEPTH; process.env.PI_SUBAGENT_DEPTH = "0"; const tempDirs: string[] = []; after(() => { for (const d of tempDirs) rmSync(d, { recursive: true, force: true }); if (originalDepth === undefined) delete process.env.PI_SUBAGENT_DEPTH; else process.env.PI_SUBAGENT_DEPTH = originalDepth; }); type Handler = (event: any, ctx: any) => unknown; function harness(o: { branch?: unknown[]; enabled?: boolean; gitFail?: (args: string[], cwd: string) => GitResult | undefined; jjWorkspace?: boolean; jjFail?: (args: string[], cwd: string) => GitResult | undefined; cwdSub?: string; sessionId?: string; model?: { provider: string; id: string }; thinkingLevel?: string; contextTokens?: number | null; telemetryWarning?: string; piVersion?: string } = {}) { const root = mkdtempSync(join(tmpdir(), "telemetry-test-")); tempDirs.push(root); mkdirSync(join(root, "doc/specs"), { recursive: true }); mkdirSync(join(root, "docs/specs"), { recursive: true }); mkdirSync(join(root, "doc/plans"), { recursive: true }); mkdirSync(join(root, ".worktrees/x/doc/specs"), { recursive: true }); const handlers = new Map(); const gitCalls: string[][] = []; const gitCwds: string[] = []; const jjCalls: string[][] = []; const readFileCalls: string[] = []; let refReads = 0; let fsCalls = 0; const recorderIo = () => ({ git: gitCalls.length, jj: jjCalls.length, fs: fsCalls, refs: refReads }); const specDirs = resolveFlowGuards({}).specDirs; const fetchedRecords = new Map(); let clock = Date.parse("2026-09-17T10:00:00Z"); let enabled = o.enabled ?? true; const deps: Deps = { telemetryRepo: () => "test/repo", readTelemetryBranch: () => { refReads++; return { records: [...fetchedRecords].map(([path, bytes]) => ({ path, bytes })) }; }, fs: { readFile: p => { fsCalls++; readFileCalls.push(p); return realFs.readFile(p); }, writeFile: (p, body) => { fsCalls++; realFs.writeFile(p, body); }, rename: (from, to) => { fsCalls++; realFs.rename(from, to); }, mkdirp: p => { fsCalls++; realFs.mkdirp(p); }, exists: p => { fsCalls++; return realFs.exists(p); }, unlink: p => { fsCalls++; realFs.unlink(p); }, }, now: () => new Date((clock += 1000)).toISOString().replace(/\.\d{3}Z$/, "Z"), git: async (args, cwd) => { gitCalls.push(args); gitCwds.push(cwd); const forced = o.gitFail?.(args, cwd); if (forced) return forced; if (args[0] === "rev-parse" && args.includes("--git-common-dir") && !args.includes("--show-toplevel")) { const primary = /^(.*\/\.worktrees\/[^/]+)(\/|$)/.test(cwd) ? cwd.replace(/\/\.worktrees\/.*$/, "") : cwd === root || cwd.startsWith(root + "/") ? root : cwd.replace(/\/(?:doc|\.pi)(?:\/.*)?$/, ""); return { code: 0, stdout: `${primary}/.git\n`, stderr: "" }; } if (args[0] === "rev-parse" && args.includes("--path-format=absolute")) { const wt = /^(.*\/\.worktrees\/[^/]+)(\/|$)/.exec(cwd)?.[1]; const primary = cwd === root || cwd.startsWith(root + "/") ? root : cwd.replace(/\/(?:doc|\.pi)(?:\/.*)?$/, ""); const toplevel = wt ?? primary; const gitDir = wt ? `${primary}/.git/worktrees/${wt.split("/").pop()}` : `${primary}/.git`; return { code: 0, stdout: `${toplevel}\n${gitDir}\n${primary}/.git\n`, stderr: "" }; } if (args[0] === "rev-parse" && args.includes("--abbrev-ref")) return { code: 0, stdout: "gh-33\n", stderr: "" }; if (args[0] === "config") return { code: 0, stdout: (args[1] === "user.name" ? "Tester" : "t@example.com") + "\n", stderr: "" }; if (args[0] === "ls-files") return { code: 0, stdout: "", stderr: "" }; return { code: 0, stdout: "", stderr: "" }; }, settings: () => ({ specDirs, planDirs: planDirsFor(specDirs), telemetry: { enabled, buckets: DEFAULT_TELEMETRY_BUCKETS, warning: o.telemetryWarning }, errors: [], agentOverrides: { implementer: { model: "p/x" } }, versions: { pi: o.piVersion ?? "0.85.1" } }), // A plain jj workspace: `jj root` answers the temp root while git rev-parse fails. jj: async (args, cwd) => { jjCalls.push(args); const forced = o.jjFail?.(args, cwd); if (forced) return forced; if (!o.jjWorkspace) return { code: 1, stdout: "", stderr: "jj stub: not a jj workspace" }; return shipJj(args, root); }, }; let branch = o.branch ?? []; const ctx = { cwd: o.cwdSub ? join(root, o.cwdSub) : root, sessionManager: { getBranch: () => branch, getSessionId: () => o.sessionId ?? "s1" }, model: o.model ?? { provider: "p", id: "m" }, thinkingLevel: o.thinkingLevel ?? "high", getContextUsage: () => ({ tokens: o.contextTokens === undefined ? 1234 : o.contextTokens, contextWindow: 200000, percent: 1 }), }; const pi = { on: (name: string, h: Handler) => handlers.set(name, [...(handlers.get(name) ?? []), h]), appendEntry: (customType: string, data: unknown) => branch.push({ type: "custom", customType, data }), }; registerTelemetry(pi as any, deps); const emit = async (name: string, event: unknown) => { const out: unknown[] = []; for (const h of handlers.get(name) ?? []) out.push(await h(event, ctx)); return out; }; const recordPath = (spec = "doc/specs/a.md") => join(ctx.cwd.replace(/\/\.worktrees\/.*$/, ""), o.jjWorkspace ? ".jj/gauntlet/telemetry" : ".git/gauntlet/telemetry", spec.replace(/\.md$/, ".yaml")); const readRecord = (spec?: string) => parseRecord(readFileSync(recordPath(spec), "utf8"))!; const gitWrites = () => gitCalls.filter((a) => a[0] === "add" || a[0] === "commit"); const excludeFile = () => join(root, ".git/info/exclude"); const bash = async (id: string, command: string, isError = false) => { await emit("tool_call", { toolName: "bash", toolCallId: id, input: { command } }); await emit("tool_result", { toolName: "bash", toolCallId: id, input: { command }, content: [], isError, details: undefined }); }; const phaseResult = (action: string, phases: Record) => emit("tool_result", { toolName: "phase_tracker", toolCallId: "pt", input: {}, content: [], isError: false, details: { action, phases } }); const writeSpec = async (rel: string, body = "# Spec\n") => { writeFileSync(join(root, rel), body); await emit("tool_call", { toolName: "write", toolCallId: "w", input: { path: rel, content: body } }); return emit("tool_result", { toolName: "write", toolCallId: "w", input: { path: rel, content: body }, content: [], isError: false, details: undefined }); }; const read = async (toolCallId: string, path: string, text: string) => { const call = { toolName: "read", toolCallId }; const input = { path }; const result = { content: [{ type: "text", text }], details: {} }; await emit("tool_execution_start", { type: "tool_execution_start", ...call, args: input }); await emit("tool_call", { type: "tool_call", ...call, input }); await emit("tool_result", { type: "tool_result", ...call, input, ...result, isError: false }); await emit("tool_execution_end", { type: "tool_execution_end", ...call, result, isError: false }); }; const planResult = async (action: string, tasks: { name?: string; status: string }[], isError = false) => { const call = { toolName: "plan_tracker", toolCallId: "pt" }; const result = { content: [], details: { action, tasks } }; await emit("tool_result", { type: "tool_result", ...call, input: { action }, ...result, isError }); await emit("tool_execution_end", { type: "tool_execution_end", ...call, result, isError }); }; return { read, recorderIo, setEnabled: (value: boolean) => (enabled = value), fetchedRecords, root, handlers, emit, gitCalls, gitCwds, jjCalls, readFileCalls, gitWrites, excludeFile, bash, recordPath, readRecord, phaseResult, planResult, writeSpec, setBranch: (b: unknown[]) => (branch = b), ctx }; } const P = (over: Record = {}) => Object.fromEntries(["brainstorm", "plan", "implement", "verify", "ship"].map((p) => [p, { status: over[p] ?? "pending" }])); const JJ_PATCH = [ "diff --git a/src/a.ts b/src/a.ts", "index 1111111..2222222 100644", "--- a/src/a.ts", "+++ b/src/a.ts", "@@ -1,4 +1,12 @@", ...Array.from({ length: 10 }, (_, i) => `+added ${i}`), "-removed 1", "-removed 2", " context", "diff --git a/test/a.test.ts b/test/a.test.ts", "new file mode 100644", "index 0000000..3333333", "--- /dev/null", "+++ b/test/a.test.ts", "@@ -0,0 +1,5 @@", ...Array.from({ length: 5 }, (_, i) => `+t ${i}`), "", ].join("\n"); function shipJj(args: string[], root: string): GitResult { if (args[0] === "root") return { code: 0, stdout: root + "\n", stderr: "" }; if (args.includes("--count")) return { code: 0, stdout: args.some((a) => a.includes('files(~glob:".pi/gauntlet/telemetry/**")')) ? "2\n" : "3\n", stderr: "" }; if (args.includes("log")) return { code: 0, stdout: "jjbase0001\n", stderr: "" }; if (args.includes("diff")) return { code: 0, stdout: JJ_PATCH, stderr: "" }; return { code: 1, stdout: "", stderr: `jj stub: unexpected call ${args.join(" ")}` }; } test("enabled: false registers handlers that no-op", async () => { const h = harness({ enabled: false }); await h.emit("session_start", { type: "session_start", reason: "startup" }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); assert.equal(existsSync(h.recordPath()), false); assert.equal(h.gitWrites().length, 0); }); test("PI_SUBAGENT_DEPTH=1 no-ops every handler", async () => { process.env.PI_SUBAGENT_DEPTH = "1"; try { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.read("child", legSkill, "ignored"); assert.equal(existsSync(h.recordPath()), false); } finally { process.env.PI_SUBAGENT_DEPTH = "0"; } }); test("bind on first spec write flushes buffered phase event and mints run_id without excludes, staging or commits", async () => { const h = harness(); await h.emit("session_start", { type: "session_start", reason: "startup" }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); assert.equal(existsSync(h.recordPath()), false); await h.writeSpec("doc/specs/a.md", "# CONTEXT DRAFT - NOT A SPEC - fully replaced at spec-writing\n"); const rec = h.readRecord(); assert.match(rec.run_id, /^[0-9a-f-]{36}$/); assert.equal(rec.spec, "doc/specs/a.md"); assert.equal(rec.branch, "gh-33"); assert.deepEqual(rec.sessions, ["s1"]); assert.deepEqual(rec.author, { name: "Tester", email: "t@example.com" }); assert.deepEqual(rec.versions, { pi: "0.85.1" }); assert.deepEqual(rec.agent_overrides, { implementer: { model: "p/x" } }); assert.equal(rec.events[0].kind, "phase"); assert.equal(rec.created_at, rec.events[0].ts); assert.equal((rec.events[0] as any).model, "p/m"); assert.equal((rec.events[0] as any).thinking, "high"); assert.equal(rec.derived.spec_writes.brainstorm?.count, 1); await h.phaseResult("substep", P({ brainstorm: "in_progress" })); await h.phaseResult("complete", P({ brainstorm: "complete" })); const rec2 = h.readRecord(); assert.equal(rec2.approved_at, rec2.events.findLast((e) => e.kind === "phase")!.ts); assert.equal(h.gitWrites().length, 0, "the recorder never runs git add or git commit"); assert.equal(existsSync(h.excludeFile()), false); await h.phaseResult("reset", P()); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md", "# Spec again\n"); assert.equal(existsSync(h.excludeFile()), false, "rebind never edits excludes"); }); test("a docs/specs write binds under the common-dir in-progress store", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); assert.equal(existsSync(h.recordPath("docs/specs/x.md")), false); await h.writeSpec("docs/specs/x.md"); assert.equal(h.recordPath("docs/specs/x.md"), join(h.root, ".git/gauntlet/telemetry/docs/specs/x.yaml")); assert.equal(h.readRecord("docs/specs/x.md").spec, "docs/specs/x.md"); await h.phaseResult("complete", P({ brainstorm: "complete" })); const rec = h.readRecord("docs/specs/x.md"); assert.equal(rec.approved_at, rec.events.findLast((e) => e.kind === "phase")!.ts); assert.equal(h.gitWrites().length, 0); }); test("bind derives approved_at from a buffered brainstorm completion", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); await h.writeSpec("doc/specs/a.md"); const rec = h.readRecord(); assert.equal(rec.events.length, 3); assert.equal(rec.approved_at, rec.events[1].ts); }); test("armed plan_check and bound shutdown flush the record without git add or commit", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); assert.equal(h.gitWrites().length, 0); await h.writeSpec("doc/specs/a.md"); assert.equal(h.gitWrites().length, 0); await h.emit("tool_result", { toolName: "plan_check", toolCallId: "pc", input: {}, content: [], isError: false, details: { status: "pass", specPath: join(h.root, "doc/specs/a.md"), planPath: join(h.root, "doc/plans/a.md") } }); assert.equal(h.gitWrites().length, 0); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); assert.equal(h.gitWrites().length, 0); const rec = h.readRecord(); assert.deepEqual(Object.keys(rec.accumulators), ["total"]); }); test("session shutdown before binding writes nothing", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); assert.equal(existsSync(join(h.root, ".pi")), false); assert.equal(h.gitWrites().length, 0); }); test("replayBranch keeps successful binding inputs in transcript order", () => { const successfulPhases = P({ brainstorm: "complete", plan: "in_progress" }); const ignoredPhases = P({ brainstorm: "complete", plan: "complete" }); const replay = replayBranch([ { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: successfulPhases } } }, { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { phases: ignoredPhases, error: "failed" } } }, { type: "message", message: { role: "toolResult", toolName: "plan_check", details: { status: "pass", specPath: "/repo/doc/specs/old.md" } } }, { type: "message", message: { role: "toolResult", toolName: "plan_check", details: { status: "fail", specPath: "/repo/doc/specs/ignored.md" } } }, { type: "message", message: { role: "toolResult", toolName: "plan_check", details: { status: "pass", specPath: "/repo/doc/specs/a.md" } } }, { type: "message", message: { role: "assistant", content: [ { type: "toolCall", id: "w1", name: "write", arguments: { path: "doc/specs/a.md" } }, { type: "toolCall", id: "w2", name: "write", arguments: { path: "README.md" } }, { type: "toolCall", id: "w3", name: "write", arguments: { path: "doc/specs/failed.md" } }, ] } }, { type: "message", message: { role: "toolResult", toolName: "write", toolCallId: "w1", isError: false } }, { type: "message", message: { role: "toolResult", toolName: "write", toolCallId: "w2", isError: false } }, { type: "message", message: { role: "toolResult", toolName: "write", toolCallId: "w3", isError: true } }, ], ["doc/specs", "docs/specs"]); assert.deepEqual(replay.phases, successfulPhases); assert.equal(replay.gauntletEntered, true); assert.deepEqual(replay.bindings.map(({ tool, path }) => [tool, path]), [ ["plan_check", "/repo/doc/specs/old.md"], ["plan_check", "/repo/doc/specs/a.md"], ["write", "doc/specs/a.md"], ]); }); test("replayBranch: candidates are only taken while armed; reset clears them; edit counts; skip brainstorm resume: binds", () => { const write = (id: string, path: string, tool = "write") => [ { type: "message", message: { role: "assistant", content: [{ type: "toolCall", id, name: tool, arguments: { path } }] } }, { type: "message", message: { role: "toolResult", toolName: tool, toolCallId: id, isError: false } }, ]; const unarmed = replayBranch([ ...write("w1", "doc/specs/a.md"), { type: "message", message: { role: "toolResult", toolName: "plan_check", details: { status: "pass", specPath: "/repo/doc/specs/a.md" } } }, ], ["doc/specs", "docs/specs"]); assert.equal(unarmed.gauntletEntered, false); assert.deepEqual(unarmed.bindings, []); const armed = replayBranch([ ...write("w0", "doc/specs/before.md"), { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, ...write("w1", "doc/specs/a.md", "edit"), ], ["doc/specs", "docs/specs"]); assert.equal(armed.gauntletEntered, true); assert.deepEqual(armed.bindings, [{ tool: "edit", path: "doc/specs/a.md", brainstorming: true }]); const reset = replayBranch([ { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, ...write("w1", "doc/specs/a.md"), { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "reset", phases: P() } } }, ], ["doc/specs", "docs/specs"]); assert.equal(reset.gauntletEntered, false); assert.deepEqual(reset.bindings, []); const resumed = replayBranch([ { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "skip", phases: { ...P({ brainstorm: "skipped" }), brainstorm: { status: "skipped", reason: "resume: /repo/doc/specs/a.md" } } } } }, ], ["doc/specs", "docs/specs"]); assert.equal(resumed.gauntletEntered, true); assert.deepEqual(resumed.bindings, [{ tool: "resume", path: "/repo/doc/specs/a.md" }]); }); test("session resume restores the last successful plan snapshot for reopen counting", async () => { const first = await boundInPlan(); const closed = [{ name: "a", status: "complete" }]; const planResult = (tasks: { name: string; status: string }[], error?: string) => ({ type: "message", message: { role: "toolResult", toolName: "plan_tracker", details: { action: "update", tasks, error } }, }); const branch = [ { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, { type: "message", message: { role: "toolResult", toolName: "plan_check", details: { status: "pass", specPath: join(first.root, "doc/specs/a.md") } } }, planResult([{ name: "a", status: "pending" }]), planResult(closed), planResult([{ name: "a", status: "failed" }], "failed"), ]; const second = harness({ sessionId: "s2", branch }); second.ctx.cwd = first.root; await second.emit("session_start", { type: "session_start", reason: "resume" }); await second.planResult("update", [{ name: "a", status: "in_progress" }]); assert.equal(second.readRecord().derived.gates.task_reopens, 1); assert.deepEqual(replayBranch(branch, ["doc/specs"]).lastPlanTasks, closed); }); test("replayBranch honors a custom spec dir list", () => { const branch = [ { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, { type: "message", message: { role: "assistant", content: [{ type: "toolCall", id: "w1", name: "write", arguments: { path: "design/specs/a.md" } }] } }, { type: "message", message: { role: "toolResult", toolName: "write", toolCallId: "w1", isError: false } }, ]; assert.equal(replayBranch(branch, ["design/specs"]).bindings[0]?.path, "design/specs/a.md"); assert.deepEqual(replayBranch(branch, ["doc/specs"]).bindings, []); }); test("session_start replay binds from plan_check details with no spec write, extends sessions, preserves run_id", async () => { const first = harness(); await first.phaseResult("start", P({ brainstorm: "in_progress" })); await first.writeSpec("doc/specs/a.md"); const runId = first.readRecord().run_id; const second = harness({ sessionId: "s2", branch: [ { type: "message", message: { role: "toolResult", toolName: "phase_tracker", toolCallId: "0", isError: false, details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, { type: "message", message: { role: "toolResult", toolName: "phase_tracker", toolCallId: "1", isError: false, details: { action: "skip", phases: P({ brainstorm: "skipped", plan: "in_progress" }) } } }, { type: "message", message: { role: "toolResult", toolName: "plan_check", toolCallId: "2", isError: false, details: { status: "pass", planPath: join(first.root, "doc/plans/a.md"), specPath: join(first.root, "doc/specs/a.md"), planSha256: "x", specSha256: "y" } } }, ] }); // point the second harness at the first root second.ctx.cwd = first.root; await second.emit("session_start", { type: "session_start", reason: "resume" }); const rec = parseRecord(readFileSync(first.recordPath(), "utf8"))!; assert.equal(rec.run_id, runId); assert.deepEqual(rec.sessions, ["s1", "s2"]); assert.equal(rec.events.filter((e) => e.kind !== "engine").length, 1, "replay emits no duplicate phase events"); assert.equal(rec.events.filter((e) => e.kind === "engine").length, 1); }); test("rename at spec-writing: write to a new spec while the bound one is a draft rebinds and moves the record", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/draft.md", "# CONTEXT DRAFT - NOT A SPEC - fully replaced at spec-writing\n"); const runId = h.readRecord("doc/specs/draft.md").run_id; await h.writeSpec("doc/specs/final.md", "# Final\n"); assert.equal(existsSync(h.recordPath("doc/specs/draft.md")), false); const rec = h.readRecord("doc/specs/final.md"); assert.equal(rec.run_id, runId); assert.equal(rec.spec, "doc/specs/final.md"); assert.ok(rec.events.some((e) => e.kind === "spec_renamed")); await h.phaseResult("complete", P({ brainstorm: "complete" })); assert.equal(h.gitWrites().length, 0); }); test("rename refuses an untracked destination collision and keeps the draft binding", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/draft.md", "# CONTEXT DRAFT - NOT A SPEC - fully replaced at spec-writing\n"); writeFileSync(h.recordPath("doc/specs/final.md"), "existing\n"); await h.writeSpec("doc/specs/final.md", "# Final\n"); const rec = h.readRecord("doc/specs/draft.md"); assert.equal(rec.spec, "doc/specs/draft.md"); assert.equal(readFileSync(h.recordPath("doc/specs/final.md"), "utf8"), "existing\n"); assert.deepEqual(rec.events.filter((e) => e.kind === "warning").map((e: any) => e.message), [`record move skipped: ${h.recordPath("doc/specs/final.md")} already exists`]); }); test("rename refuses a tracked destination collision and keeps the draft binding", async () => { const h = harness({ gitFail: (args) => args[0] === "ls-files" && args.at(-1)?.endsWith("final.yaml") ? { code: 0, stdout: args.at(-1)! + "\n", stderr: "" } : undefined }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/draft.md", "# CONTEXT DRAFT - NOT A SPEC - fully replaced at spec-writing\n"); writeFileSync(h.recordPath("doc/specs/final.md"), "tracked existing\n"); await h.writeSpec("doc/specs/final.md", "# Final\n"); const rec = h.readRecord("doc/specs/draft.md"); assert.equal(rec.spec, "doc/specs/draft.md"); assert.equal(readFileSync(h.recordPath("doc/specs/final.md"), "utf8"), "tracked existing\n"); assert.ok(rec.events.some((e) => e.kind === "warning" && (e as any).message.endsWith("already exists"))); }); test("telemetry resolver warning is emitted once", async () => { const message = "telemetry.remote must be a non-empty URL; using origin"; const h = harness({ telemetryWarning: message }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.writeSpec("doc/specs/a.md", "# Again\n"); assert.equal(existsSync(h.recordPath()), true); assert.deepEqual(h.readRecord().events.filter((e) => e.kind === "warning").map((e: any) => e.message), [message]); }); test("phase reset unbinds; the next write to the same path continues the same record", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); const runId = h.readRecord().run_id; await h.phaseResult("reset", P()); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); const rec = h.readRecord(); assert.equal(rec.run_id, runId); assert.deepEqual(rec.events.map((e) => e.kind), ["phase", "phase", "phase"]); assert.equal((rec.events[1] as any).action, "reset"); }); test("a spec outside any checkout warns once bound; nothing is written for it", async () => { const outside = mkdtempSync(join(tmpdir(), "telemetry-outside-")); tempDirs.push(outside); const outsideSpec = join(outside, "doc/specs/outside.md"); const nogit = harness({ gitFail: (args, cwd) => (args[0] === "rev-parse" && cwd.startsWith(outside) ? { code: 128, stdout: "", stderr: "fatal: not a git repository" } : undefined) }); await nogit.phaseResult("start", P({ brainstorm: "in_progress" })); await nogit.emit("tool_result", { toolName: "write", toolCallId: "outside", input: { path: outsideSpec }, content: [], isError: false, details: undefined }); await nogit.writeSpec("doc/specs/a.md"); assert.deepEqual(nogit.readRecord().events.filter((e) => e.kind === "warning").map((e: any) => e.message), [`spec ${outsideSpec} is outside a git checkout; telemetry not recorded`]); }); test("failed git-common-dir lookup never falls back to writing inside the checkout", async () => { const h = harness({ gitFail: (args) => args.includes("--git-common-dir") && !args.includes("--show-toplevel") ? { code: 1, stdout: "", stderr: "unsupported flag\nmore detail" } : undefined }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); assert.equal(existsSync(h.recordPath()), false); assert.equal(existsSync(join(h.root, ".pi/gauntlet")), false); assert.equal(existsSync(h.excludeFile()), false); }); test("jj-bound checkout: record is written with no warning, no exclude line, and no git add/commit", async () => { const h = harness({ jjWorkspace: true, gitFail: (args) => args[0] === "rev-parse" && args.includes("--path-format=absolute") ? { code: 128, stdout: "", stderr: "fatal: not a git repository" } : undefined, }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.phaseResult("complete", P({ brainstorm: "complete" })); const rec = h.readRecord(); assert.deepEqual(rec.events.filter((e) => e.kind === "warning"), []); assert.equal(h.gitWrites().length, 0); assert.equal(existsSync(h.excludeFile()), false); }); test("a sealed spec in another checkout leaves the current spec bound", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); const wt = join(h.root, ".worktrees/x"); const sealedPath = "test/repo/brainstorming/doc/specs/b.yaml"; const before = serializeRecord({ ...h.readRecord(), spec: "doc/specs/b.md", status: "shipped", shipped_at: "2026-09-17T18:00:00Z" }); h.fetchedRecords.set(sealedPath, before); await h.emit("tool_result", { toolName: "write", toolCallId: "w", input: { path: join(wt, "doc/specs/b.md"), content: "# B\n" }, content: [], isError: false, details: undefined }); await h.phaseResult("complete", P({ brainstorm: "complete" })); assert.ok(h.readRecord().events.some((e) => e.kind === "phase" && e.action === "complete")); assert.equal(h.fetchedRecords.get(sealedPath), before); }); test("worktree spec from a primary cwd uses the common-dir store without changing excludes", async () => { const h = harness(); await h.emit("session_start", { type: "session_start", reason: "startup" }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec(".worktrees/x/doc/specs/a.md"); const wt = join(h.root, ".worktrees/x"); assert.ok(existsSync(h.recordPath()), "record lives under the shared git common dir"); assert.equal(existsSync(join(wt, ".pi/gauntlet")), false); assert.equal(existsSync(join(h.root, ".pi/gauntlet")), false, "nothing written under the primary checkout"); assert.ok(h.gitCwds.every((c) => c === wt || c.startsWith(wt + "/")), "no git call runs in the primary"); assert.equal(h.gitWrites().length, 0); assert.equal(existsSync(h.excludeFile()), false); }); test("a spec in another checkout is a fresh bind, never a git mv", async () => { for (const tool of ["write", "edit"] as const) { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec(".worktrees/x/doc/specs/a.md", "# CONTEXT DRAFT - NOT A SPEC - fully replaced at spec-writing\n"); if (tool === "write") { await h.writeSpec("doc/specs/a.md"); } else { writeFileSync(join(h.root, "doc/specs/a.md"), "# Spec\n"); await h.emit("tool_result", { toolName: "edit", toolCallId: "e", input: { path: "doc/specs/a.md", oldText: "# Old\n", newText: "# Spec\n" }, content: [], isError: false, details: undefined }); } assert.ok(existsSync(h.recordPath("doc/specs/a.md")), `primary record created for ${tool}`); assert.equal(existsSync(join(h.root, ".worktrees/x/.pi/gauntlet")), false, `no worktree-local record for ${tool}`); assert.equal(h.gitCalls.some((a) => a[0] === "mv"), false); assert.equal(h.readRecord("doc/specs/a.md").events.some((e) => e.kind === "spec_renamed"), false); } }); test("second session rehydrates all derived accumulators with exactly total and live blocks", async () => { const h = harness({ contextTokens: 4321 }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); await h.emit("input", { type: "input", text: "approve", source: "interactive" }); await h.emit("message_end", { type: "message_end", message: { role: "assistant", usage: usage(10, 2) } }); await h.emit("tool_result", subagentResult([{ agent: "implementer", exitCode: 0, model: "p/i", usage: usage(20, 3) }])); await h.emit("turn_end", { type: "turn_end", turnIndex: 1, message: {}, toolResults: [] }); await h.emit("session_compact", { type: "session_compact", reason: "threshold" }); await h.writeSpec("doc/specs/a.md", "# amended\n"); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "in_progress" })); await h.planResult("update", [{ name: "done", status: "complete" }, { name: "skip", status: "skipped" }]); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); let rec = h.readRecord(); assert.deepEqual(Object.keys(rec.accumulators), ["total"]); rec.accumulators.dead = { ...emptyAccumulators(), amendments: 5 }; writeFileSync(h.recordPath(), serializeRecord(rec)); const h2 = harness({ sessionId: "s2" }); h2.ctx.cwd = h.root; await h2.phaseResult("start", P({ brainstorm: "in_progress" })); await h2.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); await h2.writeSpec("doc/specs/a.md"); rec = h2.readRecord(); assert.deepEqual(Object.keys(rec.accumulators).sort(), ["s2", "total"]); assert.equal(rec.derived.phases.plan?.user_messages, 1); assert.equal(rec.derived.phases.plan?.peak_context, 4321); assert.equal(rec.derived.phases.plan?.compactions, 1); assert.deepEqual(rec.derived.personas.implementer?.tokens, { input: 20, output: 3, cache_read: 0, cache_write: 0, cost: 0 }); assert.deepEqual(rec.derived.plan, { tasks: 2, complete: 1, failed: 0, skipped: 1 }); assert.equal(rec.derived.amendments, 7); }); test("the run records the skill and reference bytes returned in their respective phases", async () => { const h = harness({ piVersion: "1.5.0" }); const skill = join(import.meta.dirname, "../skills/brainstorming"); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); assert.equal(h.readRecord().derived.skill_read_coverage, "complete", "a live bind can measure zero"); await h.read("skill", join(skill, "SKILL.md"), "skill text"); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "in_progress" })); await h.read("reference", join(skill, "reference/legs.md"), "reference text"); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); const { brainstorm, verify } = h.readRecord().derived.phases; assert.deepEqual([brainstorm?.skill_reads, verify?.skill_reads], [ [{ path: "pi-gauntlet:skills/brainstorming/SKILL.md", bytes: 10 }], [{ path: "pi-gauntlet:skills/brainstorming/reference/legs.md", bytes: 14 }], ]); }); const entry = { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }; const legSkill = join(import.meta.dirname, "../skills/shape-ticket/SKILL.md"); const pendingRead = { bytes: 4, path: "pi-gauntlet:skills/shape-ticket/SKILL.md" }; test("a read completed between phases belongs to unphased", async () => { const h = await boundInBrainstorm(); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.read("between", legSkill, "text"); const phases = h.readRecord().derived.phases; assert.deepEqual(phases.unphased?.skill_reads, [pendingRead]); assert.equal(phases.brainstorm?.skill_reads, undefined); }); test("a tab in a skill filename survives reload without hiding reads from other phases", { skip: process.platform === "win32" }, async () => { const h = await boundInBrainstorm(); await h.read("ordinary", legSkill, "text"); await h.phaseResult("complete", P({ brainstorm: "complete" })); mkdirSync(join(h.root, "skills")); for (const name of ["chapter\ttwo.md", "chapter%09two.md"]) { const file = join(h.root, "skills", name); writeFileSync(file, "ref"); await h.read(name, file, "ref"); } const { skill_read_coverage, phases } = h.readRecord().derived; assert.equal(skill_read_coverage, "complete"); assert.deepEqual(phases.brainstorm?.skill_reads, [pendingRead]); assert.deepEqual(phases.unphased?.skill_reads?.map(r => [r.path.split("/").at(-1), r.bytes]), [["chapter%09two.md", 3], ["chapter%2509two.md", 3]]); }); test("direct and nested reads count their final results once, in the phase active at completion", async () => { const h = harness({ piVersion: "1.5.0" }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); const parent = { toolName: "codemode", toolCallId: "script" }; await h.emit("tool_call", { ...parent, input: { code: "..." } }); const calls = ["direct", "script/1", "direct-error", "script/2", "script/3"].map(toolCallId => ({ toolName: "read", toolCallId, input: { path: legSkill }, ...(toolCallId.startsWith("script/") ? { parentToolCallId: "script" } : {}), })); for (const call of calls) await h.emit("tool_call", call); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); for (const [i, call] of calls.entries()) { const intermediate = { ...call, isError: false, content: [{ type: "text", text: "original read result" }] }; await h.emit("tool_result", intermediate); const later = { content: i === 4 ? [{ type: "image", data: "AAAA", mimeType: "image/png" }] : [{ type: "text", text: i ? "ab" : "a" }], isError: i === 2 || i === 3 }; const { content, isError } = mergeToolResult(intermediate, later); const end = { ...call, input: undefined, result: { content }, isError }; await h.emit("tool_execution_end", end); await h.emit("tool_execution_end", end); } await h.emit("tool_execution_end", { ...parent, result: { content: [{ type: "text", text: "ab" }] }, isError: false }); const phases = h.readRecord().derived.phases; assert.equal(phases.brainstorm?.skill_reads, undefined); assert.deepEqual(phases.plan?.skill_reads, [1, 2].map(bytes => ({ path: pendingRead.path, bytes }))); }); test("a ranged read counts only returned UTF-8 text, including the continuation notice", async () => { const h = await boundInBrainstorm(); const path = "skills/range.md"; mkdirSync(join(h.root, "skills")); writeFileSync(join(h.root, path), "first\n\u00e9\nlast\n"); const call = { toolName: "read", toolCallId: "range", input: { path, offset: 2, limit: 1 } }; await h.emit("tool_call", call); const notice = "\n\n[1 more lines in file. Use offset=3 to continue.]"; const content = [{ type: "text", text: "\u00e9" }, { type: "image", mimeType: "image/png", data: "AAAA" }, { type: "text", text: notice }]; await h.emit("tool_execution_end", { toolName: "read", toolCallId: "range", result: { content }, isError: false }); assert.deepEqual(h.readRecord().derived.phases.brainstorm?.skill_reads?.map(r => r.bytes), [2 + notice.length]); }); test("only an entered run folds a leg skill read at plan bind; reconstruction discards unbound observations", async () => { for (const mode of ["standalone", "live", "reconstructed"] as const) { const h = harness({ branch: [entry] }); if (mode !== "standalone") await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.read("prebind", legSkill, "text"); assert.equal(existsSync(h.recordPath()), false); if (mode === "reconstructed") await h.emit("session_tree", { type: "session_tree" }); writeFileSync(join(h.root, "doc/specs/a.md"), "# Spec\n"); writeFileSync(join(h.root, "doc/plans/a.md"), "**Spec:** `doc/specs/a.md`\n"); await h.read("plan", "doc/plans/a.md", "header"); assert.equal(existsSync(h.recordPath()), mode !== "standalone"); if (mode === "standalone") continue; const { phases, skill_read_coverage } = h.readRecord().derived; assert.deepEqual(phases.brainstorm?.skill_reads, mode === "live" ? [pendingRead] : undefined); assert.equal(skill_read_coverage, mode === "live" ? "complete" : "partial"); } }); test("a disabled interval never writes and makes current and later bindings partial", async () => { const a = "doc/specs/a.md"; const b = "doc/specs/b.md"; for (const [bound, spec] of [[true, a], [false, a], [true, b]] as const) { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); if (bound) await h.writeSpec(a); const bytesBefore = bound && readFileSync(h.recordPath()); h.setEnabled(false); await h.read("disabled", legSkill, "ignored"); assert.deepEqual(existsSync(h.recordPath()) && readFileSync(h.recordPath()), bytesBefore); h.setEnabled(true); if (!bound) await h.writeSpec(a); if (spec === b) { await h.emit("turn_end", { type: "turn_end" }); assert.equal(h.readRecord(a).derived.skill_read_coverage, "partial"); await h.emit("tool_result", { toolName: "plan_check", input: {}, isError: false, details: { status: "pass", specPath: b } }); } await h.read("enabled", legSkill, "text"); await h.emit("session_shutdown", { type: "session_shutdown" }); assert.equal(h.readRecord(spec).derived.skill_read_coverage, "partial"); assert.deepEqual(h.readRecord(spec).derived.phases.brainstorm?.skill_reads, [pendingRead]); } }); test("re-enable resolves binding changes without recording disabled calls", async (t) => { for (const route of ["plan check", "spec edit", "plan read", "resume skip", "worktree write", "draft rename"] as const) { await t.test(route, async () => { const h = harness(); const a = "doc/specs/a.md"; const b = "doc/specs/b.md"; const bound = route === "plan check" || route === "worktree write" || route === "draft rename"; const rename = route === "draft rename"; await h.phaseResult("start", P({ brainstorm: "in_progress" })); if (bound) { await h.writeSpec(a, rename ? "# CONTEXT DRAFT - NOT A SPEC - fully replaced at spec-writing\n" : "# Spec\n"); await h.read("first", legSkill, "1"); } const before = bound && readFileSync(h.recordPath(a)); const io = h.recorderIo(); h.setEnabled(false); switch (route) { case "plan check": await h.emit("tool_result", { toolName: "plan_check", input: {}, isError: false, details: { status: "pass", specPath: b } }); await h.emit("tool_result", { toolName: "plan_check", input: {}, isError: false, details: { status: "fail", specPath: a } }); break; case "spec edit": writeFileSync(join(h.root, b), "# Spec\n"); await h.emit("tool_result", { toolName: "edit", input: { path: b }, isError: false }); break; case "plan read": writeFileSync(join(h.root, b), "# Spec\n"); writeFileSync(join(h.root, "doc/plans/b.md"), `**Spec:** ${b}\n`); await h.read("plan", "doc/plans/b.md", "header"); break; case "resume skip": writeFileSync(join(h.root, b), "# Spec\n"); await h.phaseResult("skip", { ...P(), brainstorm: { status: "skipped", reason: `resume: ${b}` } }); break; case "worktree write": await h.writeSpec(`.worktrees/x/${b}`); break; case "draft rename": await h.writeSpec(b); break; } if (route !== "resume skip") await h.phaseResult("complete", P({ brainstorm: "complete" })); const phase = route === "worktree write" ? "verify" : "unphased"; if (phase === "verify") { await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "in_progress" })); } await h.read("disabled", legSkill, "ignored"); assert.deepEqual(h.recorderIo(), io); assert.deepEqual(existsSync(h.recordPath(a)) && readFileSync(h.recordPath(a)), before); assert.equal(existsSync(h.recordPath(b)), false); h.setEnabled(true); await h.read("next", legSkill, "text"); const next = h.readRecord(b); assert.equal(next.derived.skill_read_coverage, "partial"); assert.deepEqual(next.derived.phases[phase]?.skill_reads, [pendingRead]); assert.deepEqual(next.derived.phases.brainstorm?.skill_reads?.map(r => r.bytes), rename ? [1] : undefined); assert.equal(next.derived.spec_writes.brainstorm?.count, rename ? 1 : undefined); assert.equal(next.derived.gates.plan_rounds, 0); assert.equal(next.events.some(e => e.kind === "spec_renamed" || (e.kind === "phase" && ["complete", "skip"].includes(e.action))), false); if (bound && !rename) { const original = h.readRecord(a).derived; assert.equal(original.skill_read_coverage, "partial"); assert.deepEqual(original.phases.brainstorm?.skill_reads?.map(r => r.bytes), [1]); assert.equal(original.phases.unphased?.skill_reads, undefined); } else assert.equal(existsSync(h.recordPath(a)), false); }); } }); test("the recorder attributes the next read using the final implement outcome", async (t) => { for (const mode of ["live bound enabled", "live bound disabled", "live error enabled", "live error disabled", "ordinary completion", "live unbound", "live unbound enabled", "replay enabled", "replay disabled"] as const) { await t.test(mode, async () => { const h = harness(); const a = "doc/specs/a.md"; const start = P({ brainstorm: "complete", plan: "complete", implement: "in_progress" }); const done = { toolName: "plan_tracker", toolCallId: "pt", isError: false, details: { tasks: [{ status: "complete" }] } }; const failed = mode.startsWith("live error"); if (mode.startsWith("live bound") || failed || mode === "ordinary completion") { await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec(a); await h.phaseResult("start", start); } const before = existsSync(h.recordPath()) && readFileSync(h.recordPath()); const io = h.recorderIo(); const enabled = mode.endsWith("enabled"); const unbound = mode.startsWith("live unbound"); h.setEnabled(enabled); if (mode.startsWith("replay")) { const result = (data: object) => ({ type: "message", message: { role: "toolResult", ...data } }); h.setBranch([ entry, result({ toolName: "plan_check", details: { status: "pass", specPath: "doc/specs/old.md" } }), result({ toolName: "plan_check", details: { status: "pass", specPath: a } }), result({ toolName: "phase_tracker", details: { action: "start", phases: start } }), result(done), ]); await h.emit("session_start", {}); } else { if (unbound) { await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.phaseResult("start", start); } if (mode === "ordinary completion") { await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete", implement: "complete" })); } else { await h.emit("tool_result", mode === "live unbound enabled" ? { ...done, details: { tasks: [{ status: "in_progress" }] } } : done); await h.emit("tool_execution_end", { toolName: done.toolName, toolCallId: done.toolCallId, isError: failed, result: { content: [], details: done.details } }); } } if (!enabled) { assert.deepEqual(h.recorderIo(), io); assert.deepEqual(existsSync(h.recordPath()) && readFileSync(h.recordPath()), before); } h.setEnabled(true); await h.read("next", legSkill, "text"); if (unbound) await h.writeSpec(a); const { derived: d, events } = h.readRecord(); assert.deepEqual(d.phases[failed ? "implement" : "unphased"]?.skill_reads, [pendingRead]); assert.equal(d.phases[failed ? "unphased" : "implement"]?.skill_reads, undefined); assert.deepEqual(d.plan, mode === "live bound enabled" ? { tasks: 1, complete: 1, failed: 0, skipped: 0 } : undefined); assert.equal(d.skill_read_coverage, enabled && !mode.startsWith("replay") ? "complete" : "partial"); assert.equal(events.filter(e => e.kind === "phase" && e.name === "implement" && e.action === "complete").length, enabled && !failed && mode.startsWith("live") ? 1 : 0); assert.equal(existsSync(h.recordPath("doc/specs/old.md")), false); }); } }); test("ship completion while disabled prevents deferred binding and disarms reads", async () => { const h = await boundInBrainstorm(); const before = readFileSync(h.recordPath()); const io = h.recorderIo(); h.setEnabled(false); await h.emit("tool_result", { toolName: "plan_check", isError: false, details: { status: "pass", specPath: "doc/specs/b.md" } }); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "complete", ship: "complete" })); assert.deepEqual(h.recorderIo(), io); assert.deepEqual(readFileSync(h.recordPath()), before); h.setEnabled(true); await h.read("after ship", legSkill, "ignored"); assert.equal(h.readRecord().derived.skill_read_coverage, "partial"); assert.deepEqual(Object.values(h.readRecord().derived.phases).flatMap(p => p.skill_reads ?? []), []); assert.equal(existsSync(h.recordPath("doc/specs/b.md")), false); }); test("disabled bindings retain the coverage gap for their checkout and spec across navigation, reset, or ship completion", async t => { const spec = "doc/specs/a.md"; const skipped = { ...P(), brainstorm: { status: "skipped", reason: `resume: ${spec}` } }; const result = (data: object) => ({ type: "message", message: { role: "toolResult", ...data } }); for (const route of [ "plan check", "spec write", "resume skip", "legacy record", "replay resume skip then reset", "live resume skip then reset", "replay resume skip then Keep As-Is", "live resume skip then Keep As-Is", ] as const) { await t.test(route, async () => { const live = route.startsWith("live "); const reset = route.endsWith("then reset"); const keep = route.endsWith("then Keep As-Is"); const original = await boundInBrainstorm(); const legacy = route === "legacy record"; if (!legacy) await original.read("first", legSkill, "text"); const record = original.readRecord(); assert.equal(record.derived.skill_read_coverage, "complete"); if (legacy) { delete record.derived.skill_read_coverage; writeFileSync(original.recordPath(), serializeRecord(record)); } const before = readFileSync(original.recordPath()); const h = harness({ enabled: false }); h.ctx.cwd = original.root; const io = h.recorderIo(); if (live) { await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.phaseResult("skip", skipped); } else { const replay: unknown[] = [entry]; if (route === "spec write") { replay.push({ type: "message", message: { role: "assistant", content: [{ type: "toolCall", id: "write", name: "write", arguments: { path: spec } }] } }); replay.push(result({ toolName: "write", toolCallId: "write" })); } else if (route === "resume skip" || reset || keep) { replay.push(result({ toolName: "phase_tracker", details: { action: "skip", phases: skipped } })); } else { replay.push(result({ toolName: "plan_check", details: { status: "pass", specPath: join(original.root, spec) } })); } h.setBranch(replay); await h.emit("session_start", {}); } await h.read("missed", legSkill, "ignored"); if (reset) await h.phaseResult("reset", P()); else if (keep) { const shipping = { ...P({ plan: "complete", implement: "complete", verify: "complete", ship: "in_progress" }), brainstorm: skipped.brainstorm }; await h.phaseResult("start", shipping); await h.phaseResult("complete", { ...shipping, ship: { status: "complete" } }); } else { h.setBranch([]); await h.emit("session_tree", {}); } assert.deepEqual(h.recorderIo(), io); assert.deepEqual(readFileSync(original.recordPath()), before); h.setEnabled(true); if (reset || keep) { await h.read("disarmed", legSkill, "ignored"); const restored = h.readRecord(); assert.equal(restored.derived.skill_read_coverage, "partial"); assert.deepEqual(restored.derived.phases.brainstorm?.skill_reads, [pendingRead]); record.derived.skill_read_coverage = "partial"; assert.deepEqual(restored, record); assert.deepEqual(readdirSync(dirname(h.recordPath())), ["a.yaml"]); return; } h.setBranch(route === "plan check" ? [entry, result({ toolName: "plan_check", details: { status: "pass", specPath: spec } })] : [entry]); await h.emit("session_tree", {}); if (route !== "plan check") { writeFileSync(join(h.ctx.cwd, "doc/plans/a.md"), `**Spec:** ${spec}\n`); await h.read("bind", "doc/plans/a.md", "header"); } const restored = h.readRecord(); assert.equal(restored.run_id, record.run_id); assert.equal(restored.derived.skill_read_coverage, legacy ? undefined : "partial"); assert.deepEqual(Object.values(restored.derived.phases).flatMap(p => p.skill_reads ?? []), legacy ? [] : [pendingRead]); }); } }); test("a disabled binding gap follows a known spec, not the next branch or checkout", async t => { const spec = "doc/specs/a.md"; const plan = "doc/plans/a.md"; const check = { toolName: "plan_check", details: { status: "pass", specPath: spec }, isError: false }; const replay = [entry, { type: "message", message: { role: "toolResult", ...check } }]; for (const route of ["replay", "live resume skip", "live spec write", "live spec edit", "live plan check", "shutdown before navigation", "enabled navigation", "disabled navigation"] as const) { await t.test(route, async () => { const original = await boundInBrainstorm(); const other = await boundInBrainstorm(); await other.emit("session_shutdown", {}); const runId = original.readRecord().run_id; const before = readFileSync(original.recordPath()); const h = harness({ branch: route === "replay" ? replay : [] }); let returnBranch: unknown[] = replay; h.ctx.cwd = original.root; if (route === "live plan check") { await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.read("pending", legSkill, "text"); } const io = h.recorderIo(); h.setEnabled(false); if (route === "replay") { await h.emit("session_start", {}); } else { if (route !== "live plan check") await h.phaseResult("start", P({ brainstorm: "in_progress" })); if (route === "live resume skip") { const phases = { ...P(), brainstorm: { status: "skipped", reason: `resume: ${spec}` } }; await h.phaseResult("skip", phases); returnBranch = [entry, { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "skip", phases } } }]; } else if (route === "live spec write" || route === "live spec edit") { await h.emit("tool_result", { toolName: route === "live spec write" ? "write" : "edit", input: { path: spec }, isError: false }); } else if (route === "live plan check") { await h.emit("tool_result", check); } else { writeFileSync(join(h.ctx.cwd, plan), `**Spec:** ${spec}\n`); await h.read("plan", plan, "header"); } } await h.read("missed", legSkill, "ignored"); if (route === "live resume skip" || route === "disabled navigation") { h.ctx.cwd = other.root; await h.emit("session_tree", {}); } assert.deepEqual(h.recorderIo(), io); assert.deepEqual(readFileSync(original.recordPath()), before); h.setEnabled(true); if (route === "shutdown before navigation") await h.emit("session_shutdown", {}); h.setBranch(returnBranch); h.ctx.cwd = other.root; await h.emit("session_tree", {}); assert.equal(other.readRecord().derived.skill_read_coverage, "complete"); assert.equal(original.readRecord().derived.skill_read_coverage, "partial"); h.ctx.cwd = original.root; await h.emit("session_tree", {}); const restored = original.readRecord(); assert.equal(restored.run_id, runId); assert.equal(restored.derived.skill_read_coverage, "partial"); assert.deepEqual(Object.values(restored.derived.phases).flatMap(p => p.skill_reads ?? []), []); }); } }); test("a disabled resume skip wins over a later same-checkout spec edit", async t => { const a = "doc/specs/a.md"; const b = "doc/specs/b.md"; const skipped = { ...P(), brainstorm: { status: "skipped", reason: `resume: ${a}` } }; const edit = { toolName: "edit", toolCallId: "edit", input: { path: b }, isError: false }; const result = (data: object) => ({ type: "message", message: { role: "toolResult", ...data } }); const branch = [entry, result({ toolName: "phase_tracker", details: { action: "skip", phases: skipped } }), { type: "message", message: { role: "assistant", content: [{ type: "toolCall", id: "edit", name: "edit", arguments: { path: b } }] } }, result(edit)]; for (const route of ["live", "replay"] as const) { await t.test(route, async () => { const original = await boundInBrainstorm(); await original.phaseResult("reset", P()); await original.phaseResult("start", P({ brainstorm: "in_progress" })); await original.writeSpec(b); await original.emit("session_shutdown", {}); const before = readFileSync(original.recordPath(b), "utf8"); const h = harness({ enabled: false, sessionId: "s2" }); h.ctx.cwd = original.root; const io = h.recorderIo(); if (route === "replay") { h.setBranch(branch); await h.emit("session_start", {}); } else { await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.phaseResult("skip", skipped); await h.emit("tool_result", edit); } await h.read("missed", legSkill, "ignored"); assert.deepEqual(h.recorderIo(), io); h.setEnabled(true); await h.emit("session_shutdown", {}); assert.equal(h.readRecord(a).derived.skill_read_coverage, "partial"); assert.equal(readFileSync(original.recordPath(b), "utf8"), before); h.setBranch([entry, result({ toolName: "plan_check", details: { status: "pass", specPath: b } })]); await h.emit("session_tree", {}); assert.equal(h.readRecord(b).derived.skill_read_coverage, "complete"); }); } }); test("disabled switches retain outgoing paths and do not reopen stale records", async () => { const first = await boundInBrainstorm(); const second = await boundInBrainstorm(); const terminal = await boundInBrainstorm(); const sealed = terminal.readRecord(); sealed.status = "shipped"; writeFileSync(terminal.recordPath(), serializeRecord(sealed)); const before = readFileSync(terminal.recordPath(), "utf8"); writeFileSync(join(first.root, "doc/plans/p.md"), "**Spec:** doc/specs/a.md\n"); const h = harness({ enabled: false }); const io = h.recorderIo(); for (const [session, checkout] of [first, second, terminal].entries()) { h.ctx.cwd = join(checkout.root, "doc"); h.ctx.sessionManager.getSessionId = () => `outgoing-${session}`; await h.emit("session_switch", {}); await h.phaseResult("start", P({ brainstorm: "in_progress" })); if (checkout === terminal) { const check = (specPath: string) => h.emit("tool_result", { toolName: "plan_check", isError: false, details: { status: "pass", specPath } }); await check("specs/a.md"); await check("specs/absent.md"); } else if (checkout === second) { await h.phaseResult("skip", { ...P(), brainstorm: { status: "skipped", reason: "resume: specs/a.md" } }); } else { await h.read("plan", "plans/p.md", "header"); } await h.read("missed", legSkill, "ignored"); } h.ctx.cwd = h.root; h.ctx.sessionManager.getSessionId = () => "incoming"; await h.emit("session_switch", {}); assert.deepEqual(h.recorderIo(), io); h.setEnabled(true); await h.emit("session_shutdown", {}); assert.equal(first.readRecord().derived.skill_read_coverage, "partial"); assert.equal(second.readRecord().derived.skill_read_coverage, "partial"); assert.equal(readFileSync(terminal.recordPath(), "utf8"), before); assert.equal(existsSync(terminal.recordPath("doc/specs/absent.md")), false); assert.equal(existsSync(h.recordPath()), false); }); test("an outgoing gap cannot change a removed, replaced, or stamped record", async t => { for (const state of ["removed", "replaced", "stamped"] as const) { await t.test(state, async () => { const h = await boundInBrainstorm(); const a = "doc/specs/a.md"; const b = "doc/specs/b.md"; await h.phaseResult("reset", P()); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec(b); const check = (specPath: string) => h.emit("tool_result", { toolName: "plan_check", isError: false, details: { status: "pass", specPath } }); await check(a); const before = readFileSync(h.recordPath(b)); h.setEnabled(false); await check(b); const io = h.recorderIo(); await h.emit("session_tree", {}); assert.deepEqual(h.recorderIo(), io); if (state === "removed") { unlinkSync(h.recordPath(a)); } else { const onDisk = h.readRecord(a); if (state === "replaced") onDisk.run_id = "replacement"; else onDisk.status = "shipped"; writeFileSync(h.recordPath(a), serializeRecord(onDisk)); } const outgoing = existsSync(h.recordPath(a)) && readFileSync(h.recordPath(a)); h.setEnabled(true); await h.emit("session_shutdown", {}); assert.deepEqual(existsSync(h.recordPath(a)) && readFileSync(h.recordPath(a)), outgoing); assert.deepEqual(readFileSync(h.recordPath(b)), before); }); } }); test("a still-bound run cannot overwrite its replacement on re-enable or shutdown", async () => { const h = await boundInBrainstorm(); const file = h.recordPath(); const replacement = { ...h.readRecord(), run_id: "replacement" }; h.setEnabled(false); await h.read("missed", legSkill, "ignored"); writeFileSync(file, serializeRecord(replacement)); const before = readFileSync(file); h.setEnabled(true); await h.emit("session_compact", {}); assert.ok(readFileSync(file).equals(before), "re-enable changed the replacement record"); const io = h.recorderIo(); await h.emit("session_shutdown", {}); assert.ok(readFileSync(file).equals(before), "shutdown changed the replacement record"); assert.deepEqual(h.recorderIo(), io, "shutdown accessed the frozen binding"); }); test("reset or navigation while disabled persists the old run's gap only on re-enable", async t => { for (const [transition, queued] of [["reset", false], ["reset", true], ["session_tree", false], ["session_tree", true]] as const) { await t.test(`${transition}, ${queued ? "queued binding" : "no queued binding"}`, async () => { const h = await boundInBrainstorm(); await h.read("first", legSkill, "text"); const original = h.readRecord(); assert.equal(original.derived.skill_read_coverage, "complete"); const before = readFileSync(h.recordPath()); const io = h.recorderIo(); h.setEnabled(false); await h.read("gap", legSkill, "ignored"); if (queued) await h.emit("tool_result", { toolName: "plan_check", isError: false, details: { status: "pass", specPath: "doc/specs/abandoned.md" } }); if (transition === "reset") await h.phaseResult("reset", P()); else await h.emit("session_tree", {}); await h.read("disabled", legSkill, "ignored"); assert.deepEqual(h.recorderIo(), io); assert.deepEqual(readFileSync(h.recordPath()), before); original.branch = "newer-on-disk"; writeFileSync(h.recordPath(), serializeRecord(original)); h.setEnabled(true); await h.read("disarmed", legSkill, "ignored"); original.derived.skill_read_coverage = "partial"; assert.deepEqual(h.readRecord(), original); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/b.md"); assert.equal(existsSync(h.recordPath("doc/specs/abandoned.md")), false); const other = h.readRecord("doc/specs/b.md").derived; assert.equal(other.skill_read_coverage, "complete"); assert.deepEqual(Object.values(other.phases).flatMap(phase => phase.skill_reads ?? []), []); }); } }); test("a resumed run retains coverage and opaque paths; a legacy run stays unmeasured until its first observation", async () => { for (const mode of ["measured", "disabled", "disabled-same-session", "legacy", "legacy-pending"]) { const h = await boundInBrainstorm(); const file = "other/skills/reference.md"; mkdirSync(join(h.root, "other/skills"), { recursive: true }); writeFileSync(join(h.root, file), "reference"); const disabled = mode.startsWith("disabled"); if (mode === "measured" || mode === "disabled-same-session") await h.read("first", file, "1"); const before = readFileSync(h.recordPath()); const io = h.recorderIo(); if (disabled) { h.setEnabled(false); await h.read("missed", file, "ignored"); } await h.emit("session_shutdown", { type: "session_shutdown" }); if (disabled) { assert.deepEqual(h.recorderIo(), io); assert.deepEqual(readFileSync(h.recordPath()), before); } const rec = h.readRecord(); assert.equal(rec.derived.skill_read_coverage, "complete"); if (mode.startsWith("legacy")) { delete rec.derived.skill_read_coverage; writeFileSync(h.recordPath(), serializeRecord(rec)); } const resumed = harness({ branch: [entry], sessionId: mode === "disabled-same-session" ? "s1" : "s2" }); resumed.ctx.cwd = h.root; await resumed.emit("session_start", { type: "session_start" }); if (mode === "legacy-pending") await resumed.read("pending", file, "1"); await resumed.writeSpec("doc/specs/a.md"); assert.equal(resumed.readRecord().run_id, rec.run_id); assert.equal(resumed.readRecord().derived.skill_read_coverage, mode === "measured" ? "complete" : mode === "legacy" ? undefined : "partial"); await resumed.read("next", file, "22"); await resumed.emit("session_shutdown", { type: "session_shutdown" }); const d = resumed.readRecord().derived; assert.equal(d.skill_read_coverage, mode === "measured" ? "complete" : "partial"); const reads = d.phases.brainstorm!.skill_reads!; assert.match(reads[0].path, /^source:[a-f0-9]{64}:skills\/reference\.md$/); assert.deepEqual(reads.map(r => [r.path, r.bytes]), (mode === "legacy" || mode === "disabled" ? [2] : [1, 2]).map(n => [reads[0].path, n])); } }); test("a clean close distinguishes an untouched session from activity during a disabled resume", async t => { for (const mode of ["clean resume", "disabled read", "disabled read before later bind"]) { await t.test(mode, async () => { const missed = mode !== "clean resume"; const laterBind = mode === "disabled read before later bind"; const branch: unknown[] = [entry]; const original = harness({ branch }); await original.phaseResult("start", P({ brainstorm: "in_progress" })); await original.writeSpec("doc/specs/a.md"); // A nested write binds live but Pi persists only the enclosing codemode result. if (!laterBind) branch.push({ type: "message", message: { role: "assistant", content: [{ type: "toolCall", name: "write", id: "w", arguments: { path: "doc/specs/a.md" } }] } }); branch.push({ type: "message", message: { role: "toolResult", toolName: laterBind ? "codemode" : "write", toolCallId: "w" } }); await original.read("covered", legSkill, "text"); await original.emit("session_shutdown", { reason: "quit" }); const closed = original.readRecord(); assert.equal(closed.derived.skill_read_coverage, "complete"); assert.deepEqual(Object.keys(closed.accumulators), ["total"]); if (missed) { const disabled = harness({ branch, enabled: false }); disabled.ctx.cwd = original.root; const io = disabled.recorderIo(); const before = readFileSync(original.recordPath()); const length = branch.length; await disabled.emit("session_start", { reason: "startup" }); await disabled.read("missed", legSkill, "not measured"); branch.push({ type: "message", message: { role: "toolResult", toolName: "read", toolCallId: "missed", content: [{ type: "text", text: "not measured" }] } }); await disabled.emit("session_shutdown", { reason: "quit" }); assert.equal(branch.length, length + 1, "disabled recorder wrote to the session"); assert.deepEqual(disabled.recorderIo(), io); assert.deepEqual(readFileSync(original.recordPath()), before); } branch.push({ type: "model_change", provider: "p", modelId: "m" }); const resumed = harness({ branch }); resumed.ctx.cwd = original.root; await resumed.emit("session_start", { reason: "startup" }); if (laterBind) await resumed.emit("tool_result", { toolName: "plan_check", details: { status: "pass", specPath: "doc/specs/a.md" } }); const rec = resumed.readRecord(); assert.equal(rec.run_id, closed.run_id); assert.equal(rec.derived.skill_read_coverage, missed ? "partial" : "complete"); assert.deepEqual(rec.derived.phases.brainstorm?.skill_reads, [pendingRead]); }); } }); test("same-session pending reads append after the observations already bound", async () => { const h = await boundInBrainstorm(); await h.read("first", legSkill, "1"); h.setBranch([entry]); await h.emit("session_tree", { type: "session_tree" }); await h.read("pending", legSkill, "22"); await h.writeSpec("doc/specs/a.md"); const d = h.readRecord().derived; assert.equal(d.skill_read_coverage, "complete"); assert.deepEqual(d.phases.brainstorm!.skill_reads!.map(r => r.bytes), [1, 2]); }); test("rebinding after an observed read leaves it with its original run", async (t) => { for (const mode of ["same record / worktree write", "valid record / worktree write", "invalid record / plan_check"]) { await t.test(mode, async () => { const h = harness(); const invalid = mode === "invalid record / plan_check"; await h.phaseResult("start", P({ brainstorm: "in_progress" })); if (invalid) { mkdirSync(dirname(h.recordPath()), { recursive: true }); writeFileSync(h.recordPath(), [ "schema: 1", "spec: doc/specs/a.md", "run_id: a", "status: in_progress", "created_at: 2026-09-17T10:00:00Z", "accumulators: { s1: { phases: { brainstorm: { skill_reads: malformed } } } }", ].join("\n")); } await h.writeSpec("doc/specs/a.md"); await h.read("first", legSkill, "text"); const original = h.readRecord(); assert.equal(original.derived.skill_read_coverage, invalid ? "invalid" : "complete"); if (mode === "same record / worktree write") { await h.writeSpec(".worktrees/x/doc/specs/a.md"); const shared = h.readRecord(); assert.equal(shared.run_id, original.run_id); assert.equal(shared.derived.skill_read_coverage, "complete"); assert.deepEqual(shared.derived.phases.brainstorm?.skill_reads, [pendingRead]); return; } if (invalid) { await h.emit("tool_result", { toolName: "plan_check", toolCallId: "pc", input: {}, content: [], isError: false, details: { status: "pass", specPath: "doc/specs/b.md" } }); } else { await h.writeSpec(".worktrees/x/doc/specs/b.md"); } const next = h.readRecord("doc/specs/b.md"); assert.equal(next.derived.skill_read_coverage, "partial"); assert.equal(next.derived.phases.brainstorm?.skill_reads, undefined); assert.deepEqual(h.readRecord().derived.phases.brainstorm?.skill_reads, invalid ? undefined : [pendingRead]); }); } }); test("an in-flight read completes exactly once on the new plan binding", async () => { const h = await boundInBrainstorm(); const call = { toolName: "read", toolCallId: "in-flight" }; await h.emit("tool_call", { ...call, input: { path: legSkill } }); await h.emit("tool_result", { toolName: "plan_check", toolCallId: "pc", input: {}, content: [], isError: false, details: { status: "pass", specPath: "doc/specs/b.md" } }); const end = { ...call, result: { content: [{ type: "text", text: "text" }] }, isError: false }; await h.emit("tool_execution_end", end); await h.emit("tool_execution_end", end); assert.equal(h.readRecord().derived.phases.brainstorm?.skill_reads, undefined); const next = h.readRecord("doc/specs/b.md").derived; assert.deepEqual(next.phases.brainstorm?.skill_reads, [pendingRead]); assert.equal(next.skill_read_coverage, "complete"); }); test("message_end appends one cache triplet per parent call; child usage is excluded from parent_calls but still summed into tokens", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "in_progress" })); const big = { input: 1, output: 1, cacheRead: 10, cacheWrite: 60000, cacheWrite1h: 60000, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }; const small = { input: 1, output: 1, cacheRead: 20, cacheWrite: 5, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }; await h.emit("message_end", { type: "message_end", message: { role: "assistant", usage: big } }); await h.emit("message_end", { type: "message_end", message: { role: "assistant", usage: small } }); await h.emit("tool_result", subagentResult([{ agent: "implementer", exitCode: 0, model: "p/i", usage: { input: 7, output: 1, cacheRead: 0, cacheWrite: 90000, totalTokens: 8, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } } }])); await h.emit("turn_end", { type: "turn_end", turnIndex: 1, message: {}, toolResults: [] }); const impl = h.readRecord("doc/specs/a.md").derived.phases.implement!; assert.deepEqual(impl.parent_calls, [[10, 60000, 60000], [20, 5, 0]]); assert.equal(impl.tokens!.cache_write, 60000 + 5 + 90000); }); const usage = (input: number, output = 0, cost = 0) => ({ input, output, cacheRead: 0, cacheWrite: 0, totalTokens: input + output, cost: { input: cost, output: 0, cacheRead: 0, cacheWrite: 0, total: cost } }); test("pre-binding observations merge into the bound record in event order", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.emit("tool_result", subagentResult([{ agent: "scout", exitCode: 0, usage: usage(10) }])); await h.emit("model_select", { type: "model_select", model: { provider: "p", id: "next" } }); await h.emit("tool_result", { toolName: "plan_check", toolCallId: "pc", input: {}, content: [], isError: false, details: { status: "fail" } }); await h.planResult("update", [{ name: "a", status: "complete" }]); await h.planResult("update", [{ name: "a", status: "in_progress" }]); await h.writeSpec("doc/specs/a.md"); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); const rec = h.readRecord(); assert.equal(rec.derived.personas.scout.dispatches, 1); assert.equal(rec.derived.personas.scout.tokens?.input, 10); assert.equal(rec.derived.gates.plan_rounds, 1); assert.equal(rec.derived.gates.task_reopens, 1); assert.equal(rec.events.filter((e) => e.kind === "gate" && (e as any).gate === "task_reopen").length, 1); assert.deepEqual(rec.events.map((e) => e.kind), ["phase", "dispatch", "config_change", "plan_check", "gate"]); }); test("pre-binding observations are discarded on shutdown without a telemetry file", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.emit("tool_result", subagentResult([{ agent: "scout", exitCode: 0, usage: usage(10) }])); await h.emit("model_select", { type: "model_select", model: { provider: "p", id: "next" } }); await h.emit("tool_result", { toolName: "plan_check", toolCallId: "pc", input: {}, content: [], isError: false, details: { status: "fail" } }); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); assert.equal(existsSync(join(h.root, ".pi/gauntlet/telemetry")), false); }); async function boundInPlan(over: Parameters[0] = {}) { const h = harness(over); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); return h; } const subagentResult = (results: unknown[], content = "") => ({ toolName: "subagent", toolCallId: "sa", input: {}, content: [{ type: "text", text: content }], isError: false, details: { results } }); async function boundInBrainstorm() { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); return h; } const memberBatch = (isError = false) => ({ ...subagentResult([ { agent: "spec-council-member", exitCode: 0, model: "p/alpha:xhigh", savedOutputPath: "/tmp/c/member-0-p-alpha.md", usage: usage(1) }, { agent: "spec-council-member", exitCode: 1, model: "p/beta:high", usage: usage(1) }, { agent: "spec-council-member", exitCode: 0, model: "p/beta:high", savedOutputPath: "/tmp/c/retry/member-1-p-beta-high.md", usage: usage(1) }, ]), isError }); const CHAIR_TEXT = "consensus: needs-work\nclusters:\n- [major] a - raised-by: [p-alpha] - x\n- [minor] b - raised-by: [p-alpha, p-beta-high] - x\nresolved:\n"; const chairResult = (text = CHAIR_TEXT, exitCode = 0) => ({ ...subagentResult([{ agent: "spec-council-synthesizer", exitCode, model: "p/chair:medium", usage: usage(1) }], text), isError: exitCode !== 0 }); const AUDIT_TEXT = "Applied: [major] a - raised-by: [p-alpha] -> edit\nDeferred: none\nRejected: [minor] b - raised-by: [p-alpha, p-beta-high] -> reason"; const assistant = (text: string) => ({ type: "message_end", message: { role: "assistant", usage: usage(1), content: [{ type: "text", text }] } }); const counts = (major = 0, minor = 0) => ({ blocker: 0, major, minor }); const EXPECTED_COUNCIL = { chair: { model: "p/chair:medium", dispatches: 1, clusters: 2, members_reported: 2 }, members: { "p/alpha:xhigh": { dispatches: 1, total: counts(1, 1), unique: counts(1), applied: counts(1), unique_applied: counts(1), deferred: counts(), rejected: counts(0, 1) }, "p/beta:high": { dispatches: 2, total: counts(0, 1), unique: counts(), applied: counts(), unique_applied: counts(), deferred: counts(), rejected: counts(0, 1) }, }, }; test("council captures failed member batch, chair, audit and persists on later flush", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch(true)); await h.emit("tool_result", chairResult()); assert.equal(h.readRecord().derived.council, undefined); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); await h.phaseResult("complete", P({ brainstorm: "complete" })); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); assert.deepEqual(h.readRecord().derived.personas["spec-council-member"], { dispatches: 3, models: ["p/alpha:xhigh", "p/beta:high"], tokens: { input: 3, output: 0, cache_read: 0, cache_write: 0, cost: 0 } }); }); for (const source of ["saved report", "inline report", "missing report"] as const) { test(`council chair artifact: ${source}`, async () => { const h = await boundInBrainstorm(); const chairPath = join(h.root, "chair.md"); if (source !== "missing report") writeFileSync(chairPath, source === "inline report" ? "consensus: sound\nclusters:\nresolved:\n" : CHAIR_TEXT); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", subagentResult([ { agent: "spec-council-synthesizer", exitCode: 0, model: "p/chair:medium", savedOutputPath: chairPath, usage: usage(1) }, ], source === "inline report" ? CHAIR_TEXT : `Output saved to: ${chairPath}`)); assert.equal(h.readFileCalls.filter((p) => p === chairPath).length, source === "inline report" ? 0 : 1); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, source === "missing report" ? undefined : EXPECTED_COUNCIL); }); } test("council accepts the applier audit before conflicting commit and chat audits", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.emit("tool_result", subagentResult([{ agent: "spec-council-applier", exitCode: 0, model: "p/applier", usage: usage(1) }], AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); const conflictingAudit = AUDIT_TEXT.replace("Applied:", "Deferred:").replace("Deferred: none", "Applied: none"); await h.bash("c1", HEREDOC_COMMIT(conflictingAudit)); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); await h.emit("message_end", assistant(conflictingAudit)); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); }); test("council ignores a failed applier audit and records different dispositions from its successful retry", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.emit("tool_result", { ...subagentResult([{ agent: "spec-council-applier", exitCode: 1, usage: usage(1) }], AUDIT_TEXT), isError: true }); assert.equal(h.readRecord().derived.council, undefined); const retryAudit = AUDIT_TEXT.replace("Applied:", "Deferred:").replace("Deferred: none", "Applied: none"); await h.emit("tool_result", subagentResult([{ agent: "spec-council-applier", exitCode: 0, usage: usage(1) }], retryAudit)); assert.deepEqual(h.readRecord().derived.council, { ...EXPECTED_COUNCIL, members: { ...EXPECTED_COUNCIL.members, "p/alpha:xhigh": { ...EXPECTED_COUNCIL.members["p/alpha:xhigh"], applied: counts(), unique_applied: counts(), deferred: counts(1) }, }, }); }); test("council defaults missing chair model and survives unrelated result before audit", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", subagentResult([{ agent: "spec-council-synthesizer", exitCode: 0, usage: usage(1) }], CHAIR_TEXT)); await h.emit("tool_result", subagentResult([{ agent: "spec-summarizer", exitCode: 0, model: "p/s", usage: usage(1) }])); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, { ...EXPECTED_COUNCIL, chair: { ...EXPECTED_COUNCIL.chair, model: "unknown" } }); }); test("council ignores plan phase", async () => { const h = await boundInPlan(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.equal(h.readRecord().derived.council, undefined); }); test("council counts failed chair retries and retains last usable clusters", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult("Failed", 1)); await h.emit("tool_result", chairResult()); await h.emit("tool_result", chairResult("no consensus here", 1)); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council!.chair, { ...EXPECTED_COUNCIL.chair, dispatches: 3 }); }); test("council retains pending on incomplete audit and ignores audit before chair", async () => { const h = await boundInBrainstorm(); await h.emit("message_end", assistant(AUDIT_TEXT)); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.emit("message_end", assistant("Applied: [major] a - raised-by: [p-alpha] -> edit\nDeferred: none\nRejected: none")); assert.equal(h.readRecord().derived.council, undefined); await h.emit("message_end", assistant("Applied: a -> edit\nDeferred: none\nRejected: none")); assert.equal(h.readRecord().derived.council, undefined); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); }); test("council reset clears pending and new member batch supersedes held chair", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.phaseResult("reset", P()); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.equal(h.readRecord().derived.council, undefined); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.emit("tool_result", memberBatch()); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.equal(h.readRecord().derived.council, undefined); await h.emit("tool_result", chairResult()); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); }); const HEREDOC_COMMIT = (body: string) => `git -C /w commit -q -F - -- doc/specs/a.md <<'EOF'\nspec: a\n\n${body}\nEOF`; const ALL_NONE = "Applied: none\nDeferred: none\nRejected: none"; const EMPTY_CHAIR = "consensus: sound\nclusters:\nresolved:\n"; test("council records the audit from the spec commit heredoc without a chat render", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.bash("c1", HEREDOC_COMMIT(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); }); test("council ignores a chat audit after the commit already recorded it", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.bash("c1", HEREDOC_COMMIT(AUDIT_TEXT)); const before = h.readRecord().derived.council; assert.deepEqual(before, EXPECTED_COUNCIL); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, before); }); test("council ignores a commit outside brainstorm or before a chair result", async () => { const planned = await boundInPlan(); await planned.emit("tool_result", memberBatch()); await planned.emit("tool_result", chairResult()); await planned.bash("c1", HEREDOC_COMMIT(AUDIT_TEXT)); assert.equal(planned.readRecord().derived.council, undefined); const early = await boundInBrainstorm(); await early.emit("tool_result", memberBatch()); await early.bash("c1", HEREDOC_COMMIT(AUDIT_TEXT)); assert.equal(early.readRecord().derived.council, undefined); }); test("council keeps pending across the grant amend and records a later chat audit", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.bash("c1", "git -C /w commit --amend --no-edit -q"); assert.equal(h.readRecord().derived.council, undefined); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); }); test("council records an all-none audit from the commit against an empty chair", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult(EMPTY_CHAIR)); await h.bash("c1", HEREDOC_COMMIT(ALL_NONE)); const zeroed = { total: counts(), unique: counts(), applied: counts(), unique_applied: counts(), deferred: counts(), rejected: counts() }; assert.deepEqual(h.readRecord().derived.council, { chair: { model: "p/chair:medium", dispatches: 1, clusters: 0, members_reported: 0 }, members: { "p/alpha:xhigh": { dispatches: 1, ...zeroed }, "p/beta:high": { dispatches: 2, ...zeroed } }, }); }); test("council ignores a non-git bash command carrying audit text", async () => { const h = await boundInBrainstorm(); await h.emit("tool_result", memberBatch()); await h.emit("tool_result", chairResult()); await h.bash("c1", `cat <<'EOF'\n${AUDIT_TEXT}\nEOF`); assert.equal(h.readRecord().derived.council, undefined); await h.emit("message_end", assistant(AUDIT_TEXT)); assert.deepEqual(h.readRecord().derived.council, EXPECTED_COUNCIL); }); test("subagent results: dispatch events, persona tokens, spec_rounds, reviews, conformance_loops; async results: [] contribute nothing", async () => { const h = await boundInPlan(); await h.emit("tool_result", subagentResult([ { agent: "implementer", exitCode: 0, model: "p/impl:high", usage: usage(100, 10, 1) }, { agent: "implementer", exitCode: 1, model: "p/impl:high", usage: usage(50, 5, 0.5) }, { agent: "spec-summarizer", exitCode: 0, model: "p/s", usage: usage(1) }, { agent: "wave-reviewer", exitCode: 0, model: "p/r", usage: usage(1) }, ], `## Outputs\n### review\n\`\`\`json\n{"structured":{"findings":{"critical":0,"moderate":1,"minor":2}}}\n\`\`\`\nOutput saved to: /tmp/review.md\n[major] duplicate [minor] duplicate`)); await h.emit("tool_result", subagentResult([])); const rec = h.readRecord(); const dispatches = rec.events.filter((e) => e.kind === "dispatch") as any[]; assert.equal(dispatches.length, 4); assert.deepEqual(dispatches[0], { ts: dispatches[0].ts, session: "s1", phase: "plan", kind: "dispatch", agent: "implementer", model: "p/impl:high" }); assert.equal(dispatches[1].exit, 1); assert.deepEqual(rec.derived.personas.implementer, { dispatches: 2, models: ["p/impl:high"], tokens: { input: 150, output: 15, cache_read: 0, cache_write: 0, cost: 1.5 } }); assert.equal(rec.derived.gates.spec_rounds, 1); assert.deepEqual(rec.derived.reviews?.["wave-reviewer"], { dispatches: 1, nonzero_exit: 0, findings: { blocker: 0, major: 1, minor: 2 } }); assert.equal(rec.derived.conformance_loops, 0); assert.deepEqual(rec.derived.phases.plan?.tokens, { input: 152, output: 15, cache_read: 0, cache_write: 0, cost: 1.5 }); }); test("zero structured findings do not fall back to inline severity tags", async () => { const h = await boundInPlan(); await h.emit("tool_result", subagentResult([{ agent: "wave-reviewer", exitCode: 0 }], '## Outputs\n### Step 1\n#### Task 1 (wave-reviewer)\n```json\n{"structured":{"findings":{"critical":0,"moderate":0,"minor":0}}}\n```\nOutput saved to: /tmp/review.md\n[Critical] duplicate')); assert.deepEqual(h.readRecord().derived.reviews?.["wave-reviewer"]?.findings, { blocker: 0, major: 0, minor: 0 }); }); for (const [name, agents] of [ ["chain", ["implementer", "wave-reviewer", "wave-reviewer"]], ["plain tasks", ["implementer", "implementer"]], ["empty results", []], ["missing results", undefined], ] as const) { test(`failed subagent ${name}: count terminal retained results once, not progress`, async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "in_progress" })); await h.planResult("init", [{ name: "repair", status: "in_progress" }]); const before = h.readRecord(); const results = agents?.map((agent, i) => ({ agent, exitCode: i === agents.length - 1 ? 1 : 0, model: `p/m${i}`, usage: usage(10, 2, 0.5) })); const terminal = { ...subagentResult(results ?? []), isError: true, details: results === undefined ? {} : { results } }; for (const partial of [results?.slice(0, 1), results]) { await h.emit("tool_execution_update", { type: "tool_execution_update", toolName: "subagent", toolCallId: "sa", args: {}, partialResult: { content: [], details: { results: partial ?? [] } } }); } assert.deepEqual(h.readRecord(), before); await h.emit("tool_result", terminal); const rec = h.readRecord(); const dispatches = rec.events.filter((e) => e.kind === "dispatch"); assert.equal(dispatches.length, agents?.length ?? 0); assert.deepEqual(dispatches.map((e) => ({ agent: e.agent, model: e.model, exit: e.exit, phase: e.phase })), results?.map((r) => ({ agent: r.agent, model: r.model, exit: r.exitCode || undefined, phase: "implement" })) ?? []); for (const agent of new Set(agents)) { const retained = results!.filter((r) => r.agent === agent); assert.deepEqual(rec.derived.personas[agent], { dispatches: retained.length, models: retained.map((r) => r.model), tokens: { input: 10 * retained.length, output: 2 * retained.length, cache_read: 0, cache_write: 0, cost: 0.5 * retained.length } }); } if (name === "chain") assert.deepEqual(rec.derived.reviews?.["wave-reviewer"], { dispatches: 2, nonzero_exit: 1 }); if (results?.length) assert.deepEqual(rec.derived.phases.implement?.tokens, { input: 10 * results.length, output: 2 * results.length, cache_read: 0, cache_write: 0, cost: 0.5 * results.length }); else assert.deepEqual(rec, before); assert.deepEqual(rec.events.filter((e) => e.kind === "phase" || e.kind === "gate"), before.events.filter((e) => e.kind === "phase" || e.kind === "gate")); assert.deepEqual(rec.derived.plan, before.derived.plan); assert.deepEqual(rec.derived.gates, before.derived.gates); }); } test("failed spec and conformance dispatches count usage and failures without advancing successful gates", async () => { const h = await boundInPlan(); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "in_progress" })); const results = ["spec-summarizer", "conformance-reviewer"].map((agent) => ({ agent, exitCode: 1, model: "p/r", usage: usage(3) })); const before = h.readRecord(); await h.emit("tool_result", { ...subagentResult(results, "Conformance verdict: GAPS\nG1:\n verdict: MISSING"), isError: true }); let rec = h.readRecord(); assert.deepEqual(rec.events.filter((e) => e.kind === "dispatch").map((e) => ({ agent: e.agent, exit: e.exit, phase: e.phase })), results.map((r) => ({ agent: r.agent, exit: 1, phase: "verify" }))); for (const { agent } of results) assert.deepEqual(rec.derived.personas[agent], { dispatches: 1, models: ["p/r"], tokens: { input: 3, output: 0, cache_read: 0, cache_write: 0, cost: 0 } }); assert.equal(rec.derived.gates.spec_rounds, before.derived.gates.spec_rounds); assert.equal(rec.derived.conformance_loops, before.derived.conformance_loops); assert.equal(rec.derived.conformance_open_gaps, before.derived.conformance_open_gaps); assert.deepEqual(rec.derived.reviews?.["conformance-reviewer"], { dispatches: 1, nonzero_exit: 1 }); assert.deepEqual(rec.derived.phases.verify?.tokens, { input: 6, output: 0, cache_read: 0, cache_write: 0, cost: 0 }); assert.deepEqual(rec.events.filter((e) => e.kind === "phase" || e.kind === "gate"), before.events.filter((e) => e.kind === "phase" || e.kind === "gate")); await h.emit("tool_result", subagentResult(results.map((r) => ({ ...r, exitCode: 0 })), "Conformance verdict: GAPS\nG1:\n verdict: MISSING")); rec = h.readRecord(); assert.equal(rec.derived.gates.spec_rounds, before.derived.gates.spec_rounds + 1); assert.equal(rec.derived.conformance_loops, 1); assert.equal(rec.derived.conformance_open_gaps, 1); assert.deepEqual(rec.derived.reviews?.["conformance-reviewer"], { dispatches: 2, nonzero_exit: 1 }); }); test("conformance-reviewer during verify counts loops and retains the latest open gap count", async () => { const h = await boundInPlan(); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "in_progress" })); await h.emit("tool_result", subagentResult([{ agent: "conformance-reviewer", exitCode: 0, usage: usage(1) }], "Conformance verdict: GAPS\nG1:\n verdict: MISSING\nG2:\n verdict: PARTIAL")); assert.equal(h.readRecord().derived.conformance_open_gaps, 2); await h.emit("tool_result", subagentResult([{ agent: "conformance-reviewer", exitCode: 2, usage: usage(1) }], "Conformance verdict: CONFORMS")); const rec = h.readRecord(); assert.equal(rec.derived.conformance_loops, 2); assert.equal(rec.derived.conformance_open_gaps, 0); assert.equal(rec.derived.reviews?.["conformance-reviewer"]?.nonzero_exit, 1); }); test("message_end sums assistant usage into the in-progress phase (unphased outside); turn_end tracks peak_context; compaction counts", async () => { const h = await boundInPlan({ contextTokens: 5000 }); await h.emit("message_end", { type: "message_end", message: { role: "assistant", usage: usage(10, 2, 0.1) } }); await h.emit("message_end", { type: "message_end", message: { role: "user" } }); await h.emit("message_end", { type: "message_end", message: { role: "assistant", usage: usage(0) } }); await h.emit("turn_end", { type: "turn_end", turnIndex: 1, message: {}, toolResults: [] }); await h.emit("session_compact", { type: "session_compact", reason: "threshold" }); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete" })); await h.emit("message_end", { type: "message_end", message: { role: "assistant", usage: usage(3) } }); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); const rec = h.readRecord(); assert.deepEqual(rec.derived.phases.plan?.tokens, { input: 10, output: 2, cache_read: 0, cache_write: 0, cost: 0.1 }); assert.equal(rec.derived.phases.plan?.peak_context, 5000); assert.equal(rec.derived.phases.plan?.compactions, 1); assert.deepEqual(rec.derived.phases.unphased?.tokens, { input: 3, output: 0, cache_read: 0, cache_write: 0, cost: 0 }); }); test("getContextUsage null tokens -> peak_context omitted", async () => { const h = await boundInPlan({ contextTokens: null }); await h.emit("turn_end", { type: "turn_end", turnIndex: 1, message: {}, toolResults: [] }); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); assert.equal("peak_context" in (h.readRecord().derived.phases.plan ?? {}), false); }); test("input counts user_messages only for plan|implement|verify|ship, only interactive non-slash text", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.emit("input", { type: "input", text: "hello", source: "interactive" }); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); await h.emit("input", { type: "input", text: "approve", source: "interactive" }); await h.emit("input", { type: "input", text: "/skill:x", source: "interactive" }); await h.emit("input", { type: "input", text: "auto", source: "extension" }); await h.emit("input", { type: "input", text: "rpc", source: "rpc" }); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); const rec = h.readRecord(); assert.equal(rec.derived.phases.plan?.user_messages, 2); assert.equal("user_messages" in (rec.derived.phases.brainstorm ?? {}), false); }); test("model_select / thinking_level_select mid-phase emit config_change", async () => { const h = await boundInPlan(); await h.emit("model_select", { type: "model_select", model: { provider: "q", id: "n" }, previousModel: undefined, source: "set" }); await h.emit("thinking_level_select", { type: "thinking_level_select", level: "low", previousLevel: "high" }); const cc = h.readRecord().events.filter((e) => e.kind === "config_change") as any[]; assert.deepEqual(cc.map((e) => [e.model, e.thinking]), [["q/n", undefined], [undefined, "low"]]); }); test("plan_tracker: reopen counts task_reopens; all-terminal snapshot auto-completes implement and records plan totals", async () => { const h = await boundInPlan(); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "in_progress" })); const pt = h.planResult; await pt("init", [{ name: "a", status: "pending" }, { name: "b", status: "pending" }]); await pt("update", [{ name: "a", status: "complete" }, { name: "b", status: "pending" }]); await pt("update", [{ name: "a", status: "in_progress" }, { name: "b", status: "pending" }]); let rec = h.readRecord(); assert.equal(rec.derived.gates.task_reopens, 1); assert.ok(rec.events.some((e) => e.kind === "gate" && (e as any).gate === "task_reopen")); await pt("update", [{ name: "a", status: "complete" }, { name: "b", status: "skipped" }]); rec = h.readRecord(); assert.deepEqual(rec.derived.plan, { tasks: 2, complete: 1, failed: 0, skipped: 1 }); assert.ok(rec.events.some((e) => e.kind === "phase" && (e as any).action === "complete" && (e as any).name === "implement")); assert.equal(rec.derived.phases.implement?.completed_at !== undefined, true); }); test("grant_fix_rounds emits a gate event and counts fix_round_grants", async () => { const h = await boundInPlan(); await h.emit("tool_result", { toolName: "phase_tracker", toolCallId: "g", input: {}, content: [], isError: false, details: { action: "grant_fix_rounds", rounds: 2, phases: P({ brainstorm: "complete", plan: "in_progress" }) } }); const rec = h.readRecord(); assert.equal(rec.derived.gates.fix_round_grants, 1); assert.deepEqual(rec.events.filter((e) => e.kind === "gate").map((e: any) => [e.gate, e.rounds]), [["fix_round_grant", 2]]); }); test("spec writes: spec_writes per phase, amendments coalesce until an input event, post-ship edits warn, banner edits count nothing, links refresh", async () => { const h = await boundInPlan(); await h.writeSpec("doc/specs/a.md", "# A\n\n> **Supersedes:** [doc/specs/old.md](./old.md)\n"); await h.writeSpec("doc/specs/a.md", "# A v3\n"); let rec = h.readRecord(); assert.equal(rec.derived.spec_writes.plan?.count, 2); assert.equal(rec.derived.spec_writes.brainstorm?.count, 1); assert.equal(rec.derived.amendments, 1, "two writes with no input between them = one amendment"); assert.equal(rec.supersedes, undefined); await h.emit("input", { type: "input", text: "change x", source: "interactive" }); await h.writeSpec("doc/specs/a.md", "# A v4\n"); assert.equal(h.readRecord().derived.amendments, 2); // predecessor banner edit on another spec: supersedes gains it, nothing else counted writeFileSync(join(h.root, "doc/specs/pred.md"), "# Pred\n\n> **Superseded by:** [doc/specs/a.md](./a.md) - fully\n"); const editInput = { path: "doc/specs/pred.md", edits: [{ oldText: "# Pred\n", newText: "# Pred\n\n> **Superseded by:** [doc/specs/a.md](./a.md) - fully\n" }] }; await h.emit("tool_result", { toolName: "edit", toolCallId: "e", input: editInput, content: [], isError: false, details: { diff: "", patch: "" } }); rec = h.readRecord(); assert.deepEqual(rec.supersedes, ["doc/specs/pred.md"]); assert.equal(rec.spec, "doc/specs/a.md"); await h.writeSpec("doc/specs/a.md", "# A without banners\n"); assert.deepEqual(h.readRecord().supersedes, ["doc/specs/pred.md"]); // post-ship edit warns without changing the sealed record rec.status = "shipped"; rec.shipped_at = "2026-09-17T12:00:00Z"; writeFileSync(h.recordPath(), serializeRecord(rec)); const before = readFileSync(h.recordPath(), "utf8"); const h2 = harness(); h2.ctx.cwd = h.root; h2.fetchedRecords.set("test/repo/brainstorming/doc/specs/a.yaml", before); await h2.phaseResult("start", P({ brainstorm: "in_progress" })); const [blocked] = await h2.emit("tool_call", { toolName: "write", toolCallId: "w", input: { path: "doc/specs/a.md" } }); assert.deepEqual(blocked, { block: true, reason: guardReason("doc/specs/a.md", rec.shipped_at, "refs/telemetry/records") }); assert.equal(readFileSync(h.recordPath(), "utf8"), before); assert.equal(h.readRecord().derived.spec_edits_after_ship, 0); }); test("a sealed record is skipped at bind and a later spec starts a fresh run", async () => { const first = harness(); await first.phaseResult("start", P({ brainstorm: "in_progress" })); await first.writeSpec("doc/specs/old.md"); const sealed = { ...first.readRecord("doc/specs/old.md"), status: "shipped" as const, shipped_at: "2026-09-17T18:00:00Z" }; writeFileSync(first.recordPath("doc/specs/old.md"), serializeRecord(sealed)); const before = readFileSync(first.recordPath("doc/specs/old.md"), "utf8"); const h = harness({ sessionId: "s2" }); h.ctx.cwd = first.root; h.fetchedRecords.set("test/repo/brainstorming/doc/specs/old.yaml", before); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/old.md"); await h.emit("tool_result", { toolName: "plan_check", input: {}, content: [], isError: false, details: { status: "pass", specPath: join(first.root, "doc/specs/old.md") } }); assert.equal(readFileSync(first.recordPath("doc/specs/old.md"), "utf8"), before); assert.equal(existsSync(h.excludeFile()), false); await h.writeSpec("doc/specs/new.md"); assert.deepEqual(parseRecord(readFileSync(first.recordPath("doc/specs/new.md"), "utf8"))!.sessions, ["s2"]); }); test("rehydration drops a body-only supersedes link removed by the new session's first write", async () => { const first = await boundInPlan(); await first.writeSpec("doc/specs/a.md", "# A\n\n> **Supersedes:** doc/specs/old.md\n"); const second = harness({ sessionId: "s2" }); second.ctx.cwd = first.root; await second.phaseResult("start", P({ brainstorm: "in_progress" })); await second.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); const body = "# A without banners\n"; writeFileSync(join(first.root, "doc/specs/a.md"), body); await second.emit("tool_call", { toolName: "write", toolCallId: "w", input: { path: "doc/specs/a.md", content: body } }); await second.emit("tool_result", { toolName: "write", toolCallId: "w", input: { path: "doc/specs/a.md", content: body }, content: [], isError: false, details: undefined }); assert.equal(second.readRecord().supersedes, undefined); }); test("rehydration does not read a traversal-valued supersedes link outside the repository", async () => { const first = await boundInPlan(); await first.writeSpec("doc/specs/a.md", "# A\n\n> **Supersedes:** ../../../../dev/zero\n"); const second = harness({ sessionId: "s2", branch: [ { type: "message", message: { role: "toolResult", toolName: "plan_check", toolCallId: "2", isError: false, details: { status: "pass", specPath: join(first.root, "doc/specs/a.md") } } }, ] }); second.ctx.cwd = first.root; await second.emit("session_start", { type: "session_start", reason: "resume" }); assert.ok(second.readFileCalls.every((p) => p === first.root || p.startsWith(first.root + "/"))); assert.deepEqual(second.readRecord().supersedes, ["../../../../dev/zero"]); }); test("rehydration retains a predecessor link that also appeared in the bound spec body", async () => { const first = await boundInPlan(); const predecessor = "# Old\n\n> **Superseded by:** [doc/specs/a.md](./a.md)\n"; writeFileSync(join(first.root, "doc/specs/old.md"), predecessor); await first.writeSpec("doc/specs/a.md", "# A\n\n> **Supersedes:** doc/specs/old.md\n"); const second = harness({ sessionId: "s2", branch: [ { type: "message", message: { role: "toolResult", toolName: "phase_tracker", toolCallId: "1", isError: false, details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: P({ brainstorm: "complete", plan: "in_progress" }) } } }, { type: "message", message: { role: "toolResult", toolName: "plan_check", toolCallId: "2", isError: false, details: { status: "pass", planPath: join(first.root, "doc/plans/a.md"), specPath: join(first.root, "doc/specs/a.md"), planSha256: "x", specSha256: "y" } } }, ] }); second.ctx.cwd = first.root; await second.emit("session_start", { type: "session_start", reason: "resume" }); const body = "# A without banners\n"; writeFileSync(join(first.root, "doc/specs/a.md"), body); await second.emit("tool_call", { toolName: "write", toolCallId: "w", input: { path: "doc/specs/a.md", content: body } }); await second.emit("tool_result", { toolName: "write", toolCallId: "w", input: { path: "doc/specs/a.md", content: body }, content: [], isError: false, details: undefined }); assert.deepEqual(second.readRecord().supersedes, ["doc/specs/old.md"]); }); test("last matched test command during verify/ship records derived.tests from the bash result", async () => { const h = await boundInPlan(); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "in_progress" })); const bash = async (id: string, command: string, isError: boolean) => { await h.emit("tool_call", { toolName: "bash", toolCallId: id, input: { command } }); await h.emit("tool_result", { toolName: "bash", toolCallId: id, input: { command }, content: [], isError, details: undefined }); }; await bash("b1", "cd x && npm test", true); assert.deepEqual(h.readRecord().derived.tests, { command: "npm test", result: "fail" }); await bash("b2", "npm test", false); await bash("b3", "ls", true); assert.deepEqual(h.readRecord().derived.tests, { command: "npm test", result: "pass" }); }); async function boundInShip() { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "complete", ship: "in_progress" })); return h; } test("ship-phase git commands leave the record untouched: no ship event, no shipped_at, no git add/commit, not frozen", async () => { const h = await boundInShip(); const before = readFileSync(h.recordPath(), "utf8"); await h.bash("s1", "git push -u origin HEAD"); await h.bash("s2", "gh pr create --fill"); await h.bash("s3", "git merge --squash gh-33"); await h.bash("d1", "git worktree remove .worktrees/gh-33 && git branch -D gh-33"); assert.equal(readFileSync(h.recordPath(), "utf8"), before); const rec = h.readRecord(); assert.equal(rec.status, "in_progress"); assert.equal(rec.shipped_at, undefined); assert.equal(rec.events.some((e) => e.kind === "ship"), false); assert.equal(h.gitWrites().length, 0); assert.deepEqual(h.jjCalls, []); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "complete", ship: "complete" })); assert.equal(h.readRecord().status, "in_progress", "git keep path no longer stamps; the seal bin owns shipped_at"); }); test("a stamped pending record freezes the recorder on shutdown", async () => { const h = await boundInShip(); const sealed = { ...h.readRecord(), status: "shipped" as const, shipped_at: "2026-09-17T18:00:00Z" }; writeFileSync(h.recordPath(), serializeRecord(sealed)); const before = readFileSync(h.recordPath(), "utf8"); await h.emit("session_compact", { type: "session_compact", reason: "manual" }); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); assert.equal(readFileSync(h.recordPath(), "utf8"), before); }); test("a branch-owned record freezes the recorder on shutdown", async () => { const h = await boundInShip(); rmSync(h.recordPath()); await h.emit("session_compact", { type: "session_compact", reason: "manual" }); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); assert.equal(existsSync(h.recordPath()), false, "flush must not recreate a published record"); }); test("a frozen record cannot bind another spec through plan_check or another checkout write", async () => { const h = await boundInShip(); writeFileSync(h.recordPath(), serializeRecord({ ...h.readRecord(), status: "shipped", shipped_at: "2026-09-17T18:00:00Z" })); const before = readFileSync(h.recordPath()); await h.emit("session_compact", { type: "session_compact", reason: "manual" }); await h.emit("tool_result", { toolName: "plan_check", toolCallId: "pc", input: {}, content: [], isError: false, details: { status: "pass", specPath: join(h.root, "doc/specs/b.md"), planPath: join(h.root, "doc/plans/b.md") } }); await h.writeSpec("doc/specs/b.md"); await h.writeSpec(".worktrees/x/doc/specs/b.md"); assert.equal(existsSync(h.recordPath("doc/specs/b.md")), false); assert.equal(existsSync(join(h.root, ".worktrees/x/.pi/gauntlet/telemetry/doc/specs/b.yaml")), false); assert.deepEqual(readFileSync(h.recordPath()), before); }); test("marker down: spec writes, plan_check passes, and phase events create no file and buffer nothing", async () => { const h = harness(); await h.writeSpec("doc/specs/a.md"); await h.emit("tool_result", { toolName: "plan_check", toolCallId: "pc", input: {}, content: [], isError: false, details: { status: "pass", specPath: join(h.root, "doc/specs/a.md"), planPath: join(h.root, "doc/plans/a.md") } }); await h.emit("tool_result", { toolName: "subagent", toolCallId: "sa", input: {}, content: [{ type: "text", text: "" }], isError: false, details: { results: [{ agent: "worker", exitCode: 0, model: "p/m", usage: { input: 10, output: 5, cacheRead: 0, cacheWrite: 0, cost: 0.1 } }] } }); assert.equal(existsSync(join(h.root, ".pi")), false); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); const rec = h.readRecord(); assert.deepEqual(rec.events.map((e) => e.kind), ["phase"], "nothing from before arming is charged to the run"); assert.deepEqual(rec.derived.personas, {}); assert.equal(rec.derived.gates.plan_rounds, 0); }); test("skip brainstorm with reason resume: binds to the existing spec and reloads its record", async () => { const first = harness(); await first.phaseResult("start", P({ brainstorm: "in_progress" })); await first.writeSpec("doc/specs/a.md"); const runId = first.readRecord().run_id; const second = harness({ sessionId: "s2" }); second.ctx.cwd = first.root; await second.phaseResult("start", P({ brainstorm: "in_progress" })); await second.phaseResult("skip", { ...P({ brainstorm: "skipped" }), brainstorm: { status: "skipped", reason: `resume: ${join(first.root, "doc/specs/a.md")}` } }); const rec = parseRecord(readFileSync(first.recordPath(), "utf8"))!; assert.equal(rec.run_id, runId); assert.deepEqual(rec.sessions, ["s1", "s2"]); assert.equal(rec.approved_at, rec.events.findLast((e) => e.kind === "phase")!.ts); }); test("reset then a new spec binds a fresh run with an empty buffer; the old record is untouched", async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); const late = { toolName: "read", toolCallId: "late" }; await h.emit("tool_call", { ...late, input: { path: legSkill } }); await h.phaseResult("reset", P()); const aBefore = readFileSync(h.recordPath("doc/specs/a.md"), "utf8"); await h.emit("tool_result", { toolName: "subagent", toolCallId: "sa", input: {}, content: [{ type: "text", text: "" }], isError: false, details: { results: [{ agent: "worker", exitCode: 0, model: "p/m", usage: { input: 10, output: 5, cacheRead: 0, cacheWrite: 0, cost: 0.1 } }] } }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/b.md"); await h.emit("tool_execution_end", { ...late, result: { content: [{ type: "text", text: "late" }] }, isError: false }); const b = h.readRecord("doc/specs/b.md"); assert.equal(b.derived.phases.brainstorm?.skill_reads, undefined); assert.deepEqual(b.events.map((e) => e.kind), ["phase"]); assert.deepEqual(b.derived.personas, {}); assert.equal(readFileSync(h.recordPath("doc/specs/a.md"), "utf8"), aBefore); }); test("session_tree flushes observations and clears abandoned plan tracker state", async () => { const h = await boundInPlan({ contextTokens: 4321 }); await h.emit("input", { type: "input", text: "review", source: "interactive" }); await h.emit("message_end", { type: "message_end", message: { role: "assistant", usage: usage(10) } }); await h.emit("turn_end", { type: "turn_end" }); await h.planResult("update", [{ status: "complete" }]); h.setBranch([]); await h.emit("session_tree", { type: "session_tree" }); const rec = h.readRecord(); assert.equal(rec.derived.phases.plan?.user_messages, 1); assert.equal(rec.derived.phases.plan?.tokens?.input, 10); assert.equal(rec.derived.phases.plan?.peak_context, 4321); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/b.md"); await h.planResult("update", [{ status: "in_progress" }]); assert.equal(h.readRecord("doc/specs/b.md").derived.gates.task_reopens, 0); }); test("session_switch reconstructs marker and binding from the target branch", async () => { const first = harness(); await first.phaseResult("start", P({ brainstorm: "in_progress" })); await first.writeSpec("doc/specs/a.md"); const h = harness({ sessionId: "s2", branch: [] }); h.ctx.cwd = first.root; await h.emit("session_start", { type: "session_start", reason: "startup" }); assert.equal(parseRecord(readFileSync(first.recordPath(), "utf8"))!.sessions.includes("s2"), false, "unarmed branch does not bind"); h.setBranch([ { type: "message", message: { role: "toolResult", toolName: "phase_tracker", toolCallId: "1", isError: false, details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, { type: "message", message: { role: "assistant", content: [{ type: "toolCall", id: "w1", name: "write", arguments: { path: join(first.root, "doc/specs/a.md") } }] } }, { type: "message", message: { role: "toolResult", toolName: "write", toolCallId: "w1", isError: false } }, ]); await h.emit("session_switch", { type: "session_switch", reason: "switch" }); assert.deepEqual(parseRecord(readFileSync(first.recordPath(), "utf8"))!.sessions, ["s1", "s2"]); }); async function boundInJjShip(jjFail?: (args: string[]) => GitResult | undefined) { const h = harness({ jjWorkspace: true, jjFail: jjFail ? (args) => jjFail(args) : undefined, gitFail: (args) => (args[0] === "rev-parse" && args.includes("--path-format=absolute") ? { code: 128, stdout: "", stderr: "fatal: not a git repository" } : undefined), }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md"); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "complete", ship: "in_progress" })); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "complete", ship: "complete" })); return h; } for (const toolName of ["write", "edit"]) { for (const nested of [false, true]) { test(`post-ship ${toolName} warning preserves results (${nested ? "nested" : "direct"})`, async () => { const h = await boundInJjShip(); const before = readFileSync(h.recordPath(), "utf8"); const event = { toolName, toolCallId: nested ? "parent/0" : "direct", ...(nested ? { parentToolCallId: "parent" } : {}), input: { path: "doc/specs/a.md", content: "# Spec\n", newText: "updated" }, content: [{ type: "text", text: "spec updated" }], isError: false, structuredContent: toolName === "write" ? { written: true } : null, details: { sentinel: "telemetry details" }, usage: { sentinel: "telemetry usage" }, }; const original = structuredClone(event); const [ret] = await h.emit("tool_result", event); let delivered = event; let merged = mergeToolResult(event, ret); if (nested) { assert.equal(ret, undefined); assert.deepEqual(merged, original); const parent = { ...event, toolName: "codemode", toolCallId: "parent", parentToolCallId: undefined, content: [{ type: "text", text: "script complete" }], structuredContent: { value: true }, }; delivered = parent; const [parentRet] = await h.emit("tool_result", parent); merged = mergeToolResult(parent, parentRet); assert.equal((await h.emit("tool_result", parent))[0], undefined); } assert.deepEqual(event, original); assert.match(merged.content[0].text, /spec shipped at .*write a follow-up spec/); assert.equal(merged.content[0].text.split("spec shipped at").length - 1, 1); assert.deepEqual(merged, { ...delivered, content: [merged.content[0], ...delivered.content] }); assert.equal(readFileSync(h.recordPath(), "utf8"), before, "sealed record stays frozen"); }); } } for (const transition of ["disabled", "session_tree"] as const) { test(`deferred shipped-spec warning survives ${transition} before its parent result`, async () => { const h = await boundInJjShip(); const before = readFileSync(h.recordPath(), "utf8"); const child = { toolName: "edit", toolCallId: "parent/0", parentToolCallId: "parent", input: { path: "doc/specs/a.md", newText: "updated" }, content: [{ type: "text", text: "updated" }], structuredContent: { edited: true }, isError: false, }; assert.equal((await h.emit("tool_result", child))[0], undefined); if (transition === "disabled") h.setEnabled(false); else await h.emit("session_tree", { type: "session_tree" }); const parent = { toolName: "codemode", toolCallId: "parent", input: {}, isError: false, content: [{ type: "text", text: "done" }], structuredContent: { value: 0 }, }; const [ret] = await h.emit("tool_result", parent); const merged = mergeToolResult(parent, ret); assert.match(merged.content[0].text, /spec shipped at .*write a follow-up spec/); assert.deepEqual(merged, { ...parent, content: [merged.content[0], ...parent.content] }); assert.equal((await h.emit("tool_result", parent))[0], undefined); assert.equal(readFileSync(h.recordPath(), "utf8"), before, "sealed record stays frozen"); }); } const diffWarnings = (h: { readRecord: () => ReturnType }) => h.readRecord()!.events.filter((e) => e.kind === "warning").map((e: any) => e.message as string).filter((m) => m.startsWith("diff omitted")); const isBaseLog = (args: string[]) => args.includes("log") && !args.includes("--count"); test("jj ship (AC 1): keep path derives modified_files and diff through jj with pinned output flags", async () => { const h = await boundInJjShip(); const rec = h.readRecord(); assert.equal(rec.status, "shipped"); assert.equal(rec.derived.gates.ship_option, "keep"); const beforeWrite = readFileSync(h.recordPath(), "utf8"); const out = await h.writeSpec("doc/specs/a.md"); assert.deepEqual(out.filter(Boolean), [{ content: [{ type: "text", text: `⚠️ spec shipped at ${rec.shipped_at}; write a follow-up spec that supersedes it` }], structuredContent: undefined }]); assert.equal(readFileSync(h.recordPath(), "utf8"), beforeWrite); assert.equal(h.readRecord().derived.spec_edits_after_ship, 0); assert.deepEqual(rec.derived.modified_files, ["src/a.ts", "test/a.test.ts"]); assert.deepEqual(rec.derived.diff, { base: "jjbase0001", commits: 2, buckets: { code: { files: 1, insertions: 10, deletions: 2 }, test: { files: 1, insertions: 5, deletions: 0 } } }); assert.deepEqual(diffWarnings(h), []); const diffSteps = h.jjCalls.filter((a) => a[0] !== "root"); assert.equal(diffSteps.length, 3); assert.ok(diffSteps.every((a) => a.includes("--color=never"))); assert.deepEqual(diffSteps[1].slice(0, 4), ["--color=never", "--config", "diff.git.show-path-prefix=true", "diff"]); assert.ok(diffSteps[0].some((a) => a.includes("fork_point(") && a.includes("~ root() & ::"))); const before = readFileSync(h.recordPath(), "utf8"); await h.emit("session_compact", { type: "session_compact", reason: "manual" }); await h.emit("session_shutdown", { type: "session_shutdown", reason: "quit" }); assert.equal(readFileSync(h.recordPath(), "utf8"), before, "sealed on disk => frozen"); }); test("jj ship (AC 2): empty mainline -> jj-named warning, both derived fields absent, record written", async () => { const h = await boundInJjShip((args) => (isBaseLog(args) ? { code: 0, stdout: "", stderr: "" } : undefined)); const rec = h.readRecord(); assert.equal(rec.status, "shipped"); assert.equal(rec.derived.diff, undefined); assert.equal(rec.derived.modified_files, undefined); assert.deepEqual(diffWarnings(h), ["diff omitted: jj mainline unresolved (trunk() is root(); no main/master bookmark)"]); }); test("jj ship: jj diff nonzero exit -> first stderr line in the warning", async () => { const h = await boundInJjShip((args) => (args.includes("diff") ? { code: 1, stdout: "", stderr: "Error: Revision `jjbase0001` doesn't exist\nHint: try jj log\n" } : undefined)); assert.deepEqual(diffWarnings(h), ["diff omitted: jj diff failed: Error: Revision `jjbase0001` doesn't exist"]); assert.equal(h.readRecord().derived.diff, undefined); }); test("jj ship: ENOENT-shaped runner result (empty stderr) -> 'unknown error'", async () => { const h = await boundInJjShip((args) => (isBaseLog(args) ? { code: 127, stdout: "", stderr: "" } : undefined)); assert.deepEqual(diffWarnings(h), ["diff omitted: jj log failed: unknown error"]); assert.equal(h.readRecord().status, "shipped"); }); test("jj ship: unparseable patch -> warning, neither field set", async () => { const h = await boundInJjShip((args) => (args.includes("diff") ? { code: 0, stdout: "not a patch\n", stderr: "" } : undefined)); assert.deepEqual(diffWarnings(h), ["diff omitted: jj diff unparseable"]); const rec = h.readRecord(); assert.equal(rec.derived.diff, undefined); assert.equal(rec.derived.modified_files, undefined); }); test("jj ship: malformed patch header -> warning, neither field set, record shipped", async () => { const h = await boundInJjShip((args) => (args.includes("diff") ? { code: 0, stdout: "diff --git c/x b/x\n", stderr: "" } : undefined)); assert.deepEqual(diffWarnings(h), ["diff omitted: jj diff unparseable"]); const rec = h.readRecord(); assert.equal(rec.status, "shipped"); assert.equal(rec.derived.diff, undefined); assert.equal(rec.derived.modified_files, undefined); }); test("jj ship: --count failure -> jj log warning, no derived diff", async () => { const h = await boundInJjShip((args) => (args.includes("--count") ? { code: 1, stdout: "", stderr: "Error: invalid revset\nHint: check the mainline\n" } : undefined)); assert.deepEqual(diffWarnings(h), ["diff omitted: jj log failed: Error: invalid revset"]); const rec = h.readRecord(); assert.equal(rec.status, "shipped"); assert.equal(rec.derived.diff, undefined); assert.equal(rec.derived.modified_files, undefined); }); test("jj ship: unparseable --count -> warning with the raw output", async () => { const h = await boundInJjShip((args) => (args.includes("--count") ? { code: 0, stdout: "abc\n", stderr: "" } : undefined)); assert.deepEqual(diffWarnings(h), ["diff omitted: jj log --count unparseable: abc"]); assert.equal(h.readRecord().derived.diff, undefined); }); test("guard: write to a shipped spec during brainstorm is blocked before any state change; edit passes; other phases pass", async () => { const h = await boundInShip(); writeFileSync(h.recordPath(), serializeRecord({ ...h.readRecord(), status: "shipped", shipped_at: "2026-09-17T18:00:00Z" })); const before = readFileSync(h.recordPath(), "utf8"); const h2 = harness(); h2.ctx.cwd = h.root; h2.fetchedRecords.set("test/repo/brainstorming/doc/specs/a.yaml", before); await h2.phaseResult("start", P({ brainstorm: "in_progress" })); const [blocked] = await h2.emit("tool_call", { toolName: "write", toolCallId: "w1", input: { path: "doc/specs/a.md", content: "# again" } }); assert.deepEqual(blocked, { block: true, reason: guardReason("doc/specs/a.md", h.readRecord().shipped_at!, "refs/telemetry/records") }); assert.equal(readFileSync(h.recordPath(), "utf8"), before, "shipped record byte-identical"); const [editPass] = await h2.emit("tool_call", { toolName: "edit", toolCallId: "e1", input: { path: "doc/specs/a.md", edits: [] } }); assert.equal(editPass, undefined); const [otherPass] = await h2.emit("tool_call", { toolName: "write", toolCallId: "w2", input: { path: "doc/specs/b.md", content: "# new" } }); assert.equal(otherPass, undefined); const h3 = harness(); h3.ctx.cwd = h.root; await h3.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); const [planPass] = await h3.emit("tool_call", { toolName: "write", toolCallId: "w3", input: { path: "doc/specs/a.md", content: "# again" } }); assert.equal(planPass, undefined); }); for (const { engine, example } of [ { engine: "loop", example: false }, { engine: "waves", example: false }, { engine: undefined, example: false }, { engine: "waves", example: true }, { engine: undefined, example: true }, ]) { test(`brainstorm complete records bound spec engine ${engine ?? "legacy"}${example ? " with fenced loop example" : ""}, unrelated phases do not`, async () => { const h = harness(); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md", `# Spec\n${engine ? `**Engine:** ${engine}\n` : ""}${example ? "\n## Design\n```markdown\n**Engine:** loop\n```\n" : ""}`); await h.writeSpec("doc/specs/other.md", "**Engine:** waves\n"); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "in_progress" })); const rec = h.readRecord(); assert.deepEqual(rec.events.filter((e) => e.kind === "engine").map((e) => e.engine), [engine ?? "waves"]); assert.equal(rec.derived.engine, engine ?? "waves"); h.setBranch([ { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, { type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: "complete", phases: P({ brainstorm: "complete" }) } } }, { type: "message", message: { role: "toolResult", toolName: "plan_check", details: { status: "pass", specPath: "doc/specs/a.md" } } }, ]); for (const event of ["session_start", "session_switch", "session_fork", "session_tree"]) { await h.emit(event, {}); assert.deepEqual(h.readRecord().events.filter((e) => e.kind === "engine"), rec.events.filter((e) => e.kind === "engine")); } }); } test("resume skip reads engine from bound spec live and on branch replay", async () => { for (const replay of [false, true]) { const h = harness(); writeFileSync(join(h.root, "doc/specs/a.md"), "# Spec\n**Engine:** loop\n"); const started = P({ brainstorm: "in_progress" }); const skipped = { ...P({ brainstorm: "skipped" }), brainstorm: { status: "skipped", reason: "resume: doc/specs/a.md" } }; if (replay) { h.setBranch([started, skipped].map((phases, i) => ({ type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: i ? "skip" : "start", phases } } }))); await h.emit("session_start", {}); } else { await h.phaseResult("start", started); await h.phaseResult("skip", skipped); } assert.equal(h.readRecord().derived.engine, "loop"); assert.equal(h.readRecord().events.filter((e) => e.kind === "engine").length, 1); h.setBranch([started, skipped].map((phases, i) => ({ type: "message", message: { role: "toolResult", toolName: "phase_tracker", details: { action: i ? "skip" : "start", phases } } }))); const events = h.readRecord().events.filter((e) => e.kind === "engine"); for (const event of ["session_start", "session_switch", "session_fork", "session_tree"]) { await h.emit(event, {}); assert.deepEqual(h.readRecord().events.filter((e) => e.kind === "engine"), events); } } }); test("any successful tracker action counts named transitions once and ignores retries and amendments", async () => { const h = await boundInPlan(); const pt = h.planResult; const closed = [{ name: "a", status: "complete" }, { name: "b", status: "complete" }]; const reopened = [{ name: "b", status: "failed" }, { name: "a", status: "in_progress" }]; await pt("init", closed); await pt("refresh", reopened, true); assert.equal(h.readRecord().derived.gates.task_reopens, 0); await pt("refresh", reopened); await pt("refresh", reopened); await pt("status", [{ name: "a", status: "in_progress" }, { name: "b", status: "in_progress" }]); assert.equal(h.readRecord().derived.gates.task_reopens, 2); assert.equal(h.readRecord().events.filter((e) => e.kind === "gate" && e.gate === "task_reopen").length, 1); await pt("add", closed); await pt("refresh", [{ name: "a", status: "skipped" }, { name: "b", status: "pending" }]); await pt("clear", []); await pt("init", reopened); assert.equal(h.readRecord().derived.gates.task_reopens, 2); }); test("verification calls count from implement start through seal, reload and flush", async () => { const h = await boundInPlan(); await h.bash("before", "npm test"); assert.equal(h.readRecord().derived.verification_runs, undefined); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "in_progress" })); assert.equal(h.readRecord().derived.verification_runs, 0); await h.bash("implement", "npm test", true); assert.equal(h.readRecord().derived.verification_runs, 1); assert.equal(h.readRecord().derived.tests, undefined); await h.phaseResult("start", P({ brainstorm: "complete", plan: "complete", implement: "complete", verify: "in_progress" })); await h.bash("verify1", "npm test"); await h.bash("verify2", "cd x && npm test", true); await h.bash("unmatched", "echo verification: custom-check"); assert.equal(h.readRecord().derived.verification_runs, 3); assert.deepEqual(h.readRecord().derived.tests, { command: "npm test", result: "fail" }); const branch = [{ type: "message", message: { role: "toolResult", toolName: "phase_tracker", isError: false, details: { action: "start", phases: P({ brainstorm: "in_progress" }) } } }, { type: "message", message: { role: "toolResult", toolName: "phase_tracker", isError: false, details: { action: "skip", phases: { ...P({ brainstorm: "skipped", plan: "complete", implement: "complete", verify: "in_progress" }), brainstorm: { status: "skipped", reason: "resume: doc/specs/a.md" } } } } }]; const resumed = harness({ branch, sessionId: "s2" }); resumed.ctx.cwd = h.root; await resumed.emit("session_start", {}); await resumed.bash("resumed", "npm test"); assert.equal(resumed.readRecord().derived.verification_runs, 4); await resumed.phaseResult("start", P({ brainstorm: "skipped", plan: "complete", implement: "complete", verify: "complete", ship: "in_progress" })); await resumed.bash("ship", "npm test"); process.env.PI_SUBAGENT_DEPTH = "1"; try { await resumed.bash("child", "npm test"); } finally { process.env.PI_SUBAGENT_DEPTH = "0"; } await resumed.emit("session_shutdown", {}); assert.equal(resumed.readRecord().derived.verification_runs, 5); const sealed = { ...resumed.readRecord(), status: "shipped" as const, shipped_at: "2026-09-17T18:00:00Z" }; writeFileSync(resumed.recordPath(), serializeRecord(sealed)); const before = readFileSync(resumed.recordPath(), "utf8"); await resumed.bash("sealed", "npm test"); assert.equal(readFileSync(resumed.recordPath(), "utf8"), before); const unbound = harness(); await unbound.bash("unbound", "npm test"); assert.equal(existsSync(unbound.recordPath()), false); }); test("sealed loop fixture keeps dispatch iterations, skipped plan rows and verification runs", async () => { const h = harness({ jjWorkspace: true, gitFail: (args) => args[0] === "rev-parse" ? { code: 1, stdout: "", stderr: "not git" } : undefined }); await h.phaseResult("start", P({ brainstorm: "in_progress" })); await h.writeSpec("doc/specs/a.md", "# Spec\n\n**Engine:** loop\n\n## Scope\nSynthetic run.\n"); await h.phaseResult("complete", P({ brainstorm: "complete" })); await h.phaseResult("start", P({ brainstorm: "complete", plan: "skipped", implement: "in_progress" })); await h.planResult("refresh", [{ name: "row 1", status: "skipped" }, { name: "row 2", status: "complete" }, { name: "row 3", status: "in_progress" }]); await h.emit("tool_result", { toolName: "subagent", input: {}, details: { results: [{ agent: "loop-implementer", exitCode: 0 }] }, content: [], isError: false }); await h.planResult("refresh", [{ name: "row 1", status: "skipped" }, { name: "row 2", status: "complete" }, { name: "row 3", status: "complete" }]); await h.phaseResult("start", P({ brainstorm: "complete", plan: "skipped", implement: "complete", verify: "in_progress" })); await h.bash("verify", "npm test"); await h.phaseResult("start", P({ brainstorm: "complete", plan: "skipped", implement: "complete", verify: "complete", ship: "in_progress" })); await h.phaseResult("complete", P({ brainstorm: "complete", plan: "skipped", implement: "complete", verify: "complete", ship: "complete" })); const rec = h.readRecord(); assert.equal(rec.status, "shipped"); assert.equal(rec.derived.engine, "loop"); assert.equal(rec.derived.iterations, 1); assert.equal(rec.events.filter((e) => e.kind === "dispatch" && e.agent === "loop-implementer").length, 1); assert.deepEqual(rec.derived.plan, { tasks: 3, complete: 2, failed: 0, skipped: 1 }); assert.equal(rec.derived.verification_runs, 1); assert.equal(rec.derived.events_dropped, 0); });