// acceptance-evidence.ts — THE WORKER'S ACCEPTANCE ARTIFACT (KEY: AUTOPILOT-108). // // WHY THIS EXISTS. The review contract used to ask the reviewer to RE-RUN the // acceptance suite. In practice it USUALLY COULD NOT: a deps-less worktree // (AUTOPILOT-103: 43 environmental failures from `Cannot find package 'typebox'` // vs 903 pass / 0 fail on main WITH deps), a read-only review scope (no branch // tree to materialise), or a permission layer refusing `bun run` outright // (AUTOPILOT-101). Each reviewer improvised a DIFFERENT substitute — and every // substantive finding actually came from a DIFFERENT method (a byte-for-byte // projection compare, a Python re-implementation of a drift check, per-file // typecheck arithmetic, an importer grep). Re-running the same suite added cost // without adding that independence. // // So the LOAD-BEARING test evidence is the WORKER'S, captured as an AUDITABLE // ARTIFACT rather than prose: // * the EXACT command(s) run, // * the TREE (commit sha) they ran against, // * the RAW output (not a summary), // * the ENVIRONMENT — whether dependencies were present, the single most // common confounder (an environmental failure count is NOT a verdict on the // change), and // * for every LOAD-BEARING test claim, a NEGATIVE CONTROL: evidence that the // test FAILS without the change. (The ACTUATOR-78 redo did exactly this — // its leak test returned closes=0 with the fix reverted, which is why that // claim was trustworthy.) // // A claim with no artifact is UNVERIFIED — the reviewer must record it as // COULD NOT CHECK (src/review-contract.ts), never accept it by declaration. // // This module is the ONE definition of the artifact shape: the worker prompt // (agents/worker/canonical.md), the worker task text (workerTask in // framework/auto-dispatch.ts), and the harness's capture on worker completion // (core.ts) all reference it, so the format cannot drift between the instruction // and the reader. import type { CompletionEvent } from "./types.ts"; import { candidateTexts } from "./verdict.ts"; /** The block delimiters, one spelling shared by the worker instruction and the * parser (the same marker discipline as the spec-pass scope). */ export const ACCEPTANCE_EVIDENCE_START = "=== ACCEPTANCE EVIDENCE START ==="; export const ACCEPTANCE_EVIDENCE_END = "=== ACCEPTANCE EVIDENCE END ==="; /** The label introducing each REQUIRED field. Case-insensitive on read; the * prompt spells them in this exact form. */ export const ACCEPTANCE_FIELDS = { commands: "COMMAND", treeSha: "TREE", environment: "ENVIRONMENT", output: "OUTPUT", negativeControls: "NEGATIVE CONTROL", } as const; const LABEL_RE = /^(COMMAND|TREE|ENVIRONMENT|OUTPUT|NEGATIVE[ _]CONTROL)\s*:\s*(.*)$/i; /** A parsed acceptance artifact. All fields come from the worker's own output — * nothing here is inferred from the diff. */ export interface AcceptanceEvidence { /** The exact command(s) run (`COMMAND:` lines, in order). */ commands: string[]; /** The tree/commit the commands ran against (`TREE:` value). */ treeSha: string; /** Whether dependencies were present (`ENVIRONMENT:` value) — the confounder * that explains an environmental failure count. */ environment: string; /** The RAW output verbatim (`OUTPUT:` block), never a summary. */ output: string; /** One `NEGATIVE CONTROL:` line per load-bearing claim: evidence the test * FAILS without the change. */ negativeControls: string[]; } /** The text strictly between the START/END markers, or null when the markers * are missing/out of order/empty between them. Strict like the spec-pass scope * parse: a malformed block is no artifact, never a guessed one. */ function pickAcceptanceBlock(out: string): string | null { const startIdx = out.indexOf(ACCEPTANCE_EVIDENCE_START); if (startIdx === -1) return null; const afterStart = startIdx + ACCEPTANCE_EVIDENCE_START.length; const endIdx = out.indexOf(ACCEPTANCE_EVIDENCE_END, afterStart); if (endIdx === -1) return null; const body = out.slice(afterStart, endIdx).trim(); return body ? body : null; } /** Parse one block body into its labeled fields. `OUTPUT:` runs until the next * recognized label or the end of the block, so multi-line raw output survives * intact (everything else is a single trimmed value). */ function fieldsOf(body: string): AcceptanceEvidence { const lines = body.split(/\r?\n/); const commands: string[] = []; const negativeControls: string[] = []; const output: string[] = []; let treeSha = ""; let environment = ""; let current: "commands" | "treeSha" | "environment" | "output" | "negativeControls" | null = null; const push = (line: string): void => { if (line.trim()) { if (current === "commands") commands.push(line.trim()); else if (current === "negativeControls") negativeControls.push(line.trim()); else if (current === "output") output.push(line); } }; for (const raw of lines) { const label = raw.match(LABEL_RE); if (label) { const key = label[1].toUpperCase().replace(/[ _]/, " "); const value = label[2].trim(); if (key === "COMMAND") { current = "commands"; if (value) commands.push(value); } else if (key === "TREE") { current = "treeSha"; treeSha = value; } else if (key === "ENVIRONMENT") { current = "environment"; environment = value; } else if (key === "OUTPUT") { current = "output"; if (value) output.push(value); } else { current = "negativeControls"; if (value) negativeControls.push(value); } continue; } push(raw); } return { commands, treeSha, environment, output: output.join("\n").trim(), negativeControls }; } /** * The acceptance artifact in a completion event, read through the SAME * candidate order as verdict parsing (worker agents first, then every result, * then the event summary). Returns null when no correctly-marked block exists — * the caller then treats the worker's claims as UNVERIFIED rather than guessing. */ export function parseAcceptanceEvidence(ev: CompletionEvent, workerAgents: string[] = []): AcceptanceEvidence | null { for (const out of candidateTexts(ev, workerAgents)) { const body = pickAcceptanceBlock(out); if (body) return fieldsOf(body); } return null; } /** The REQUIRED fields an artifact is missing. Empty array = every required * field is present. null (no artifact at all) is one deficiency, never a pass. */ export function validateAcceptanceEvidence(ev: AcceptanceEvidence | null): string[] { if (!ev) { return [ "no acceptance-evidence artifact was captured (the worker must emit the " + `${ACCEPTANCE_EVIDENCE_START} … ${ACCEPTANCE_EVIDENCE_END} block)`, ]; } const missing: string[] = []; if (!ev.commands.length) missing.push(`${ACCEPTANCE_FIELDS.commands} (the exact command(s) run)`); if (!ev.treeSha.trim()) missing.push(`${ACCEPTANCE_FIELDS.treeSha} (the tree/commit sha they ran against)`); if (!ev.environment.trim()) missing.push(`${ACCEPTANCE_FIELDS.environment} (whether dependencies were present)`); if (!ev.output.trim()) missing.push(`${ACCEPTANCE_FIELDS.output} (the RAW output, not a summary)`); if (!ev.negativeControls.length) { missing.push(`${ACCEPTANCE_FIELDS.negativeControls} (evidence each load-bearing test FAILS without the change)`); } return missing; } /** APPEND-ONLY RECORD CAP, justified: queue.json is re-read by every tick and * panel render, so the artifact must stay a bounded addition per item. The * stored OUTPUT is a verbatim (never summarized) excerpt; when the raw output * is longer it is TRUNCATED with an explicit marker and the full text remains * in the worker run's own artifacts. */ export const ACCEPTANCE_EVIDENCE_CHARS = 2000; /** A bounded, persisted projection of a parsed artifact (what rides the item). */ export interface StoredAcceptanceEvidence extends AcceptanceEvidence { /** ISO timestamp of the capture. */ at: string; /** true when any field was cut at the cap. */ truncated: boolean; } /** Cap `text` to `limit`, marking a cut with a trailing ellipsis. */ function cap(text: string, limit: number): { text: string; truncated: boolean } { if (text.length <= limit) return { text, truncated: false }; return { text: text.slice(0, Math.max(1, limit - 1)) + "…", truncated: true }; } /** * Bound a parsed artifact for persistence: the small fields are kept whole, the * RAW OUTPUT is truncated at the cap (verbatim, not summarized), and the result * says so. Never throws. */ export function storedAcceptanceEvidence( ev: AcceptanceEvidence, at: string, maxChars = ACCEPTANCE_EVIDENCE_CHARS, ): StoredAcceptanceEvidence { const output = cap(ev.output, maxChars); return { ...ev, output: output.text, at, truncated: output.truncated, }; } /** The worker-facing instruction, ONE spelling shared by the worker prompt * (agents/worker/canonical.md) and the pi-subagents worker task text * (workerTask in framework/auto-dispatch.ts) — the two channels that reach the * two hosts. It states the artifact as REQUIRED and shows the literal block. */ export const WORKER_EVIDENCE_INSTRUCTION = [ "## Acceptance evidence (REQUIRED — KEY: AUTOPILOT-108)", "Your completion report MUST include a captured acceptance artifact between these EXACT marker lines, so the reviewer can AUDIT your test evidence instead of re-running the suite:", ACCEPTANCE_EVIDENCE_START, ` ${ACCEPTANCE_FIELDS.commands}: `, ` ${ACCEPTANCE_FIELDS.treeSha}: `, ` ${ACCEPTANCE_FIELDS.environment}: `, ` ${ACCEPTANCE_FIELDS.output}:`, " ", ` ${ACCEPTANCE_FIELDS.negativeControls}: `, ACCEPTANCE_EVIDENCE_END, "A claim with NO artifact is UNVERIFIED — do not assert a green suite you did not capture. If the environment cannot run the suite at all, say so in ENVIRONMENT; the reviewer will re-derive cheap static facts by other means and mark the rest COULD NOT CHECK.", ].join("\n"); /** The one-line marker an operator/the reviewer sees: either the captured * artifact summary or a plain statement that none was captured. Shared by the * review task text so the reviewer is never left guessing whether an artifact * exists. */ export function acceptanceEvidenceStatus(ev: StoredAcceptanceEvidence | null | undefined): string { if (!ev) return "NO acceptance-evidence artifact was captured from the worker's completion."; const truncated = ev.truncated ? " (output truncated in this excerpt)" : ""; return ( `worker's acceptance evidence (captured ${ev.at})${truncated}:\n` + ` TREE: ${ev.treeSha}\n` + ` ENVIRONMENT: ${ev.environment}\n` + ` COMMAND(S): ${ev.commands.join(" | ")}\n` + ` NEGATIVE CONTROL(S): ${ev.negativeControls.join(" | ")}\n` + ` OUTPUT:\n${ev.output}` ); }