/** * pi-goal-list-loop-audit — v0.2.0 * extensions/goal-loop-shield.ts * * regression_shield — pure, dependency-free enforcement logic. * * When a goal has a verification contract, an verdict is only * accepted if the auditor's report carries an section that * references every contract item. This kills the "auditor ran bash true and * approved" class of bamboozle that pi-goal-x's author explicitly documented * as a known hole. * * Kept free of pi imports so unit tests can exercise it under plain node. */ import { resolveCanonicalRunnerCommand } from "./goal-loop-backoff.js"; import * as fs from "node:fs"; import * as path from "node:path"; /** Split a verification contract into its individual checkable items. */ export function contractItems(contract: string): string[] { return contract .split("\n") .map((l) => l.trim()) .map((l) => l.replace(/^(?:done when|verify|verified when|verification|done)\s*:\s*/i, "")) .map((l) => l.replace(/^[-*•]\s+/, "").replace(/^\d+[.)]\s+/, "")) .filter((l) => l.length > 0) // Boundary lines ("Out of scope: ...") constrain the auditor's judgment; // they are not deliverables and have no evidence to quote (v0.22.6). .filter((l) => !/^out of scope\b/i.test(l)) // Preamble lines are not checkable items (v0.23.4, darklord field bug: // "Done when ALL of the following are true:" survived as an "item" — // the prefix strip only fires when a colon directly follows "done // when" — and the shield then blocked TWO genuine approvals forever, // because no evidence can reference a preamble). Two mechanical // predicates: a line still ending in a colon introduces a list, and a // "(done when) (all of) the following ..." line IS the introducer. .filter((l) => !l.endsWith(":")) .filter((l) => !/^(?:done when\s+)?(?:all of\s+)?the following\b/i.test(l)); } export interface RegressionShieldResult { passed: boolean; missingItems: string[]; hasEvidenceBlock: boolean; } /** Strip prose punctuation glued to a token ("file/element." → "file/element"). * v0.34.77 (GitHub #5): Unicode-aware — \p{L}\p{N} with the /u flag keeps * CJK letters. The old ASCII-only class treated every Chinese character as * punctuation, so a pure-Chinese token like 调研报告文件 shrank to nothing. */ function stripEdgePunct(w: string): string { return w.replace(/^[^\p{L}\p{N}]+/u, "").replace(/[^\p{L}\p{N}/_.-]+$/u, ""); } /** * Is a candidate token present in the report? Compound tokens joined by * "-" or "/" (left-cropped, file/element, Phaser/Svelte) count as present * when ALL their segments (len >= 3) appear — a good-faith report writes * "no cropped strip on the left", not the contract's literal compound. */ function tokenPresent(candidate: string, reportLower: string): boolean { const c = candidate.toLowerCase(); if (reportLower.includes(c)) return true; // v0.34.77 (GitHub #5): Han (CJK) tokens match by exact substring only — // Chinese words have no compound-segment decomposition, so the ASCII // segment rule below would wrongly reject a quoted 章节 line. if (/\p{Script=Han}/u.test(c)) return reportLower.includes(c); const segments = c.split(/[-/]+/).filter((s) => s.length >= 3); return segments.length > 1 && segments.every((s) => reportLower.includes(s)); } /** v0.34.77 (GitHub #5): punctuation-edge-normalized lowercase for the * no-candidate fallback — a verbatim quote that drops the item's trailing * full-width colon (章节: → 章节) still counts as a reference. */ function normalizeForMatch(s: string): string { return s.toLowerCase().replace(/^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu, "").replace(/\s+/g, " "); } /** * Check an approved auditor report against the verification contract. * Rules (deliberately simple + auditable): * 1. The report must contain an ... block. * 2. Every contract item must be referenced inside the report by ANY of * its top-3 longest tokens (>= 5 chars, edge punctuation stripped; * compounds match via their segments). v0.22.6: the previous * single-longest-word rule false-rejected genuine approvals when the * longest word was contract-only vocabulary ("left-cropped") or had * prose punctuation glued on ("file/element.") — three real approved * audits on hegemon were converted to disapprovals that way. */ export function checkRegressionShield(report: string, contract: string): RegressionShieldResult { const evidenceMatch = /[\t\n\r ]*([\s\S]*?)<\/evidence>/i.exec(report); const hasEvidenceBlock = evidenceMatch !== null; const items = contractItems(contract); const missingItems: string[] = []; // Contract references must live inside the evidence block. Searching the // whole report let an auditor satisfy the shield by repeating the contract // in prose while leaving the evidence block empty or unrelated. const evidenceLower = (evidenceMatch?.[1] ?? "").toLowerCase(); for (const item of items) { // v0.34.77 (GitHub #5): Unicode-aware token split — \p{L}\p{N} treats // CJK characters as letters, so a pure-Chinese contract line is ONE // candidate token instead of a pile of delimiters. const candidates = item .split(/[^\p{L}\p{N}_.\-/]+/u) .map(stripEdgePunct) .filter((w) => w.length >= 5) .sort((a, b) => b.length - a.length) .slice(0, 3); const addressed = candidates.length > 0 ? candidates.some((c) => tokenPresent(c, evidenceLower)) : evidenceLower.includes(normalizeForMatch(item)); if (!addressed) missingItems.push(item); } return { passed: hasEvidenceBlock && missingItems.length === 0, missingItems, hasEvidenceBlock, }; } /** * v0.24.2: pure auditor-verdict parser (approved / disapproved / impossible). * Lives here (not goal-loop-auditor.ts) so tests can import it without * dragging in the auditor's relative .js imports. The final nonblank line * is the only authoritative verdict location; prose tags are not verdicts. */ export function parseAuditorVerdict(output: string): { approved: boolean; disapproved: boolean; impossible: boolean; impossibleReason?: string } { // A few RPC/test transports serialize newlines as literal `\\n` text; // normalize that wire representation without relaxing the final-line gate. const normalizedOutput = output.replaceAll("\\n", "\n").replaceAll("\\r", "\r"); const finalLine = normalizedOutput.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).at(-1) ?? ""; const impossibleMatch = /^([\s\S]*?)<\/impossible>$/i.exec(finalLine); // The final line is the only authoritative verdict location. return { approved: /^$/i.test(finalLine), disapproved: /^$/i.test(finalLine), impossible: impossibleMatch !== null, impossibleReason: impossibleMatch?.[1]?.trim().slice(0, 300) || undefined, }; } /** * v0.35.7: Extract mechanical shell command gates from a verification contract. * Captures explicit commands (e.g. `npm test`, `tsc --noEmit`, `cargo test`) * for deterministic fast-fail pre-auditing before spawning the heavy LLM worker. */ export function extractMechanicalCheckCommands(contract: string): string[] { if (!contract) return []; const items = contractItems(contract); const commands: string[] = []; for (const item of items) { const backtickMatch = /`([^`]+)`/.exec(item); let candidate = backtickMatch ? backtickMatch[1]!.trim() : item.trim(); if (!backtickMatch) { // v0.35.24: "completes successfully" / "succeeds" joined the strip list // after a field incident (2026-08-22): the contract line // "bun run build completes successfully" survived stripping, was run // verbatim, and vite parsed "completes" as its root directory — // `Could not resolve entry module "completes/index.html"` — fast-failing // an audit whose real gate (bun run build) was green. candidate = candidate.replace(/\s+(?:passes(?:\s+cleanly|\s+with\s+zero\s+errors)?|exits\s+0|returns\s+0|cleanly|completes(?:\s+successfully)?|succeeds(?:\s+cleanly)?|successfully).*$/i, "").trim(); } if (/^(?:npm\s+(?:test|run\s+[\w:-]+)|bun\s+(?:test|run\s+[\w:-]+)|pnpm\s+(?:test|run\s+[\w:-]+)|yarn\s+(?:test|[\w:-]+)|tsc\b|cargo\s+(?:test|check|build)|pytest\b|python\s+-m\s+unittest|go\s+test|vitest\b|jest\b|make\s+test|git\s+diff|test\s+-[a-z])/i.test(candidate)) { commands.push(candidate); } } return commands; } export interface MechanicalCheckResult { passed: boolean; failedCommand?: string; output?: string; exitCode?: number; /** v0.35.20: set when the first attempt failed transiently and the single * bounded retry passed — honest evidence of the wobble, not a silent mask. */ recoveredRetryNote?: string; } /** * Mechanical checks are intentionally a small, shell-free command language. * Contract text is not trusted input: accepting `npm test; ...` and passing it * to a shell would turn the verification gate into arbitrary code execution. */ const SAFE_MECHANICAL_COMMAND = /^[A-Za-z0-9_./:@=+,-]+(?:[ \t]+[A-Za-z0-9_./:@=+,-]+)*$/; export function isSafeMechanicalCommand(command: string): boolean { return SAFE_MECHANICAL_COMMAND.test(command.trim()); } /** v0.35.7: Execute mechanical pre-audit checks deterministically. * * v0.35.16: default timeout 60s → 10min. The old 60s ceiling killed * LEGITIMATE long gates mid-run: this repo's own contract command, * `npm run release:check`, needs ~3 minutes (full suite + tsc + Jiti smoke * + pack), so every deterministic pre-audit fast-failed with a truncated * head-of-output report that showed startup logs instead of any failure — * twice in the field (2026-08-21 14:17 and 16:01 disapprovals), burning two * auditor rounds on a gate that could never pass inside its own bound. The * timeout still bounds genuinely hung commands; it no longer bounds honest * slow ones. Failed output now keeps the TAIL, not the head — the end of a * killed/failed run shows what was actually happening at death. */ export function runMechanicalPreAuditChecks(cwd: string, commands: string[], timeoutMs = 600_000): MechanicalCheckResult { if (!commands || commands.length === 0) return { passed: true }; const { execFileSync } = require("node:child_process"); for (const rawCommand of commands) { const cmd = rawCommand.trim(); if (!isSafeMechanicalCommand(cmd)) { return { passed: false, failedCommand: rawCommand, output: "Rejected unsafe mechanical command syntax; only a single executable followed by literal arguments is allowed.", exitCode: 126, }; } const [rawProgram] = cmd.split(/[ \t]+/); // v0.35.18: a contract that names a raw runner ("bun test") must run the // project's CANONICAL invocation of it — package.json scripts encode the // flags the suite requires (serialization/isolation/timeouts). Running // the bare runner ignores that configuration and fails spuriously while // the real gate is green (field: fourth audit round, 2026-08-21). let scripts: Record = {}; try { const pkg = JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf8")); if (pkg && typeof pkg.scripts === "object" && pkg.scripts) scripts = pkg.scripts; } catch { /* no package.json — nothing to resolve */ } const { program, args } = resolveCanonicalRunnerCommand(cmd, scripts); if (!program) { return { passed: false, failedCommand: rawCommand, output: "Empty mechanical command.", exitCode: 126 }; } // v0.35.20: ONE bounded automatic retry per failed mechanical command. // Field (sixth audit round, 2026-08-21): the gate died MID-RUN under // machine load ~30 (output ends inside a passing file, no runner // summary, exit 1) while the identical tree passed green twice in // isolation — resource contention, not a red suite. A deterministic // contract command that genuinely fails stays red on both attempts; // only transient deaths get a second chance. The retry is bannered in // the returned evidence so the auditor sees it happened. let firstFailure: { output: string; exitCode: number } | null = null; let passed = false; for (let attempt = 1; attempt <= 2 && !passed; attempt++) { try { // v0.35.25: pass an explicit maxBuffer. The default is 1 MB, and when // a child's output exceeds it Node kills the child with SIGTERM and // throws ENOBUFS — which this function then mislabels via the // signal==="SIGTERM" banner as "killed after 600s" even though the // child died in seconds. Field incident (2026-08-23, five auditor // rounds on hellhunter): the gate `bun test src/lib/game` emits // ~1.17 MB of (all-passing!) output and was unpassable by // construction — both attempts died at ~1 MB with exit 1 while the // identical tree passed green from an interactive shell 19/19 // times. 64 MB covers any realistic suite tail without enabling // runaway memory. execFileSync(program, args, { cwd, timeout: timeoutMs, stdio: ["ignore", "pipe", "pipe"], encoding: "utf-8", maxBuffer: 64 * 1024 * 1024 }); passed = true; } catch (err: any) { const exitCode = typeof err.status === "number" ? err.status : (typeof err.code === "number" ? err.code : 1); const stdout = err.stdout ? String(err.stdout) : ""; const stderr = err.stderr ? String(err.stderr) : ""; const combined = (stdout + "\n" + stderr).trim() || err.message || "Command failed"; const killed = err.killed === true || err.signal === "SIGTERM" && err.code !== "ENOBUFS"; const banner = killed ? `[mechanical check killed after ${Math.round(timeoutMs / 1000)}s — output tail below]` : ""; const body = combined.length > 4000 ? "…[truncated head]\n" + combined.slice(-4000) : combined; const failureOutput = (banner ? banner + "\n" : "") + body; if (attempt === 1) { firstFailure = { output: failureOutput, exitCode }; } else { return { passed: false, failedCommand: rawCommand, output: firstFailure ? `[mechanical check retried once after a failed first attempt (exit ${firstFailure.exitCode}); second attempt also failed — output tail below]\n` + failureOutput : failureOutput, exitCode, }; } } } if (passed && firstFailure) { // First attempt failed, second PASSED — recoverable transience; pass, // but leave an honest trace of the wobble in the result. return { passed: true, recoveredRetryNote: `[mechanical check: first attempt failed (exit ${firstFailure.exitCode}); automatic retry passed]` }; } } return { passed: true }; }