/**
* pi-goal-list-loop-audit — v0.2.0
* extensions/goal-loop-shield.ts
*
* regression_shield — pure, dependency-free enforcement logic.
*
* When a goal has a verification contract, an verdict is only
* accepted if the auditor's report carries an section that
* references every contract item. This kills the "auditor ran bash true and
* approved" class of bamboozle that pi-goal-x's author explicitly documented
* as a known hole.
*
* Kept free of pi imports so unit tests can exercise it under plain node.
*/
import { resolveCanonicalRunnerCommand } from "./goal-loop-backoff.js";
import * as fs from "node:fs";
import * as path from "node:path";
/** Split a verification contract into its individual checkable items. */
export function contractItems(contract: string): string[] {
return contract
.split("\n")
.map((l) => l.trim())
.map((l) => l.replace(/^(?:done when|verify|verified when|verification|done)\s*:\s*/i, ""))
.map((l) => l.replace(/^[-*•]\s+/, "").replace(/^\d+[.)]\s+/, ""))
.filter((l) => l.length > 0)
// Boundary lines ("Out of scope: ...") constrain the auditor's judgment;
// they are not deliverables and have no evidence to quote (v0.22.6).
.filter((l) => !/^out of scope\b/i.test(l))
// Preamble lines are not checkable items (v0.23.4, darklord field bug:
// "Done when ALL of the following are true:" survived as an "item" —
// the prefix strip only fires when a colon directly follows "done
// when" — and the shield then blocked TWO genuine approvals forever,
// because no evidence can reference a preamble). Two mechanical
// predicates: a line still ending in a colon introduces a list, and a
// "(done when) (all of) the following ..." line IS the introducer.
.filter((l) => !l.endsWith(":"))
.filter((l) => !/^(?:done when\s+)?(?:all of\s+)?the following\b/i.test(l));
}
export interface RegressionShieldResult {
passed: boolean;
missingItems: string[];
hasEvidenceBlock: boolean;
}
/** Strip prose punctuation glued to a token ("file/element." → "file/element").
* v0.34.77 (GitHub #5): Unicode-aware — \p{L}\p{N} with the /u flag keeps
* CJK letters. The old ASCII-only class treated every Chinese character as
* punctuation, so a pure-Chinese token like 调研报告文件 shrank to nothing. */
function stripEdgePunct(w: string): string {
return w.replace(/^[^\p{L}\p{N}]+/u, "").replace(/[^\p{L}\p{N}/_.-]+$/u, "");
}
/**
* Is a candidate token present in the report? Compound tokens joined by
* "-" or "/" (left-cropped, file/element, Phaser/Svelte) count as present
* when ALL their segments (len >= 3) appear — a good-faith report writes
* "no cropped strip on the left", not the contract's literal compound.
*/
function tokenPresent(candidate: string, reportLower: string): boolean {
const c = candidate.toLowerCase();
if (reportLower.includes(c)) return true;
// v0.34.77 (GitHub #5): Han (CJK) tokens match by exact substring only —
// Chinese words have no compound-segment decomposition, so the ASCII
// segment rule below would wrongly reject a quoted 章节 line.
if (/\p{Script=Han}/u.test(c)) return reportLower.includes(c);
const segments = c.split(/[-/]+/).filter((s) => s.length >= 3);
return segments.length > 1 && segments.every((s) => reportLower.includes(s));
}
/** v0.34.77 (GitHub #5): punctuation-edge-normalized lowercase for the
* no-candidate fallback — a verbatim quote that drops the item's trailing
* full-width colon (章节: → 章节) still counts as a reference. */
function normalizeForMatch(s: string): string {
return s.toLowerCase().replace(/^[\p{P}\p{S}]+|[\p{P}\p{S}]+$/gu, "").replace(/\s+/g, " ");
}
/**
* Check an approved auditor report against the verification contract.
* Rules (deliberately simple + auditable):
* 1. The report must contain an ... block.
* 2. Every contract item must be referenced inside the report by ANY of
* its top-3 longest tokens (>= 5 chars, edge punctuation stripped;
* compounds match via their segments). v0.22.6: the previous
* single-longest-word rule false-rejected genuine approvals when the
* longest word was contract-only vocabulary ("left-cropped") or had
* prose punctuation glued on ("file/element.") — three real approved
* audits on hegemon were converted to disapprovals that way.
*/
export function checkRegressionShield(report: string, contract: string): RegressionShieldResult {
const evidenceMatch = /[\t\n\r ]*([\s\S]*?)<\/evidence>/i.exec(report);
const hasEvidenceBlock = evidenceMatch !== null;
const items = contractItems(contract);
const missingItems: string[] = [];
// Contract references must live inside the evidence block. Searching the
// whole report let an auditor satisfy the shield by repeating the contract
// in prose while leaving the evidence block empty or unrelated.
const evidenceLower = (evidenceMatch?.[1] ?? "").toLowerCase();
for (const item of items) {
// v0.34.77 (GitHub #5): Unicode-aware token split — \p{L}\p{N} treats
// CJK characters as letters, so a pure-Chinese contract line is ONE
// candidate token instead of a pile of delimiters.
const candidates = item
.split(/[^\p{L}\p{N}_.\-/]+/u)
.map(stripEdgePunct)
.filter((w) => w.length >= 5)
.sort((a, b) => b.length - a.length)
.slice(0, 3);
const addressed = candidates.length > 0
? candidates.some((c) => tokenPresent(c, evidenceLower))
: evidenceLower.includes(normalizeForMatch(item));
if (!addressed) missingItems.push(item);
}
return {
passed: hasEvidenceBlock && missingItems.length === 0,
missingItems,
hasEvidenceBlock,
};
}
/**
* v0.24.2: pure auditor-verdict parser (approved / disapproved / impossible).
* Lives here (not goal-loop-auditor.ts) so tests can import it without
* dragging in the auditor's relative .js imports. The final nonblank line
* is the only authoritative verdict location; prose tags are not verdicts.
*/
export function parseAuditorVerdict(output: string): { approved: boolean; disapproved: boolean; impossible: boolean; impossibleReason?: string } {
// A few RPC/test transports serialize newlines as literal `\\n` text;
// normalize that wire representation without relaxing the final-line gate.
const normalizedOutput = output.replaceAll("\\n", "\n").replaceAll("\\r", "\r");
const finalLine = normalizedOutput.split(/\r?\n/).map((line) => line.trim()).filter(Boolean).at(-1) ?? "";
const impossibleMatch = /^([\s\S]*?)<\/impossible>$/i.exec(finalLine);
// The final line is the only authoritative verdict location.
return {
approved: /^$/i.test(finalLine),
disapproved: /^$/i.test(finalLine),
impossible: impossibleMatch !== null,
impossibleReason: impossibleMatch?.[1]?.trim().slice(0, 300) || undefined,
};
}
/**
* v0.35.7: Extract mechanical shell command gates from a verification contract.
* Captures explicit commands (e.g. `npm test`, `tsc --noEmit`, `cargo test`)
* for deterministic fast-fail pre-auditing before spawning the heavy LLM worker.
*/
export function extractMechanicalCheckCommands(contract: string): string[] {
if (!contract) return [];
const items = contractItems(contract);
const commands: string[] = [];
for (const item of items) {
const backtickMatch = /`([^`]+)`/.exec(item);
let candidate = backtickMatch ? backtickMatch[1]!.trim() : item.trim();
if (!backtickMatch) {
// v0.35.24: "completes successfully" / "succeeds" joined the strip list
// after a field incident (2026-08-22): the contract line
// "bun run build completes successfully" survived stripping, was run
// verbatim, and vite parsed "completes" as its root directory —
// `Could not resolve entry module "completes/index.html"` — fast-failing
// an audit whose real gate (bun run build) was green.
candidate = candidate.replace(/\s+(?:passes(?:\s+cleanly|\s+with\s+zero\s+errors)?|exits\s+0|returns\s+0|cleanly|completes(?:\s+successfully)?|succeeds(?:\s+cleanly)?|successfully).*$/i, "").trim();
}
if (/^(?:npm\s+(?:test|run\s+[\w:-]+)|bun\s+(?:test|run\s+[\w:-]+)|pnpm\s+(?:test|run\s+[\w:-]+)|yarn\s+(?:test|[\w:-]+)|tsc\b|cargo\s+(?:test|check|build)|pytest\b|python\s+-m\s+unittest|go\s+test|vitest\b|jest\b|make\s+test|git\s+diff|test\s+-[a-z])/i.test(candidate)) {
commands.push(candidate);
}
}
return commands;
}
export interface MechanicalCheckResult {
passed: boolean;
failedCommand?: string;
output?: string;
exitCode?: number;
/** v0.35.20: set when the first attempt failed transiently and the single
* bounded retry passed — honest evidence of the wobble, not a silent mask. */
recoveredRetryNote?: string;
}
/**
* Mechanical checks are intentionally a small, shell-free command language.
* Contract text is not trusted input: accepting `npm test; ...` and passing it
* to a shell would turn the verification gate into arbitrary code execution.
*/
const SAFE_MECHANICAL_COMMAND = /^[A-Za-z0-9_./:@=+,-]+(?:[ \t]+[A-Za-z0-9_./:@=+,-]+)*$/;
export function isSafeMechanicalCommand(command: string): boolean {
return SAFE_MECHANICAL_COMMAND.test(command.trim());
}
/** v0.35.7: Execute mechanical pre-audit checks deterministically.
*
* v0.35.16: default timeout 60s → 10min. The old 60s ceiling killed
* LEGITIMATE long gates mid-run: this repo's own contract command,
* `npm run release:check`, needs ~3 minutes (full suite + tsc + Jiti smoke
* + pack), so every deterministic pre-audit fast-failed with a truncated
* head-of-output report that showed startup logs instead of any failure —
* twice in the field (2026-08-21 14:17 and 16:01 disapprovals), burning two
* auditor rounds on a gate that could never pass inside its own bound. The
* timeout still bounds genuinely hung commands; it no longer bounds honest
* slow ones. Failed output now keeps the TAIL, not the head — the end of a
* killed/failed run shows what was actually happening at death.
*/
export function runMechanicalPreAuditChecks(cwd: string, commands: string[], timeoutMs = 600_000): MechanicalCheckResult {
if (!commands || commands.length === 0) return { passed: true };
const { execFileSync } = require("node:child_process");
for (const rawCommand of commands) {
const cmd = rawCommand.trim();
if (!isSafeMechanicalCommand(cmd)) {
return {
passed: false,
failedCommand: rawCommand,
output: "Rejected unsafe mechanical command syntax; only a single executable followed by literal arguments is allowed.",
exitCode: 126,
};
}
const [rawProgram] = cmd.split(/[ \t]+/);
// v0.35.18: a contract that names a raw runner ("bun test") must run the
// project's CANONICAL invocation of it — package.json scripts encode the
// flags the suite requires (serialization/isolation/timeouts). Running
// the bare runner ignores that configuration and fails spuriously while
// the real gate is green (field: fourth audit round, 2026-08-21).
let scripts: Record = {};
try {
const pkg = JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf8"));
if (pkg && typeof pkg.scripts === "object" && pkg.scripts) scripts = pkg.scripts;
} catch { /* no package.json — nothing to resolve */ }
const { program, args } = resolveCanonicalRunnerCommand(cmd, scripts);
if (!program) {
return { passed: false, failedCommand: rawCommand, output: "Empty mechanical command.", exitCode: 126 };
}
// v0.35.20: ONE bounded automatic retry per failed mechanical command.
// Field (sixth audit round, 2026-08-21): the gate died MID-RUN under
// machine load ~30 (output ends inside a passing file, no runner
// summary, exit 1) while the identical tree passed green twice in
// isolation — resource contention, not a red suite. A deterministic
// contract command that genuinely fails stays red on both attempts;
// only transient deaths get a second chance. The retry is bannered in
// the returned evidence so the auditor sees it happened.
let firstFailure: { output: string; exitCode: number } | null = null;
let passed = false;
for (let attempt = 1; attempt <= 2 && !passed; attempt++) {
try {
// v0.35.25: pass an explicit maxBuffer. The default is 1 MB, and when
// a child's output exceeds it Node kills the child with SIGTERM and
// throws ENOBUFS — which this function then mislabels via the
// signal==="SIGTERM" banner as "killed after 600s" even though the
// child died in seconds. Field incident (2026-08-23, five auditor
// rounds on hellhunter): the gate `bun test src/lib/game` emits
// ~1.17 MB of (all-passing!) output and was unpassable by
// construction — both attempts died at ~1 MB with exit 1 while the
// identical tree passed green from an interactive shell 19/19
// times. 64 MB covers any realistic suite tail without enabling
// runaway memory.
execFileSync(program, args, { cwd, timeout: timeoutMs, stdio: ["ignore", "pipe", "pipe"], encoding: "utf-8", maxBuffer: 64 * 1024 * 1024 });
passed = true;
} catch (err: any) {
const exitCode = typeof err.status === "number" ? err.status : (typeof err.code === "number" ? err.code : 1);
const stdout = err.stdout ? String(err.stdout) : "";
const stderr = err.stderr ? String(err.stderr) : "";
const combined = (stdout + "\n" + stderr).trim() || err.message || "Command failed";
const killed = err.killed === true || err.signal === "SIGTERM" && err.code !== "ENOBUFS";
const banner = killed
? `[mechanical check killed after ${Math.round(timeoutMs / 1000)}s — output tail below]`
: "";
const body = combined.length > 4000 ? "…[truncated head]\n" + combined.slice(-4000) : combined;
const failureOutput = (banner ? banner + "\n" : "") + body;
if (attempt === 1) {
firstFailure = { output: failureOutput, exitCode };
} else {
return {
passed: false,
failedCommand: rawCommand,
output: firstFailure
? `[mechanical check retried once after a failed first attempt (exit ${firstFailure.exitCode}); second attempt also failed — output tail below]\n` + failureOutput
: failureOutput,
exitCode,
};
}
}
}
if (passed && firstFailure) {
// First attempt failed, second PASSED — recoverable transience; pass,
// but leave an honest trace of the wobble in the result.
return { passed: true, recoveredRetryNote: `[mechanical check: first attempt failed (exit ${firstFailure.exitCode}); automatic retry passed]` };
}
}
return { passed: true };
}