/** * The settlement audit: checking a worker's own account of its work against the * pull request it actually pushed. * * The report's `changed:` line — the one fact a reviewer needs to use a * settlement report as a summary — is no longer hand-written at all. It is * derived at settlement from the pull request's own file list: {@link * deriveChangedLine} composes it and {@link withDerivedChangedLine} rewrites it * into the report, so the disclosure is the diff's, never a model's memory of * work that happened dozens of turns earlier (#488). Only the *narrative* — what * changed and why — remains the worker's to write. * * What this module still checks by reading a report is content, not disclosure. * `state: pushed-green` stopped being believed in #85 — a claim is now checked * against GitHub by `verifyPushedGreenClaim` rather than taken at face value — * and the epic's ranked item 7, "don't weaken tests you didn't write, checked * by diff review", is the other half: a diff-level check of the test files a PR * touched, where until now the reviewer was the worker reviewing itself * (#128). * * Everything here is **advisory**. A flag never changes a run's state, never * blocks a merge and never fails a run. The judgement a flag invites — "was * deleting that test correct?" — needs context this code does not have, so the * code's whole job is to put the evidence in front of whoever does. * * That posture fixes the accuracy bar, and it is asymmetric. A missed weakening * costs one unflagged PR that a reviewer might still catch. A false flag costs * attention on a *clean* run, and an audit that cries wolf gets ignored — at * which point it is worth less than nothing, because the fleet believes it is * being checked. So **every heuristic below resolves ambiguity towards * silence**, and each one names the blind spot it accepts to stay quiet. * * Pure by construction: the analyser takes a parsed diff and the dispatching * issue's text, and returns flags. Fetching the diff is the tracker's job * (`Tracker.prDiff`) — the same split `prStateFrom`/`prState` already use — so * every rule here is pinned by a recorded diff rather than by a live * repository. */ import type { FileLane, PrDiff, PrDiffFile, SettlementFlag, SettlementFlagKind } from "./types.ts"; // ------------------------------------------------------------------ diff parse const FILE_HEADER = "diff --git "; /** * Unquote a git path. Paths containing a space, a quote, or a non-ASCII byte are * emitted C-quoted (`"a/my file.ts"`), and only the escapes git actually writes * are undone — an unrecognised escape keeps its backslash rather than being * silently eaten, because a wrong path here is a flag naming a file that does * not exist. */ function unquote(raw: string): string { if (!raw.startsWith('"') || !raw.endsWith('"') || raw.length < 2) return raw; return raw .slice(1, -1) .replaceAll(/\\([\\"nt])/g, (_, ch: string) => ch === "n" ? "\n" : ch === "t" ? "\t" : ch, ); } /** * One path out of a diff header, or undefined for `/dev/null` (how git spells * "this side has no file"). * * `prefixed` is not cosmetic. `---`, `+++` and `diff --git` carry git's `a/` * and `b/` prefixes; `rename from` and `rename to` do not. Stripping two * characters off a rename header turns `a/old.ts` into `old.ts` — a path that * exists nowhere, on a flag that tells a reviewer to go and look at it. */ function diffPath(raw: string, prefixed: boolean): string | undefined { // Some producers append a tab and a timestamp; git does not, but a diff that // travelled through another tool might. const path = unquote(raw.split("\t")[0]?.trim() ?? ""); if (path === "" || path === "/dev/null") return undefined; return prefixed && (path.startsWith("a/") || path.startsWith("b/")) ? path.slice(2) : path; } /** * Both paths off a `diff --git` line. Only a fallback: the `---`/`+++` and * `rename to` lines are unambiguous and are preferred everywhere they exist. * This line is not parseable in general — `diff --git a/x b/y b/z` is genuinely * ambiguous when a path contains ` b/` — so it is used only for the header * shapes that carry nothing else (binary and mode-only changes). */ function headerPaths(line: string): { a?: string; b?: string } { const rest = line.slice(FILE_HEADER.length); const quoted = /^("(?:[^"\\]|\\.)*") ("(?:[^"\\]|\\.)*")$/.exec(rest); if (quoted?.[1] !== undefined && quoted[2] !== undefined) { return { a: diffPath(quoted[1], true), b: diffPath(quoted[2], true) }; } const mid = rest.indexOf(" b/"); if (mid < 0) return {}; return { a: diffPath(rest.slice(0, mid), true), b: diffPath(rest.slice(mid + 1), true) }; } function fileFrom(section: string[]): PrDiffFile | undefined { let status: PrDiffFile["status"] = "modified"; let previousPath: string | undefined; let renameTo: string | undefined; let minus: string | undefined; let plus: string | undefined; let hunkAt = -1; for (let i = 1; i < section.length; i++) { const line = section[i] ?? ""; // Header lines run until the first hunk; everything after is content, and // content can legitimately start with `--- ` or `+++ `. if (line.startsWith("@@")) { hunkAt = i; break; } if (line.startsWith("new file mode")) status = "added"; else if (line.startsWith("deleted file mode")) status = "removed"; else if (line.startsWith("rename from ")) { previousPath = diffPath(line.slice("rename from ".length), false); status = "renamed"; } else if (line.startsWith("rename to ")) { renameTo = diffPath(line.slice("rename to ".length), false); status = "renamed"; } else if (line.startsWith("--- ")) minus = diffPath(line.slice(4), true); else if (line.startsWith("+++ ")) plus = diffPath(line.slice(4), true); } const header = headerPaths(section[0] ?? ""); const path = plus ?? renameTo ?? (status === "removed" ? minus : undefined) ?? header.b ?? header.a; if (path === undefined) return undefined; const file: PrDiffFile = { path, status }; if (previousPath !== undefined && previousPath !== path) file.previousPath = previousPath; // The trailing newline of the diff itself lands in the last file's section; // it is not part of any hunk and would show up in every fixture comparison. if (hunkAt >= 0) file.hunks = section.slice(hunkAt).join("\n").replace(/\n+$/, ""); return file; } /** * Split a unified diff into per-file records. * * Kept here rather than in the GitHub adapter because the format is git's, not * GitHub's: a Gitea or local-file tracker producing the same text gets the same * parser, and the analyser below is pinned by raw diffs a human can read. */ export function parsePrDiff(raw: string, truncated = false): PrDiff { const files: PrDiffFile[] = []; let section: string[] | undefined; const close = (): void => { if (section === undefined) return; const file = fileFrom(section); if (file !== undefined) files.push(file); }; for (const line of raw.split("\n")) { if (line.startsWith(FILE_HEADER)) { close(); section = [line]; continue; } section?.push(line); } close(); return { files, truncated }; } /** One added or removed line of a hunk, with the line number of the side it * came from: the post-image for `+`, the pre-image for `-`. */ interface ChangedLine { added: boolean; text: string; line: number; } function* changedLines(hunks: string): Generator { let oldLine = 0; let newLine = 0; for (const raw of hunks.split("\n")) { const header = /^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(raw); if (header !== null) { oldLine = Number(header[1]); newLine = Number(header[2]); continue; } // A blank context line is a single space, but tooling that trims trailing // whitespace turns it into an empty string — treated as context so the // line numbers downstream stay aligned with the file. if (raw === "") { oldLine++; newLine++; continue; } const sign = raw[0]; if (sign === "+") { yield { added: true, text: raw.slice(1), line: newLine }; newLine++; } else if (sign === "-") { yield { added: false, text: raw.slice(1), line: oldLine }; oldLine++; } else if (sign === " ") { oldLine++; newLine++; } // `\ No newline at end of file` and any other marker advances neither side. } } // ------------------------------------------------------------- path vocabulary function basename(path: string): string { return path.slice(path.lastIndexOf("/") + 1); } /** * Files whose contents are data the audit must not read as code. A recorded * fixture, a golden file or a snapshot legitimately *contains* the text * `it.skip(` — that is what makes it a fixture — and this module's own test * suite is the first thing that would trip over it. */ const DATA_DIR = /(?:^|\/)(?:__fixtures__|fixtures|testdata|test-data|__snapshots__|snapshots|golden|recordings|cassettes)\//; /** Code this repository did not write, so weakening it is not this fleet's * "tests you didn't write" case — it is a vendored upstream. */ const VENDORED = /(?:^|\/)(?:node_modules|vendor|third_party|dist|build)\//; /** * Test files, by the naming conventions of the languages this loop has actually * dispatched work in. Deliberately conservative: a file this misses is a file * the weakening rules stay silent about, which is the safe direction. */ const TEST_FILE_NAME = /(?:^|\/)(?:test_[^/]+\.py|conftest\.py|[^/]+_test\.(?:py|go|rb|rs|ts|tsx|js|jsx)|[^/]+\.(?:test|spec)\.[cm]?[jt]sx?)$/; const TEST_DIR = /(?:^|\/)(?:tests?|__tests__|specs?)\//; function isTestPath(path: string): boolean { return TEST_FILE_NAME.test(path) || TEST_DIR.test(path); } // --------------------------------------------------------- derived disclosure /** * The `changed:` line is composed from the PR's own diff — every file it * touched, lockfiles included. Nothing is exempted, because nothing is being * held against the worker any more: the list is the diff's, so completeness * costs nothing. */ /** The `changed:` line of the settlement report, as the rewrite targets it. */ const CHANGED_LINE = /^[ \t>]*changed:[ \t]*(.*)$/im; /** * Any other report field heading: `word:` opening a line, possibly indented or * blockquote-prefixed (`> next:`). A `changed:` disclosure never continues * past one — the rewrite below ends its old-list scan at a following field, * so `next:`/`gates:` values stay attached to the report (#435). */ const FIELD_LINE = /^[ \t>]*[A-Za-z_][A-Za-z0-9_-]*\s*:/; /** * A token that reads as a path — the shape a hand-written file list is made * of. The rewrite scans a wrapped old disclosure line by line and stops where * the lines stop looking like paths: `omp/src/foo.ts` is a path, `(new)` and * `e.g` are prose. The last clause is not fussiness: without it `e.g`, `i.e` * and `0.3.18` would all count as paths and a wrapped list's boundary would * land one line too late. */ const CLAIMED_PATH = /^(?:[\w.@~+-]+\/)+[\w.@~*+-]*$|^[\w.@~+-]*\.[A-Za-z][A-Za-z0-9]{1,7}$/; /** A Python module name: `services.rule_purge`, `omp.routing` — a legitimate * way a report names a file by import path rather than location (#224), and * still a path-shaped token for the old-list scan. Every segment is at least * two characters so prose abbreviations (`e.g`, `i.e.`, `a.k.a`) stay * commentary. */ const DOTTED_MODULE = /^[A-Za-z_]\w+(?:\.[A-Za-z_]\w+)+$/; /** A numeric diff statistic — `+1/-1`, `+12/-3`, `1/-1`, a bare `+1`. It is * slash-shaped enough to look like a path — every segment is a signed * integer, which CLAIMED_PATH's character classes accept — but it is a count, * not a path: the wrapped-list scan must cut at it, exactly where it appears * in real reports (`omp/package.json (…, +1/-1; …)`, the #388 report, * verbatim). */ const DIFF_STAT = /^[+-]?\d+(?:\/[+-]?\d+)?$/; /** One-level brace expansion: `a/{b,c}.py` → [`a/b.py`, `a/c.py`]. Braces are * how a report compresses adjacent files into one token; each comma-separated * alternative is substituted into the skeleton. Nesting is deliberately * unsupported — one level matches the spellings reports actually produce. */ function expandBraces(token: string): string[] { const group = /\{[^{}]*\}/.exec(token); if (group === null) return [token]; const alternatives = group[0].slice(1, -1).split(",").filter((a) => a.length > 0); if (alternatives.length === 0) return [token]; const prefix = token.slice(0, group.index); const suffix = token.slice(group.index + group[0].length); return alternatives.map((alt) => prefix + alt + suffix); } /** A trailing slash-separated extension list: * `scripts/a.sh/.py` → [`scripts/a.sh`, `scripts/a.py`]. */ function expandExtensionAlternation(token: string): string[] { const match = /^(.+?)\.([A-Za-z][A-Za-z0-9]{1,7})((?:\/\.[A-Za-z][A-Za-z0-9]{1,7})+)$/.exec(token); if (match === null) return [token]; const [, base, first, rest] = match; if (base === undefined || first === undefined || rest === undefined) return [token]; return [first, ...rest.split("/.").filter(Boolean)].map((extension) => `${base}.${extension}`); } /** One cleaned token of a `changed:` line, as claim parsing sees it: leading * quotes and brackets stripped, trailing punctuation stripped, empties, the * word `none`, and numeric diff statistics dropped. */ function claimTokens(line: string): string[] { return line .split(/[\s;]+|,(?![^{}]*\})/) .map((raw) => raw.replace(/^[`'"([*-]+/, "").replace(/[`'")\].,:;]+$/, "")) .filter((token) => token !== "" && token !== "none" && !DIFF_STAT.test(token)); } /** Whether a cleaned token names at least one path once braces and extension * alternatives are expanded — the same filter the disclosure rewrite applies * to every continuation line it considers part of the old file list. */ function tokenNamesAPath(token: string): boolean { const normalised = token.replace(/^\.?\//, ""); return expandBraces(normalised).some((expanded) => expandExtensionAlternation(expanded).some((path) => CLAIMED_PATH.test(path) || DOTTED_MODULE.test(path)), ); } /** * The file list the PR's own diff reports, in diff order, comma-separated — * the settlement report's `changed:` line. Derived rather than asked for, so a * worker that never writes a file list still settles with a correct one, and a * report that contradicts the diff is rewritten to say what the diff says * (never the other way round). */ export function deriveChangedLine(diff: PrDiff): string { return diff.files.map((file) => file.path).join(", "); } /** * Rewrite the worker's hand-written `changed:` line with the derived one, * keeping the rest of the report — the narrative — verbatim. * * The old line and any wrapped continuation that still reads as a path list * (#435) are removed wholesale and replaced by a single derived line. A report * with no `changed:` line at all has one inserted before `next:`, so a settled * report always carries the disclosure. */ export function withDerivedChangedLine(report: string, changedLine: string): string { const lines = report.split("\n"); const start = lines.findIndex((line) => CHANGED_LINE.test(line)); const disclosure = `changed: ${changedLine}`; if (start === -1) { const next = lines.findIndex((line) => /^[ \t>]*next:/.test(line)); lines.splice(next === -1 ? lines.length : next, 0, disclosure); return lines.join("\n"); } // The old disclosure is the `changed:` line plus whatever following lines // still read as a wrapped path list: indented, non-blank, not a field heading, // not prose. Everything past it belongs to the report and survives. let end = start + 1; while (end < lines.length) { const line = lines[end] ?? ""; if (line.trim() === "") break; if (!/^[ \t>]/.test(line)) break; if (FIELD_LINE.test(line)) break; const tokens = claimTokens(line); if (tokens.length === 0 || !tokenNamesAPath(tokens[0] ?? "")) break; end++; } const prefix = /^[ \t>]*/.exec(lines[start] ?? "")?.[0] ?? ""; lines.splice(start, end - start, `${prefix}${disclosure}`); return lines.join("\n"); } // ------------------------------------------------------ test-weakening lexicon /** * Strip string literals and the trailing line comment, so a marker that is only * *mentioned* is not read as a marker that was *added*. `it("skip the retry")` * and `# it.skip(...) — see #91` are both silent after this. * * Single-line only, which is the accepted blind spot: a diff shows added lines * without the file around them, so a line inside a multi-line template literal * is indistinguishable from code. {@link DATA_DIR} covers the case that * actually occurs — recorded fixtures — and the anchored patterns below cover * most of the rest. */ function splitCode(line: string, hashComments: boolean): { code: string; comment: string } { let code = ""; let quote: string | undefined; for (let i = 0; i < line.length; i++) { const ch = line[i]; if (quote !== undefined) { if (ch === "\\") i++; else if (ch === quote) quote = undefined; continue; } if (ch === '"' || ch === "'" || ch === "`") { quote = ch; // A placeholder, not nothing: `it.skip("x", fn)` must still parse as a // call, and `describe(name)` must not collapse into `describe()`. code += '""'; continue; } if (ch === "/" && line[i + 1] === "/") return { code, comment: line.slice(i + 2) }; if (hashComments && ch === "#") return { code, comment: line.slice(i + 1) }; code += ch; } return { code, comment: "" }; } /** `#` opens a comment here, and `#[ignore]` does not exist. Rust and TypeScript * are excluded on purpose: `#[ignore]` and `#private` are code. */ const HASH_COMMENTS = /\.(?:py|rb|sh|bash|yml|yaml|toml|cfg|pl)$/; /** * A disabled or focused test. Anchored on a runner identifier at the start of * the statement, which is what keeps `iterator.skip(3)` and `{ skip: false }` * out — an unanchored `\.skip\(` flags both. * * The marker must then be *called* (`it.skip(`) or chained into one * (`test.skip.each(`). Accepting a bare `.` after it looked harmless and was * not: `it("x", () => { plan.todo.push(1) })` reads as a todo-marked test. */ const DISABLED_TEST: readonly RegExp[] = [ /^(?:await\s+)?(?:it|test|describe|context|suite|bench)\b.*?\.\s*(?:skip|only|todo|failing)\s*(?:\(|\.\s*(?:each|for)\b)/, /^(?:await\s+)?[xf](?:it|test|describe|context)\s*\(/, /^@(?:pytest\.mark\.(?:skip|skipif|xfail)|unittest\.skip)/, /^#\[ignore\b/, // Go's is a statement anywhere in a line — `if testing.Short() { t.Skip() }` // is the idiomatic form — and the `t.` receiver is specific enough to anchor // on instead. /\bt\.Skip(?:f|Now)?\s*\(/, ]; /** * An assertion, as a whole statement. Anchored for the same reason as above: a * mid-line `expect` is usually a helper's name or a comment about one. */ const ASSERTION = /^(?:await\s+)?(?:expect|assert|assert_[a-z_]+|assertEquals?|assertTrue|assertFalse|assertThat|assertRaises|assertRaisesRegex|self\.assert[A-Za-z]*|should|chai\.|t\.(?:Error|Fatal)f?|require\.[A-Z][A-Za-z]*|Expect)\s*[.(]/; /** * A named timeout and its value, but only on the test runner's own timeout * surface ({@link onRunnerSurface}) — a matching key handed to the code under * test is a domain parameter, not a runner deadline (#1062). Only a *raised* * one is a finding — a brand-new timeout on a new test is not a weakening — so * the value is compared against the same key on the pre-image side and silence * is the answer whenever the key appears on only one side. */ const TIMEOUT = /\b(timeout|timeoutMs|timeout_ms|timeoutSeconds|deadline|maxDuration|wallClock(?:Ms)?|setTimeout|jest\.setTimeout|retries)\b\s*([:=(])\s*(\d[\d_]*)/gi; /** Identifiers that name the test runner itself when a timeout-shaped key is * called on or passed to them. `t` is the test context (vitest, node:test, a * Go `*testing.T` helper); `pytest` names the Python runner module. */ const RUNNER_BINDINGS: Record = { test: true, it: true, describe: true, context: true, suite: true, bench: true, jest: true, t: true, pytest: true, }; /** * Whether a {@link TIMEOUT} match sits on the runner's own timeout surface of * its line, as opposed to an argument handed to the code under test. * * The audit sees hunks, not files, so the judgement is structural on the * changed line alone, and anything it cannot vouch for stays silent: a flag is * advisory, and a false positive costs the trust in every flag after it * (measured: #614/#896 read a raised `timeoutMs` passed to `arm(...)` — the * challenge window of the code under test — as a weakened test). * * Two positions qualify, both on the runner's own call: * * - the key is called on the runner — `jest.setTimeout(...)`, * `t.timeout(...)`, `test.setTimeout(...)`, `pytest.mark.timeout(...)` — * the dotted receiver before the key starts with a runner binding; * - the key is a property of an object literal that is a direct argument of a * runner call — the trailing per-test configuration: `test("x", fn, { * timeout: 45_000 })`, `test.use({ retries: 3 })`, * `describe.configure({ retries: 3 })`. * * Everything else is the code under test: `await arm({ timeoutMs: 60_000 })`, * `const opts = { timeout: 45000 }`, `server.setTimeout(30_000)`. A shape * whose decisive frame is on another line — `}, { timeout: 45000 });` after a * multi-line `test(` — cannot be vouched for from the line alone and also * stays silent. */ function onRunnerSurface(code: string, match: RegExpMatchArray): boolean { const key = match[1]?.toLowerCase() ?? ""; if (RUNNER_BINDINGS[key.split(".")[0] ?? ""] === true) return true; const before = code.slice(0, match.index); // The dotted receiver the key is called on: `t.timeout(5000)` reads `t.`, // `pytest.mark.timeout(500)` reads `pytest.mark.`. No receiver — a bare // `timeout: 5000` property — falls through to the config-object rule. let at = before.length - 1; while (at >= 0 && /[A-Za-z0-9_$.]/.test(before[at] ?? "")) at--; const receiver = before.slice(at + 1).replace(/\.$/, "").split(".")[0]; if (RUNNER_BINDINGS[receiver ?? ""] === true) return true; // A property key (`key: value`) needs the object around it to be the // runner's own configuration; a `key = value` assignment or a bare // `key(5000)` call on the line is never that. if (match[2] !== ":") return false; return inRunnerConfigObject(before); } type RunnerFrame = { kind: "call"; base: string | undefined } | { kind: "obj" }; /** Whether the key sits in an object literal that is a direct argument of a * call on a runner binding — the trailing per-test configuration: * `test("x", fn, { timeout: 45_000 })`, `test.use({ retries: 3 })`, * `describe.configure({ retries: 3 })`. The frames are walked over the line * prefix only, with strings already stripped by {@link splitCode}, so the * object's nesting inside the call — a direct argument versus a property of * a nested object or of the callback's own body — decides the verdict. */ function inRunnerConfigObject(before: string): boolean { const frames: RunnerFrame[] = []; for (let at = 0; at < before.length; at++) { const ch = before[at]; if (ch === "(") { // The dotted name the call was opened on, if any: `test.use(` reads // `test.use`. No name — an arrow's parameter list, or a call on a // previous call's result — means the object below it is not a runner // surface. let end = at; while (end > 0 && /[A-Za-z0-9_$.]/.test(before[end - 1] ?? "")) end--; const chain = before.slice(end, at); frames.push({ kind: "call", base: chain.length > 0 && !chain.endsWith(".") ? chain : undefined, }); continue; } if (ch === "{" || ch === "[") { frames.push({ kind: "obj" }); continue; } if (ch === ")" || ch === "}" || ch === "]") frames.pop(); } const object = frames.at(-1); if (object?.kind !== "obj") return false; const enclosing = frames.at(-2); if (enclosing?.kind !== "call") return false; const firstSegment = enclosing.base?.split(".")[0]; return firstSegment !== undefined && RUNNER_BINDINGS[firstSegment] === true; } function timeouts(text: string): { key: string; value: number }[] { const found: { key: string; value: number }[] = []; for (const match of text.matchAll(TIMEOUT)) { const key = match[1]?.toLowerCase(); const raw = match[3]?.replaceAll("_", ""); if (key === undefined || raw === undefined) continue; if (!onRunnerSurface(text, match)) continue; const value = Number(raw); if (Number.isSafeInteger(value)) found.push({ key, value }); } return found; } // ------------------------------------------------------------------- attribute /** * Whether the dispatching issue named this file. A flag on a file the issue * never mentions is the "tests you didn't write" case and is louder, so the * match is lenient in the direction that produces *fewer* loud flags: the bare * basename counts, and a common one like `index.test.ts` will match almost any * issue that mentions a test at all. * * Deliberately *not* the containing directory. That rule was tried and it * attributes everything: this package's entire source lives under `omp/src`, * so any issue naming one file there would vouch for every test file in the * repository, and the mark would stop distinguishing anything. */ function namedByIssue(issueText: string, file: PrDiffFile): boolean { const haystack = issueText.toLowerCase(); for (const path of [file.path, file.previousPath]) { if (path === undefined) continue; const lower = path.toLowerCase(); if (haystack.includes(lower) || haystack.includes(basename(lower))) return true; } return false; } /** Evidence text is a source line, and a source line can be minified. */ const EVIDENCE_LIMIT = 120; function evidence(text: string): string { const trimmed = text.trim(); return trimmed.length <= EVIDENCE_LIMIT ? trimmed : `${trimmed.slice(0, EVIDENCE_LIMIT - 1)}…`; } // -------------------------------------------------------------------- analyser export interface SettlementAudit { /** The dispatching issue's title and body — the attribution source. */ issueText: string; diff: PrDiff; /** The pull request body, or undefined when the tracker could not read one. * The claims audited below live here, so absent means no proof commands are * extracted — the flag family stays silent, never cries wolf on nothing. */ prBody?: string; /** The run's session transcript (the raw JSONL), or undefined when the run * had none or it could not be read. Absent means no claimed command can be * checked — same silence, for the same reason. */ transcript?: string; /** * The effective file lane admission resolved for this run at dispatch * (`effectiveLane` — a pre-dispatch comment declaration supersedes an older * body one) and persisted on the row (#744/#758): the exact snapshot the * gate enforced and the worker brief rendered. Absent means the issue was * admitted with no declaration (fail open) — never "empty lane" — and no * diff can then be outside it. */ lane?: FileLane; } /** * Every advisory finding about one settled pull request, in a stable order * (test weakening in diff order). * * The file-list disclosure is not audited here at all: it is derived from * `diff` itself ({@link deriveChangedLine}), so the `undisclosed-file` / * `unmatched-claim` reconciliation flags are structurally gone. What remains * is the diff-content check a human is worst placed to do by hand. * * An empty array is the expected result for an honest run, and is load-bearing: * the settlement report says nothing at all when this returns nothing. */ export function analyseSettlement(audit: SettlementAudit): SettlementFlag[] { const flags: SettlementFlag[] = []; detectWeakening(audit, flags); detectClaimedProof(audit, flags); detectLaneEscape(audit, flags); return flags; } /** * The one finding a settlement can carry without a diff at all. `changed-line-missing` * used to mean "the worker's report disclosed nothing"; with the list derived * from the PR's own diff it can only mean the tree could not be read — a real * fault worth surfacing, never an empty list that reads as a clean bill. */ export const UNREADABLE_TREE_FLAG: SettlementFlag = { kind: "changed-line-missing", file: "(report)", detail: "the settlement could not read the PR's diff, so no `changed:` file list could be derived", }; // ---------------------------------------------------------- declared file lane /** * Whether a diff path counts as inside the declared lane: an explicitly * declared path, the co-located test of one, or a descendant of a declared * directory (#1091) — admission's grammar accepts a trailing-`/` entry * ("everything under `src/`"), so settlement honours the same contract rather * than narrowing it back to exact files. The co-located allowance is the * "obviously intended" case: `foo.ts` vouches for `foo.test.ts`. Other test * shapes (`.spec.ts`, pytest's `test_` prefix) are not vouched for: the rule * is the shape the fleet actually uses, and a lane that wants a * differently-shaped sibling declares it. Deliberately one-directional: a lane * that declares a *test* file does not vouch for its source, because declaring * the test alone is a narrower promise and widening it silently is exactly * what this flag exists to name. The directory rule keeps its trailing-slash * boundary explicit: the entry already carries the separator, so `foo/` * covers `foo/bar.ts` at any depth and can never vouch for `foobar/x.ts`. */ function withinLane(path: string, declared: readonly string[]): boolean { if (declared.includes(path)) return true; for (const d of declared) { if (d.endsWith("/") && path.startsWith(d)) return true; if (coLocatedTest(d) === path) return true; } return false; } /** `omp/src/verbs/server.ts` → `omp/src/verbs/server.test.ts`; undefined when * the declared path has no file extension to splice before, since the * co-located-test shape is defined by an extension. */ function coLocatedTest(declared: string): string | undefined { const dot = declared.lastIndexOf("."); const slash = declared.lastIndexOf("/"); if (dot <= slash + 1) return undefined; return `${declared.slice(0, dot)}.test${declared.slice(dot)}`; } /** * A diff that escapes its issue's declared file lane (#739). * * The lane is the effective declaration admission resolved at dispatch — * `effectiveLane(body, comments)`, so a pre-dispatch comment beats an older * body declaration — and the flag uses that resolved snapshot, never a re-parse * of the body. Each finding names the delivered files outside it, which is the * part a reviewer is worst placed to notice: the diff's own file list is the * only surface that shows the escape, and reading PR file lists by hand is * exactly what nothing else in the loop does. * * An escape splits into up to two findings because it is up to two different * events (#1096): an undeclared production module is scope escaping into code, * while an undeclared test file is usually coverage arriving beside the * behaviour it pins — the #1064/#1080 shape, required work the declaration * could not have named. Naming the class on each finding keeps the second * readable as what it usually is without letting it disguise the first. The * declared source's own `.test` twin is admitted by {@link withinLane}, so * the obvious case raises nothing — and the rule stays one-directional, so a * lane declaring only the test still names its source when the worker edits * it. * * Fail-open, like admission: an issue with no lane declaration has nothing to * escape, so no flag — a flag on every undeclared run would be noise within a * day, worse than no flag. Advisory like every other flag here: a widened lane * is often legitimate, and the orchestrator is the judge this exists to brief. */ function detectLaneEscape(audit: SettlementAudit, flags: SettlementFlag[]): void { const lane = audit.lane; if (lane === undefined || lane.files.length === 0) return; const outside = audit.diff.files .map((f) => f.path) .filter((path) => !withinLane(path, lane.files)); if (outside.length === 0) return; // Production first: the rarer, heavier event leads the report block. const production = outside.filter((path) => !isTestPath(path)); const otherTests = outside.filter((path) => isTestPath(path)); if (production.length > 0) { flags.push({ kind: "lane-escape", file: "(lane)", detail: `PR diff touches production files outside the declared file lane: ${production.join(", ")}`, }); } if (otherTests.length > 0) { flags.push({ kind: "lane-escape", file: "(lane)", detail: `PR diff touches unrelated test files outside the declared file lane: ${otherTests.join(", ")}`, }); } } function detectWeakening(audit: SettlementAudit, flags: SettlementFlag[]): void { // A test file that left one path and arrived at another is a move, not a // deletion. `status: renamed` covers the renames git detected; the basename // pairing covers the ones it did not, because a move plus a heavy edit is // emitted as an unrelated delete and add. It can hide a genuine deletion that // happens to share a basename with an added file — accepted, as a false // negative for a false positive on every legitimate move. const arrived = new Set( audit.diff.files .filter((f) => f.status === "added" && isTestPath(f.path)) .map((f) => basename(f.path)), ); for (const file of audit.diff.files) { // A recorded fixture legitimately *contains* `it.skip(`; that is what makes // it a fixture. Vendored trees are somebody else's tests entirely. if (DATA_DIR.test(file.path) || VENDORED.test(file.path)) continue; if (!isTestPath(file.path) && !isTestPath(file.previousPath ?? "")) continue; const unattributed = namedByIssue(audit.issueText, file) ? undefined : true; const mark = (flag: Omit): void => { flags.push(unattributed === undefined ? flag : { ...flag, unattributed }); }; if (file.status === "removed") { if (arrived.has(basename(file.path))) continue; mark({ kind: "test-file-deleted", file: file.path, detail: "test file deleted, and no rename in this PR accounts for it", }); // Its every line is a removal; listing them as separate findings would // bury the one that matters. continue; } if (file.hunks === undefined) continue; scanHunks(file, file.hunks, mark); } } function scanHunks( file: PrDiffFile, hunks: string, mark: (flag: Omit) => void, ): void { const hashComments = HASH_COMMENTS.test(file.path); const disabledBefore = new Set(); const disabledAfter: { line: number; text: string }[] = []; const commentedAssertions: { line: number; text: string }[] = []; const timeoutBefore = new Map(); const timeoutAfter = new Map(); let assertionsAdded = 0; let assertionsRemoved = 0; let firstRemovedAssertion: { line: number; text: string } | undefined; for (const change of changedLines(hunks)) { const { code, comment } = splitCode(change.text, hashComments); const statement = code.trim(); const disabled = DISABLED_TEST.some((re) => re.test(statement)); if (disabled) { if (change.added) disabledAfter.push({ line: change.line, text: change.text }); // Normalised, so a marker that only moved or was re-indented is not new. else disabledBefore.add(statement.replaceAll(/\s+/g, " ")); } if (ASSERTION.test(statement)) { if (change.added) assertionsAdded++; else { assertionsRemoved++; firstRemovedAssertion ??= { line: change.line, text: change.text }; } } // A commented-out assertion is unambiguous — nobody does it by accident — // but only when the comment body is a whole statement. `// expect() is // called by the helper` is prose and must stay silent, so the body has to // end the way a statement ends. const commented = comment.trim(); if (change.added && ASSERTION.test(commented) && /[;)]$/.test(commented)) { commentedAssertions.push({ line: change.line, text: commented }); } for (const { key, value } of timeouts(code)) { if (change.added) { const prior = timeoutAfter.get(key); if (prior === undefined || value > prior.value) { timeoutAfter.set(key, { value, line: change.line, text: change.text }); } } else { timeoutBefore.set(key, Math.max(timeoutBefore.get(key) ?? 0, value)); } } } for (const found of disabledAfter) { if (disabledBefore.has(splitCode(found.text, hashComments).code.trim().replaceAll(/\s+/g, " "))) { continue; } mark({ kind: "test-disabled", file: file.path, line: found.line, detail: `added \`${evidence(found.text)}\``, }); } for (const found of commentedAssertions) { mark({ kind: "assertions-removed", file: file.path, line: found.line, detail: `assertion commented out: \`${evidence(found.text)}\``, }); } // Net, not gross. A rewritten test moves its assertions, and counting removals // alone flags every honest refactor — measured against this package's own // history, that is most test edits. if (assertionsRemoved > assertionsAdded && firstRemovedAssertion !== undefined) { mark({ kind: "assertions-removed", file: file.path, line: firstRemovedAssertion.line, detail: `${assertionsRemoved - assertionsAdded} more assertion(s) removed than added ` + `(first: \`${evidence(firstRemovedAssertion.text)}\`)`, }); } for (const [key, after] of timeoutAfter) { const before = timeoutBefore.get(key); if (before === undefined || after.value <= before) continue; mark({ kind: "test-timeout-raised", file: file.path, line: after.line, detail: `\`${key}\` raised from ${before} to ${after.value}`, }); } } // ------------------------------------------------------- claimed proof commands /** * The proof claims a PR body makes, checked against the run's transcript. * * A worker's Verified section is what the orchestrator merges on, and until * this check nothing compared its claims against what the session actually ran * (#582). The audit is the same family as the test-weakening rules: advisory, * never a state change, and resolving ambiguity towards silence. Every PR * body carries backticked prose, so the false-positive pressure here is real — * an extractor that reads any backtick span in the narrative as a claim would * flag every run within a day. * * Four tolerances make the matcher honest rather than decorative: * * - Claims are extracted only from a verified section — a `## Verified`-style * heading or an inline `Verified:` label — never from the narrative. This is * the second named fake of the issue (one-heading parsing), asserted for * both shapes below. * - Matching is on meaningful shell tokens, not string equality: a claimed * `bun test src/worker.test.ts` is satisfied by a transcript invocation of * `cd …/omp && bun test src/worker.test.ts src/session-host.test.ts`, * because a `cd` pair, shell operators and wrappers are dropped from both * sides and the claim's tokens must then be a subset of one invocation's. * Flag order is thereby free, which string equality could never be. * - A claimed command whose invocation the transcript shows was refused by the * shared-host guard is `claimed-proof-blocked`, not `claimed-proof-missing`: * the worker *tried* and the guard stopped it (the honest #566/#570 shape, * whose own words are "CI owns them"), which is not the same finding as a * claim with no attempt at all. * - Claims are positions before they are texts (#1102): a unit of the region * claims what it opens with — a bullet or bare line whose lead is the * backticked command, or a `**Verified:**` label declaring its whole row — * never a span quoted mid-sentence. The soft-wrapped prose that quoted * `issue is closed` inside #1097's Verified bullet is commentary whatever * words it quotes, and the structural rule says so without a runner * vocabulary that would silently stop auditing the first PR proved by * anything else. */ /** The verified-section markers a claim must live under: a heading whose text * contains a `veri*`/`proof` root (`## Verified`, `## Verification`, * `## How verified`, `## Verification & proof`), or an inline * `Verified:` / `**Verified:**` label row. `## Tests` is deliberately not * one — a section describing the diff's test impact is narrative, not proof. */ const VERIFIED_MARKER = /^(?:#{1,6}\s+.*\b(?:verif|proof)\w*\b.*$|^\s*(?:\*\*)?(?:Verif(?:ied|ication)|Proof)(?:\*\*)?\s*[:.])/im; /** * The slice of a PR body that can carry proof claims: from the first * verified-section marker to the end of its block. Claims live under backticks, * so the region is what keeps narrative code spans out of the audit — a * backticked `retry.modelFallback: true` in a design explanation is commentary, * never a command somebody claimed to run. */ function claimRegion(body: string): string { const marker = VERIFIED_MARKER.exec(body); if (marker === null) return ""; const start = marker.index; const rest = body.slice(start); // The next-heading scan starts after the marker's own match, or `## Verified` // would cut its own block to nothing — the marker line is a heading too. const after = rest.slice(marker[0].length); const next = /^#{1,6}\s+/m.exec(after); return next === null ? rest : rest.slice(0, marker[0].length + next.index); } /** * An executable token: the shape a claimed command's first word must have to * be a program somebody could have run. Either an ordinary command name — an * ASCII letter or `_` first, then letters, digits, `_`, `.`, `+`, `-` (`bun`, * `bash`, `cd`, `export`, `omp-conductor`, `python3.12`) — or a `./`-relative * executable path built from those same characters plus `/` * (`./scripts/deploy.sh`). * * Nothing else is a command, and that exclusion is the whole point (#1039): * this predicate used to accept any first word that merely lacked a `/`, so * the malformed-JSON fixture `{not json` described in a PR's verified prose * became a "claimed" command with no transcript match, and a green PR settled * with a false `claimed-proof-missing`. The grammar therefore refuses * punctuation-led fragments (`{not`, `{"kind":`, `[1,`), quote-led values * (`"some value"`), tokens carrying `:` or `=` (config expressions like * `retry.modelFallback: true`, env assignments), digit-led data literals * (timestamps like `2026-08-24 09:14Z`) and flag-led spans (`--json`). An * absolute path stays excluded as it always was: it carries `/` without the * leading `./`. */ const EXECUTABLE_TOKEN = /^(?:[A-Za-z_][A-Za-z0-9_.+-]*|\.\/[A-Za-z0-9_.+\-/]+)$/; /** One backticked span that reads as a whole command: at least two shell words * and a first word that is an {@link EXECUTABLE_TOKEN}. * A span that names a single file or a config key is not a command claim. */ function claimedCommand(span: string): boolean { const words = span.split(/\s+/).filter((word) => word.length > 0); if (words.length < 2) return false; return EXECUTABLE_TOKEN.test(words[0] ?? ""); } /** A markdown list marker opening a line: `- item`, `* item`, `1. item`, * `2) item`. The required trailing whitespace is what keeps a bold * `**Verified:**` row or an italic `*span*` from reading as a bullet. */ const LIST_MARKER = /^[ \t]*(?:[-*+]|\d+[.)])[ \t]+/; /** The inline verified label a claims row can open with — `**Verified:**`, * `Verification:`, `proof.` — mirroring the label branch of * {@link VERIFIED_MARKER}, so the very row that opens the region also parses * as claims-bearing. */ const CLAIMS_ROW_LABEL = /^\s*(?:\*\*)?(?:Verif(?:ied|ication)|Proof)(?:\*\*)?\s*[:.]\s*/i; /** Every backticked span in `text`, trimmed. */ function backtickSpans(text: string): string[] { return [...text.matchAll(/`([^`\n]+)`/g)].map((match) => (match[1] ?? "").trim()); } /** The one backticked span `text` opens with, when it opens with one. */ function leadingBacktickSpan(text: string): string | undefined { const match = /^`([^`\n]+)`/.exec(text); const span = (match?.[1] ?? "").trim(); return span === "" ? undefined : span; } /** * The logical line-units of the verified region. A heading or a list marker * always opens a unit, a blank line ends one, and every other non-blank line * is the markdown lazy continuation of the unit above it. That continuation * rule is the foundation of the #1102 fix: a bullet soft-wrapped across * physical lines stays ONE unit, so its wrapped second line — exactly where * #1097's quoted log line sat — can never pose as a fresh claim-bearing line. * The cost is deliberate and stated: bare claim lines stacked without markers * or blank lines merge into one unit, and only their first extracts. This * fleet writes claims as bullets, so the lost shape is the rarer one. */ function* claimUnits(region: string): Generator { let unit: string[] = []; for (const line of region.split("\n")) { if (/^#{1,6}\s/.test(line) || LIST_MARKER.test(line)) { if (unit.length > 0) yield unit; unit = [line]; } else if (line.trim() === "") { if (unit.length > 0) yield unit; unit = []; } else { unit.push(line); } } if (unit.length > 0) yield unit; } /** * The claim candidates one unit makes — a matter of position, not vocabulary * (#1102). A unit claims what it OPENS with: * * - a verified-label row (`**Verified:** …`, the #749/#731 semicolon-joined * shape) declares every span it carries a claim — the label is what marks * the whole row as proof; * - any other unit contributes only its leading span — `- \`cd omp && bun test …\` * — 42 pass, 0 fail.` or a bare `\`./scripts/deploy.sh\` — deployed.` line — * whose tail is commentary about the result. * * A span embedded after prose is neither: `…assert exactly one log line naming * \`issue is closed\`` quotes the transcript's own output, and quotation is * commentary whatever the quoted words spell. */ function unitClaimSpans(unit: string[]): string[] { const lead = (unit[0] ?? "").replace(LIST_MARKER, ""); if (CLAIMS_ROW_LABEL.test(lead)) { return backtickSpans([lead.replace(CLAIMS_ROW_LABEL, ""), ...unit.slice(1)].join("\n")); } const span = leadingBacktickSpan(lead); return span === undefined ? [] : [span]; } /** * The commands a PR body claims as proof, each deduplicated by its exact span. * Extraction is structural twice over: only spans inside the verified region * count ({@link claimRegion}), and only spans in a claim-bearing position * within it ({@link unitClaimSpans}) — the two filters together are what keep * `## Verified`-island prose out of the audit. */ function claimedProofCommands(body: string): string[] { const claims: string[] = []; const seen = new Set(); for (const unit of claimUnits(claimRegion(body))) { for (const span of unitClaimSpans(unit)) { if (seen.has(span) || !claimedCommand(span)) continue; seen.add(span); claims.push(span); } } return claims; } /** * Split a shell command into its meaningful words: whitespace-separated, quote * characters removed so `bun test "src/my file.ts"` stays one word. Deliberately * not a shell parser — the wide net is fine because the comparison is a subset * in one direction only (see {@link satisfiedBy}). */ function shellWords(command: string): string[] { const words: string[] = []; let current = ""; let quote: string | undefined; for (const ch of command) { if (quote !== undefined) { if (ch === quote) quote = undefined; else current += ch; continue; } if (ch === '"' || ch === "'") { quote = ch; continue; } if (/\s/.test(ch)) { if (current !== "") { words.push(current); current = ""; } continue; } current += ch; } if (current !== "") words.push(current); return words; } /** A token that is shell plumbing rather than a command word: `&&`, `|`, * `;`, redirects, grouping — anything made only of the metacharacter * alphabet. `2>&1` is kept (it carries a number), so stripping never eats a * word that discriminates one command from another. */ const SHELL_PUNCT = /^[&|;<>()$!`]+$/; /** * The meaningful tokens of one command: shell punctuation dropped, and a * `cd ` pair consumed as binding context rather than a command word — * the tolerated-variant half of the acceptance criteria. Wrapper verbs * (`timeout 300`, `env -i`) survive tokenisation because a *transcript* side * carrying them never hurts a subset check; only a claimed side spelling them * would, which is a shape this fleet's bodies do not produce. */ function commandTokens(command: string): string[] { const words = shellWords(command); const tokens: string[] = []; for (let i = 0; i < words.length; i++) { const word = words[i] ?? ""; if (word === "cd") { i++; // its target is binding context too continue; } if (SHELL_PUNCT.test(word)) continue; tokens.push(word); } return tokens; } /** * The shell-executing tools whose invocations can satisfy a claimed command. * The shared-host guard's own invocation prefixes, minus `source` — a dot * script has no command text of its own worth matching. */ const SHELL_TOOLS = new Set(["bash", "sh", "zsh"]); /** * Whether a shell command ran in parse-only mode — a leading `bash -n`, * `sh -n` after the `cd`-stripping, before the first non-flag token. This is * the one how-vs-whether distinction the incident itself turned on: the * shared-host guard explicitly permits `bash -n` on a guarded script while * executing it is refused, and the *path token appears in both forms*, so a * subset match alone would let a parse invocation satisfy a claim of having * run the suite. The shared-host refusal wording makes the same explicit * ("parsing never executes"). */ function parseOnly(tokens: string[]): boolean { const first = tokens[0]; if (first === undefined || !SHELL_TOOLS.has(first)) return false; return tokens.slice(1).includes("-n"); } /** Whether one claimed command is satisfied by one transcript invocation: the * claim's tokens are a subset (order-free, multiplicity-checked) of the * invocation's. Subset rather than subsequence is what makes differing flag * order harmless; multiplicity is what keeps `bun test a a` from being * satisfied by `bun test a`. * * The one semantic guard on top of the subset: a claim that the worker *ran* * a command is never satisfied by an invocation that only parsed it * (`bash -n`), for the reason {@link parseOnly} states. A parse claim is * still satisfied by a parse invocation. */ function satisfiedBy(claimTokens: string[], invocationTokens: string[]): boolean { if (claimTokens.length === 0 || invocationTokens.length === 0) return false; if (parseOnly(invocationTokens) && !parseOnly(claimTokens)) return false; const remaining = [...invocationTokens]; for (const token of claimTokens) { const at = remaining.indexOf(token); if (at < 0) return false; remaining.splice(at, 1); } return true; } interface TranscriptCall { /** The tool-call id, present when the record carries one — links an * invocation to its tool result for the refused-detection below. */ id?: string; /** The command text of the invocation, or undefined when the record's shape * carries none (a synthetic/aborted call, a record this parser does not * recognise). */ command?: string; } /** The command text a content block or top-level record carries, whatever the * harness's spelling for it — `input` (tool_use), `arguments` (toolCall) or a * bare `command` — and undefined when the shape is unrecognised. */ function commandField(record: { readonly [key: string]: unknown }): string | undefined { for (const key of ["input", "arguments"] as const) { const field = record[key]; if (field !== null && typeof field === "object") { // The harness's unvalidated transcript shapes; read as unknowns, like the // daemon's other transcript readers. const args = field as { readonly [key: string]: unknown }; const command = args["command"]; if (typeof command === "string") return command; } } const direct = record["command"]; return typeof direct === "string" ? direct : undefined; } /** * Every shell invocation a transcript records, plus every tool-result that a * shared-host guard refusal produced. Defensive, in the module header's sense: * the transcript is the harness's file, so both shapes this package already * reads (a top-level `{toolName}` event and a `tool_use`/`toolCall` content * block inside an assistant message) are recognised, and an unknown shape is * skipped rather than TypeErroring the audit. */ export function transcriptCalls(text: string): { calls: TranscriptCall[]; refusals: Set } { const calls: TranscriptCall[] = []; const refusals = new Set(); for (const line of text.split("\n")) { if (line.length === 0) continue; let row: unknown; try { row = JSON.parse(line) as unknown; } catch { continue; } if (row === null || typeof row !== "object") continue; const rec = row as { readonly [key: string]: unknown }; const toolName = rec["toolName"]; if (typeof toolName === "string") { if (SHELL_TOOLS.has(toolName)) { const id = rec["id"]; const command = commandField(rec); calls.push({ ...(typeof id === "string" ? { id } : {}), ...(command === undefined ? {} : { command }), }); } continue; } if (rec["type"] !== "message") continue; const message = rec["message"]; if (message === null || typeof message !== "object") continue; const m = message as { readonly [key: string]: unknown }; const content = m["content"]; if (Array.isArray(content) && m["role"] === "assistant") { for (const part of content) { if (part === null || typeof part !== "object") continue; const p = part as { readonly [key: string]: unknown }; const name = p["name"]; if (typeof name !== "string" || !SHELL_TOOLS.has(name)) continue; const id = p["id"]; const command = commandField(p); calls.push({ ...(typeof id === "string" ? { id } : {}), ...(command === undefined ? {} : { command }), }); } continue; } // A shared-host refusal arrives as a tool result carrying the guard's // own marker. Keyed by the tool-call id so the refused invocation is the // one identified, never its neighbour. if (m["role"] === "toolResult") { const toolCallId = m["toolCallId"]; if (typeof toolCallId !== "string") continue; const parts = Array.isArray(content) ? content : []; const refused = parts.some((part): boolean => { if (part === null || typeof part !== "object") return false; const block = part as { readonly [key: string]: unknown }; const text = block["text"]; return typeof text === "string" && text.includes("sharedHostPolicy"); }); if (refused) refusals.add(toolCallId); } } return { calls, refusals }; } /** * The claimed-proof half of the settlement audit: one finding per claimed * command whose transcript is silent, and a distinct finding (the honest * shape) for one the guard refused. The presence direction — a claim satisfied * by any invocation — produces nothing, which is the whole point of the flag * existing: silence is what a true Verified section earns. */ function detectClaimedProof(audit: SettlementAudit, flags: SettlementFlag[]): void { if (audit.prBody === undefined || audit.transcript === undefined) return; const claims = claimedProofCommands(audit.prBody); if (claims.length === 0) return; const { calls, refusals } = transcriptCalls(audit.transcript); const invocations = calls .filter((call) => call.command !== undefined) .map((call) => { const tracked = call as { id?: string; command: string }; return { ...tracked, tokens: commandTokens(tracked.command) }; }); for (const claim of claims) { const claimTokens = commandTokens(claim); if (claimTokens.length === 0) continue; const matched = invocations.filter((call) => satisfiedBy(claimTokens, call.tokens)); if (matched.length > 0) { // A claimed command the guard refused is an attempted-and-blocked report, // never a fabricated one: the worker's own words were "CI owns them", // and a reviewer should be pointing at the guard, not the worker. if (matched.some((call) => call.id !== undefined && refusals.has(call.id))) { flags.push({ kind: "claimed-proof-blocked", file: "(transcript)", detail: `\`${evidence(claim)}\` was attempted, but the shared-host guard refused it (CI owns it)`, }); } continue; } flags.push({ kind: "claimed-proof-missing", file: "(transcript)", detail: `\`${evidence(claim)}\` is claimed as proof, but no command matching it appears in the run's transcript`, }); } } // -------------------------------------------------------------------- printing /** Beyond this the report is a wall of text nobody reads; the count still tells * the truth about how much was suppressed. */ const RENDERED_FLAGS = 20; const HEADING = "settlement audit"; /** * The kinds that mean coverage got weaker — the quiet signals the audit * exists to deliver, and the family mining.ts mines. Presentation keeps them * ahead of everything else and lane escapes last (#1096): a check that fires * on half of all merges gets skimmed, and a skimmed finding must not sit * between the reviewer and these. Kept in step with `WEAKENING_KINDS` in * mining.ts, which is the same four kinds spelled for mining. */ const TEST_INTEGRITY_KINDS: Partial> = { "test-file-deleted": true, "test-disabled": true, "assertions-removed": true, "test-timeout-raised": true, }; /** * The flag block appended to a settlement report, or no lines at all. * * Silence is the design: an honest run must add nothing to its report, or the * block becomes furniture and stops being read. The one exception is a diff * that could not be read in full — then the absence of flags is not evidence of * anything, and saying so is the honest report. */ export function formatSettlementFlags( flags: readonly SettlementFlag[], diff: { truncated: boolean } = { truncated: false }, ): string[] { if (flags.length === 0) { return diff.truncated ? [ `${HEADING}: no flags, but the PR diff was too large to read in full` + " — this is not a clean bill", ] : []; } const lines = [ `${HEADING}: ${flags.length} advisory flag(s) — the run's state is unchanged by them` + (diff.truncated ? ", and the PR diff was too large to read in full" : ""), ]; // Presentation order is structural, not incidental: test-integrity flags // first, then everything else, lane escapes last (#1096). The analyser // emits them in roughly this order already, but this surface also renders // stored rows and post-settlement appends, whose order nobody chose. const rank = (flag: SettlementFlag): number => TEST_INTEGRITY_KINDS[flag.kind] === true ? 0 : flag.kind === "lane-escape" ? 2 : 1; const ordered = flags .map((flag, index) => ({ flag, index })) .sort((a, b) => rank(a.flag) - rank(b.flag) || a.index - b.index) .map((entry) => entry.flag); for (const flag of ordered.slice(0, RENDERED_FLAGS)) { if (flag.kind === "pr-adopted") { lines.push(` ${flag.kind} — ${flag.detail}`); continue; } lines.push( ` ${flag.kind} ${flag.file}${flag.line === undefined ? "" : `:${flag.line}`} — ${flag.detail}` + (flag.unattributed === true ? " [unattributed: the dispatching issue never names this file]" : ""), ); } if (flags.length > RENDERED_FLAGS) { lines.push(` … and ${flags.length - RENDERED_FLAGS} more`); } return lines; } /** The one-line form for `omp-conductor status`, where a flagged run has to be * visible long after its escalation was delivered and deduplicated. Lane * escapes are presented apart from the test-integrity family (#1096), so the * noisy check cannot bury the quiet one in one comma join. */ export function settlementFlagSummary(flags: readonly SettlementFlag[] | undefined): string | undefined { if (flags === undefined || flags.length === 0) return undefined; const integrity = new Set(); const lane = new Set(); const rest = new Set(); for (const { kind } of flags) { if (TEST_INTEGRITY_KINDS[kind] === true) integrity.add(kind); else if (kind === "lane-escape") lane.add(kind); else rest.add(kind); } const segments: { label?: string; kinds: SettlementFlagKind[] }[] = []; if (integrity.size > 0) segments.push({ label: "test-integrity", kinds: [...integrity] }); if (rest.size > 0) segments.push({ kinds: [...rest] }); if (lane.size > 0) segments.push({ label: "lane", kinds: [...lane] }); // One family alone reads exactly as before — a label with nothing to // separate is furniture. const parts = segments.map((segment) => segments.length > 1 && segment.label !== undefined ? `${segment.label}: ${segment.kinds.join(", ")}` : segment.kinds.join(", "), ); const loud = flags.some((f) => f.unattributed === true) ? ", some unattributed" : ""; return `${HEADING}: ${flags.length} flag(s) — ${parts.join("; ")}${loud}`; }