/** * The admission half of the dispatcher (`daemon.ts` composes; this owns the * gates). * * Extracted from `daemon.ts` so the most contested region — who gets a worker * this tick — is a lane of its own: #283, #419, #420, #421, #507 and #555 all * live here, and a diff touching only admission must not have to touch the * composition root. Behavior is deliberately untouched by the move; a later * slice reworks the seams. * * What admission is allowed to consult is the point of the module. It reads a * structural slice of the daemon's deps — never the whole `Deps` and never the * module that owns them, which would be the circular import the split exists * to rule out. Adding a field to the daemon's deps must not break these tests. */ import { join } from "node:path"; import { log, errText, safeEscalate } from "./log.ts"; import { oauthFenceVerdict, type CredentialClassProbeResult } from "./credential-class.ts"; import type { Caps, EffectiveModel, Escalation, FileLane, IssueComment, IssueSnapshot, LaneDeclaration, ModelDeclaration, OpenCloser, ProjectConfig, RunRecord, Tracker, Store, AdmissionHoldReason, } from "./types.ts"; import { readPlanUsage, type PlanUsageStatus, type UsageSource } from "./usage.ts"; import type { CriticalBaseProbe, CriticalBaseVerdict, LaneFile, LaneSource, RunLaneProbe } from "./gitops.ts"; import { repoSlugFor } from "./gitops.ts"; import { LIVE_STATES } from "./store.ts"; import { branchName, type Routed } from "./routing.ts"; import { parseDependsOn, buildDependencyCycles } from "./depends-on.ts"; import type { DependencyCycle, DependencyNodeReader } from "./depends-on.ts"; /** Fleet-wide escalations still need an issue number in the payload; 0 is the * sentinel that reads as "no issue" in every renderer. */ const NO_ISSUE = 0; /** * The slice of the daemon's `Deps` admission reads. Defined here rather than * imported from `daemon.ts` (that would be the cycle this module exists to * avoid); every `daemon.test.ts` fake already satisfies it, so the seam costs * nothing. */ export interface AdmissionDeps { project: ProjectConfig; caps: Caps; tracker: Tracker; store: Store; usage: UsageSource; escalate(e: Escalation): Promise; probeCriticalBase?: CriticalBaseProbe; probeWorktreeLane?: RunLaneProbe; /** * Reads one issue's tracker state in a repository the admission tracker is * NOT bound to — the cross-repo half of the Depends-on interlock (#420). * The tracker bound to `project.tracker.repo` can only answer same-repo * references; a `owner/repo#n` prerequisite naming a different routed repo * goes through this. Undefined means "could not tell" (same posture as * {@link Tracker.issueSnapshot}); a routed prerequisite it cannot answer * fails that candidate closed. Unset, admission fails a routed cross-repo * prerequisite closed too — the daemon always wires it. */ probeIssueIn?: (repo: string, issue: number) => Promise; /** * Reads one issue's BODY in a repository the admission tracker is NOT bound * to — the cross-repo half of the dependency-graph cycle pass (#421). The * tracker bound to `project.tracker.repo` already answers same-repo bodies * via {@link Tracker.issueBody}; a reachable prerequisite body in a routed * repo goes through this. Undefined means "could not tell"; unset, a routed * reachable body fails that branch closed (no false cycle), exactly like * {@link Tracker.issueBody}. The daemon always wires it. */ probeBodyIn?: (repo: string, issue: number) => Promise; /** * Reads one provider's current credential class from the harness (#852). * * Injected because the production reader spawns a subprocess against the live * credential store, and admission's tests must be able to drive every verdict * — including "the probe could not answer" — without one. Unset, the fence * cannot verify anything, so a project that declares * `requireOauthProviders` is held rather than dispatched: the whole point is * that unverified and wrong cost the same. A project that declares nothing * never calls it. */ probeCredentialClass?: (provider: string) => Promise; } /** `stops` are the operational ends that each require one resume. */ export function hasContinuationBudget(stops: number, maxContinuations: number): boolean { return stops <= maxContinuations; } /** True while unspent failed-implementation attempts remain. This is the * dispatcher's admission gate: once every `maxAttemptsPerIssue` slot is * spent, the issue is held as `failed-attempts` forever, and the `unblock` * verb withholds the queue label on the same predicate (#348). */ export function hasFailedAttemptBudget(failures: number, maxAttempts: number): boolean { return failures < maxAttempts; } /** * Local midnight, matching how a human reads "today". * * ponytail: a rolling 24h window would be fairer to a run that started at * 23:50, but midnight is what someone checking a morning spend report expects. * Upgrade path is a `capWindow: "day" | "rolling24h"` config key. * * Lives here, with the spend gates that read it, and is imported by the * composition root rather than the other way round: one definition of "today" * for the fleet-stop check, the reservation gate and the status line, because * three midnights would eventually disagree. */ export function startOfToday(): number { const d = new Date(); d.setHours(0, 0, 0, 0); return d.getTime(); } /** * What one run is allowed to spend, and therefore what admission reserves for * it before launching anything (#851). * * `null` means unbounded: no daily cap and no explicit per-run ceiling, so * there is nothing to reserve and nothing to enforce. * * An explicit `maxRunSpendUsd` always wins. Otherwise the allowance is derived * as the daily cap divided by the worker cap — the largest per-run reservation * that still lets the fleet run at its configured concurrency. Reserving the * WHOLE daily cap per run would bound overshoot just as well, but it would * also serialise every fleet on defaults (a $25 cap and two workers would only * ever admit one run), turning a spend guard into a concurrency change nobody * asked for. Dividing keeps both promises: the day's total is still bounded by * the cap, because N reservations of cap/N are the cap. * * A zero worker cap has no runs to allow, and is guarded rather than divided * by. */ export function runSpendAllowanceUsd( caps: Pick, ): number | null { if (caps.maxRunSpendUsd !== null) return caps.maxRunSpendUsd; if (caps.dailySpendUsd === null) return null; const workers = Math.max(1, caps.maxConcurrentWorkers); return caps.dailySpendUsd / workers; } /** A candidate cleared for dispatch, with the attempt number it will run as. * `lane` is the effective file-lane declaration admission resolved for it * (#608): the exact snapshot the overlap gate enforced. Dispatch renders this * value into the worker brief, so a changed or failed second comment read can * neither hide nor reword the lane admission held — the gate and the * worker-visible brief are one value, not two reads of the same thread. * `model` is the effective `Model:` declaration resolved from the same read * (#535): not a gate, but the same one-value contract — dispatch launches on * this selector and the brief echoes it, so a second read can neither hide * the model admission held nor drift from it. */ export interface Admission { r: Routed; attempt: number; lane?: FileLane; model?: EffectiveModel; } export interface AdmissionHold { issue: number; reason: AdmissionHoldReason; /** Human-readable "what" for the surface the reason group alone cannot say — * the overlapping file and the run holding it for `file-lane`. Survives only * far enough to be rendered; it never leaves the current pass's summary. */ detail?: string; } export interface AdmissionPass { admitted: Admission[]; holds: AdmissionHold[]; } /** * What a held plan-usage gate says to a human, if anything. * * Three different problems hide behind one hold, and they want different * tiers. Reaching the threshold is the guard *working*: tier 1, because the * fleet resumes on its own at the provider's reset and nobody needs to get * out of bed. Everything else — a window nothing reports, a window that * resolves to two allowances, a meter that has been unreadable for half an * hour — is dispatch stopped with no self-recovery, which is tier 2. * * Each summary carries the fact that will change when the situation does (the * reset instant, the configured id, the date), because the escalation ledger * dedupes on the summary: a stable one pages once and then goes quiet, which * is right for a repeated tick and wrong for the next window. */ function planUsageEscalation(project: string, plan: PlanUsageStatus): Escalation | undefined { const base = { project, issue: NO_ISSUE }; if (plan.state === "at-cap") { const window = plan.window?.id ?? plan.cap?.windowId ?? "the configured window"; const resets = plan.resetsAt === undefined ? new Date().toISOString().slice(0, 10) : new Date(plan.resetsAt).toISOString(); return { ...base, tier: 1, summary: `Plan allowance cap reached on ${window} — ${project} is not claiming new work (window ${resets})`, detail: [ plan.detail, "Running workers finish normally; only new claims are held.", "Dispatch resumes by itself once the provider reports the window reset or usage below the threshold —", "no `resume` needed. Raise `caps.planUsage.maxUsedFraction` only if you mean to spend the rest.", ].join("\n"), }; } if (plan.state === "blind") { return { ...base, tier: 2, category: "fleet-stopped", // Dated: a meter that breaks again next month is a new incident, not a // repeat of this one. summary: `Plan usage source unreadable — ${project} is not claiming new work (${new Date().toISOString().slice(0, 10)})`, detail: [ plan.detail, "The guard admitted work while the failure looked transient and has now stopped.", "Check `omp usage --json` on the fleet host, or set `caps.planUsage` to null if this fleet is unmetered.", ].join("\n"), }; } if ( plan.state === "window-missing" || plan.state === "window-ambiguous" || plan.state === "window-uncomparable" ) { return { ...base, tier: 2, category: "fleet-stopped", summary: `Plan usage cap names an unusable window "${plan.cap?.windowId ?? "?"}" — ${project} is not claiming new work`, detail: [ plan.detail, "Run `omp usage --json` and copy an allowance `id` into `caps.planUsage.windowId`,", "or set `caps.planUsage` to null if this fleet is unmetered.", ].join("\n"), }; } return undefined; } /** * The candidate half of the file-lane interlock: the machine-readable list of * files an issue declares it will touch (#555). * * The orchestrator already writes exactly this list into every promotion brief * in prose; this parses that same declaration out of the issue body — or out of * a pre-dispatch comment, which is the same declaration on a supported brief * surface since #517 — so the interlock is load-bearing rather than advisory. * Two spellings are accepted, and both work on every surface: * * - The inline line: a line beginning with "file lane" (case-insensitive, * optional bold/heading markers) whose rest carries the paths as * backtick-delimited spans (the brief form), with a bare * comma/space-separated fallback that keeps tokens that look like relative * paths. The paths are the *leading* backticked run: a comma/space-separated * list at the line's start, and any backticked path later on the line is * prose, not a write target (#1073). * - The write-lane section (#825): a markdown heading "Exact write lane" (or * "Write lane" / "write-lane") whose immediately following bullet items * carry the paths — the package-floor decomposition format, where a groomed * child names the files it writes as one backticked path per bullet. Only * the contiguous bullet run under that heading is read: a following * paragraph (like a "Read only:" caveat) or a later section ends it, so * read-only entry points, proof commands and acceptance bullets elsewhere in * the issue are never captured. Each bullet contributes its leading * backticked path run alone — a filename named in a bullet's prose (say, to * note that it does not exist) is prose, never a lane entry (#1073). * * Inline wins when both are present — the machine-shaped sentence has held * since #555, and the mediated label echo shows exactly which declaration * admission enforces. A line or section whose tokens are not pathlike (like * `File lane: none`) is not a declaration. Anything else — including an * absent declaration, which is the default — is an empty lane: the issue is * admitted exactly as today (`fail open`), and the gate never refuses work for * wanting a lane. A clearly delimited write-lane section that parsed nothing * is NOT an absent declaration: the promotion verb refuses it with an * actionable syntax error ({@link writeLaneSectionHeading}) rather than * echoing a fail-open the section's own heading contradicts. */ export function laneDeclaration(text: string): LaneDeclaration | undefined { return inlineLaneDeclaration(text) ?? sectionLaneDeclaration(text); } /** One declared surface's paths (#1073). The canonical spelling is a leading * run of backticked paths — a comma/space-separated list at the surface's * start. A backticked path later in the same surface is prose, not part of * the lane: a parenthetical like "(there is no `settle-pass.test.ts`)" is a * sentence for the reader, and reading it as a write target makes a correct * issue permanently refuse. A surface with no leading backticked path keeps * the fallback: every backticked pathlike token, else bare * comma/space-separated pathlike tokens (the unbackticked spelling). */ function writeLanePaths(text: string): string[] { const fromRun = leadingBacktickedRun(text).filter(isPathLike); if (fromRun.length > 0) return fromRun; const backticked = [...text.matchAll(/`([^`]+)`/g)] .map((m) => m[1]!.trim()) .filter(isPathLike); return backticked.length > 0 ? [...new Set(backticked)] : [...new Set(text.split(/[,\s]+/).map((s) => s.trim()).filter(isPathLike))]; } /** The leading backticked run of a surface (or bullet): consecutive * backtick-delimited tokens separated by commas or whitespace, starting at * the first character. A run is a *list*; the first non-backtick, * non-separator character ends it, so "`a.ts`, `b.ts`" yields both while * "`a.ts` — see `b.ts`" yields only `a.ts`. Empty when the surface does not * open with a backtick. */ function leadingBacktickedRun(text: string): string[] { const trimmed = text.trim(); if (!trimmed.startsWith("`")) return []; const run: string[] = []; let pos = 0; while (true) { if (trimmed[pos] !== "`") break; const close = trimmed.indexOf("`", pos + 1); if (close < 0) break; const token = trimmed.slice(pos + 1, close).trim(); if (token === "") break; run.push(token); pos = close + 1; const sep = /^[,\s]+/.exec(trimmed.slice(pos)); if (sep === null) break; pos += sep[0].length; } return run; } /** The inline `File lane:`/`File-lane=` sentence grammar (see {@link laneDeclaration}). */ function inlineLaneDeclaration(text: string): LaneDeclaration | undefined { const match = text.match( /^\s*(?:[#>*-]\s*)*file[- ]lane\s*[:=]\s*(?:(?:\*\*|\*|__|_)\s*)?([^\n]*)$/im, ); if (match === null) return undefined; const rest = match[1] ?? ""; const files = writeLanePaths(rest); if (files.length === 0) return undefined; return { files, source: match[0].trim() }; } /** * A markdown heading that opens a clearly delimited write-lane section * (#825): `## Exact write lane`, `## Write lane`, or a hyphenated spelling, * optional trailing colon. Returns the heading line verbatim, or undefined. * The promotion verb uses this to refuse a section that parsed to nothing — * the echo must never claim fail-open for a heading that plainly tried to * declare one. */ export function writeLaneSectionHeading(text: string): string | undefined { const match = text.match( /^[ \t]*#{1,6}[ \t]+(?:exact[ \t]+)?write[-\s]lane[ \t]*[:.]?[ \t]*$/im, ); return match === null ? undefined : match[0].trim(); } /** The write-lane section grammar (see {@link laneDeclaration}): the heading * plus the contiguous bullet run beneath it. A non-bullet line ends the run, * so a caveat paragraph or the next section bounds the declaration. */ function sectionLaneDeclaration(text: string): LaneDeclaration | undefined { const lines = text.split("\n"); let start = -1; for (let i = 0; i < lines.length; i++) { if (writeLaneSectionHeading(lines[i]!) !== undefined) { start = i; break; } } if (start < 0) return undefined; const files: string[] = []; const bullets: string[] = []; for (let i = start + 1; i < lines.length; i++) { const line = lines[i]!; if (line.trim() === "") continue; // A markdown bullet marker (`- `, `* `, `+ `, `1. `), with an optional // task checkbox. Anything else ends the section's bullet run. const marker = line.match(/^[ \t]*(?:[-*+]|\d+[.)])[ \t]+(?:\[[ xX]\][ \t]+)?/); if (marker === null) break; const rest = line.slice(marker[0].length); // One bullet contributes its leading backticked path run — what it // declares it writes, not every backticked token its prose mentions. // A bullet without a leading backticked path keeps the fallback // (bare pathlike tokens), so unbackticked lanes are unaffected. files.push(...writeLanePaths(rest)); bullets.push(line.trim()); } if (files.length === 0) return undefined; return { files: [...new Set(files)], source: [lines[start]!.trim(), ...bullets].join("\n") }; } /** Who holds one file's lane this admission pass, and which read proved it. * `"declared"` covers both occupancy halves: an admitted candidate's own * lane for the rest of its pass, and — since #744 — a live run's persisted * declaration on every later pass, without any probe read. */ export type LaneHolder = { issue: number; source: LaneSource | "declared" }; /** Resolves the effective lane across the body and the whole comment thread. */ export function effectiveLane(body: string, comments: IssueComment[]): FileLane | undefined { let current: FileLane | undefined; const bodyDecl = laneDeclaration(body); if (bodyDecl !== undefined) current = { ...bodyDecl, at: "body" }; for (let i = 0; i < comments.length; i++) { const decl = laneDeclaration(comments[i]!.body); if (decl !== undefined) current = { ...decl, at: i }; } return current; } /** The paths of an issue body's declared lane (empty for no declaration). * Exported so the format is pinned independent of admission. */ export function declaredLane(body: string): string[] { return laneDeclaration(body)?.files ?? []; } /** * The model half of a promotion brief (#535): the selector an issue declares * it will dispatch on, read from the body or a pre-dispatch comment — the * same two surfaces `File lane:` reads. Conductor names a selector and omp * resolves what it means, so the declaration is one token in omp's own model * syntax — a role alias (`@slow`) or a concrete `provider/model`, either * optionally `:thinking`-suffixed — passed through verbatim as the session's * `modelPattern`. A line whose rest is not exactly one selector (blank, * `none`, multi-word prose) is not a declaration: dispatch fails open to the * project's `workerModel`, exactly as a lane-less issue is admitted today. * This is the one grammar for all surfaces; there is no second comment-only * spelling. */ export function modelDeclaration(text: string): ModelDeclaration | undefined { const match = text.match( /^\s*(?:[#>*-]\s*)*model\s*[:=]\s*([^\n]*)$/im, ); if (match === null) return undefined; const rest = (match[1] ?? "").trim().replace(/^[*`\s]+|[*`\s]+$/g, "").trim(); if (rest === "" || /^none$/i.test(rest) || /\s/.test(rest)) return undefined; return { model: rest, source: match[0].trim() }; } /** Resolves the effective model across the body and the whole comment thread. */ export function effectiveModel(body: string, comments: IssueComment[]): EffectiveModel | undefined { let current: EffectiveModel | undefined; const bodyDecl = modelDeclaration(body); if (bodyDecl !== undefined) current = { ...bodyDecl, at: "body" }; for (let i = 0; i < comments.length; i++) { const decl = modelDeclaration(comments[i]!.body); if (decl !== undefined) current = { ...decl, at: i }; } return current; } /** * The file lane as one line, for surfaces where a human asserts the claim * (#724): the parsed file list exactly as admission will enforce it, or the * explicit fail-open note when nothing parsed. Verbatim parser output only — * a paraphrase would hide the very over-parse (#720) or under-parse (#684) * this echo exists to surface. */ export function laneEcho(lane: LaneDeclaration | undefined): string { if (lane === undefined) return "no lane declared (fail open)"; return lane.files.join(", "); } /** A plausible relative path: it has a `.` extension or a directory separator. */ function isPathLike(token: string): boolean { return token !== "" && !/\s/.test(token) && (token.includes("/") || /\.[A-Za-z0-9]{1,10}$/.test(token)); } /** * Where a `owner/repo` cross-repo reference can be read. Only repositories * routed in THIS fleet are resolvable (#420): the issue source repo * (`project.tracker.repo`) is answered by the existing tracker, and any routed * work repo by {@link AdmissionDeps.probeIssueIn}. Anything else is unresolved * — the caller fails closed rather than guess which repo was meant, and never * silently resolves the reference against the candidate's own repo. */ function crossRepoTarget(project: ProjectConfig, ownerRepo: string): "tracker" | "routed" | undefined { if (ownerRepo === project.tracker.repo) return "tracker"; for (const routed of Object.values(project.routing.repos)) { if (repoSlugFor(routed) === ownerRepo) return "routed"; } return undefined; } /** * Which routed candidates get a worker this tick — in queue order, never more * than `slots` of them. Every non-admission receives a stable reason code. * * Exported so the admission rules can be pinned without spawning a worker. * Every one of them exists because of a live incident, and each guards a * different way the same issue gets worked twice — including epic siblings * racing onto the same files (#48). * * Takes the slice of `Deps` it actually reads rather than the whole thing: what * admission is allowed to consult is the point of the function, and a `Deps` * that grows a field has no business breaking these tests. */ export async function admitCandidates( d: AdmissionDeps, routed: Routed[], slots: number, ): Promise { const { project, caps, tracker, store } = d; const activeRuns = store.activeRuns(project.name); const busyIssues = activeRuns.map((r) => r.issue); const busy = new Set(busyIssues); // issue -> its active run rows, for the pushed-green admission bypass (#175): // only a worker-free pushed-green row may be bypassed, and only when *every* // active run for the issue is worker-free. A live (claimed/running) row still // holds unconditionally. const activeByIssue = new Map(); for (const run of activeRuns) { const list = activeByIssue.get(run.issue); if (list === undefined) activeByIssue.set(run.issue, [run]); else list.push(run); } // Live worker count per repo, seeded from live runs and incremented as this // same pass admits — so two same-repo candidates can never both clear the // per-repo cap in one tick (#186). const liveByRepo = new Map(); for (const run of store.liveRuns(project.name)) { liveByRepo.set(run.repo, (liveByRepo.get(run.repo) ?? 0) + 1); } const holds: AdmissionHold[] = []; const hold = (issue: number, reason: AdmissionHoldReason, detail?: string): void => { holds.push({ issue, reason, ...(detail === undefined ? {} : { detail }) }); }; // Runs holding an ACTIVE MUTATION LEASE (#899): the only rows whose files a // candidate must not touch, because they are the only rows something is // currently writing through. // // Deliberately narrower than `activeRuns`, and this is the whole of the fix. // A `pushed-green` / `pushed-pending` row is worker-free by construction — // its PR is live but no process owns its branch — and seeding lane occupancy // from it meant a settled artifact held its entire diff forever: measured on // this fleet, worker-free #782 / PR #870 held `omp/src/daemon.ts` and idled // slots against unrelated candidates that merely overlapped it. // // What does NOT change: `activeRuns` still answers same-issue ownership, the // sibling/parent occupancy and the open-PR gate, so releasing the *lane* // never lets the artifact's own issue be re-implemented (the named silent // fake of #899). Only file occupancy narrows. // // A dispatched, unsettled review revision is a lease too. Its run row is // `running` the moment `claimRunForReview` fires, so it is normally already // in `LIVE_STATES`; it is unioned here anyway because the claim and the // launch are not the same instant, and a lane the gate briefly forgets is a // lane two writers can share. const leasedRuns = store.leasedRuns(project.name); // One run's actual lane: the files it has touched relative to base, read from // its worktree and/or its branch in the mirror. Fails OPEN by construction — // an unreadable probe yields no files — because this side of the interlock is // automatic and continuous, and a gate that refused well-formed work whenever // a git read hiccuped would idle the fleet on a transient. (The deliberate // counterpart is `conductor_pr_recover`, which is a single human-initiated // call with a retryable refusal, and therefore fails closed — #925.) const probeRunFiles = async (run: RunRecord): Promise => { if (d.probeWorktreeLane === undefined) return []; const base = project.routing.repos[run.repo]?.defaultBranch ?? "main"; try { return await d.probeWorktreeLane({ worktree: run.worktree, baseRef: `refs/remotes/origin/${base}`, ...(run.branch === "" ? {} : { branchRef: `refs/heads/${run.branch}` }), mirror: join(project.mirrorRoot, `${run.repo}.git`), }); } catch { return []; } }; // The file-lane interlock (#555): repo -> file -> the issue holding that file. // Seeded from what a leaseholder actually *occupies*, never from its issue // body: the probed worktrees/branches of runs holding a mutation lease (#288, // #899) unioned — since #744 — with the lane declaration admission persisted // on the row at dispatch, which is the exact enforced snapshot, not a // re-parse. The declared half is what survives across passes: a run admitted // with a lane it has not yet written still occupies while it holds its lease, // and the declared lanes are extended as this pass admits candidates, so two // overlapping candidates cannot both clear the gate in one tick. Keyed by // repo because a path only collides within its own checkout: a `daemon.ts` on // the api repo and one on the web repo are different files. Built lazily and // once, only when the first candidate that declares a lane reaches the gate, // so a queue of laneless issues pays nothing for it. let laneOccupancy: Map> | undefined; const ensureLaneOccupancy = async (): Promise>> => { if (laneOccupancy !== undefined) return laneOccupancy; const occupied = new Map>(); // Shared per-file first-read-wins: a file a run both declares and has // actually touched is tagged by the concrete probe read, and across runs // whichever live run comes first in the active set is the holder named — // both are legitimate for the gate, which holds on any overlap. const occupy = (run: RunRecord, file: string, source: LaneSource | "declared"): void => { let perRepo = occupied.get(run.repo); if (perRepo === undefined) { perRepo = new Map(); occupied.set(run.repo, perRepo); } if (!perRepo.has(file)) perRepo.set(file, { issue: run.issue, source }); }; for (const run of leasedRuns) { // The probed actual half first, so the concrete read outnames the // declaration for a file the run has already written. for (const { file, source } of await probeRunFiles(run)) occupy(run, file, source); // The declared durable half (#744): the lane admission persisted at // dispatch. Occupies even before the run has written anything, on every // pass — the declaration lives on the row, not in a per-pass memo. // Released by the lease filter exactly like probed occupancy: a run that // settles or is terminal stops holding a lease and its files stop // occupying here (#899). if (run.lane !== undefined && run.lane.files.length > 0) { for (const file of run.lane.files) occupy(run, file, "declared"); } } laneOccupancy = occupied; return occupied; }; // The candidate half of the file-lane interlock (#555): the machine-readable // lane an issue declares. Since #517 a promotion brief may live in a // pre-dispatch comment rather than the body, so the lane is read from both // surfaces — the same two the worker brief renders — through the tracker's // existing comment port, never a second comment reader. Body and comments // share the one `laneDeclaration` grammar, and the brief's "later correction // visibly supersedes" contract holds: the effective lane is the latest // declaration among the body and the WHOLE thread, so a correction posted as // a comment replaces an earlier body lane instead of widening it (#608). // // Reading every comment — not just the ones the brief's discussion budget // renders — is deliberate: this is the same `effectiveLane` the brief // renders, and the dispatch side guarantees the winning declaration appears // in the brief verbatim even when it sits beyond the budget. The gate and // the worker therefore agree on one lane, which is the #608 defect's shape. // // Comments are read at most once per issue per pass — the lane feeds both // the gate and the same-pass sibling occupancy below, and the model rides // the same read for its one-value contract with dispatch (#535), so the // cache is what stops one candidate costing two comment reads. Unreadable // comments fail open to the body declarations (a body lane stays // load-bearing, and a body model stays the one dispatch follows), and with // no readable declaration at all the #555 fail-open admission is unchanged. // // The resolved `FileLane` — not just its paths — is what an admitted // candidate carries into dispatch (#608): the gate and the brief must agree // on the *same declaration* (paths, verbatim source line, and which surface // it came from), so dispatch renders this cached value rather than reading // the thread a second time and hoping it did not change. The resolved // `EffectiveModel` is carried the same way and for the same reason: dispatch // launches on exactly this selector and the brief echoes it. const laneCache = new Map(); const modelCache = new Map(); const declarationsFor = async (r: Routed): Promise => { const issue = r.issue.number; if (laneCache.has(issue)) return; let lane: FileLane | undefined; let model: EffectiveModel | undefined; try { const comments = await tracker.listComments(issue); lane = effectiveLane(r.issue.body, comments); model = effectiveModel(r.issue.body, comments); } catch (err) { const bodyDecl = laneDeclaration(r.issue.body); lane = bodyDecl === undefined ? undefined : { ...bodyDecl, at: "body" }; const bodyModel = modelDeclaration(r.issue.body); model = bodyModel === undefined ? undefined : { ...bodyModel, at: "body" }; log(`#${issue} comments unreadable at admission; the body's declarations stand: ${errText(err)}`); } laneCache.set(issue, lane); modelCache.set(issue, model); }; const laneFor = (r: Routed): Promise => declarationsFor(r).then(() => laneCache.get(r.issue.number)); const modelFor = (r: Routed): Promise => declarationsFor(r).then(() => modelCache.get(r.issue.number)); // The plan allowance is a fleet-wide question, so it is asked once per pass // and answers for every candidate — unlike every gate below it, which is // per-issue. It sits here rather than beside the spend cap in `tick` for one // reason: the spend cap *pauses the daemon* and waits for a human, and a // weekly plan window resets by itself. A guard that demanded `resume` after // every rollover would cost more operator attention than the guard saves // (#110). Already-running workers are untouched and settle normally. // // Placed after the cheap local busy-set read and before the first tracker // call, so a held fleet spends no GitHub API budget discovering it is held. // The credential-class fence (#852), asked once per pass for the same reason // the plan allowance is: it is a property of the fleet's credentials, not of // any candidate, and its answer is identical for all of them. Placed // immediately before the plan gate so both fleet-wide questions sit together, // and so a fleet billing to the wrong credential spends no tracker budget // discovering it is held. // // It clears by itself: the operator re-authenticates and the next pass passes. // No `resume` is required and no escalation is raised here — a held candidate // renders its reason in `status`, and inventing a page for a condition an // operator fixes in a minute is how a guard costs more attention than it saves // (the #110 reasoning, applied to the same shape of gate). const requiredOauth = project.requireOauthProviders ?? []; if (requiredOauth.length > 0 && routed.length > 0) { const fence = await oauthFenceVerdict( requiredOauth, d.probeCredentialClass ?? (async (provider) => ({ ok: false, reason: `no credential probe is wired for ${provider}`, })), ); if (!fence.ok) { for (const r of routed) hold(r.issue.number, "credential-class", fence.reason); log(`credential-class gate holding ${String(routed.length)} candidate(s): ${fence.reason}`); store.reconcileGrooming(project.name, holds); return { admitted: [], holds }; } } const plan = await readPlanUsage(caps.planUsage, d.usage); if (plan.blocking) { for (const r of routed) hold(r.issue.number, "plan-usage-cap"); log(`plan usage gate holding ${String(routed.length)} candidate(s): ${plan.detail}`); const escalation = planUsageEscalation(project.name, plan); if (escalation !== undefined) await safeEscalate(d, escalation); // Every pass exit reconciles the durable blocked inventory against this // pass's holds, so a hold that stopped applying clears itself (#735). store.reconcileGrooming(project.name, holds); return { admitted: [], holds }; } // parent -> repo name -> blocking issue. Seeded from active runs (including // pushed-green), then extended by candidates admitted earlier in this same // pass so two siblings of one epic never both clear the gate in one tick. // A busy issue whose run row cannot be resolved occupies the sentinel repo // "" — treated as matching every repo, failing toward holding (#197). const occupiedParents = new Map>(); const parentCache = new Map(); const resolveParent = async (issue: number): Promise => { if (parentCache.has(issue)) return parentCache.get(issue); const parent = await tracker.parentOf(issue); parentCache.set(issue, parent); return parent; }; // Bounded by concurrent workers, not queue depth. A failed lookup here cannot // mark an epic occupied; candidates still fail closed on their own parentOf. for (const issue of busyIssues) { try { const parent = await resolveParent(issue); if (parent === undefined) continue; // The runs table records which repo each attempt worked in, and sibling // holds are now per-repo, so a busy child only occupies its epic under // that repo's name (same spelling as `createRun` writes from // `r.repo.name`). A busy issue with no resolvable run row occupies the // sentinel "" instead — matching every repo, failing toward holding. const repo = store.latestRun(project.name, issue)?.repo ?? ""; const siblings = occupiedParents.get(parent); if (siblings === undefined) { occupiedParents.set(parent, new Map([[repo, issue]])); } else if (!siblings.has(repo) && !siblings.has("")) { siblings.set(repo, issue); } } catch (err) { log(`#${issue} parent lookup failed while seeding epic occupancy (${errText(err)})`); } } const admitted: Admission[] = []; // Phase A — dependency-graph cycle pass (#421). Built ONCE per pass, before // any per-candidate open-prerequisite hold, so a candidate sitting on a cycle // is held as `dependency-cycle` instead of having an open first edge mask the // group. Readers follow the same tracker/probe seams as the per-candidate // interlock: same-repo refs via `tracker`, routed refs via `probeIssueIn` / // `probeBodyIn`. Traversal is bounded to the tracker/routed repos and every // node is read at most once; a tracker failure fails closed (no edge) rather // than ever synthesising a cycle. const readDepNode: DependencyNodeReader = async (repo, issue) => { const snap = await (async () => { try { if (repo === project.tracker.repo) return await tracker.issueSnapshot(issue); return d.probeIssueIn === undefined ? undefined : await d.probeIssueIn(repo, issue); } catch { return undefined; } })(); if (snap === undefined) return undefined; if (snap.state === "closed") return { state: "closed", body: "" }; const body = await (async () => { try { if (repo === project.tracker.repo) return await tracker.issueBody(issue); return d.probeBodyIn === undefined ? undefined : await d.probeBodyIn(repo, issue); } catch { return undefined; } })(); return body === undefined ? undefined : { state: "open", body }; }; const graph = await buildDependencyCycles( routed.map((r) => ({ repo: project.tracker.repo, issue: r.issue.number, body: r.issue.body })), project.tracker.repo, (ownerRepo) => crossRepoTarget(project, ownerRepo), readDepNode, ); // candidate issue -> the cycle path that holds it, for status/digest. const memberCycle = new Map(); for (const cycle of graph.cycles) { for (const member of cycle.members) { const [repo, issueStr] = member.split("#") as [string, string]; if (repo !== project.tracker.repo) continue; const n = Number(issueStr); // Prefer the cycle anchored at this member (its "own" cycle) when a // candidate sits on more than one. if (!memberCycle.has(n) || cycle.anchorIssue === n) memberCycle.set(n, cycle.display); } // One material event per stable cycle identity: the escalation ledger // dedupes on project/issue/tier/summary, and the summary carries the // canonical path, so a stable cycle pages once and a changed cycle emits a // new event (the rotated/queue-ordered spelling is canonicalised away). await safeEscalate(d, { tier: 1, project: project.name, issue: cycle.anchorIssue, summary: `Dependency cycle detected: ${cycle.display}`, detail: [ "None of the issues on this cycle can be dispatched until one edge is broken.", `Cycle: ${cycle.display}`, "Break it by closing or reordering a `Depends-on:` reference, and the affected", "candidates become claimable again on the next tick.", ].join("\n"), }); } // The spend reservation (#851). Read once per pass: what today has already // cost, plus what every live run has committed but not yet spent. The daily // cap on its own is a post-spend stop — it compares spend-to-date at // admission and pauses the fleet on the NEXT pass — so one expensive run // could cross it by its entire cost before anything objected. On 2026-08-21 // one `@slow` run spent an estimated $25.16 against a $25.00 cap and the // fleet paused only after it had settled. const runAllowanceUsd = runSpendAllowanceUsd(caps); const dailyCapUsd = caps.dailySpendUsd; const budgeted = runAllowanceUsd !== null && dailyCapUsd !== null; const spentTodayUsd = budgeted ? store.spendSince(project.name, startOfToday()) : 0; const liveReservedUsd = budgeted ? store.reservedSpendUsd(project.name) : 0; // Candidates admitted earlier in THIS pass have no run row yet (the claim // happens in dispatch), so the pass counts its own reservations or it would // admit a whole slot's worth of runs against one slot's budget. let passReservedUsd = 0; for (const r of routed) { const issue = r.issue.number; const cyclePath = memberCycle.get(issue); if (cyclePath !== undefined) { hold(issue, "dependency-cycle", cyclePath); log(`#${issue} held (dependency-cycle): ${cyclePath}`); continue; } if (admitted.length >= slots) { hold(issue, "capacity"); continue; } if (busy.has(issue)) { // A pushed-green row is worker-free by definition (it is not in // LIVE_STATES): its PR is live but no process is writing to its branch. // So an issue whose active runs are ALL pushed-green is not actually // occupied — the corrective attempt the operator unblocked may be // admitted as a continuation of that PR, and the open-PR gate below // decides the identity. Any live row still holds (#175). const allWorkerFree = (activeByIssue.get(issue) ?? []).every((r) => r.state === "pushed-green"); if (!allWorkerFree) { hold(issue, "issue-active"); continue; } } // Per-repo concurrency: the mirror, branch-protection staleness and shared // CI egress are all per-repo collision domains, so extra slots should land // on other repos rather than stacking workers into the same one (#186). const liveInRepo = liveByRepo.get(r.repo.name) ?? 0; if (liveInRepo >= caps.maxConcurrentWorkersPerRepo) { hold(issue, "repo-active"); log(`#${issue} skipped: ${liveInRepo} live worker(s) already in ${r.repo.name} (cap ${caps.maxConcurrentWorkersPerRepo})`); continue; } // Reserve before launching, never after settling: the whole allowance is // taken out of the day's budget up front, so the worst case is bounded by // what was reserved rather than by what a run turns out to cost. A run // that settles for less releases the remainder simply by leaving the live // set, which is why the next pass can admit again without any release // bookkeeping. Distinct from the fleet-stopping `daily-spend-cap`: this // holds one candidate while admitted work is still running. if (budgeted && spentTodayUsd + liveReservedUsd + passReservedUsd + runAllowanceUsd > dailyCapUsd) { const remaining = dailyCapUsd - spentTodayUsd - liveReservedUsd - passReservedUsd; const detail = `reserving $${runAllowanceUsd.toFixed(2)} for this run would exceed today's $${dailyCapUsd.toFixed(2)} ` + `budget: $${spentTodayUsd.toFixed(2)} spent, $${(liveReservedUsd + passReservedUsd).toFixed(2)} reserved by ` + `runs in flight, $${Math.max(0, remaining).toFixed(2)} left`; hold(issue, "spend-reservation", detail); log(`#${issue} held (spend-reservation): ${detail}`); continue; } const priorRuns = store.attemptsFor(project.name, issue); const failures = store.failuresFor(project.name, issue); if (!hasFailedAttemptBudget(failures, caps.maxAttemptsPerIssue)) { hold(issue, "failed-attempts"); await safeEscalate(d, { tier: 1, project: project.name, issue, summary: `#${issue} has used all ${caps.maxAttemptsPerIssue} failed attempts`, detail: [ r.issue.title, r.issue.url, "Another implementation attempt almost always means the issue itself is underspecified.", "Rewrite the acceptance criteria, or take it off the queue.", ].join("\n"), }); continue; } const continuations = store.continuationsFor(project.name, issue); if (!hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) { hold(issue, "continuations"); await safeEscalate(d, { tier: 1, project: project.name, issue, summary: `#${issue} exceeded its ${caps.maxContinuationsPerIssue}-continuation budget`, detail: [ r.issue.title, r.issue.url, "Repeated cap kills, daemon orphans, or answered blocks need an operator to inspect progress.", ].join("\n"), }); continue; } // Fail closed on work that exists only in a run repo. `addRunRepo` clears // the tree at / before it provisions, so admitting // this issue is what finally destroys the copy the salvage could not save // (#118). Nothing here can recover it — git already refused once — so the // only safe move is to refuse the claim and keep saying why until an // operator has looked and run `unblock --force`. const newest = store.latestRun(project.name, issue); if (newest?.salvageError !== undefined && newest.salvageAckAt === undefined) { hold(issue, "unsalvaged-wip"); await safeEscalate(d, { tier: 1, project: project.name, issue, summary: `#${issue} is holding unsalvaged work and will not be re-claimed`, detail: [ r.issue.title, r.issue.url, `Attempt ${newest.attempt} could not commit its uncommitted changes: ${newest.salvageError}`, `The only copy is the worktree ${newest.worktree === "" ? "(path not recorded)" : newest.worktree}.`, "Dispatch is held because claiming this issue removes that tree.", "Recover it by hand, then `omp-conductor unblock --force` to release the hold.", ].join("\n"), }); continue; } // #428 half (a): a preserved continuation that predates a configured // critical-base/safety marker must not be reattached. A base safety fix // protects only branches forked after it landed — a continuation forked // before it still carries the dangerous test/runtime code, and re-running // it on the shared host is what SIGTERMed the production daemon. Fail // closed: only a probe that proves every marker is in the reattach // source's ancestry admits, and a project that names a marker but has no // probe wired (never happens outside tests) holds. Both the hold and the // escalation are durable across restart and orphan recovery because this // gate runs every admission pass; the branch is re-admitted automatically // once the operator updates it to contain the marker, without losing work. const markers = project.criticalBase ?? []; if (markers.length > 0) { const branch = branchName(r.issue); let verdict: CriticalBaseVerdict; if (d.probeCriticalBase === undefined) { verdict = { state: "unknown", error: "no critical-base probe is wired in this deployment" }; } else { try { verdict = await d.probeCriticalBase(r.repo, markers, branch); } catch (err) { verdict = { state: "unknown", error: errText(err) }; } } if (verdict.state === "stale") { hold(issue, "stale-base"); log( `#${issue} held (stale-base): continuation branch ${branch} predates critical-base marker ${verdict.marker}`, ); await safeEscalate(d, { tier: 1, project: project.name, issue, summary: `#${issue} continuation branch predates a critical base safety commit and is held (stale-base)`, detail: [ r.issue.title, r.issue.url, `The retained branch ${branch} does not contain critical-base marker ${verdict.marker}.`, ...(verdict.range.length > 0 ? [`Base commits the branch is missing: ${verdict.range.join(", ")}`] : []), "Recovery: merge current base into the branch so it contains the marker, and the next", "admission pass re-admits it automatically without losing the branch's work; or review", "the branch by hand and clear the hold once the fix is present.", ].join("\n"), }); continue; } if (verdict.state === "unknown") { // Fail closed: a branch that cannot be *proven* to contain the marker // is refused. This is a verification/lookup failure — the mirror fetch // failed, the marker did not resolve, or no probe is wired — never // evidence that the branch predates the marker. Hold with a distinct // reason so status and the friction rollup do not read a provider // outage as branch staleness, and prescribe no branch change: the next // admission pass re-runs the probe and admits the unchanged branch // once verification succeeds. hold(issue, "critical-base-verify-error"); log( `#${issue} held (critical-base-verify-error): continuation branch ${branch} could not be verified ` + `against critical-base marker(s) ${markers.join(", ")} (${verdict.error})`, ); await safeEscalate(d, { tier: 1, project: project.name, issue, summary: `#${issue} continuation branch could not be verified against a critical base safety commit and is held (critical-base-verify-error)`, detail: [ r.issue.title, r.issue.url, `The retained branch ${branch} could not be verified against critical-base marker(s) ${markers.join(", ")}: ${verdict.error}`, "The branch is not claimed to predate the marker: this is a verification failure.", "No branch change is prescribed. The next admission pass retries the critical-base", "verification automatically and admits the unchanged branch once the probe succeeds.", ].join("\n"), }); continue; } } // Soft concurrency per epic, per repository: at most one in-flight child of // a given parent in each repo. Children of one epic in *different* repos // parallelise freely — `repo-active` / `maxConcurrentWorkersPerRepo` owns // the same-repo collision domain (#197). The "" sentinel matches every // repo. No parent means today's concurrent admission. Cheap local filters // already ran; this sits before the open-PR API call so a held sibling // frees the slot for unrelated work without spending a closers query. let parent: number | undefined; try { parent = await resolveParent(issue); } catch (err) { hold(issue, "parent-lookup-error"); log(`#${issue} held: parent check failed (${errText(err)}) — retrying next tick`); continue; } if (parent !== undefined) { const occupied = occupiedParents.get(parent); const blocker = occupied?.get(r.repo.name) ?? occupied?.get(""); // The gate serializes siblings under one epic: a held candidate must not // proceed while a *different* child of the parent is occupied. But a // candidate's own worker-free pushed-green row is exactly the work it is // continuing, not a rival — the unblocked continuation of that same // issue must not be rejected by its own occupancy, or the retained // continuation deadlocks forever with the PR open. if (blocker !== undefined && blocker !== issue) { hold(issue, "sibling-active"); log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent} in ${r.repo.name}`); continue; } } // The busy set is built from run rows, so it can only speak for work this // database recorded. Work pushed before this store existed — a migration, a // wiped or relocated state dir, a restore onto a new host — looks exactly // like fresh work, and a worker sent at it re-implements a finished PR. The // tracker is the only party that remembers, so it is asked. The cost is // bounded by free slots, not by queue depth: the call sits behind the two // cheap local filters and candidates beyond capacity skip it. let closer: OpenCloser | undefined; try { closer = await tracker.openCloserFor(issue); } catch (err) { // Fail closed, per candidate. An API error means "unknown whether // finished work exists", and admitting on unknown recreates precisely the // duplicate-work failure this guard exists to kill: the worst case of // holding is a five-minute delay, the worst case of admitting is a burned // attempt and a second PR on the same issue. Holding one candidate rather // than aborting the loop keeps a transient GitHub failure from deadlocking // the whole dispatcher; the next tick retries by itself. hold(issue, "open-pr-lookup-error"); log(`#${issue} held: open-PR check failed (${errText(err)}) — retrying next tick`); continue; } if (closer !== undefined) { const latest = store.latestRun(project.name, issue); // Terminality is the first half of the test and is not negotiable: while a // run is live its worker is still pushing to that branch, and a second // worker sent at the same PR is exactly the duplicate-work failure this // guard exists to kill. Only a run that has stopped can be continued. const retained = latest?.state === "blocked" || latest?.state === "failed" || latest?.state === "killed" || latest?.state === "orphaned" || latest?.state === "pushed-green" ? latest : undefined; // The second half asks "is this open PR our retained work", and accepts // two identities for it, because the branch is the durable artefact of a // retained run and the PR is not. A cap kill can end a run before any PR // exists: veltro#324 attempt 1 was killed at the turns cap on // 2026-08-09T00:47Z before its worker opened one, so the row kept `branch` // and `prUrl` stayed NULL. chad#438 was opened from that exact branch // afterwards, and URL equality — the only test 0.3.20 had — can never match // a URL the terminal run never recorded, so every tick held #324 as // `open-pr` until an operator closed recoverable work to free the branch // (#50). An ordinary issue whose open PR is unrelated still fails both // identities and stays ineligible, and an empty `headRefName` (a reply that // did not carry the field) is never a match: unknown is not identity. let resume: string | undefined; if (retained !== undefined) { if (retained.prUrl === closer.url) { resume = `from ${retained.state} run (matched recorded PR URL)`; } else if (closer.headRefName !== "" && retained.branch === closer.headRefName) { resume = `from ${retained.state} run (matched retained branch ${closer.headRefName})`; } } if (resume === undefined) { hold(issue, "open-pr"); log(`#${issue} skipped: open PR ${closer.url} already closes it`); continue; } // The continuation reacquire (#925). A continuation resumes writing the // retained branch, so the files that matter are the ones that branch // ACTUALLY carries — not the lane its issue declares. The declared gate // below cannot stand in for this: a continuation's declaration may be // narrower than its predecessor's diff (or absent entirely), and the // whole point of #899 is that the retained branch's files were released // the moment its worker finished, so an unrelated worker may legitimately // hold them by the time the continuation is admitted. // // Its own predecessor can never block it: a retained row is terminal or // `pushed-*`, so it holds no lease, and the holder check ignores this // issue's own rows in any case. if (retained !== undefined) { const carried = await probeRunFiles(retained); if (carried.length > 0) { const occupied = await ensureLaneOccupancy(); const perRepo = occupied.get(r.repo.name); const collision = carried .map(({ file }) => ({ file, holder: perRepo?.get(file) })) .find(({ holder }) => holder !== undefined && holder.issue !== issue); if (collision !== undefined) { const detail = `${collision.file} held by run #${collision.holder!.issue} (${collision.holder!.source})` + ` — carried by retained branch ${retained.branch}`; hold(issue, "file-lane", detail); log(`#${issue} held (file-lane): ${detail}`); continue; } } } log(`#${issue} continuing retained PR ${closer.url} ${resume}`); } // The queue comes from GitHub's eventually-consistent search index. Re-read // state and labels directly at the last possible moment so a just-closed or // explicitly dequeued issue cannot turn a stale candidate into another // attempt (#247). let snapshot: IssueSnapshot | undefined; try { snapshot = await tracker.issueSnapshot(issue); } catch { snapshot = undefined; } if (snapshot === undefined) { hold(issue, "issue-state-lookup-error"); log(`#${issue} held: issue snapshot check failed — retrying next tick`); continue; } if (snapshot.state === "closed") { hold(issue, "issue-closed"); log(`#${issue} skipped: issue is closed (search index lag)`); continue; } if (!snapshot.labels.includes(project.queueLabel)) { hold(issue, "issue-dequeued"); log(`#${issue} skipped: queue label ${project.queueLabel} was removed (search index lag)`); continue; } // Park beats queue here too (#734): the candidate list was fetched from // the search index before the operator parked the issue, so the fresh // snapshot is the last gate. A park never touches a live run — it only // stops a claim. if (snapshot.labels.includes(project.stateLabels.backlog)) { hold(issue, "issue-parked"); log(`#${issue} skipped: park label ${project.stateLabels.backlog} was applied since listing`); continue; } // The Depends-on interlock (#419/#420): a candidate declares the issues it // must not be dispatched before — same-repo (`#123`) or cross-repo // (`owner/repo#123`) — and any *open* prerequisite holds it until every // referenced issue is closed. Prerequisite state is read fresh from the // tracker at claim time, every pass — never cached across ticks — so a // prerequisite that reopens re-holds on the next tick. Cross-repo // references resolve only against repositories routed in this fleet; // graph cycles are a later slice (#421). const dependsOn = parseDependsOn(r.issue.body); if (dependsOn.malformed.length > 0) { // A marker line with no strict `#` reference (e.g. `Depends-on: // #abc`) is ignored as a gate — it never holds, and it never crashes — // but it is flagged so the issue is never *silently* dispatched with a // dependency nobody read. Emitted once: the escalation ledger dedupes on // the stable project/issue/tier/summary key, so repeated ticks do not // re-page the orchestrator for the same malformed body. await safeEscalate(d, { tier: 1, project: project.name, issue, summary: `#${issue} has a malformed Depends-on declaration and its undeclared prerequisite was not held`, detail: [ r.issue.title, r.issue.url, ...dependsOn.malformed.map((line) => `Malformed declaration: ${line}`), "Depends-on references must be bare issue numbers (`#`). Groom the body", "so the dependency is honoured, or the issue is dispatched without it.", ].join("\n"), }); log(`#${issue} flagged: malformed depends-on declaration(s): ${dependsOn.malformed.join("; ")}`); } if (dependsOn.refs.length > 0) { let blocking: number | undefined; let unreadable: number | undefined; for (const ref of dependsOn.refs) { let state: IssueSnapshot | undefined; try { state = await tracker.issueSnapshot(ref); } catch { state = undefined; } if (state?.state === "open") { blocking = ref; break; } if (state === undefined) { unreadable = ref; break; } } // Fail closed on an unreadable prerequisite: admitting on unknown would // dispatch work whose prerequisite may still be open, and the next tick // rereads the tracker itself, so a transient failure just defers. if (blocking !== undefined) { hold(issue, "depends-on", `blocked by #${blocking}`); log(`#${issue} held (depends-on): prerequisite #${blocking} is open`); continue; } if (unreadable !== undefined) { hold(issue, "depends-on", `prerequisite #${unreadable} state unreadable`); log(`#${issue} held (depends-on): prerequisite #${unreadable} state unreadable`); continue; } } if (dependsOn.crossRefs.length > 0) { // The cross-repo half (#420): each `owner/repo#n` is read from the repo // it names. The fully qualified form travels in the `detail` string, so // status/digest render it as `blocked by owner/repo#123` while same-repo // refs stay compact `blocked by #5`. let blocking: string | undefined; let unreadable: string | undefined; let unresolved: string | undefined; for (const ref of dependsOn.crossRefs) { const where = crossRepoTarget(project, ref.repo); const full = `${ref.repo}#${ref.issue}`; if (where === undefined) { unresolved = full; break; } let state: IssueSnapshot | undefined; try { if (where === "tracker") { // The qualified form of the candidate's own tracker repo: the // same state read the same-repo interlock uses. state = await tracker.issueSnapshot(ref.issue); } else if (d.probeIssueIn === undefined) { // No resolver wired — fail closed, never guess the issue's state. state = undefined; } else { state = await d.probeIssueIn(ref.repo, ref.issue); } } catch { state = undefined; } if (state?.state === "open") { blocking = full; break; } if (state === undefined) { unreadable = full; break; } } // Same fail-closed posture as same-repo refs: an open prerequisite // holds, an unreadable one holds (the next tick rereads it fresh), and // an unresolved repo holds because the dispatcher cannot verify the // dependency — it is never silently treated as closed. Only the // unresolved repo also surfaces one material event for grooming, and it // follows the malformed-declaration escalation's stable-summary // dedupe, so repeated ticks page once, not per tick. if (blocking !== undefined) { hold(issue, "depends-on", `blocked by ${blocking}`); log(`#${issue} held (depends-on): cross-repo prerequisite ${blocking} is open`); continue; } if (unreadable !== undefined) { hold(issue, "depends-on", `prerequisite ${unreadable} state unreadable`); log(`#${issue} held (depends-on): cross-repo prerequisite ${unreadable} state unreadable`); continue; } if (unresolved !== undefined) { hold(issue, "depends-on", `blocked by ${unresolved} — repo not routed in this fleet`); log(`#${issue} held (depends-on): cross-repo prerequisite ${unresolved} is not a routed repo`); await safeEscalate(d, { tier: 1, project: project.name, issue, summary: `#${issue} depends on ${unresolved}, a repo not routed in this fleet — held until groomed`, detail: [ r.issue.title, r.issue.url, `Unroutable prerequisite: ${unresolved}`, "Depends-on cross-repo references must name a repository routed in this project", "(`routing.repos` or the tracker repo). Groom the body or route the repo.", ].join("\n"), }); continue; } } // The file-lane interlock (#555): a candidate whose declared lane overlaps // a live run's *actual* lane is held until that run's work has merged, so // no second worker is sent at files another worker is still writing. The // gate is the mechanical version of the prose rule that failed three times // in one day. Only the machine-readable lane participates — read from the // body and pre-dispatch comments alike, so the gate sees the same surface // the worker brief renders (#608) — and a candidate without one is // admitted exactly as today (fail open), while a candidate's own retained // run never holds it — that is the continuation it continues. const lane = await laneFor(r); if (lane !== undefined && lane.files.length > 0) { const occupied = await ensureLaneOccupancy(); const perRepo = occupied.get(r.repo.name); let blocked = false; for (const file of lane.files) { const holder = perRepo?.get(file); if (holder !== undefined && holder.issue !== issue) { // The source names the probe read that proved the occupancy, so a // hold reads as authored work — worktree or branch — rather than the // base-reconciliation noise #684 filters out of the probe entirely. const detail = `${file} held by run #${holder.issue} (${holder.source})`; hold(issue, "file-lane", detail); log(`#${issue} held (file-lane): ${detail}`); blocked = true; break; } } if (blocked) continue; } admitted.push({ r, attempt: priorRuns + 1, lane, model: await modelFor(r) }); if (budgeted) passReservedUsd += runAllowanceUsd; liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1); // Same-pass sibling occupancy for the file-lane gate: once admitted, a // candidate's declared lane occupies for the rest of the pass, so a later // overlapping candidate is held rather than both clearing in one tick. if (lane !== undefined && lane.files.length > 0) { const occupied = await ensureLaneOccupancy(); let perRepo = occupied.get(r.repo.name); if (perRepo === undefined) { perRepo = new Map(); occupied.set(r.repo.name, perRepo); } for (const file of lane.files) { if (!perRepo.has(file)) perRepo.set(file, { issue, source: "declared" }); } } if (parent !== undefined) { // Extend the epic's occupancy under this repo (slot empty by construction // here — the gate above would have held the candidate otherwise) so a // same-repo sibling later in this pass does not clear the gate (#197). let siblings = occupiedParents.get(parent); if (siblings === undefined) { siblings = new Map(); occupiedParents.set(parent, siblings); } if (!siblings.has(r.repo.name)) siblings.set(r.repo.name, issue); } } // Every pass exit reconciles the durable blocked inventory against this // pass's holds, so a lane/dependency hold that stopped applying clears // itself on the very next pass (#735). store.reconcileGrooming(project.name, holds); return { admitted, holds }; }