/** * Why a run ended badly, and what to do about it (#132). * * Half of this fleet's spend produced no merged PR, and every one of those runs * ended at a human who then re-derived the same triage by hand: read the row, * read the PR's checks, decide whether to requeue, re-run, settle or escalate — * and then threw the conclusion away. The daemon already knew *that* a run ended * badly; it never recorded *why*, so a mechanical recovery cost human attention * and an implementation budget it had not spent. * * This module is the "why", and nothing else: pure, synchronous, and given the * facts rather than fetching them. The caller gathers what the row needs (a PR * state, a mergeability, a check list), classifies, then performs the one * recovery the class names. Keeping the decision table here is what makes "one * green test per class" possible without a daemon, a tracker or a network. */ import { DEFAULT_CAPS, type Caps, type FailureClass, type RecoveryAction, type RunRecord } from "./types.ts"; /** Facts the caller fetched, each only for the rows that need it. * * `pr` can also be `"missing"`: the tracker has proven the claimed PR * definitively does not exist (#779). It is a fact, the same way `"closed"` * is — a REST 404 the adapter corroborated with a same-repository pulls-list * read — never a transient "could not tell", which stays `undefined`. */ export interface ClassifyFacts { pr?: "open" | "merged" | "closed" | "missing"; mergeable?: "conflicting" | "clean" | "unknown"; checks?: { name: string; state: string; link?: string }[]; /** Full session error recovered from the transcript. Kept as a fact so a * classification retry does not lose an HTTP status the run row cannot store. */ sessionError?: { status?: number; message: string }; /** Tail (ANSI-stripped) of the first failed check's log, when one was * reachable. Lets the table tell an infrastructure outage (#177) from a * deterministic test failure by the log's own words. */ failingLog?: string; /** The PR's changed-file list, derived at the settle call site from the * diff the settlement already fetches (#1059). Lets the table tell a * Compose dependency-startup failure no PR could have caused from one a * PR's own diff did — a PR that touches no container configuration cannot * have broken `docker compose up`. Absent means the diff could not be * read (or was cut short), which is "could not tell": the conditional * signature below then stays silent and the attempt is charged. */ changedFiles?: string[]; } export interface Classification { cls: FailureClass; recovery: RecoveryAction; /** One line naming the signals that matched, carried into the escalation. */ evidence: string; } /** * Check states GitHub reports for a run that never produced a verdict. * * Infrastructure, not code: a cancelled or timed-out check says nothing about * the diff, so re-running it is free information and charging an implementation * attempt for it is simply wrong. Compared lowercased because `gh pr checks * --json state` has emitted both spellings across versions. */ const INFRA_CHECK_STATES: Record = { cancelled: true, timed_out: true, startup_failure: true, stale: true, skipped: true, }; /** States that mean "this check has a verdict and it is good". */ const SUCCESS_CHECK_STATES: Record = { success: true, neutral: true }; /** * The runner's own sentence for "the agent under this job went away": written * when the runner service is stopped, auto-updates, or is torn down under a * live job. Named once because two different accountings quote it — the * implementation-attempt waiver below ({@link INFRA_LOG_SIGNATURES}, #177) and * the review-correction waiver ({@link runnerInfraFailure}) — and a second * spelling of it would waive one while charging the other. */ const RUNNER_SHUTDOWN_SIGNAL = "the runner has received a shutdown signal"; /** * Substrings in a failed check's log that prove the failure was infrastructure, * not the diff (#177). Each is a registry/docker/runner fault a worker cannot * have introduced: a rate limit, an image-manifest resolution failure, a runner * a runner being torn down under the job, or a DNS failure. Matched lowercased against * the log tail. GitHub writes the rate-limit sentence "Response status code does * not indicate success: 429 (Too Many Requests)" where Docker writes "429 Too * Many Requests"; the full sentence is its own exact entry rather than a blanket * paren-strip of the log — stripping every parenthesis would let a deterministic * log like "failed to resolve source (metadata for fixture)" slip past the * `failed to resolve source metadata for` signature and waive an attempt it * genuinely spent (#177). * * Deliberately *not* matching bare `failed to solve:` — a docker build failure * often prints it with a real resolution error, so the closing words carry the * signal, not the phrase. */ const INFRA_LOG_SIGNATURES = [ "429 too many requests", // GitHub's setup/action-download sentence for the same rate limit — the full // lowercased phrase, never the bare parenthesized status: an application log // like "expected 200, got 429 (Too Many Requests)" is a product verdict, not // the runner's, and must not be waived (#637, #639). "response status code does not indicate success: 429 (too many requests)", "failed to resolve source metadata for", RUNNER_SHUTDOWN_SIGNAL, "could not resolve host", ]; /** * The closed infra signature a failed check log must contain for the failure * to be infrastructure rather than the diff, or `undefined` when the log * carries none (#177). Exported so the historical reconciliation recognises * exactly what the forward classifier does — one definition of "the log * proves infra", or a repaired row and a fresh row would diverge (#638). */ export function infraLogSignature(log: string): string | undefined { const lower = log.toLowerCase(); return INFRA_LOG_SIGNATURES.find((signature) => lower.includes(signature)); } /** * Runner-infrastructure wordings: a red check whose own text says the *runner* * failed, not the diff (Phase 3, #1043 lane). Matched lowercased as substrings * against the failing job's log — the same surface {@link infraLogSignature} * reads — and each entry is the real GitHub Actions sentence, verified against * the runner's own reports rather than invented: * * Kept as its own list rather than folded into {@link INFRA_LOG_SIGNATURES} * for two reasons. The consumers differ: that list waives an *implementation * attempt* for a settled run (#177), this one waives a *correction round* for * a live pull request's red check. And {@link infraSignatureVersion} is a * persisted cursor fingerprint — growing that list restarts the historical * reconciliation, which recognising a runner-lost red has no business doing. * The one overlap, {@link RUNNER_SHUTDOWN_SIGNAL}, is shared by reference so * the two accountings can never disagree about that wording. */ const RUNNER_INFRA_SIGNATURES = [ // "The self-hosted runner: lost communication with the server." and // its hosted sibling "The hosted runner: lost communication with the // server." — the runner process was killed, starved or cut off mid-job // (actions/runner#3539, community#84877, community#173431). The name varies, // so only the invariant tail is matched. "lost communication with the server", // The runner service was stopped or auto-updated under the job: an // infrastructure cancellation, not a verdict on the diff. RUNNER_SHUTDOWN_SIGNAL, // "The job was not acquired by Runner of type hosted even after multiple // attempts" — the job never reached a runner at all, so nothing in the diff // was ever executed (community#186216, community#165287). "was not acquired by runner", // "The hosted runner encountered an error while running your job. (Error // Type: Failure)." — GitHub's own statement that its runner broke // (community#126539). The parenthesised error type varies; the sentence does // not. "the hosted runner encountered an error while running your job", ]; /** * The runner-infrastructure wording a red CI check's text carries, or * `undefined` when it carries none. * * Exists so the review-correction accounting can decline to charge a * correction round for a red that the worker's diff did not cause: a lost * runner, a runner shut down under the job, or a job no runner ever picked up. * The return value is the matched wording, so whatever waives a round can say * *which* sentence waived it instead of asserting "infra" unexplained. * * Fails closed in both directions, and deliberately narrower than it could be: * * - Unclassifiable, empty or unread text is `undefined` — an unexplained red * still counts, and a log-fetch failure hands this function nothing to match, * so it never waives. * - "The operation was canceled." is *not* here: it is what a failed `needs:` * dependency and an operator's own cancel both print, so matching it would * waive rounds for ordinary red. * - "…has exceeded the maximum execution time of N minutes" is *not* here * either: a job that ran until the ceiling is usually a hanging test or an * infinite loop, which is exactly the diff's problem to fix. */ export function runnerInfraFailure(log: string): string | undefined { const lower = log.toLowerCase(); return RUNNER_INFRA_SIGNATURES.find((signature) => lower.includes(signature)); } /** * The cap kills that produced nothing: the worker reached a ceiling with no PR, * no observed head and no salvage commit. * * Named here — beside the table that assigns them — because two callers need * exactly the same set and neither may guess at it: the escalate recovery * quotes the transcript's last tool calls for these classes, and the one-shot * model escalation (#807) fires only for these. Their `-progress` siblings are * deliberately absent: a cap kill *with* work to continue from is a * decomposition verdict, and retrying it on a stronger tier would hide an * oversized slice behind a capability problem. */ export const SPINNING_CAP_CLASSES: readonly FailureClass[] = [ "turn-cap-spinning", "wall-clock-cap-spinning", ]; /** How much of the wall-clock ceiling the default stall window may take (#1086). */ export const WALL_CLOCK_STALL_SHARE = 3; /** * The transcript-silence window a stalled run is settled at (#1086): the * configured `workerStallSilenceMs`, or a third of the wall-clock ceiling when * it is null or unconfigured. Shared by the daemon's progress watch (which * settles at this threshold) and the classifier (which recognises a row killed * at it), so the two can never disagree about what a stall is. */ export function stallSilenceMs( caps: | { workerStallSilenceMs?: number | null; workerWallClockMs: number; } | undefined, ): number { const wallClock = caps?.workerWallClockMs ?? DEFAULT_CAPS.workerWallClockMs; return caps?.workerStallSilenceMs ?? Math.floor(wallClock / WALL_CLOCK_STALL_SHARE); } /** * How long a run sat silent before its stall kill, from the row's own facts * (#1086). Defined only when the progress watch had observed the run AND the * silence reached the configured window — the exact predicate that keeps every * other killed-under-ceiling row (a drain, an operator stop) reading as * `admin-kill`. */ export function stallSilence( run: RunRecord, caps: { workerStallSilenceMs?: number | null; workerWallClockMs: number } | undefined, ): number | undefined { if (run.lastProgressAt === undefined || run.endedAt === undefined) return undefined; const silent = run.endedAt - run.lastProgressAt; if (silent < stallSilenceMs(caps)) return undefined; return silent; } /** * A stable fingerprint of the infrastructure signature list (#638). The * historical reconciliation persists a per-project review cursor stamped with * this version so a bounded pass resumes where the last one stopped; when the * classifier learns a new signature, the fingerprint changes, the stored * cursor is stale, and the pass restarts from the newest row so the newly * recognised evidence is never skipped past. Two values are equal exactly when * the signature set is — a join cannot collide, because each entry is a * distinct delimiter-free token sequence. */ export function infraSignatureVersion(): string { return INFRA_LOG_SIGNATURES.join("|"); } /** * Compose's own sentence for a dependency that failed its healthcheck during * `docker compose up`: "dependency failed to start: container * chad-postgres-1 is unhealthy" (#1059). Matched with its trailing context, * never a bare `is unhealthy` — that phrase is common enough in application * logs to be unsafe on its own. Shared with the settle call site so the diff * fetch and the classifier can never disagree about which log sentence needs * the changed-file list. */ export const COMPOSE_DEPENDENCY_STARTUP_SIGNATURE = "dependency failed to start: container "; /** * A changed path that configures the containers Compose jobs run, or the * workflow that invokes them: a compose file — `docker-compose*.ya?ml`, * Compose v2's default `compose.ya?ml`, or any `*.ya?ml` directly inside a * `compose/` directory (#1071) — anywhere in the tree, a `Dockerfile*`, or * anything under `.github/workflows/`. A PR touching any of these can have * broken `docker compose up` itself — that is what turns the signature below * from infrastructure into a charged, deterministic failure. Matched * case-insensitively: the price of missing a container file is waiving a * genuine attempt, while a false positive (a path merely *named* like one) * only charges one. */ const CONTAINER_CONFIG_PATH = /(?:^|\/)(?:docker-compose[^/]*\.ya?ml|compose\.ya?ml|compose\/[^/]+\.ya?ml|Dockerfile[^/]*)$|(?:^|\/)\.github\/workflows\//i; /** * Evidence that a failed check's log names a Compose dependency-startup * failure, or `undefined` when the log does not name one or the PR's own diff * could have caused it (#1059). * * Deliberately conditional on the PR's changed-file list, and deliberately * left out of {@link INFRA_LOG_SIGNATURES}: a job that runs `docker compose` * can be broken by the diff itself — a bad `docker-compose.yml` in a PR * produces this exact sentence — so the blanket signature would waive the * implementation attempt for a real defect. Only when the diff touches no * compose file, no Dockerfile and no workflow is the fault one the PR cannot * have introduced, and the recovery (`rerun-checks`) is bounded by the strike * cap either way. * * Fails closed: `undefined` changedFiles is "could not read the diff", never * "clean", and stays silent — an unknown or truncated diff must not waive an * attempt. */ export function composeDependencyStartup( log: string, changedFiles: string[] | undefined, ): string | undefined { if (changedFiles === undefined) return undefined; const lower = log.toLowerCase(); if (!lower.includes(COMPOSE_DEPENDENCY_STARTUP_SIGNATURE)) return undefined; if (changedFiles.some((path) => CONTAINER_CONFIG_PATH.test(path))) return undefined; return COMPOSE_DEPENDENCY_STARTUP_SIGNATURE.trim(); } /** Lowercased check state — `gh pr checks` has emitted both `failure` and * `FAILURE` across versions, and the classifier's callers must agree on one * spelling so log selection and classification see the same set of checks. */ export function normalise(state: string): string { return state.trim().toLowerCase(); } /** * Classify one terminal run. * * First match wins, and the order is the contract: the earlier a row matches, * the less its raw state matters. `settlement-stuck` leads because a merged PR * makes every other reading of the row wrong — that is the #362 case, where a * `pushed-green` row outlived its own PR and held the issue out of dispatch. */ /** * Harness start failures, matched on the text it prints. * * A small closed list rather than "any error at turn 0": a run that ended at * turn 0 with an error nobody has named is exactly what `unknown` is for, and * declaring that an environment fault would waive an attempt that may have been * genuinely spent. A mirror fetch that failed is not a session that never * started. */ const START_FAILURE_SIGNATURES = [ "no model selected", "no model configured", "invalid api key", "authentication failed", "could not load its peer dependency", // The pre-launch identity gate (#798/#828): the account is missing, or the // harness binding a worker resolves through is not established. The daemon // refuses before it spawns anything, so no attempt was spent — and the // symptom this replaced (the child dying on the peer import) already read as // a start failure, so charging one here would be a regression dressed as a fix. "worker identity unavailable", // The 2026-08-2x host faults (#986), each recorded verbatim in `lastError` // by the launcher or the worker identity gate — all before the session took // a turn: // - the child died before it could connect to the dispatcher; // - node could not resolve a package out of the worker harness tree; // - the worker's settings file was unreadable (the EACCES outage). "exited 1 before connecting", "cannot find package", "enoent while resolving package", "failed to read settings config", ] as const; /** * The first line of a harness start failure, or undefined when this error is not * one. Exported so the store's one-time repair recognises exactly what the * classifier does — two definitions of "the session never started" would drift. */ export function startFailure(lastError: string | undefined): string | undefined { if (lastError === undefined) return undefined; const text = lastError.toLowerCase(); const hit = START_FAILURE_SIGNATURES.find((signature) => text.includes(signature)); return hit === undefined ? undefined : lastError.split("\n")[0]?.trim(); } /** * Provider messages that mean "this key is out of money", matched on the text * the provider itself prints. */ const CREDIT_REFUSAL_SIGNATURES = [ "requires more credits", "insufficient credits", "insufficient_quota", "exceeded your current quota", "credit balance is too low", ] as const; /** * Evidence that a model provider refused this run for credit, or `undefined` * when it did not. * * Two independent signals, either sufficient: HTTP 402, which means exactly * this and nothing else, and the refusal text providers print when a key is out * of allowance without using 402. Deliberately narrow — a generic 429 is rate * limiting, which is a different failure with a different remedy. */ export function providerCreditRefusal(error: { status?: number; message: string; }): string | undefined { const text = error.message.toLowerCase(); const named = CREDIT_REFUSAL_SIGNATURES.some((signature) => text.includes(signature)); if (error.status !== 402 && !named) return undefined; return error.message.split("\n")[0]?.trim() ?? error.message; } /** Provider text that names a per-request stream fault. Deliberately narrow: * a 429 is rate limiting and a 402 is credit — different remedies (#220). * * The structural `kind` that `readSessionError` marks outranks this list: a * transcript record the harness itself attributed to a provider abort is a * per-request stream fault whatever the provider called it, so no vendor * prose needs enumerating here (#743). The list remains for transcripts that * only ever carried prose, like the original "stream stalled" stall. */ const TRANSIENT_FAULT_SIGNATURES = ["stream stalled"] as const; export function providerTransientFault(error: { status?: number; message: string; kind?: "provider-stream"; }): string | undefined { if (error.kind === "provider-stream") { return error.message.split("\n")[0]?.trim() ?? error.message; } const text = error.message.toLowerCase(); if (!TRANSIENT_FAULT_SIGNATURES.some((s) => text.includes(s))) return undefined; return error.message.split("\n")[0]?.trim() ?? error.message; } /** * How many in-session provider 429s read as *sustained* rate limiting rather * than a retried blip (#573). * * A single 429 the harness retried and recovered from is noise — counting it * would reclassify every run that ever hit one throttle as a provider fault * and let real failures escape the failed-attempt budget. Only a run that kept * hitting the throttle never gets clear of it, and that is the signature the * classifier needs. Three matches the "three strikes" motif this file already * uses (`DISPATCH_INFRA_MAX_STRIKES`, `PROVIDER_TRANSIENT_MAX_STRIKES`). */ const SUSTAINED_RATE_LIMIT_COUNT = 3; /** * Evidence that this run drowned in in-session provider rate limiting, or * `undefined` when it did not. * * Reads the count the worker recorded while the run was live — never a string * match over the transcript, and never `lastError`: omp retried the 429s * internally, so they never surfaced one `lastError`, which is exactly why the * run was previously unclassifiable (#573). 429 is deliberately its own class: * a generic 429 is rate limiting, a 402 is credit, a stream abort is * transient — three failures with three remedies (#220). */ export function providerCapacityFault(count: number | undefined): string | undefined { if (count === undefined || count < SUSTAINED_RATE_LIMIT_COUNT) return undefined; return `${count} in-session provider rate limits (HTTP 429) — the harness retried and was exhausted`; } /** * Evidence that this run never started, or `undefined` when something did happen. * * Two shapes, and the second is the one the reference fleet actually produced. * * 1. Turn zero with a recognised harness start error — the session said why. * 2. Turn zero, **no artifacts and no error text at all**. The row records * nothing: no turn, no head, no salvage, not even a message. On the reference * fleet every worker died at turn 0 with `No model selected`, and that string * went into the tier-1 escalation rather than into `lastError` — so the rows * for issues 373, 359, 343 and 319 carry no error whatsoever and were logged * `unknown`, each charging an implementation attempt (#152). * * Shape 2 is a deliberate judgement, not a loosening. There is no evidence an * attempt was spent, so charging one is charging for a run whose only trace is * that it existed; and the two ways of being wrong are not symmetric — being * generous grants an issue one more attempt a human will see, while being strict * leaves it permanently undispatchable for a fault that was never in its code. * A turn-0 run carrying an *unrecognised* error still reads `unknown`, because * there the row does say something happened. * * Shared with the store's one-time repair so history and future agree. */ export function neverStarted( run: Pick, ): string | undefined { if (run.turns !== 0) return undefined; const detail = startFailure(run.lastError); if (detail !== undefined) return `the session never started: ${detail}`; const produced = run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined; if (!produced && run.lastError === undefined) { return "the session never started: no turns, no commit and no error were recorded"; } return undefined; } /** * Evidence that the conductor's own git path failed before the worker's first * turn, or `undefined` when this row is something else. * * A mirror ref-lock (#168), a dead mirror, a transiently unreachable host: the * worker never touched the issue, so none of this should charge an * implementation attempt. `lastError` of this shape is written only by the * dispatch catch (`errText(err)` = the stack, which prefixes the thrown * "git … exited N: …" message with "Error: ") and by state messages — never by * a worker — so it is proof of the daemon's own git rather than a worker's * failed push. * * Exported for the same reason {@link neverStarted} is: the store's one-time * repair has to recognise exactly what the classifier does, or history and * future disagree about which rows were the daemon's fault. */ export function dispatchInfra( run: Pick, ): string | undefined { if (run.turns !== 0) return undefined; if (run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined) return undefined; if (run.lastError === undefined || !DISPATCH_GIT_ERROR.test(run.lastError)) return undefined; return `dispatch failed in the conductor's own git path before turn 1: ${run.lastError.split("\n")[0]}`; } // See {@link dispatchInfra}: the "Error: " prefix is what `errText` puts in // front of the thrown message, so a bare /^git / would never match a real row. const DISPATCH_GIT_ERROR = /^(?:Error: )?git .+ exited \d+/s; /** * Evidence that one review round's resumed session never got as far as a turn, * or `undefined` when the round actually ran (#903). * * The measurement is the round's OWN session, never the run row's cumulative * counters: a revision reuses its run, so `run.turns` already carries every * turn the original attempt spent and would never read zero. * * Deliberately wider than {@link neverStarted}: any zero-turn round that * produced nothing counts, whatever the error text says. The reference * incident is why. During the 2026-08-21/22 worker-UID outage every revision * dispatch died at turn 0 on `EACCES .../config.yml` — an error string no * signature list contained — and each death still consumed one of the PR's * three review rounds, until the ceiling refused any further round and the PR * could not be finished at all. The discriminator that matters is "did the * resumed session take a turn", not which errno the host produced; matching on * errno text is how the next outage spells its failure differently and lands * back in the same deadlock. * * Two things keep the widening honest, and both are the caller's: * - it decides only whether a round is CHARGED, never whether a run is * classified — run-level classification is untouched, so an unrecognised * turn-0 run still reads `unknown` and still spends its attempt; * - a retried round is bounded by {@link REVIEW_ROUND_INFRA_MAX_RETRIES}, so * a permanently broken host escalates instead of retrying forever. * * A round that produced a PR, a head or a salvage did work no matter what its * turn counter says, and is never this. */ export function reviewRoundNeverWorked(round: { turns: number; prUrl?: string; headSha?: string; salvageSha?: string; }): string | undefined { if (round.turns !== 0) return undefined; if (round.prUrl !== undefined || round.headSha !== undefined || round.salvageSha !== undefined) { return undefined; } return "the resumed session never took a turn: no turn, no commit and no pushed head"; } /** The prefix the draining/restarting process writes to a run it killed. */ const ADMIN_RESTART_MARKER = "admin restart:"; /** * The actor a draining restart recorded on a run it killed, or `undefined` * when the row carries no such attribution. Read from `lastError` for the * same reason {@link dispatchInfra} is read from it: it is the one surface a * run's own row already persists, and the restarting process writes it the * way `recordOperatorStop` writes "operator stopped: …". Recognised as an * administrative kill, never a charging failure, because nothing the worker * did ended it — #512's gap was that a killed session left no attribution. */ export function adminRestartAttribution(lastError: string | undefined): string | undefined { if (lastError === undefined) return undefined; const marker = lastError.indexOf(ADMIN_RESTART_MARKER); if (marker === -1) return undefined; const firstLine = lastError.slice(marker + ADMIN_RESTART_MARKER.length).trim().split("\n")[0]; if (firstLine === undefined || firstLine.trim() === "") return undefined; return firstLine.trim(); } /** The launcher's own statement that the session child died mid-run, before * the session ended. Written by `omp.ts` when the child exits while the * dispatcher still owns it; the exit code is the child's own, so a signal * death reads >=128 (143 = SIGTERM, 137 = SIGKILL) — an external kill, * never anything the worker's work decided (#986). */ const CHILD_EXIT_MARKER = /session child exited (\d+) before the session ended/; /** * Evidence that this run's session child was terminated by a signal, or * `undefined` when it was not. * * Read from the surfaces a run persists — `lastError` first, then `report`, * because the prompt-throw path records the launcher's message as the report * and leaves `lastError` empty. Deliberately narrowed to signal-range codes: * a clean nonzero exit is some process's verdict nobody has named yet, and * naming it `admin-kill` would waive an attempt that may have been spent. */ export function childSignalDeath( run: Pick, ): string | undefined { const text = run.lastError ?? run.report; if (text === undefined) return undefined; const match = CHILD_EXIT_MARKER.exec(text); if (match === null) return undefined; const code = Number(match[1]); if (!Number.isFinite(code) || code < 128) return undefined; return match[0]; } /** The reason the settle sweep writes beside a reviewer-closed PR * (`settlementFor`), matched as recorded evidence so rows written before * that sweep stamped a class read the same way (#986). */ const REVIEWER_CLOSED_MARKER = "closed without merging"; /** * Evidence that a reviewer closed this run's pushed work without merging it, * or `undefined` when nothing says so. Two independent signals, either * sufficient: the tracker fact (`facts.pr === "closed"`), or the settle * sweep's own recorded reason in `lastError`. A review decision is not a * worker failure, which is exactly what `returned-for-revision` means; the * only new reading here is applying it to rows whose PR closed while the row * itself had already gone terminal-failed. */ export function reviewerClosed( run: Pick, facts: ClassifyFacts, ): string | undefined { if (run.lastError !== undefined && run.lastError.includes(REVIEWER_CLOSED_MARKER)) { return run.lastError.split("\n")[0]?.trim(); } if (facts.pr === "closed" && run.prUrl !== undefined) { return `${run.prUrl} closed without merging`; } return undefined; } /** The harness's own name for giving up on a provider that kept answering with * empty assistant turns: it retries, then ends the session on the retry cap * and says so in its maintenance routing (`route: "empty-stop-retry-cap"`, * beside `Assistant returned empty stop after retry cap; try switching * models`). A closed marker written by the harness, not vendor prose. */ const EMPTY_STOP_MARKER = "empty-stop-retry-cap"; /** * Evidence that the provider answered with empty turns until the harness's own * retry cap ended the session, or `undefined` when nothing recorded says so * (#986). * * Read from the terminal evidence the settlement sweep recorded, because this * marker is written by the harness's session log rather than by the transcript * or the row — the run itself has no error, no verdict and no idea it died, * which is precisely why two of these read as `unknown` on 2026-08-23 at * 100/180 and 54/180 turns while a third run on the same model finished green * in the same window. * * Provider flake, not worker logic: the recovery continues from whatever the * attempt pushed rather than charging an implementation attempt for it. */ export function modelEmptyStop( run: Pick, ): string | undefined { if (run.terminalEvidence === undefined) return undefined; if (!run.terminalEvidence.includes(EMPTY_STOP_MARKER)) return undefined; return run.terminalEvidence; } /** * Evidence that this run ended without ever delivering a settlement verdict, * or `undefined` when something on the row says otherwise (#986). * * The shape: state `failed`, turns taken, no error anywhere, and a final * report whose words are mid-delivery narration — "Now I'll execute …", "Type * check passes. Now the test suite:" — with no `state:`/`status:` line the * result parser could read. The session reached its worker and died between * turns without handing over; measured over two weeks of the reference fleet, * this one shape was half of all `unknown`. * * Deliberately narrow at every edge: turn zero belongs to * {@link neverStarted}, any error text keeps the row `unknown` (an * unrecognised error says something happened), and an absent report leaves * genuinely nothing to name. The class charges its attempt exactly as * `unknown` did — only the name becomes specific. */ export function noVerdictExit(run: RunRecord): string | undefined { if (run.state !== "failed") return undefined; if (run.turns <= 0) return undefined; // Blank counts as absent: a row whose `lastError` is an empty string carries // no error information, and reading it as "something happened" is what kept // four real rows unnamed. Any non-blank text, recognised or not, still wins — // an unrecognised error says something happened. if (run.lastError !== undefined && run.lastError.trim() !== "") return undefined; if (run.report === undefined || run.report.trim() === "") return undefined; // Mirror the two verdict spellings the result path reads (`worker.ts` // STATE_LINE_PATTERN for prose settlements, `renderSettlement`'s leading // `status:` line for structured yields). Either one present means a verdict // was delivered and this function must stay silent. if (/^state:\s*\S+\s*$/im.test(run.report)) return undefined; if (/^status:\s*\S+\s*$/im.test(run.report)) return undefined; const lastWords = run.report.split("\n")[0]?.trim() ?? ""; return `the session ended without delivering a settlement verdict; last words: "${lastWords.slice(0, 80)}"`; } /** A reliable PR reference inside a blocker's prose: a `PR #N`/ * `pull request #N` mention or a GitHub `…/pull/N` URL. Named once so the * evidence's "the next act is an observation of this PR" claim and the code * that finds the PR stay one pattern (#1068). */ const PR_REFERENCE_PATTERN = /(?:PR|pull request)\s+#?\s*\d+|github\.com\/[^\s/]+\/[^\s/]+\/pull\/\d+/i; /** * The `blockers:` list of a blocked settlement's stored report, in order * (#1068). The worker's structured yield renders one item per blocker as * ` - ` under a `blockers:` heading, and prose settlements that use the * same heading parse the same way. `undefined` when the report carries no * such section — a report that names no blockers and a missing report read * identically, which is honest: the worker did not name a condition to * observe. */ function blockedSettlementBlockers(report: string | undefined): string[] | undefined { if (report === undefined) return undefined; const blockers: string[] = []; let inBlockers = false; for (const line of report.split("\n")) { if (inBlockers) { const item = /^ {2,}-\s+(.*)$/.exec(line); if (item === null) break; blockers.push(item[1]!.trim()); continue; } if (/^blockers:\s*$/i.test(line)) inBlockers = true; } return blockers.length === 0 ? undefined : blockers; } /** * An explicit question in the blocked row's own words, or `undefined` when the * row names none (#1068). The worker's question travels in the pre-existing * `lastError` slot, or as a `blockers:` item that ends interrogative; a * blocker that names a condition to observe ("PR #1067 checks still * pending…") is not a question. Quoted verbatim into the escalation evidence, * because the orchestrator answers the worker's own words rather than a * paraphrase. */ function blockedQuestion(run: RunRecord): string | undefined { if (run.lastError !== undefined && run.lastError.trim() !== "") return run.lastError; return blockedSettlementBlockers(run.report)?.find((b) => /[??]\s*$/.test(b)); } /** * The report's first substantive line: skips the `status:`/`pr:`/`branch:`/ * `head:` headers and blank lines a structured settlement puts on top, and * bounds the quote so a prose dump cannot balloon an evidence line. `undefined` * when the report is empty or carries nothing but scaffold. */ function firstReportContentLine(report: string | undefined): string | undefined { if (report === undefined) return undefined; for (const raw of report.split("\n")) { const line = raw.trim(); if (line === "") continue; if (/^(?:status|pr|branch|head):\s*\S/.test(line)) continue; return line.length > 200 ? `${line.slice(0, 200)}…` : line; } return undefined; } /** * What the wall-clock ceiling actually buys on this fleet right now (#1063). * * `caps.workerMaxTurns` and `caps.workerWallClockMs` are independent config * values, but they meet at an implicit pace: 0.5 minutes per turn at the * shipped defaults. A model slower than that can never reach the turn ceiling, * so the budget a groomer sized a slice against is fiction — the run dies of * the clock first. This observation is derived from *completed* runs (never * from the configured numbers, whose ratio is a constant and tells nobody * anything), so `status` and the settlement report can name the real number. */ export interface ObservedTurnBudget { /** Aggregate elapsed minutes ÷ aggregate turns across the sample. */ minutesPerTurn: number; /** Completed runs the figure was derived from. */ sampleSize: number; /** ⌊wall clock ÷ minutesPerTurn⌋ — what the clock buys at that pace. */ effectiveTurns: number; } /** How many completed runs the effective-budget sample spans. */ export const EFFECTIVE_BUDGET_SAMPLE_RUNS = 20; /** A wall-clock kill below this share of its turn ceiling is a latency * verdict; at or above it, both ceilings were nearly exhausted and size is * the honest reading (#1063). */ export const WALL_CLOCK_SIZE_VERDICT_SHARE = 0.7; /** Strips omp selector suffixes (`:thinking`, `:max`) so only the model path * is compared. */ function modelBase(selector: string): string { const colon = selector.indexOf(":"); return colon === -1 ? selector : selector.slice(0, colon); } /** * Whether one model attribution names the same model as the configured * selector, compared as path suffixes: config says * `openrouter/stealth/ox-alpha:max` while the transcript's resolved model * reads `stealth/ox-alpha`, and strict equality would empty every sample. * Role aliases (`@slow`) have no concrete spelling to resolve to from here, so * they match themselves only — no sample, rather than an invented one. */ function sameModel(configured: string, observed: string): boolean { if (configured.startsWith("@") || observed.startsWith("@")) return configured === observed; const c = modelBase(configured); const o = modelBase(observed); return c === o || c.endsWith(`/${o}`) || o.endsWith(`/${c}`); } /** * Derives {@link ObservedTurnBudget} from completed-run rows, newest-first * callers' ordering irrelevant. Rows without turns or elapsed time carry no * pace and are skipped; when a configured worker model is named, rows written * by another model are skipped with them (`resolvedModel` preferred — what * actually wrote the messages — falling back to the dispatch record). * `undefined` when nothing qualifying remains: absence is the honest answer, * never a guess from the configured constants. */ export function observeTurnBudget( samples: readonly Pick[], caps: Pick, configuredModel?: string, excludeRunId?: string, ): ObservedTurnBudget | undefined { let minutes = 0; let turns = 0; let used = 0; for (const s of samples) { if (s.id === excludeRunId) continue; if (s.turns <= 0 || s.endedAt === undefined || s.endedAt <= s.startedAt) continue; if (configuredModel !== undefined) { const attribution = s.resolvedModel ?? s.model; if (attribution === undefined || !sameModel(configuredModel, attribution)) continue; } minutes += (s.endedAt - s.startedAt) / 60_000; turns += s.turns; used += 1; } if (used === 0 || turns <= 0) return undefined; const minutesPerTurn = minutes / turns; if (!Number.isFinite(minutesPerTurn) || minutesPerTurn <= 0) return undefined; return { minutesPerTurn, sampleSize: used, effectiveTurns: Math.floor(caps.workerWallClockMs / 60_000 / minutesPerTurn), }; } export function classifyRun( run: RunRecord, facts: ClassifyFacts, caps?: Pick & { workerStallSilenceMs?: number | null }, observedBudget?: ObservedTurnBudget, ): Classification { const hasArtifacts = run.prUrl !== undefined || run.headSha !== undefined || run.salvageSha !== undefined; const providerError = facts.sessionError ?? (run.lastError === undefined ? undefined : { message: run.lastError }); // The PR landed while the row says otherwise. Whatever else is true about this // run, it succeeded, and the recovery is bookkeeping. if (run.prUrl !== undefined && facts.pr === "merged") { return { cls: "settlement-stuck", recovery: "settle", evidence: `row state ${run.state} but ${run.prUrl} is merged`, }; } // A green PR whose base moved under it. Mechanical to fix and it never spent // an implementation attempt: veltro#365 lost a continuation to exactly this. if (run.state === "pushed-green" && facts.pr === "open" && facts.mergeable === "conflicting") { return { cls: "merge-conflict", recovery: "continue", evidence: `${run.prUrl ?? "the PR"} is open and conflicting — the base moved under a green PR`, }; } // The launcher's own statement that the session child died on a signal // mid-run: an external kill — a drain, a restart, the host — not anything // the worker's work decided (#986). Ahead of the provider branches on // purpose: a transcript can carry an older, long-recovered stream fault, // and the launcher's terminal statement outranks whatever the session said // earlier. Requeued free like every other administrative kill. if (run.state === "failed") { const marker = childSignalDeath(run); if (marker !== undefined) { return { cls: "admin-kill", recovery: "requeue", evidence: `killed externally (${marker}) — the session child was terminated, not finished`, }; } } // A billing state, not an implementation failure. Its own class because it is // the one outage an operator fixes with a card rather than a diagnosis, and // because burying a self-describing provider error in `unknown` erodes what // `unknown` means (#220). Ahead of `neverStarted` on purpose: a 402 on the // first request produces a turn-0 row with no artifacts, which // `env-start-failure` would otherwise absorb and lose the cause. if (run.state === "failed" || run.state === "killed") { const credit = providerError === undefined ? undefined : providerCreditRefusal(providerError); if (credit !== undefined) { return { cls: "provider-credit", recovery: "requeue", evidence: credit }; } } // A per-request stream fault: the provider aborted mid-stream — the session // records "OpenAI responses stream stalled while waiting for the next event", // but the attempt never produced a verdict and 0 tokens were billed. Not the // work's fault, so it must not charge an implementation attempt; requeued, but // bounded by PROVIDER_TRANSIENT_MAX_STRIKES so a provider that keeps aborting // escalates to a human instead of looping (#220). Ahead of `neverStarted` for // the same reason as the credit branch: a turn-0 stall would otherwise be // absorbed by `env-start-failure` and lose its cause. if (run.state === "failed" || run.state === "killed") { const transient = providerError === undefined ? undefined : providerTransientFault(providerError); if (transient !== undefined) { return { cls: "provider-transient", recovery: "requeue", evidence: transient }; } } // Sustained in-session provider rate limiting: the run drowned in HTTP 429s // the harness retried and never surfaced one as `lastError`, so neither the // credit nor the transient branch sees it and it used to fall through to // `unknown`, spending an implementation attempt on the provider's capacity // (#573). Only the worker-recorded count — above a *sustained* threshold, so // a single retried 429 on an otherwise healthy run stays untouched — reads as // this. Distinct from the two provider classes on purpose and requeued free, // bounded by its own strike cap in the daemon so a permanently throttled // provider escalates instead of looping. Ahead of `neverStarted` for the same // reason as the credit branch: a turn-0 run that immediately drowned in 429s // would otherwise be absorbed by `env-start-failure` and lose its cause. if (run.state === "failed" || run.state === "killed") { const capacity = providerCapacityFault(run.provider429Count); if (capacity !== undefined) { return { cls: "provider-capacity", recovery: "requeue", evidence: capacity }; } } // A run that never started is the most classifiable failure there is, and the // least deserving of an implementation attempt: the session did not get as far // as reading the issue. See {@link neverStarted} for the two shapes and why the // second one counts. if (run.state === "failed" || run.state === "killed") { const evidence = neverStarted(run); if (evidence !== undefined) { return { cls: "env-start-failure", recovery: "escalate", evidence }; } } // Dispatch infra: the conductor's own git path failed before the worker's // first turn. See {@link dispatchInfra} — the worker never touched the issue, // so this must not charge an implementation attempt. if (run.state === "failed") { const evidence = dispatchInfra(run); if (evidence !== undefined) { return { cls: "dispatch-infra", recovery: "requeue", evidence }; } } // A run the draining/restarting process attributed before it restarted the // session host: killed under its own ceilings by an administrative action, // not by anything the worker did. Costs no budget and requeues, naming the // actor (#512). A killed session used to leave no attribution and fell // through to `unknown`, charging a failure for an operator's restart. if (run.state === "failed" || run.state === "killed") { const actor = adminRestartAttribution(run.lastError); if (actor !== undefined) { return { cls: "admin-kill", recovery: "requeue", evidence: `killed by an administrative restart (${actor}) — an operator's restart, not the run's work`, }; } } // A reviewer closed this run's pushed work without merging it: a review // decision, not a worker failure (#986). Two rows reach here — the settle // sweep's own recorded reason in `lastError` on a row that predates the // class, and a live `closed` tracker fact on a row whose worker died after // pushing but before claiming. Recovery `none` mirrors the settle sweep // exactly: the queue label is the remedy, and no action is performed here. if (run.state === "failed") { const closed = reviewerClosed(run, facts); if (closed !== undefined) { return { cls: "returned-for-revision", recovery: "none", evidence: closed }; } } if (run.state === "blocked") { // `blocked` is overloaded (#1068). A worker that finishes its work and // stops because it is waiting on an observable condition — almost always // its PR's checks — records that condition as `blockers:` in its // settlement report; a worker that stops to ask records an actual // question. Only the second shape is a human escalation: the first names // the next act (observe, then the sweep settles the row the way it // settles `settlement-stuck` when the PR resolves), so escalating it // wakes the orchestrator for a question that does not exist and the act // that *is* required appears nowhere. The evidence never claims the // worker "left no report" — it quotes the question or the first blocker, // and when the report genuinely is empty it says which fields are. const question = blockedQuestion(run); if (question !== undefined) { return { cls: "question", recovery: "escalate", evidence: question }; } const blockers = blockedSettlementBlockers(run.report); if (blockers !== undefined) { // Prefer the blocker that names a PR: the reader's next act is an // observation of that PR and a merge, and the evidence must say so. const named = blockers.find((b) => PR_REFERENCE_PATTERN.test(b)) ?? blockers[0]!; return { cls: "awaiting-observation", recovery: "observe", evidence: `waiting on ${named}`, }; } const reportEmpty = run.report === undefined || run.report.trim() === ""; const firstContent = firstReportContentLine(run.report); return { cls: "question", recovery: "escalate", evidence: reportEmpty ? `the worker stopped without a run report (run.report is empty${ run.lastError === undefined || run.lastError.trim() === "" ? "; lastError is empty too" : "" })` : firstContent === undefined ? "the worker stopped without asking a question or naming a blocker (run.report is non-empty but has no readable content)" : `the worker stopped without asking a question or naming a blocker — ${firstContent}`, }; } if (run.state === "orphaned") { // Dirty means the worktree holds the only copy: re-claiming would delete it, // which is the #319 data-loss shape. The existing unsalvaged-WIP admission // hold already fails dispatch closed, so recording the class is the whole // recovery — an operator has to look either way. if (run.salvageError !== undefined && run.salvageAckAt === undefined) { return { cls: "orphan-dirty", recovery: "hold", evidence: `uncommitted work in ${run.worktree} is the only copy: ${run.salvageError}`, }; } return { cls: "orphan-clean", recovery: "requeue", evidence: `daemon died holding the claim; nothing uncommitted (${run.branch})`, }; } if (run.state === "killed") { // A stall the daemon settled (#1086), recognised from the row's own facts // — the watch observed a transcript write instant, and the silence to the // kill reached the configured window. Checked before the cap branches on // purpose: a hung session's turns never move, so this row is under its // ceilings, but even if a config change made it *reach* one mid-hang, the // silence is the operative fact about why it died. The same split as the // caps: artifacts continue, nothing shows escalates. const silentMs = stallSilence(run, caps); if (silentMs !== undefined) { const minutes = Math.round(silentMs / 60_000); const silent = minutes < 60 ? `${minutes}m` : `${Math.floor(minutes / 60)}h${minutes % 60 === 0 ? "" : `${minutes % 60}m`}`; return hasArtifacts ? { cls: "progress-stall", recovery: "continue", evidence: `transcript silent ${silent} at turn ${run.turns}/${run.maxTurns}; the session never came back — work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`, } : { cls: "progress-stall", recovery: "escalate", evidence: `transcript silent ${silent} at turn ${run.turns}/${run.maxTurns}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`, }; } if (run.turns >= run.maxTurns) { return hasArtifacts ? { cls: "turn-cap-progress", recovery: "continue", evidence: `turns ${run.turns}/${run.maxTurns} with work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`, } : { cls: "turn-cap-spinning", recovery: "escalate", evidence: `turns ${run.turns}/${run.maxTurns}, no PR, no commits — $${run.spendUsd.toFixed(2)} spent`, }; } // A run killed at the wall-clock cap is a cap kill too, even though its // turns ceiling never tripped — the harness would have kept burning a // worker slot until the ticks ran out (#490). The same split as the turns // path: artifacts mean the next attempt resumes from real work; nothing to // show means a human re-scopes, naming the time and spend consumed. // // #1063: which ceiling was ever reachable? The two caps meet at an implied // pace (wall clock ÷ max turns), so a run killed at the clock far under // its turn budget died of *latency* — re-attempting it as an oversized // slice spends a continuation on the wrong diagnosis. The evidence names // the run's own pace, the latency/size verdict the share of the turn // ceiling implies, and — when the caller supplied one — the effective // turn budget observed across completed runs on this model. const wallClockCap = caps?.workerWallClockMs; const wallClockElapsed = run.endedAt === undefined ? undefined : run.endedAt - run.startedAt; if (wallClockCap !== undefined && wallClockElapsed !== undefined && wallClockElapsed >= wallClockCap) { const clock = `${Math.round(wallClockElapsed / 60_000)}m of ${Math.round(wallClockCap / 60_000)}m`; const pace = run.turns > 0 ? ` at ${((wallClockElapsed / 60_000) / run.turns).toFixed(2)} min/turn` : ""; const share = run.maxTurns > 0 ? run.turns / run.maxTurns : 1; const verdict = share < WALL_CLOCK_SIZE_VERDICT_SHARE ? `latency verdict: ${run.turns}/${run.maxTurns} turns before the clock — the slice fits, the model is slow` : `size verdict: ${run.turns}/${run.maxTurns} turns — both ceilings nearly exhausted`; const observed = observedBudget === undefined ? "" : `; effective ~${observedBudget.effectiveTurns} turns at ${observedBudget.minutesPerTurn.toFixed(2)} min/turn (last ${observedBudget.sampleSize} runs)`; return hasArtifacts ? { cls: "wall-clock-cap-progress", recovery: "continue", evidence: `wall clock ${clock}${pace} — ${verdict}${observed}; work to continue from (${run.prUrl ?? run.salvageSha ?? run.headSha})`, } : { cls: "wall-clock-cap-spinning", recovery: "escalate", evidence: `wall clock ${clock}${pace} — ${verdict}${observed}; no PR, no commits — $${run.spendUsd.toFixed(2)} spent`, }; } // Below its own ceilings, so nothing this worker did ended it: a daemon // restart, a drain, an operator. Costs no budget and needs no human. return { cls: "admin-kill", recovery: "requeue", evidence: `killed at ${run.turns}/${run.maxTurns} turns — under its own ceiling, so not a cap kill`, }; } if (run.state === "failed" && facts.pr === "open") { const checks = facts.checks ?? []; const unresolved = checks.filter((c) => !SUCCESS_CHECK_STATES[normalise(c.state)] === true); if (checks.length > 0 && unresolved.length === 0) { // The PR is open and every check is green: the strongest mechanical // success evidence there is short of merge. The row's `failed` state is // stale — whatever the worker recorded, its work is passing — so this is // not a failure of any class. `none` recovery restores it to // `pushed-green` in classifyAndRecover instead of escalating an // `[unknown]` that charges an attempt (#766). An empty check list is // "could not tell", not all-green, and still falls through to `unknown` // below. return { cls: "unknown", recovery: "none", evidence: `${run.prUrl ?? "the PR"} is open and every check is green`, }; } if (checks.length > 0 && unresolved.length > 0) { // A failed check whose *log* smells like infrastructure — a registry 429, // a runner shutdown, a DNS failure (#177). The check has a verdict, so // the check-state table above cannot call it infra; only the log can. It // beats ci-deterministic because charging an implementation attempt for a // rate limit is exactly the waste that class exists to prevent. if (facts.failingLog !== undefined) { // Matched exactly, without paren-stripping: GitHub's full setup/codeload // sentence is its own closed entry above, and a generic paren-strip would // broaden the other signatures into waiving real deterministic failures // (#177). A bare "429" or "too many requests" (parenthesized or not) // still never matches without that sentence. const signature = infraLogSignature(facts.failingLog); if (signature !== undefined) { const check = checks.find((c) => normalise(c.state) === "failure"); return { cls: "ci-infra", recovery: "rerun-checks", evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${signature}"`, }; } // A Compose dependency-startup failure is infra only when the PR's // diff could not have caused it (#1059): a job that runs `docker // compose` is broken by a PR that breaks its own compose file, and // that attempt is genuinely spent. The settlement hands the // classifier the PR's changed-file list, so "did this diff touch any // container configuration" is a fact, not a guess from the log alone. // Absent list (or a log without the sentence) stays silent and the // row reads `ci-deterministic`. const compose = composeDependencyStartup(facts.failingLog, facts.changedFiles); if (compose !== undefined) { const check = checks.find((c) => normalise(c.state) === "failure"); return { cls: "ci-infra", recovery: "rerun-checks", // Both halves, so the evidence says *which* sentence matched and // *why* it was not the diff's fault — the changed-file list // showed no container configuration. evidence: `infrastructure signature in ${check?.name === undefined ? "failed check" : check.name} log: "${compose}" — the PR diff touches no container configuration, so the PR cannot have failed this check`, }; } } const failing = unresolved.filter((c) => normalise(c.state) === "failure"); if (failing.length > 0) { return { cls: "ci-deterministic", recovery: "escalate", evidence: `checks failed: ${failing .map((c) => `${c.name}${c.link === undefined ? "" : ` (${c.link})`}`) .join(", ")}`, }; } if (unresolved.every((c) => INFRA_CHECK_STATES[normalise(c.state)] === true)) { return { cls: "ci-infra", recovery: "rerun-checks", evidence: `no check reached a verdict: ${unresolved.map((c) => `${c.name} ${normalise(c.state)}`).join(", ")}`, }; } } } // The provider answered with empty turns until the harness gave up (#986). // Below every branch that reads a real error on purpose: a row carrying its // own provider fault is that fault, and this marker only decides rows that // would otherwise be `unknown`. Continues rather than requeues, because the // attempt's pushed work is the honest place to resume from, and a provider // flake must not charge an implementation attempt (#1001). `killed` rows // never reach here — a cap kill is the operative fact about a run that hit // its ceiling, whatever the provider did earlier in the session. if (run.state === "failed") { const emptyStop = modelEmptyStop(run); if (emptyStop !== undefined) { return { cls: "model-empty-stop", recovery: "continue", evidence: `the provider returned empty turns until the harness's retry cap ended the session (${emptyStop}) — turns ${run.turns}/${run.maxTurns}`, }; } } // The session reached its worker and died between turns without handing over // a verdict (#986): the largest single shape inside `unknown`. Named rather // than requeued blind — an operator reading `status` learns the run stopped // mid-delivery, which is a different question from "nothing is known". if (run.state === "failed") { const narration = noVerdictExit(run); if (narration !== undefined) { return { cls: "no-verdict", recovery: "escalate", evidence: narration }; } } // Deliberately escalate rather than retry. An unrecognised shape is a gap in // this table, and a silent requeue would spend a budget on a cause nobody has // named — the exact behaviour #132 exists to end. return { cls: "unknown", recovery: "escalate", evidence: `state ${run.state}, turns ${run.turns}/${run.maxTurns}, pr ${facts.pr ?? "unknown"}${ run.lastError === undefined ? "" : `, last error: ${run.lastError}` }`, }; }