/** * The status-rendering half of the fleet operator surface (`fleet.ts` * composes; this owns the rendered text). * * Extracted from `fleet.ts` so the region every issue that surfaces something * an operator reads must touch is a lane of its own: #459, #582, #318, #283 * and #484 all render a new line here, and a diff touching only this module * must not have to touch the composition root. Behavior is deliberately * untouched by the move — this is a move, not a rewrite: the module takes a * data structure and returns text, and reads nothing itself. * * What status rendering is allowed to consult is the point of the module. It * does not import `fleet.ts` back, and it owns the structural input it * consumes (`FleetLayers`, `TelegramHealth`, `FleetDaemonProbe`) rather than * reaching into the module that builds them — the same seam #580 used for * `AdmissionDeps`. Adding a field to `fleetLayers` must not break these * functions. */ import { formatZonedMinute } from "./availability.ts"; import { spendTelemetryDetail, type SpendTelemetryVerdict } from "./spend-telemetry.ts"; import type { DaemonStop, GroomingRecord, InstallSurfaceObservation, ReviewCorrectionRound, RunRecord, } from "./types.ts"; import { settlementFlagSummary } from "./diff-flags.ts"; import type { CodeGraphHealth } from "./graph-health.ts"; import { formatDigestBacklog, formatOpenReports } from "./reports.ts"; import { formatDownDuration, formatOrchestratorDown } from "./orchestrator-down.ts"; import { planUsageLine } from "./usage.ts"; import { HEALTH_TIMEOUT_MS, SYSTEMD_UNIT, type UnitOwnership } from "./lifecycle.ts"; import type { WorkerPauseView } from "./fleet.ts"; import { formatRss, rssBytesFromHealthz } from "./host.ts"; // One constant, two readers: the durable in-flight to-spec launch marker // (#777) is written by the daemon's grooming launcher and read here, so the // reason string is imported rather than hand-copied — the copy that used to // live in this file was a second source of truth for a row shape nobody // audits. import { TO_SPEC_IN_FLIGHT_REASON } from "./groom.ts"; import { formatBaseHealth, formatDispatchSummary, formatFreezes, formatReleaseGrants, isPaused, pausedAt, pauseProvenance, type StatusSnapshot, } from "./daemon.ts"; import { pidAlive } from "./escalate.ts"; import { formatQuarantinedRuns, formatSalvagedRuns } from "./settlement.ts"; import { formatVerbLedger } from "./verbs/ledger.ts"; // layered status // --------------------------------------------------------------------------- export type DispatchLayer = "running" | "paused" | "stopped"; export type TicksLayer = | "armed" | "disarmed" | "no-heartbeat-config" | "invalid-heartbeat-config" | "ungated"; export type PaneLayer = "live" | "missing" | "unknown"; /** `unpinnable`: no tick config, so FLEET_CWD — the only path recovery reads — is unknown. */ export type RecoveryLayer = "pinned" | "clear" | "unpinnable"; /** * A token count a human reads at a glance: `102.1k`, not `102105`. * * Thousands only — a run's output tokens are the one count here, and they land * between thousands and low millions, where a single unit reads unambiguously. */ function formatCount(n: number): string { return n < 1_000 ? String(n) : `${(n / 1_000).toFixed(1)}k`; } export type HerdrLayer = "active" | "inactive" | "unknown"; export type TelegramLayer = "ok" | "degraded" | "down" | "unconfigured" | "unprobed"; export interface TelegramHealth { kind: TelegramLayer; detail?: string; } export interface FleetLayers { dispatch: DispatchLayer; ticks: TicksLayer; ticksDetail?: string; nextTickAt?: string; pane: PaneLayer; paneDetail?: string; recovery: RecoveryLayer; recoveryDetail?: string; herdr: HerdrLayer; herdrDetail?: string; armedPath?: string; tickConfigPath?: string; paneHaltPath?: string; paused: boolean; daemon: { running: boolean; pid?: number; port?: number }; } export function formatCodeGraphHealth(graph: CodeGraphHealth, now = Date.now()): string | undefined { if (!graph.configured) return undefined; const indexed = graph.repos.filter((repo) => repo.index === "present").length; let refresh: string = graph.refresh.result; if (graph.refresh.lastSuccessAt !== undefined) { const ageMs = Math.max(0, now - Date.parse(graph.refresh.lastSuccessAt)); refresh = `${graph.refresh.lastSuccessAt} (${Math.max(1, Math.ceil(ageMs / 60_000))}m ago)`; } return [ `code graph ${graph.status} ${indexed}/${graph.repos.length} repos indexed`, ` indexer ${graph.prerequisites.indexer}`, ` MCP mount ${graph.prerequisites.mcpMount}`, // The runtime half of the mount finding (#726): what dispatched sessions // actually held in their registries. Rendered only when a run recorded // it — "observed" is never claimed without evidence. ...(graph.session.recorded === 0 ? [] : [ ` session observed in ${graph.session.recorded} run(s): present ${graph.session.present} / absent ${graph.session.absent}`, ]), ` timer ${graph.timer.enabled} / ${graph.timer.active}`, ` refresh ${refresh}`, ...graph.reasons.map((reason) => ` - ${reason}`), ].join("\n"); } /** * Project-scoped reading of a living daemon record + `/healthz` body. * * Board and `status` share this so a host running one daemon for several * projects cannot be called healthy for a project it does not serve (#379). * A third interpretation convention is forbidden. */ export type DaemonProjectHealth = | { kind: "stopped" } | { kind: "ok" } | { kind: "unreachable" } | { kind: "unresponsive" } | { kind: "other-project"; serves?: string }; /** * Facts `formatFleetStatus` needs about the living daemon. Pure input — the * caller probes; the formatter never shells out (#379). */ export type FleetDaemonProbe = { project: DaemonProjectHealth; /** `/healthz` body when the probe is project-ok (rss / overlays). */ body?: string; /** systemd ownership of the living record pid; omit when unprobed. */ unit?: UnitOwnership; }; /** * #685: the healthz timeout is a probe outcome, never a death verdict — the * same discipline the indexer probe uses when it reports `probe timed out` * instead of declaring the indexer missing. */ const UNRESPONSIVE_HEALTHZ = `unresponsive (healthz timed out after ${HEALTH_TIMEOUT_MS / 1000}s)`; /** One line of a possibly-multi-line record field, bounded for a status row: * the evidence and disposition an adjudicator writes are prose, and status is * a scannable board — the full text lives in the durable row and the ledger. */ function firstLine(text: string, limit = 120): string { const line = text.split("\n").find((candidate) => candidate.trim().length > 0)?.trim() ?? ""; return line.length <= limit ? line : `${line.slice(0, limit - 1)}…`; } function formatDaemonHealthz(probe: FleetDaemonProbe | undefined): string { if (probe === undefined) return "unprobed"; switch (probe.project.kind) { case "stopped": return "stopped"; case "ok": return "ok"; case "unresponsive": return UNRESPONSIVE_HEALTHZ; case "unreachable": return "unreachable — the process is up but not serving"; case "other-project": return probe.project.serves === undefined ? "other-project" : `other-project — serves ${probe.project.serves}`; } } /** * Unit line for a living record. The bare unit name alone used to imply * systemd ownership that was never checked (#379); every branch states the * probe result, and `unknown` is never rendered as owned. */ function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): string { if (unit === undefined) return ` unit ${SYSTEMD_UNIT} unprobed`; switch (unit.kind) { case "active": if (unit.pid === recordPid) { return ` unit ${SYSTEMD_UNIT} systemd-owned`; } // Live unit, different MainPID — the record pid is not systemd's. return ` unit ${SYSTEMD_UNIT} unmanaged — MainPID ${unit.pid}`; case "failed": return ` unit ${SYSTEMD_UNIT} unmanaged — unit failed`; case "inactive": return ` unit ${SYSTEMD_UNIT} inactive`; case "unknown": return ` unit ${SYSTEMD_UNIT} unknown — ${unit.reason}`; } } /** * The newest daemon stop/restart provenance as status lines (#378). * * Rendered under the daemon block whether the daemon is down or has since * been restarted — the whole point is that the debrief survives the restart. * An unattributed record says so explicitly: the external-signal fallback * records what the receiving daemon knew (that it was unattributed, when, * which projects it served with live counts) rather than guessing a caller. */ function formatLastStop(lastStop: DaemonStop | undefined): string[] { if (lastStop === undefined) return []; const caller = lastStop.unattributed ? "unattributed — no mediated request (external signal)" : `pid ${lastStop.callerPid ?? "?"}` + (lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) + (lastStop.role === undefined ? "" : ` (${lastStop.role})`); return [ ` last stop ${new Date(lastStop.at).toISOString()} ${lastStop.controlPath}`, ` caller ${caller}`, ` scope ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`, ` affects ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`, ` reason ${lastStop.reason}`, ]; } // durable to-spec grooming lifecycle (#809) // --------------------------------------------------------------------------- /** * The durable mechanical-hold reasons admission's reconcile writes as blocked * rows (#735) — the lane/dependency holds that clear by themselves. Mirrors * `BLOCKED_GROOMING_HOLDS` in `store.ts`; the status reads the stored reason * strings so both sides of the contract stay on the store vocabulary. */ const MECHANICAL_GROOMING_REASONS: Record = { "file-lane": true, "depends-on": true, }; /** * The to-spec refusal classes persisted as blocked rows (#772, #1064) — a * result that failed validation, or a pass that produced no answer at all, is * a mechanical block, never a verdict. Mirrors the failure kinds of * `ToSpecFailure` in `to-spec.ts`. */ const REFUSED_GROOMING_REASONS: Record = { malformed: true, "no-answer": true, "missing-source": true, "stale-source": true, }; /** * One project's durable grooming state as status lines, or nothing when there * is nothing to report (#809). * * Renders the state already owned by the `grooming_verdicts` table plus the * dispatch snapshot's parked count — each category below is derived from the * project-scoped durable rows the caller passes, never from parsing evidence * strings and never from a second cache: * * - `awaiting to-spec` — the routable queue candidates the last dispatch * pass saw that no durable row covers: `routed − rows`, clamped at zero so * stale rows (candidates since removed from the queue) cannot push it * negative. `routed` is the same denominator the tick's grooming trigger * reads (`summary.routed >= groomBelow`; under `groomBelow: "always"`, * #988, the duty never clears on volume), and the fleet's queue-digest * arithmetic already treats durable rows as current-queue facts * (`claimable = routed − knownBlocked`), so this is the operator-facing * half of the same subtraction. Parent/epic exclusions and backlog * candidates that never entered the queue are tracker-side and not * store-knowable; the row lines below carry the durable results either * way. * - `promotable` / `considered` / `blocked` — completed to-spec results * (`blocked` rows whose reason is a groomer verdict or a product-judgement * label, e.g. `needs-product-decision`). * - `mechanically blocked` — admission's lane/dependency holds. * - `refused` — to-spec passes that produced no usable verdict (`malformed`, * `no-answer`, `missing-source`, `stale-source`), told apart from the * holds so an operator sees whether the runway cannot move or a result * cannot be trusted. Each row's line names its own class (`#19 malformed`, * `#22 no-answer`), so a run of identical refusals is visible as a pattern * rather than a wall of one word. * - `in-flight` — a launched batch is running (#777). * - `operator-parked` — the dispatch snapshot's parked count (#507). * * Rendering is read-only: nothing here reconciles, promotes, retries, or * mutates grooming records. Empty categories are omitted — zero-value rows * would be noise, and an absent block means a fleet with no grooming state. */ export interface GroomingStatusInput { /** Project-scoped durable rows (`Store.groomingVerdicts`), issue-ascending. */ records: readonly GroomingRecord[]; /** Routable queue candidates from the last dispatch pass * (`DispatchSummary.routed`). */ routed: number; /** Operator-parked candidates from the last dispatch pass * (`DispatchSummary.parked`). */ parked: number; } /** How many issue identifiers one category line may carry — enough to act on, * never a queue dump. Mirrors the dispatch hold sampling. */ const GROOMING_STATUS_SAMPLE = 5; /** One category line: label, count, and a bounded sample of issue identifiers * with their durable reasons where the reason differs from the line label * (e.g. `#13 already-done` under `considered`, `#19 malformed` under * `refused`). Absent at zero. */ function groomingStatusLine( label: string, rows: readonly GroomingRecord[], ): string | undefined { if (rows.length === 0) return undefined; const sample = rows .slice(0, GROOMING_STATUS_SAMPLE) .map((row) => (row.reason === label ? `#${row.issue}` : `#${row.issue} ${row.reason}`)); const more = rows.length > GROOMING_STATUS_SAMPLE ? ", …" : ""; return ` ${label.padEnd(21)}${rows.length} ${sample.join(", ")}${more}`; } export function formatGroomingStatus(input: GroomingStatusInput): string | undefined { const { records, routed, parked } = input; const promotable = records.filter((row) => row.verdict === "promotable"); const considered = records.filter((row) => row.verdict === "considered"); const blockedResults = records.filter( (row) => row.verdict === "blocked" && MECHANICAL_GROOMING_REASONS[row.reason] !== true && REFUSED_GROOMING_REASONS[row.reason] !== true && row.reason !== TO_SPEC_IN_FLIGHT_REASON, ); const mechanical = records.filter( (row) => row.verdict === "blocked" && MECHANICAL_GROOMING_REASONS[row.reason] === true, ); const refused = records.filter( (row) => row.verdict === "blocked" && REFUSED_GROOMING_REASONS[row.reason] === true, ); const inFlight = records.filter( (row) => row.verdict === "blocked" && row.reason === TO_SPEC_IN_FLIGHT_REASON, ); const awaiting = Math.max(0, routed - records.length); const lines: string[] = []; if (awaiting > 0) { lines.push(` ${"awaiting to-spec".padEnd(21)}${awaiting} (of ${routed} routable)`); } for (const line of [ groomingStatusLine("promotable", promotable), groomingStatusLine("considered", considered), groomingStatusLine("blocked", blockedResults), groomingStatusLine("mechanically blocked", mechanical), groomingStatusLine("refused", refused), groomingStatusLine("in-flight", inFlight), ]) { if (line !== undefined) lines.push(line); } if (parked > 0) lines.push(` ${"operator-parked".padEnd(21)}${parked}`); if (lines.length === 0) return undefined; return ["grooming (to-spec)", ...lines].join("\n"); } /** * The one install-surface fault the cheap surfaces are allowed to name (#919): * two different releases live on one host at the same time. * * Deliberately narrower than `doctor`'s finding. An absent surface, a `local:` * herdr link and a pin whose release could not be verified are all real * observations, and all of them are *steady states* someone chose — a warning * repeated every fifteen minutes would train an operator to ignore the row that * matters. A CLI and an omp plugin on different versions is neither steady nor * chosen: it is the 2026-08-22 shape, where the plugin sat on a withdrawn * 0.18.1 beside a 0.18.0 daemon for a day with nothing saying so. * * Pure, so the tick and `status` read the same recorded row and agree by * construction rather than by two copies of a rule. */ export function installSurfaceMismatch( observation: InstallSurfaceObservation | undefined, ): string | undefined { if (observation === undefined) return undefined; const { cliVersion, ompVersion } = observation; if (ompVersion === undefined || ompVersion === cliVersion) return undefined; return ( `installed surfaces disagree: omp plugin ${ompVersion}, CLI/daemon ${cliVersion}` + `${observation.herdrSource === undefined ? "" : `, herdr ${observation.herdrSource}`}` + ` — run \`omp-conductor upgrade --to ${cliVersion}\` so every surface carries one identity` ); } /** The `status` row for the recorded observation (#919): nothing on agreement, * the mismatch when there is one, and an honest "not observed yet" before the * first dispatch pass has looked. Never a probe — the whole point of the * recorded row is that reading it spawns nothing. */ export function installSurfaceStatusLine( observation: InstallSurfaceObservation | undefined, ): string | undefined { if (observation === undefined) return "surfaces not observed yet (the next dispatch pass records them)"; const mismatch = installSurfaceMismatch(observation); return mismatch === undefined ? undefined : `surfaces ${mismatch}`; } /** * The `status` row for the recorded `omp-telegram` versions (#961). * * Same discipline as the row above: rendered from what the dispatch pass * already recorded, so reading it spawns nothing and touches no registry — and * silent on agreement, because a row that always prints is a row nobody reads. * * Deliberately its own line rather than a clause on the `telegram` row. That * row's severity ladder carries one remedy at a time (token, then inbound, then * approval surface, then profile), and a stale install is a different kind of * fact about a different thing: the bot can be perfectly healthy while the * plugin driving it predates the contract the brief mandates. Folding it in * would either bury it under a louder rung or displace one. */ export function telegramPluginStatusLine( observation: InstallSurfaceObservation | undefined, ): string | undefined { if (observation === undefined) return undefined; const { telegramInstalled, telegramDaemon, telegramPublished } = observation; // Nothing recorded at all: this pass predates the read, or none answered. // Silent rather than "unverified" — a fleet that does not use Telegram must // not grow a permanent row about a plugin it does not have. if (telegramInstalled === undefined) return undefined; if (telegramPublished !== undefined && telegramPublished !== telegramInstalled) { return ( `tg-plugin omp-telegram ${telegramInstalled} installed, ${telegramPublished} published` + `${telegramDaemon === undefined || telegramDaemon === telegramInstalled ? "" : `, daemon ${telegramDaemon}`}` + " — install it and restart its daemon (#882's target ladder ships in the newer one)" ); } if (telegramDaemon !== undefined && telegramDaemon !== telegramInstalled) { return ( `tg-plugin the running omp-telegram daemon is ${telegramDaemon}, installed is ${telegramInstalled}` + " — restart it so it serves the installed code" ); } return undefined; } /** * The `status` row qualifying the spend figure above it (#970). * * Silent when telemetry is healthy, for the same reason the install-surface row * is: a row that always prints is a row nobody reads. Loud when the cap's input * is going absent, because `spend today $0.64 / $35.00` reads as headroom and * the only surface that said otherwise was an on-demand `doctor` run — the exact * gap #919 closed for install surfaces, on a control that stops the fleet * spending money. * * The wording is the shared one, so this row and `doctor`'s finding cannot state * the same fact differently. */ export function spendTelemetryStatusLine( verdict: SpendTelemetryVerdict | undefined, ): string | undefined { if (verdict === undefined) return undefined; const detail = spendTelemetryDetail(verdict); return detail === undefined ? undefined : ` spend telemetry ${detail}`; } export function formatFleetStatus( s: StatusSnapshot, layers: FleetLayers, daemon: FleetDaemonProbe | undefined = undefined, telegram: TelegramHealth = { kind: "unprobed" }, now = Date.now(), codeGraph: CodeGraphHealth = { configured: false }, brief: string | undefined = undefined, decisions: string | undefined = undefined, failureClasses: string | undefined = undefined, workerPhases: ReadonlyMap = new Map(), intake: string | undefined = undefined, lastStop: DaemonStop | undefined = undefined, siblings: { project: string; live: number }[] = [], grooming: string | undefined = undefined, /** One line naming the flat-chat deliveries this project's stale pin caused * (#1094), or undefined when there is nothing outstanding. */ misrouteNote: string | undefined = undefined, ): string { const tickLine = layers.ticksDetail === undefined ? `ticks ${layers.ticks}` : `ticks ${layers.ticks} (${layers.ticksDetail})`; let nextTickLine: string | undefined; if (layers.nextTickAt !== undefined) { const delta = Date.parse(layers.nextTickAt) - now; const minutes = Math.max(1, Math.ceil(Math.abs(delta) / 60_000)); nextTickLine = `next tick ${layers.nextTickAt} ` + `(${delta >= 0 ? `in ${minutes}m` : `overdue by ${minutes}m`})`; } const paneLine = layers.paneDetail === undefined ? `pane ${layers.pane}` : `pane ${layers.pane} (${layers.paneDetail})`; const recoveryLine = layers.recoveryDetail === undefined ? `recovery ${layers.recovery}` : `recovery ${layers.recovery} (${layers.recoveryDetail})`; const herdrLine = layers.herdrDetail === undefined ? `herdr ${layers.herdr}` : `herdr ${layers.herdr} (${layers.herdrDetail})`; const telegramLine = telegram.detail === undefined ? `telegram ${telegram.kind}` : `telegram ${telegram.kind} (${telegram.detail})`; let daemonBlock: string; if (!layers.daemon.running || layers.daemon.pid === undefined) { daemonBlock = "daemon not running"; } else { // Only trust rss from a project-ok body — a foreign payload's bytes are // not this project's daemon facts (#379). const rss = daemon?.project.kind === "ok" ? rssBytesFromHealthz(daemon.body) : undefined; // Three states, never two (#685): `not running` is reserved for a pid // that is actually gone, and a timed-out `/healthz` is its own state — // a healthy fleet that answers slowly must not read as stopped. const state = daemon?.project.kind === "unresponsive" ? UNRESPONSIVE_HEALTHZ : "running"; daemonBlock = [ `daemon ${state}`, ` pid ${layers.daemon.pid}`, ` port ${layers.daemon.port ?? "?"}`, ...(rss === undefined ? [] : [` rss ${formatRss(rss)}`]), ` healthz ${formatDaemonHealthz(daemon)}`, formatDaemonUnit(daemon?.unit, layers.daemon.pid), ].join("\n"); } const graphBlock = formatCodeGraphHealth(codeGraph, now); // Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause, // a halt) answers "who stopped the fleet" without opening a file (#185). An // unparseable sentinel — paused but with no datable line 1 — is itself news: // it means a run admitted before an *unknown* pause cannot prove innocence // (#174), so completion mutations fail closed while release gates remain usable. const dispatchLine = layers.dispatch === "paused" ? (() => { const prov = pauseProvenance(s.project); if (prov !== undefined) { const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`; // A fence that declared an owning process is judged against it // (#938): a setup transaction's fence outliving the setup is a // stalled fleet that reads exactly like a deliberate hold. Never // cleared here — status reports, and the operator decides. const owner = prov.owner === undefined ? "" : pidAlive(prov.owner) ? `, held by a live process (pid ${prov.owner})` : `, ABANDONED — its process (pid ${prov.owner}) is gone; clear it with \`omp-conductor resume${ s.project === undefined ? "" : ` --project ${s.project}` }\``; return `dispatch paused (source: ${prov.source}${reason}${owner})`; } return isPaused(s.project) && pausedAt(s.project) === undefined ? "dispatch paused (unparseable sentinel — new work and completion mutations refused; release gates remain available)" : "dispatch paused"; })() : `dispatch ${layers.dispatch}`; const upgradeRecoveryBlock = s.upgradeRecovery === undefined ? undefined : [ `upgrade recovery required for failed omp-conductor@${s.upgradeRecovery.failedVersion} (${s.upgradeRecovery.phase})`, ` journal ${s.upgradeRecovery.journalPath}`, ` owed ${s.upgradeRecovery.verificationOwed}`, " release automatic after successful verification; operator acknowledgement: `omp-conductor resume`", ].join("\n"); return [ dispatchLine, ...(upgradeRecoveryBlock === undefined ? [] : [upgradeRecoveryBlock]), tickLine, ...(nextTickLine === undefined ? [] : [nextTickLine]), paneLine, recoveryLine, herdrLine, telegramLine, ...(misrouteNote === undefined ? [] : [misrouteNote]), ...(brief === undefined ? [] : [brief]), ...(decisions === undefined ? [] : [decisions]), ...(failureClasses === undefined ? [] : [failureClasses]), ...(intake === undefined ? [] : [intake]), ...(grooming === undefined ? [] : [grooming]), ...(graphBlock === undefined ? [] : [graphBlock]), daemonBlock, ...formatLastStop(lastStop), "", formatProjectBody(s, workerPhases, now, siblings), ].join("\n"); } function formatAvailabilityStatus(s: StatusSnapshot): string[] { const availability = s.availability; if (availability === undefined) return []; if (availability.mode === "always") { return ["availability 24-hour interrupts (no weekly window)"]; } const mode = availability.nextTransitionAt === undefined || availability.timezone === undefined ? `${availability.mode}; next transition could not be calculated` : `${availability.mode} until ${formatZonedMinute(availability.nextTransitionAt, availability.timezone)}`; const bypass = availability.bypass.length === 0 ? "none" : availability.bypass.join(", "); return [`availability ${mode}; quiet-hours bypass ${bypass}`]; } function formatDigestScheduleStatus(s: StatusSnapshot): string[] { const schedule = s.digestSchedule; if (schedule === undefined) return []; if (schedule.mode === "disabled") return ["next digest disabled"]; if (schedule.mode === "per-tick") return ["next digest every tick"]; if (schedule.mode === "due") return ["next digest due now"]; return [ schedule.nextAt === undefined ? "next digest could not be calculated" : `next digest ${formatZonedMinute(schedule.nextAt, schedule.timezone)}`, ]; } /** The reporting row: which categories interrupt the operator's phone, and * where everything else goes. It reads the snapshot's effective summary — * the legacy preset name is a label, never the source of truth: an explicit * policy without `scopePreset` renders truthfully from `interruptOn` and the * digest cadence, so a routine outcome under `decisions` explains itself as * digest-only instead of reading like a broken Telegram (#633). */ function formatReportingStatus(s: StatusSnapshot): string[] { const r = s.reporting; if (r === undefined) return []; const interrupt = r.interruptOn.length === 0 ? "nothing interrupts" : `${r.interruptOn.join(", ")} interrupt`; const label = r.scopePreset === undefined ? "reporting —" : `reporting ${r.scopePreset} —`; let consequence: string; if (r.interruptOn.includes("material")) { consequence = "no outcome waits for the digest"; } else if (r.digest.cadence === "daily") { consequence = "material outcomes wait for the daily digest"; } else if (r.digest.cadence === "none") { consequence = "material outcomes are held; no digest is configured"; } else { consequence = "material outcomes wait for the per-tick digest"; } return [`${label} ${interrupt}; ${consequence}`]; } /** * The shared-daemon visibility row (#545). When the daemon also serves sibling * projects, tell the reader how many runs each has live, so they can tell "my * fleet is idle" from "the process I am about to stop is busy" — the count is * the point, so a "shared daemon" line that omits it is the bug re-expressed. */ function formatSiblingLive(siblings: { project: string; live: number }[]): string | undefined { if (siblings.length === 0) return undefined; const counts = siblings.map((s) => `${s.project} (${s.live} live)`).join(", "); return `shared daemon also serves ${siblings.length} other project(s): ${counts}`; } // review-correction provenance (#1048) // --------------------------------------------------------------------------- /** * How many rounds of one run's correction chain print. A review ceiling is low * single digits, so this bounds a pathological row rather than routinely * truncating — and the last line printed says how many rounds it stands for, * because a silently shortened chain is how a repeating pattern hides. */ const CORRECTION_CHAIN_LINES = 3; /** * A session lineage as a status row carries it. A ref is a session file path, * and the leading directories are the same for every session in a fleet: the * identifying half is the run and the file, so the line prints those and marks * what it dropped with `…/` — the same honesty `firstLine` uses above, and the * full ref stays in the durable row this line is projected from. Two lineages * plus two models on one row is already at the width `omp-conductor status` * can be read at; a full 60-character path twice is not. * * An absent ref reads `unrecorded`: rows written before #1045 know nothing * about their own launch, and saying so is the only honest projection of one. */ function correctionRef(ref: string | undefined): string { const value = ref?.trim() ?? ""; if (value === "") return "unrecorded"; const segments = value.split("/").filter((segment) => segment !== ""); const tail = segments.slice(-2).join("/"); return firstLine(segments.length > 2 ? `…/${tail}` : value, 60); } /** * The models of one correction round: what the daemon asked for and what the * harness actually resolved. Those two differing is the whole finding — in * #1035 the fleet had already switched worker models while every correction * round replayed the exhausted session under the old one — so the arrow form is * kept whenever they differ and collapsed only when they agree. * * A resumed round is worded differently on purpose. `resolvedModel` is written * only by `Store.recordReviewCorrectionSession`, which refuses a round decided * `resume-original` (a resumed round *is* its origin session), so a resume * durably has a request and no resolution. Rendering that through the fresh * wording — `requested X → ran unrecorded` — would read as a launch that lost * its model, when what actually happened is that the daemon asked the harness * to continue an existing transcript on X and nothing re-recorded the * resolution. * * Every branch reads the round's OWN recorded values. The project config is * deliberately not a parameter: rendering it would make a round dispatched * under a since-replaced model report the replacement, which is exactly the * silent fake #1048 names. */ function correctionModels(round: ReviewCorrectionRound): string { const { requestedModel: requested, resolvedModel: resolved } = round; if (round.launchMode === "resume-original") { if (resolved !== undefined) return requested === undefined || requested === resolved ? `ran ${resolved}` : `ran ${resolved} (asked ${requested})`; if (requested !== undefined) return round.state === "pending" ? `to continue on ${requested}, not launched yet` : `asked to continue on ${requested}`; return "model unrecorded"; } if (requested !== undefined && resolved !== undefined) return requested === resolved ? `model ${resolved} as requested` : `requested ${requested} → ran ${resolved}`; if (requested !== undefined) return round.state === "pending" ? `requested ${requested}, not launched yet` : `requested ${requested} → ran unrecorded`; // A resolution with no request: the harness named what it ran, the decision // row that asked for it predates #1045. Still not a guess in either direction. if (resolved !== undefined) return `ran ${resolved} (request unrecorded)`; return "model unrecorded"; } /** * One run's review-correction chain as status lines (#1048), newest round * first, six-space indented like every other run annotation. * * Before this, a correction round rendered as `review-revision N` and a * dispatch age: an operator could not tell whether conductor had resumed the * diagnosed implementation transcript or launched a fresh correction, nor under * which model — which is why #1035 read as idle while an exhausted session * consumed every round the fleet's new model was never asked to run. * * The per-line `current`/`prior` marker is where the runtime/history split * lives: `current` is the newest round that has not settled, the one whose * elapsed time and cumulative turns the run line above reports; every other * line is chain history and carries no runtime at all. That is deliberately a * property of each line rather than a header, because a header explaining how * to read the following lines is a line nobody reads. * * Exported so the projection can be asserted directly on hand-built rounds: * the interesting cases (a differing requested/resolved pair, a legacy row with * no provenance) are properties of one row, not of a whole fleet. */ export function formatReviewCorrections(rounds: readonly ReviewCorrectionRound[]): string[] { if (rounds.length === 0) return []; const newestFirst = [...rounds].sort((a, b) => b.round - a.round); const shown = newestFirst.slice(0, CORRECTION_CHAIN_LINES); const hidden = newestFirst.length - shown.length; return shown.map((round, index) => { const current = index === 0 && round.state !== "settled"; // The round's lifecycle as one phrase. `queued` covers both a round nobody // has dispatched yet and one a restart returned to the queue — durably they // are the same row, and status must not invent a distinction the store does // not keep. const state = round.state === "settled" ? `settled ${round.outcome ?? "outcome unrecorded"}` : round.state === "pending" ? "queued" : "dispatched"; const provenance = round.launchMode === undefined ? // A row written before #1045 recorded no launch decision. Status says // so and stops: the alternative — reading today's configuration back // as though it had been this round's — is a fabricated provenance, // and an operator auditing #1035 would have believed it. "provenance unknown (pre-#1045 round)" : round.launchMode === "resume-original" ? // A resume continues one lineage, so there is one session to name: // the implementation transcript the review diagnosed. `resume-original resumed session ${correctionRef(round.originSessionRef)} ${correctionModels(round)}` : // A fresh correction has two: the transcript that was reviewed, and // the new session doing the correcting. Naming only one of them is // how "fresh" became indistinguishable from "resumed". `fresh-correction new session ${correctionRef(round.correctionSessionRef)} (diagnosed ${correctionRef( round.originSessionRef, )}) ${correctionModels(round)}`; const tail = index === shown.length - 1 && hidden > 0 ? ` (+${hidden} earlier round(s))` : ""; return ` correction round ${round.round} ${current ? "current" : "prior"} (${state}) ${provenance}${tail}`; }); } /** * The #1063 effective-turn-budget annotation for the `new worker turns` row: * shown whenever observed per-turn latency makes the configured ceiling * unreachable at all — the break-even pace is exactly wall clock ÷ max turns, * so anything slower is material, anything faster is noise. */ function effectiveTurnBudgetSuffix(s: StatusSnapshot): string { const observed = s.workerTurnBudgetObserved; if (observed === undefined || observed.effectiveTurns >= s.caps.workerMaxTurns) return ""; return ` (effective ~${observed.effectiveTurns} at ${observed.minutesPerTurn.toFixed(2)} min/turn, last ${observed.sampleSize} runs)`; } function formatProjectBody( s: StatusSnapshot, workerPhases: ReadonlyMap = new Map(), now = Date.now(), siblings: { project: string; live: number }[] = [], ): string { const siblingLine = formatSiblingLive(siblings); const lines = [ `project ${s.project}${s.paused ? " (PAUSED)" : ""}`, ...(s.pauseReason === undefined ? [] : [`paused ${s.pauseReason}`]), // Structured-exposure half of the drain surface (#484): a plain row for // the renderer, so the active drain is visible in `status` before the // human-wording work of the later #484 child lands. ...(s.drain === undefined ? [] : [ `drain active — expires ${new Date(s.drain.expiresAt).toISOString()}, ${s.drain.remainingRuns} run(s) remaining`, ]), `config ${s.configPath}`, `state ${s.stateDir}`, ...(siblingLine === undefined ? [] : [siblingLine]), // The orchestrator-down degrade row: first-class in the body, present only // while the incident is open, so recovery drops it (#288). ...(s.orchestratorDown === undefined ? [] : formatOrchestratorDown(s.orchestratorDown, now)), // The recorded install-surface observation (#919): silent on agreement, // loud on a proven mismatch, honest before the first pass has looked. ...(installSurfaceStatusLine(s.installSurfaces) === undefined ? [] : [installSurfaceStatusLine(s.installSurfaces) as string]), // Same recorded pass, same silence-on-agreement rule (#961). ...(telegramPluginStatusLine(s.installSurfaces) === undefined ? [] : [telegramPluginStatusLine(s.installSurfaces) as string]), ...formatReportingStatus(s), ...formatAvailabilityStatus(s), ...formatDigestScheduleStatus(s), "", "caps", ` workers ${s.liveWorkers} / ${s.caps.maxConcurrentWorkers}`, ` issues today ${s.runsToday}`, // "estimated" is not hedging (#851): this figure is OMP's own local price // for the tokens the session recorded, not the provider's bill. On // 2026-08-21 the local estimate read $25.16 while roughly $16 of provider // credit actually moved, so a row that called it "spend" was asserting // something conductor cannot see. The reservation is shown beside it // because that, not spend-to-date, is what the next admission subtracts. s.caps.dailySpendUsd === null ? ` spend today $${s.spendTodayUsd.toFixed(2)} estimated (no daily cap)` : ` spend today $${s.spendTodayUsd.toFixed(2)} estimated / $${s.caps.dailySpendUsd.toFixed(2)}` + // A dollar ceiling over a subscription-billed fleet can never fire, so // showing it bare asserts headroom that is not the constraint (#984). // The cap stays visible because it IS configured — what changes is that // the row stops implying it governs anything. (s.spendTelemetry?.kind === "subscription" ? " (inert — subscription-billed, see below)" : "") + (s.reservedSpendUsd === undefined || s.reservedSpendUsd === 0 ? "" : ` (+$${s.reservedSpendUsd.toFixed(2)} reserved by runs in flight)`), // What that figure is built on (#970). Silent when telemetry is healthy; // its own row when it is not, because the number above reads as headroom // and the cap subtracts exactly it. Measured 2026-08-22: 9 of 12 working // runs reported $0.00 and the only surface that could have said so was an // on-demand `doctor` run — whose predicate would not have fired either. ...(spendTelemetryStatusLine(s.spendTelemetry) === undefined ? [] : [spendTelemetryStatusLine(s.spendTelemetry) as string]), // Its own row beside the spend row, never folded into it: they are two // independent controls and an operator has to see which one stopped the // fleet (#110). ` plan usage ${planUsageLine(s.planUsage)}`, // The tracker's API budget, when the renderer could read it. Absent on a // broken `gh`: one missing row, never a broken report (#188). Next to it, // the refusals the tracker actually observed, so a budget that looks // healthy next to every call being refused is still visible (#198). ...(s.github === undefined ? [] : [ ` github graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` + (s.ghRefusals === undefined || s.ghRefusals.count === 0 ? "" : ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`), ]), // The daemon's own observed github traffic today (#198): a separate row // from the polled budget above, so one missing `status` read never hides // the other. `onCall` counts every spawn including ones that end 304, so // `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈ // difference, and the polled budget row stays the authority (#203). ` github calls daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` + ((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0 ? "" : ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`), // Label-projection ops the tracker has not applied yet (#201): GitHub is // behind what the store decided, and the operator can see the lag instead // of discovering it as a stale label or a missing one. ...(s.labelOps === undefined ? [] : [ ` labels projection ${s.labelOps.pending} pending (oldest ${ s.labelOps.oldestAgeMs >= 60_000 ? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m` : `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s` })`, ]), ` new worker turns ${s.caps.workerMaxTurns}${effectiveTurnBudgetSuffix(s)}`, ...s.turnOverrides.map( ({ issue, maxTurns }) => ` turn override #${issue} → ${maxTurns} (next attempt)`, ), ` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`, ` failed attempts ${s.caps.maxAttemptsPerIssue}`, ` continuations ${s.caps.maxContinuationsPerIssue}`, "", ...formatReleaseGrants(s.releaseGrants), // The effective review policy (#678), read from the same snapshot surface // as the grants: the level is what Duty 1 judges findings against, and the // ceiling is its hard bound — an orchestrator reading `review-revision N` // below has to see both without opening the config. "review", ` strictness ${s.review.strictness}`, ` max rounds/PR ${s.review.maxRounds}`, "", formatDispatchSummary(s.dispatch), "", ]; // The two lifecycle kinds are rendered as two blocks (#898). An operator // looking at "a branch holding a file" has to be able to tell "a worker is // editing this right now" from "durable work nobody is touching, waiting on // review, merge or recovery" — the same distinction #899 made load-bearing in // admission, where only the first kind occupies files. One block for both // read as though every preserved branch had a live editor, which is exactly // the misreading that let a settled artifact look like a busy worker. // // Membership comes from the snapshot's `leasedRunIds` (`Store.leasedRuns`), // never from a state check here: a dispatched review revision is a lease on a // row whose state still says `pushed-green`, so a renderer that classified by // state alone would call a live rewrite "preserved". // // Membership is the UNION of the run's own live state and the recorded ids, // never the ids alone. A live worker is a lease by definition, so the field // exists only to ADD the case a state read cannot see — a dispatched revision // on a row that still says `pushed-green`. Reading it as the whole truth would // let an absent or stale field print a live worker under "worker-free", which // is the exact misreading this block exists to stop, and the union can never // over-report either: `leasedRuns` is itself live rows plus those revisions. // // The two state names are spelled here rather than importing `LIVE_STATES`: // this module is a pure renderer and does not pull in the store (and its // sqlite binding) to read two string literals. const leased = new Set(s.leasedRunIds ?? []); const isLease = (r: RunRecord): boolean => r.state === "claimed" || r.state === "running" || leased.has(r.id); const leases = s.activeRuns.filter((r) => isLease(r)); const preserved = s.activeRuns.filter((r) => !isLease(r)); // The per-run annotations both blocks carry: they are facts about the run, // not about which lifecycle kind it is, and a flagged or merge-blocked // artifact is if anything MORE interesting once nobody is editing it. const annotations = (r: RunRecord): string[] => { const out: string[] = []; // The orchestrator's Duty 1 reads this command, and a flagged run's // escalation is deduplicated after one delivery — so this is where a // flagged PR stays visible for as long as it is still open (#128). const flagged = settlementFlagSummary(r.settlementFlags); if (flagged !== undefined) out.push(` ${flagged}`); // The exact-head merge blocker (#888): a green PR whose head carries // unresolved durable review evidence would otherwise read as merge-ready // while `conductor_pr_merge` refuses it. Same rows the verb reads, // rendered where Duty 1 already looks. // The review-ceiling adjudication for this run's exact head (#874). Matched // on PR + head, never on the run: a verdict describes the diff it read, so a // run that has since pushed a corrected head must not appear to carry the // older head's decision. const adjudication = r.prUrl === undefined || r.headSha === undefined ? undefined : s.reviewAdjudications?.find( (a) => a.prUrl === r.prUrl && a.headSha.toLowerCase() === r.headSha?.toLowerCase(), ); if (adjudication !== undefined) { // The role asked for AND the model that actually ran it: those differ // exactly when something is misconfigured, and then the state is // `unavailable-model` and the difference is the whole finding (#875). const launched = adjudication.provenance === undefined ? `role ${adjudication.role} (not launched)` : `role ${adjudication.role} → ${adjudication.provenance.model}${ adjudication.provenance.provider === undefined ? "" : ` (${adjudication.provenance.provider})` }`; out.push( ` adjudication ${adjudication.state} ${launched}` + (adjudication.evidence === undefined ? "" : ` ${firstLine(adjudication.evidence)}`) + (adjudication.disposition === undefined ? "" : ` → ${firstLine(adjudication.disposition)}`), ); } const blocked = s.mergeBlockers?.[r.id]; if (blocked !== undefined) out.push( ` merge blocked: review round ${blocked.round} ${ blocked.state === "pending" ? "queued at this head" : blocked.state === "crashed" ? "dispatched and unfinished at this head" : "failed at this head" } — unresolved findings; push a corrected head, or record a conductor_pr_review_clear for this exact head, before merge`, ); // The correction chain (#1048), rendered from the same annotation helper as // the merge blocker above so it reaches BOTH surfaces that show a run: the // live lease and the preserved artifact. A round whose worker has finished // leaves the lease list, and that is precisely when an operator asks what // the last correction actually did. out.push(...formatReviewCorrections(s.reviewCorrections?.[r.id] ?? [])); return out; }; if (s.activeRuns.length === 0) { lines.push("mutation leases (none)"); } else { lines.push(leases.length === 0 ? "mutation leases (none)" : "mutation leases"); for (const r of leases) { const pauseView = workerPhases.get(r.issue); const phase = pauseView?.phase; // A run in a live review round reads `review-revision N`, distinct from // a failure and from an ordinary continuation, with the round number // from the durable revision row (#692). The live pause overlay still // wins: an operator pause is the newer fact about the same session. // The overlay names its source and age when the daemon recorded them // (#997): `paused (board, 4m)` answers "who and since when" at the // surface where the question actually gets asked. const paused = phase === "pausing" || phase === "paused"; const round = s.reviewRounds?.[r.id]; const pauseDetail = paused && (pauseView?.source !== undefined || pauseView?.pausedAtMs !== undefined) ? ` (${[ ...(pauseView.source === undefined ? [] : [pauseView.source]), ...(pauseView.pausedAtMs === undefined ? [] : [formatDownDuration(Math.max(0, now - pauseView.pausedAtMs))]), ].join(", ")})` : ""; const state = paused ? `${phase}${pauseDetail}` : round !== undefined ? `review-revision ${round.round}` : r.state; // Turn rate and cap projection (#730/#767): a stalled run and a fast one // used to render identically as a bare turn count. The rate is the // lifetime average over the elapsed shown — the snapshot carries no // checkpoint from which a recent-interval rate could be derived — so it // is labelled `avg`, and `elapsed` is elapsed since claim, never // "remaining wall clock": a paused worker banks its budget, which the // snapshot cannot know. #767 adds the projection #518 asked for: the // same claim-elapsed rate held forward to the wall-clock budget // (`caps.workerWallClockMs`, rendered in the caps block) as the turn // count the run would reach at that cap — trajectory math over // elapsed-so-far, so reading it against the run's `maxTurns` tells // which cap fires first. A paused run gets neither number, because its // elapsed includes banked pause time and its state already says // `paused`. const elapsedMs = Math.max(0, now - r.startedAt); // A review revision resumes the SAME run and session, so `turns` and // `startedAt` are cumulative over the whole attempt — and reading them as // this round's is how a revision recorded two minutes ago was read as a // worker stuck for two hours (#802, #797 on 2026-08-19). Nothing is reset: // the attempt totals are the cap evidence and stay exactly as recorded. // They are labelled `cumulative` instead, and the round's own age is shown // beside them from the durable dispatch instant. // // The rate and cap projection are deliberately dropped for a live round: // both divide cumulative turns by cumulative elapsed, so neither describes // the phase the line is naming, and a number attributed to the wrong phase // is worse than no number. The attempt's own cap remains visible as // `N/M turns cumulative`. // Transcript silence (#1086), from the progress watch's recorded write // instant — read off the row, never probed at render time (#919). This // is where a stall is visible forming, before the pass settles it. // Skipped under pause, whose silence is the pause working (#938), and // for rows no longer live, whose transcripts have stopped forever. const silent = paused || r.state !== "running" || r.lastProgressAt === undefined ? "" : ` silent ${formatDownDuration(Math.max(0, now - r.lastProgressAt))}`; const progress = paused ? "" : round !== undefined ? ` cumulative ${formatDownDuration(elapsedMs)} elapsed cumulative round ${ round.round } dispatched ${formatDownDuration(Math.max(0, now - round.dispatchedAt))} ago` : ` ${formatDownDuration(elapsedMs)} elapsed ${( (r.turns * 60_000) / Math.max(elapsedMs, 1_000) ).toFixed(1)} turns/min avg` + ` projects ${Math.round( (r.turns * s.caps.workerWallClockMs) / Math.max(elapsedMs, 1_000), )} turns at cap` + silent; lines.push( ` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` + `${r.turns}/${r.maxTurns} turns${progress} ${r.spendUsd.toFixed(2)} ${r.branch}` + (r.prUrl ? ` ${r.prUrl}` : ""), ); // What the lease actually holds, named where the operator is already // looking: the run this is (an issue can have several attempts, and only // one of them holds the lease), how long it has held it, the read the // interlock uses to prove occupancy, and the files it occupies. // // The source is not a guess: `probeRunLane` reads a live checkout when the // row has one and the mirror branch when it does not, which is exactly // this condition — a dispatched revision on a settled row has no worktree, // and its lane comes from the branch. // // The file list is the durable declaration admission persisted at dispatch // (#744) — the half that occupies before anything is written. What the run // has additionally touched is a git read, deliberately not taken here: // status is a recorded answer, never a subprocess per render (#919). A // lease with no declaration says so, because "no files listed" and "no // declaration made" invite different actions. lines.push( ` lease run ${r.id.slice(0, 8)} ${formatDownDuration(Math.max(0, now - r.startedAt))} held ` + `${r.worktree === "" ? "branch" : "worktree"} ` + (r.lane === undefined || r.lane.files.length === 0 ? "no declared lane (probe-only occupancy)" : `occupies ${r.lane.files.join(", ")}`), ); // Whether the operator can watch this worker, named rather than absent // (#841). A live run with no pane is a fleet running blind, and silence // reads identical to "there is nothing to see" — so degraded says so and // carries the reason the launch or the last reconcile recorded. lines.push( r.paneId === undefined ? ` pane degraded${r.paneUnavailable === undefined ? "" : ` — ${r.paneUnavailable}`}` : ` pane ${r.paneId}${r.paneLabel === undefined ? "" : ` ${r.paneLabel}`}`, ); // The fact that explains an expensive-in-time, cheap-in-dollars run // (#518). Shown only when the share is disproportionate, because a run // reasoning normally needs no annotation and a line printed for every run // is a line nobody reads. The threshold is deliberately high: at 80% the // turn cap is already unreachable and the wall clock is the only ceiling // that can fire, which is the operator's actual decision. if (r.outputTokens !== undefined && r.outputTokens > 0 && r.reasoningTokens !== undefined) { const share = r.reasoningTokens / r.outputTokens; if (share >= 0.8) { lines.push( ` tokens ${Math.round(share * 100)}% of ${formatCount(r.outputTokens)} output tokens ` + `were reasoning — deliberating, not stalled; wall clock is the binding cap`, ); } } lines.push(...annotations(r)); } } // Preserved artifacts: durable work with no worker and no revision behind it. // Rendered without a turn rate or a cap projection on purpose — both describe // a session in progress, and printing them beside work nobody is running is // how a settled artifact came to read as a busy one. What matters here is how // long it has been waiting and on what: a PR to review and merge, or (with no // PR) a branch `conductor_pr_recover` can still publish. if (preserved.length > 0) { lines.push("preserved artifacts (worker-free — nothing is editing these)"); for (const r of preserved) { const since = r.endedAt ?? r.startedAt; lines.push( ` #${r.issue} ${r.repo} ${r.state} attempt ${r.attempt} ` + `${formatDownDuration(Math.max(0, now - since))} waiting ${r.turns} turns spent ` + `${r.spendUsd.toFixed(2)} ${r.branch}` + (r.prUrl === undefined ? " no PR — recoverable with conductor_pr_recover" : ` ${r.prUrl}`), ); lines.push(...annotations(r)); } } lines.push(...formatBaseHealth(s.baseHealth)); lines.push(...formatFreezes(s.freezes)); lines.push(...formatSalvagedRuns(s.salvagedRuns)); lines.push(...formatQuarantinedRuns(s.quarantinedRuns)); lines.push(...formatOpenReports(s.openReports)); const withdrawals = s.handoffWithdrawals ?? []; if (withdrawals.length > 0) { lines.push("handoff withdrawals"); for (const withdrawal of withdrawals) { lines.push( ` ${withdrawal.id} ${withdrawal.target} ${new Date(withdrawal.at).toISOString()} ` + `by ${withdrawal.actor} — ${withdrawal.reason}; ${withdrawal.summary}`, ); } } lines.push(...formatDigestBacklog(s.digestBacklog)); // The mediated-verb ledger (#972). Absent from this renderer since the block // was written (#133) — it was wired into `daemon.ts`'s copy, which was // already not the live renderer, so the read below was paid for on every // status call and dropped. It belongs here: the count is how an operator sees // that a mediated act was refused without going to the sqlite file, and // #968's refusal class (41% of this fleet's refusals) went unnoticed for a // fortnight precisely because nothing on this surface said so. Self-silencing // on an empty ledger, so a fresh fleet gains no row. lines.push(...formatVerbLedger(s.verbLedger)); if (s.liveWorkers > 0) { lines.push( "", `deploy ${s.liveWorkers} live worker(s) — restart salvages dirty trees then orphans the rows; ` + `hold and wait for workers 0/${s.caps.maxConcurrentWorkers} when you can drain instead`, ); } return lines.join("\n"); }