/** * The orchestrator-down incident: how the daemon learns — and tells its * operator — that the embedded orchestrator session is gone. * * A failed orchestrator used to degrade quietly: the daemon logged a warning * and kept running, but the tier-1 escalations that then fell back to issue * comments were exactly the "nobody reads it until morning" path the * orchestrator exists to avoid. Nothing paged, and `status` never said the * operator channel was degraded (the pending issue is #288). * * This module makes three facts durable and observable: * * 1. **The incident.** Start failure (no handle) and unexpected session death * (a handle whose `alive()` is false) each open one durable * `orchestrator-incident` row, re-derived across restarts so a daemon * restarted while still down rediscovers it. The row survives and is * removed only on recovery. * 2. **One page per incident.** Open, and every subsequent down tick, page a * tier-2 urgent escalation whose key is anchored on the incident's `since` * timestamp — the notification ledger pages once, never per tick, and a * *new* incident (a flapping orchestrator recovering then dying again) gets * a new anchor and pages again. * 3. **The degrade row.** While down, `status` names the mode, the since- * moment and how many tier-1 escalations were diverted to issue comments. * Recovery closes the row and pages one closing notice on the same anchor, * with downtime and the diverted count. * * The session liveness driving this is event-driven (the proxy's `session_exit` * terminal event), never a timer: a wedged-but-alive session stays "alive" * here and belongs to the `.conductor-stalled` watchdog, which this module * deliberately does not touch. */ import type { OrchestratorHandle } from "./orchestrator.ts"; import type { Escalation, OrchestratorDownMode, OrchestratorIncident, ProjectConfig, Store, } from "./types.ts"; /** Fleet-scoped pages carry issue `0` — there is no tracker issue for this. */ const NO_ISSUE = 0; export interface OrchestratorDownDeps { project: ProjectConfig; store: Store; /** The embedded session handle; `undefined` when it failed to start. */ orchestrator?: OrchestratorHandle; /** * The daemon's escalation transport. Errors are caught here (the call is * best-effort and ledger-deduped), so a transport that is itself down never * takes the tick down with it. */ escalate(e: Escalation): Promise; /** * Start-failure cause, only meaningful when a new incident is opened with no * handle. Carried so the page and `status` tell the operator *why*, not just * that the orchestrator is gone. */ startCause?: string; now?(): number; log?(msg: string): void; } function iso(at: number): string { return new Date(at).toISOString(); } /** Compact, human-scale duration for downtime and ages. */ export function formatDownDuration(ms: number): string { if (ms < 60_000) return `${Math.max(0, Math.round(ms / 1_000))}s`; if (ms < 3_600_000) return `${Math.round(ms / 60_000)}m`; if (ms < 86_400_000) return `${Math.round(ms / 3_600_000)}h`; return `${Math.round(ms / 86_400_000)}d`; } function warn(msg: string): void { process.stderr.write(`[conductor ${new Date().toISOString()}] ${msg}\n`); } function errText(e: unknown): string { return e instanceof Error ? e.message : String(e); } /** * Best-effort page, mirroring the daemon's `safeEscalate`: a transport failure * is logged, never thrown. The recipient (the daemon's escalator) owns the * notification-ledger dedupe, which is what makes "page once per incident" a * property of the store rather than of this loop. */ async function deliver(deps: OrchestratorDownDeps, e: Escalation): Promise { try { await deps.escalate(e); return true; } catch (err) { (deps.log ?? warn)(`orchestrator-down escalation could not be delivered: ${errText(err)}`); return false; } } /** * The down page. Keyed on the incident's `since` so a still-open incident never * pages again (the ledger dedupes on the summary), while each *new* incident — * a fresh `since` — pages once. */ export function downEscalation(project: string, incident: OrchestratorIncident): Escalation { return { tier: 2, category: "confirmed-failure", urgent: true, project, issue: NO_ISSUE, summary: `Orchestrator down since ${iso(incident.since)} — tier-1 escalations diverting to issue comments (${project})`, detail: [ `Mode: ${incident.mode}.`, ...(incident.cause === undefined ? [] : [`Cause: ${incident.cause}.`]), "", "Tier-1 escalations still land — as issue comments, the path this orchestrator", "exists to avoid. Dispatch, settlement and release are unaffected; the operator", "channel is degraded. Recovery sends one closing notice on this same incident.", ].join("\n"), }; } /** The closing notice, referencing the same incident anchor. */ export function recoveryEscalation( project: string, closed: OrchestratorIncident, now: number, ): Escalation { const downtime = formatDownDuration(Math.max(0, now - closed.since)); return { tier: 2, category: "confirmed-failure", urgent: true, project, issue: NO_ISSUE, summary: `Orchestrator recovered for ${project} ` + `(down since ${iso(closed.since)}) — tier-1 escalations resume as injected prompts`, detail: [ `Mode: ${closed.mode}. Downtime: ${downtime}.`, `Tier-1 escalations diverted to issue comments while down: ${closed.diverted}.`, "", "The orchestrator is back and ticking. The .conductor-stalled wedge watchdog is", "unchanged and still covers a session that stops draining its queue.", ].join("\n"), }; } /** * Reconcile the embedded orchestrator's liveness against the durable incident. * Idempotent, so it is safe to call at startup (after the escalator exists) and * on every tick: opening, paging, divert-counting and recovering all converge * on the store. * * External orchestrators return early — their down state is a human-owned pane, * not conductor's to page, and the stall watchdog still covers their wedge. */ export async function reconcileOrchestratorDown(deps: OrchestratorDownDeps): Promise { const { project, store } = deps; const now = deps.now ?? Date.now; const log = deps.log ?? warn; if (project.escalation.orchestrator === "external") return; const handle = deps.orchestrator; const up = handle !== undefined && handle.alive(); const existing = store.orchestratorIncident(project.name); if (up) { if (existing === undefined) return; // A session is (re)started and alive: close the incident from before and // page one closing notice on the same anchor. const closed = store.closeOrchestratorIncident(project.name, now()); if (closed === undefined) return; // raced with another daemon const delivered = await deliver(deps, recoveryEscalation(project.name, closed, now())); log( `orchestrator recovered — closed incident from ${iso(closed.since)} ` + `(${closed.diverted} diverted)${delivered ? "" : " (page unsent)"}`, ); return; } // Down. Open once (the store's INSERT OR IGNORE is the dedupe), then page — // the page itself is ledger-deduped so later ticks re-derive the world // without paging again. if (existing === undefined) { const mode: OrchestratorDownMode = handle === undefined ? "start-failed" : "crashed"; if ( !store.openOrchestratorIncident({ project: project.name, mode, ...(mode === "start-failed" ? deps.startCause === undefined ? {} : { cause: deps.startCause } : { cause: "session died after a healthy start" }), since: now(), }) ) { return; // raced with another daemon opening it } } const incident = store.orchestratorIncident(project.name); if (incident === undefined) return; await deliver(deps, downEscalation(project.name, incident)); log( `ERROR: orchestrator ${incident.mode} down since ${iso(incident.since)} — ` + `tier-1 escalations diverting to issue comments`, ); } /** * The first-class degrade row for `status`: mode, since-when, and how many * tier-1 escalations have been diverted to issue comments. Absent entirely * (empty array) when the orchestrator is healthy, so recovery drops the row. */ export function formatOrchestratorDown(row: OrchestratorIncident, now = Date.now()): string[] { return [ `orchestrator DEGRADED ${row.mode} since ${iso(row.since)} ` + `(${formatDownDuration(Math.max(0, now - row.since))} ago)`, ` diverting tier-1 escalations to issue comments` + (row.diverted === 0 ? " (none yet)" : ` — ${row.diverted} escalated issue${row.diverted === 1 ? "" : "s"} diverted`), ...(row.cause === undefined ? [] : [` cause ${row.cause}`]), ]; }