/** * The sweep — the one thing that makes alerting run by itself. * * Everything else in services/alerting/ is inert until this runs: monitors are * rows nobody executes, alerts never leave `pending`, escalation never fires. * The dispatcher invokes `celilo alerts sweep` on `timer.tick.5m`, and this is * what that becomes. * * Order matters, and each step depends on the previous one having settled: * * 1. run every DUE monitor → alerts created / refreshed / resolved * 2. promote past-grace alerts → pending becomes firing * 3. re-evaluate suppression → at notify time, never at fire time (S1) * 4. notify what is still due → escalation decides whether and whom * * Step 3 sits after 1 and 2 deliberately. Monitors do not run in a guaranteed * order, so a module's check can fail seconds before the machine check that * explains it; deciding suppression during step 1 would page for the symptom * moments before the cause lands. */ import type { DbClient } from '../../db/client'; import type { Alert, Monitor } from '../../db/schema'; import { MONITOR_INTERVAL_FLOOR_MINUTES } from '../cadence'; import { isScheduled, loadModuleHealthCadences } from './health-cadence'; import { type DroppedAlert, deleteMonitor } from './monitors'; import type { NotifyDeps, NotifyOutcome } from './notifier'; import { deliverDeferred, notifyAlert } from './notifier'; import { type MonitorRunDeps, runOneMonitor } from './run-monitor'; import { clearDeferral, deferNotification, dueDeferrals, loadAllLiveAlerts, markSuppressed, markUnsuppressed, promoteReadyAlerts, recordStepTaken, } from './store'; import { type SuppressionTopology, findSuppressor } from './suppression'; import { selectDueMonitors } from './sweep'; export interface SweepDeps { monitorDeps: MonitorRunDeps; /** Topology for suppression — read once per sweep, not per alert. */ loadTopology(): SuppressionTopology; /** Modules currently inside a deploy window. */ loadDeployWindowModules(): Set; /** Ids of modules currently PAUSED — a pause explains its own module's alerts. */ loadPausedModules(): Set; /** Whether the monitor owning an alert may be suppressed at all. */ isSuppressible(alert: Alert): boolean; /** Compose the per-alert notification context. Null when nothing can page. */ notifyDepsFor(alert: Alert): NotifyDeps | null; now(): Date; } export interface SweepReport { monitorsRun: number; monitorsErrored: number; promoted: number; suppressed: number; unsuppressed: number; notified: number; deferred: number; /** Messages held over quiet hours and delivered now that the window ended. */ deferredDelivered: number; failed: number; /** * Live alerts nobody is configured to be told about — no escalation policy on * the monitor, so `notifyDepsFor` returns null. Each carries the monitor that * owns it, which is the thing an operator has to assign a policy TO. * * Identified rather than merely counted: `1 no-policy` told an operator that * something was unreachable but not WHICH thing, so the remedy the sweep * printed alongside it could not be aimed at anything. Assigning the policy to * all eighteen monitors then changed nothing observable (#481). A bare count * is only half a step better than the silence it replaced. */ noPolicy: { alertKey: string; monitor: string }[]; /** * Deliveries escalation declined — the reason AND the alert it applies to. * * `notifyAlert` returns the reason precisely so the caller can record it: its * own contract says a silent skip is indistinguishable from a bug, and * dropping it here is what made a firing-but-undelivered alert undebuggable * (#450). * * The alert key is carried too, because a bare `within_grace×2` still does not * answer "why was I not paged" for the alert the operator is actually looking * at — they cannot tell which of their live alerts each count refers to. Same * reasoning `noPolicy` already applies, and the same failure it was fixing. * Counts are derived at render time so there is one source for both. */ skipped: { alertKey: string; reason: string }[]; /** * Why each failed delivery failed, as `: `. * * A count alone does not answer the only question that matters after a page * did not arrive. The transport is loaded lazily *inside* `notifyAlert`'s try * block, so "the capability would not load" and "Signal rejected the message" * both surface here and nowhere else — there is no daemon-side log for the * former, because nothing ever reached the daemon. */ failures: string[]; /** * Modules whose `module_hook` monitor was dropped because the module itself * is gone, and the live alerts each one took with it. * * The alerts are DELETED by the monitor's cascade, not resolved, so a firing * check and the coverage of it disappear in the same instant with nothing * else recording either. A bare module name is not enough to act on: the row * that mattered on the fleet was holding `Cannot reach router: Router login * failed`, and `1 stranded-dropped (greenwave)` would not have told anyone * that a router had stopped being checked. Same reason `noPolicy`, `skipped` * and `failures` all name their subject rather than counting it. */ strandedDropped: { moduleId: string; alerts: DroppedAlert[] }[]; } /** * Run one sweep. * * Never throws for a single bad monitor or a single failed send: the sweep is * the fleet's only heartbeat, and one broken module must not stop every other * alert from being evaluated. Failures are counted and returned. */ export async function runSweep( db: DbClient, monitors: Monitor[], deps: SweepDeps, ): Promise { const report: SweepReport = { monitorsRun: 0, monitorsErrored: 0, promoted: 0, suppressed: 0, unsuppressed: 0, notified: 0, deferred: 0, deferredDelivered: 0, failed: 0, noPolicy: [], skipped: [], failures: [], strandedDropped: [], }; // 1. Run due monitors. // // A `module_hook` monitor's cadence and whether it is watched at all come // from the module's effective health-check cadence, NOT from its row: the row // was seeded once at first deploy and never reconsulted, so a corrected // manifest could never reach an existing install (design.md D2/D8). A // `builtin_check` has no module and no manifest, so its row is the config. const cadences = loadModuleHealthCadences(db); // Reconcile away monitors whose module is gone, before anything tries to run // them. `loadModuleHealthCadences` holds every module, so a miss here means // no module row — and a `module_hook` monitor without one is unschedulable // forever: its cadence resolves to null, `isScheduled` says false, and the // sweep never selects it again. Anything it left firing would sit there with // no path back to `resolved`, suppressing every alert it is an ancestor of // (celilo#1029). // // The removal path deletes the monitor itself, so this only fires for rows a // celilo without that fix left behind, or a removal that died between the two // deletes. Same stance `setMonitorEnabled` already takes: a monitor that will // never report again must not hold live alerts. const stranded = monitors.filter((m) => m.kind === 'module_hook' && !cadences.has(m.target)); for (const monitor of stranded) { report.strandedDropped.push({ moduleId: monitor.target, alerts: deleteMonitor(db, monitor.id), }); } const active = stranded.length > 0 ? monitors.filter((m) => !stranded.includes(m)) : monitors; const due = selectDueMonitors( active.map((m) => { if (m.kind !== 'module_hook') { return { id: m.id, intervalMinutes: m.intervalMinutes, enabled: m.enabled, lastRunAt: m.lastRunAt, monitor: m, }; } const cadence = cadences.get(m.target)?.cadence ?? null; return { id: m.id, // Unscheduled monitors are filtered out by `enabled` below, so this // value is never used to decide due-ness for them. intervalMinutes: isScheduled(cadence) ? cadence.minutes : MONITOR_INTERVAL_FLOOR_MINUTES, enabled: isScheduled(cadence), lastRunAt: m.lastRunAt, monitor: m, }; }), deps.now(), ); for (const entry of due) { try { const summary = await runOneMonitor(db, entry.monitor, deps.monitorDeps); report.monitorsRun++; if (summary.outcome === 'error') report.monitorsErrored++; } catch { // A monitor that throws outside its own error handling still must not // stop the sweep — the other monitors are the rest of the fleet. report.monitorsErrored++; } } // 2. Promote alerts past their grace window. report.promoted = promoteReadyAlerts(db, deps.now()); // 3. Re-evaluate suppression across everything live, now that this sweep's // monitors have all reported. const live = loadAllLiveAlerts(db); const firingKeys = new Set( live.filter((a) => a.state === 'firing' || a.state === 'acked').map((a) => a.key), ); const topology = deps.loadTopology(); const deployWindows = deps.loadDeployWindowModules(); const paused = deps.loadPausedModules(); for (const alert of live) { const suppressor = findSuppressor({ key: alert.key, firingKeys, suppressible: deps.isSuppressible(alert), modulesInDeployWindow: deployWindows, pausedModules: paused, topology, }); const wasSuppressed = alert.state === 'suppressed'; if (suppressor && !wasSuppressed) { markSuppressed(db, alert.id, { alertId: suppressor.kind === 'alert' ? suppressor.key : undefined, // A pause is recorded on the same column as a deploy window: both are // "a module-scoped condition explains this", and the operator reads the // module id either way. windowId: suppressor.kind === 'deploy_window' || suppressor.kind === 'paused' ? suppressor.moduleId : undefined, }); report.suppressed++; } else if (!suppressor && wasSuppressed) { // Lifting suppression sets awaitingConfirmation, so this does NOT page // now — it pages after a later run confirms the problem survived (S2). markUnsuppressed(db, alert.id, deps.now()); report.unsuppressed++; } } // 4. Flush anything held over quiet hours whose window has now ended. Runs // BEFORE new notifications so an overnight page arrives ahead of whatever // this morning's sweep decides. for (const alert of dueDeferrals(db, deps.now())) { const notifyDeps = deps.notifyDepsFor(alert); const route = alert.deferredRouteId ? notifyDeps?.routeDetails.get(alert.deferredRouteId) : undefined; // A deferral whose route was deleted overnight is dropped rather than // retried forever — the escalation clock already moved past that step. clearDeferral(db, alert.id); if (!notifyDeps || !route) continue; try { const outcome = await deliverDeferred(alert, route, notifyDeps); if (outcome.result === 'sent') report.deferredDelivered++; else if (outcome.result === 'failed') { report.failed++; report.failures.push(`${alert.key} (deferred): ${outcome.error}`); } } catch (error) { report.failed++; report.failures.push( `${alert.key} (deferred): ${error instanceof Error ? error.message : String(error)}`, ); } } // 5. Notify. Re-read: the steps above changed state under us. const monitorTargets = new Map(monitors.map((m) => [m.id, m.target])); for (const alert of loadAllLiveAlerts(db)) { const notifyDeps = deps.notifyDepsFor(alert); if (!notifyDeps) { report.noPolicy.push({ alertKey: alert.key, monitor: monitorTargets.get(alert.monitorId) ?? '(unknown monitor)', }); continue; } let outcome: NotifyOutcome; try { outcome = await notifyAlert(alert, notifyDeps); } catch (error) { // notifyAlert already converts transport errors into a `failed` outcome; // reaching here means something above the transport broke. report.failed++; report.failures.push( `${alert.key}: ${error instanceof Error ? error.message : String(error)}`, ); continue; } if (outcome.result === 'sent') { recordStepTaken(db, alert.id, outcome); report.notified++; } else if (outcome.result === 'deferred') { // The step counts as taken even though nothing was sent (D13), so the // chain keeps moving and a later step can reach someone who is awake. recordStepTaken(db, alert.id, { stepIndex: alert.escalationStep, nextStepDueAt: alert.nextEscalationAt, }); deferNotification(db, alert.id, { routeId: outcome.routeId, until: outcome.until }); report.deferred++; } else if (outcome.result === 'failed') { report.failed++; report.failures.push(`${alert.key}: ${outcome.error}`); } else if (outcome.result === 'skipped') { report.skipped.push({ alertKey: alert.key, reason: outcome.reason }); } } return report; }