/** * `celilo alerts sweep` — one pass of the alerting loop. * * Invoked by the event-bus dispatcher on `timer.tick.5m`, not by a human. It * is a normal command so the dispatcher's existing subprocess isolation, * retry, and timeout apply unchanged — the sweep gets those for free rather * than reimplementing them. * * Thin adapter (Rule 10.5): compose the real dependencies, call runSweep, * report counts. */ import { hostname } from 'node:os'; import { getDb } from '../../db/client'; import { modules } from '../../db/schema'; import { readJailMode } from '../../hooks/jail'; import { runBuiltinCheckForMonitor } from '../../services/alerting/builtin-source'; import { loadModuleCoverage } from '../../services/alerting/coverage-source'; import { modulesInDeployWindow } from '../../services/alerting/deploy-hooks'; import { listMonitors } from '../../services/alerting/monitors'; import { buildNotifyDeps } from '../../services/alerting/notify-deps'; import type { SuppressionTopology } from '../../services/alerting/suppression'; import { runSweep } from '../../services/alerting/sweep-runner'; import { getModuleSystems } from '../../services/deployed-systems'; import { runModuleHealthCheck } from '../../services/health-runner'; import { listPausedModules } from '../../services/module-pause'; import type { CommandResult } from '../types'; /** Grace window before a newly-fired alert may notify. */ const DEFAULT_GRACE_MS = 60_000; /** * Deployment topology for suppression, read once per sweep. * * Derived from what celilo already records — asking an operator to restate it * would guarantee it drifts, and a stale suppression graph hides real * failures rather than merely being untidy. */ function loadTopology(db: ReturnType): SuppressionTopology { const moduleRows = db.select({ id: modules.id }).from(modules).all(); const moduleSystems = moduleRows.flatMap((module) => getModuleSystems(module.id, db).map((system) => ({ moduleId: module.id, hostname: system.hostname, zone: system.zone, infraType: system.infrastructure.type, })), ); // Zone-capability providers are wired in a follow-up; an empty list means // only the machine edge is active, which is the MVP scope. return { moduleSystems, zoneProviders: [] }; } export async function handleAlertsSweep(): Promise { const db = getDb(); const monitorRows = listMonitors(db); const suppressibleByMonitor = new Map(monitorRows.map((m) => [m.id, m.suppressible])); const report = await runSweep(db, monitorRows, { monitorDeps: { runModuleCheck: (moduleId: string) => runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }), runBuiltinCheck: (category) => runBuiltinCheckForMonitor(category, db), loadModuleCoverage: () => loadModuleCoverage(db), loadJailState: () => ({ record: readJailMode(), host: hostname() }), now: () => new Date(), graceMs: DEFAULT_GRACE_MS, }, loadTopology: () => loadTopology(db), loadDeployWindowModules: () => modulesInDeployWindow(db), loadPausedModules: () => new Set(listPausedModules(db).map((m) => m.id)), isSuppressible: (alert) => suppressibleByMonitor.get(alert.monitorId) ?? true, notifyDepsFor: (alert) => buildNotifyDeps(db, alert, new Date()), now: () => new Date(), }); const parts = [ `${report.monitorsRun} run`, `${report.monitorsErrored} errored`, `${report.promoted} promoted`, `${report.suppressed} suppressed`, `${report.unsuppressed} unsuppressed`, `${report.notified} notified`, ]; // Only shown when non-zero: a quiet sweep should stay quiet. But a delivery // that failed, was deferred, or was declined must never render as `0 notified` // and nothing else — that is indistinguishable from "nothing needed sending", // which is exactly how a transport that has stopped paging looks like a quiet // night (#450). if (report.deferred > 0) parts.push(`${report.deferred} deferred`); if (report.deferredDelivered > 0) parts.push(`${report.deferredDelivered} deferred-delivered`); if (report.failed > 0) parts.push(`${report.failed} FAILED`); if (report.noPolicy.length > 0) parts.push(`${report.noPolicy.length} no-policy`); // A monitor whose module is gone is dropped here rather than left to sit // unschedulable forever (celilo#1029). if (report.strandedDropped.length > 0) { const names = report.strandedDropped.map((d) => d.moduleId).join(', '); parts.push(`${report.strandedDropped.length} stranded-dropped (${names})`); } const lines = [`alert sweep: ${parts.join(', ')}`]; // The reason escalation declined is the single most useful fact when someone // asks "why was I not paged", so name it rather than aggregating it away. if (report.skipped.length > 0) { // Aggregate first — the shape of a sweep at a glance — then name each alert. // A bare `within_grace×2` does not answer "why was I not paged" for the // alert someone is actually looking at, and that question is the whole // reason the reason is reported at all (#450). const byReason = new Map(); for (const { reason } of report.skipped) { byReason.set(reason, (byReason.get(reason) ?? 0) + 1); } const counts = [...byReason].sort(([, a], [, b]) => b - a); lines.push(` not delivered: ${counts.map(([r, n]) => `${r}×${n}`).join(', ')}`); for (const { alertKey, reason } of report.skipped) { lines.push(` ${alertKey} (${reason})`); } } // A dropped monitor's alerts are deleted with it, not resolved — nothing else // anywhere records that they existed. Name each one and what it said: the // failure it was reporting is real and is now unwatched, which is the fact an // operator has to act on and the one a module name alone does not carry. for (const { moduleId, alerts: dropped } of report.strandedDropped) { if (dropped.length === 0) continue; lines.push(` ${moduleId} is gone; its monitor was holding ${dropped.length} live alert(s):`); for (const alert of dropped) { lines.push(` ${alert.key}: ${alert.message}`); } lines.push(` nothing checks ${moduleId} any more, so these will not be reported again.`); } // The error itself, not just a count: the transport is loaded lazily inside // the send, so a capability that will not load produces no other record // anywhere — nothing ever reaches the transport's own logs. for (const failure of report.failures) { lines.push(` FAILED ${failure}`); } // Name the alert AND the monitor that owns it. The previous wording printed a // count and a template command, which is unusable: the operator cannot tell // which of their live alerts is the unrouted one, so the only way to act on it // is to assign a policy to every monitor and hope (#481). // // The wording says "no policy, or a policy with no steps" because that is // genuinely all the sweep knows here — both produce a null NotifyDeps. Naming // only the first would send an operator whose policy is merely empty chasing // an assignment they have already made, which is the failure being fixed. if (report.noPolicy.length > 0) { lines.push( ` ${report.noPolicy.length} live alert(s) reach nobody — no escalation policy, or a policy with no steps:`, ); for (const { alertKey, monitor } of report.noPolicy) { lines.push(` ${alertKey} (monitor: ${monitor})`); lines.push(` celilo escalation-policy assign ${monitor}`); } lines.push(' Which policy each monitor uses: celilo monitor list'); lines.push(' Whether that policy has steps: celilo escalation-policy list'); } return { success: true, message: lines.join('\n') }; }