/** * Alert reconciliation — turning one monitor run into alert state changes. * * This is the most consequential logic in the alerting system, and it is pure * so that it can be tested exhaustively without a database, a bus, or an SSH * connection. The caller applies the returned actions. * * The whole design rests on one asymmetry: * * a run that SUCCEEDED tells you about every key it did not report * → absent means resolved (set difference) * * a run that FAILED TO EXECUTE tells you nothing about any key * → absent means unknown, and nothing may be resolved * * Both produce an empty failing set. Treating them alike is not a subtle bug: * it emits "✅ RESOLVED" for every outstanding problem at the exact moment a * machine goes offline, and then goes quiet — the worst possible behaviour for * an alerting system, which is confident false reassurance. * * See openspec/changes/add-alerting/design.md D5. */ import type { AlertSeverity, MonitorRunOutcome } from '../../db/schema'; import type { FailingKey } from './keys'; /** An alert that has not yet resolved, as the reconciler needs to see it. */ export interface LiveAlert { id: string; key: string; } export interface ReconcileInput { /** Every non-resolved alert currently owned by this monitor. */ liveAlerts: LiveAlert[]; outcome: MonitorRunOutcome; /** * The complete set of currently-failing keys. Meaningful ONLY when * `outcome` is `success`; ignored otherwise, because a run that could not * execute has no opinion about what is failing. */ failingKeys: FailingKey[]; /** * The key meaning "this monitor could not run" — `module:` for a module * hook, `builtin:` for a built-in check. */ monitorLevelKey: string; /** Populated when `outcome` is `error`; becomes the monitor-level message. */ errorMessage?: string; /** Severity carried by the monitor-level alert. */ monitorSeverity: AlertSeverity; now: Date; /** Grace window before a newly-fired alert may notify. */ graceMs: number; } export type ReconcileAction = | { type: 'create'; key: string; severity: AlertSeverity; message: string; details?: string; graceUntil: Date; } | { type: 'refresh'; alertId: string; key: string; severity: AlertSeverity; message: string; details?: string; } | { type: 'resolve'; alertId: string; key: string }; /** * Reconcile one monitor run against the alerts that monitor currently owns. * * Returns the actions to apply, in no particular order. Applying them is the * caller's job — keeping that separate is what lets every branch below be * asserted directly. */ export function reconcile(input: ReconcileInput): ReconcileAction[] { return input.outcome === 'error' ? reconcileErroredRun(input) : reconcileSuccessfulRun(input); } /** * A run that executed. The failing set is authoritative: what is present is * firing, what is absent has recovered. */ function reconcileSuccessfulRun(input: ReconcileInput): ReconcileAction[] { const actions: ReconcileAction[] = []; const byKey = new Map(input.liveAlerts.map((alert) => [alert.key, alert])); const reported = new Set(); for (const failing of input.failingKeys) { reported.add(failing.key); const existing = byKey.get(failing.key); if (existing) { actions.push({ type: 'refresh', alertId: existing.id, key: failing.key, severity: failing.severity, message: failing.message, details: failing.details, }); } else { actions.push({ type: 'create', key: failing.key, severity: failing.severity, message: failing.message, details: failing.details, graceUntil: new Date(input.now.getTime() + input.graceMs), }); } } // Set difference: anything live that this run did NOT report has recovered. // The monitor-level alert falls out of this naturally — the run executed, so // "could not run" is no longer true and it is never in `failingKeys`. for (const alert of input.liveAlerts) { if (!reported.has(alert.key)) { actions.push({ type: 'resolve', alertId: alert.id, key: alert.key }); } } return actions; } /** * A run that could not execute. It reports exactly one thing — that it could * not run — and says nothing whatsoever about the individual checks. * * Every existing alert is therefore left FROZEN: not refreshed, not resolved, * not touched. They are neither confirmed nor cleared, which is the honest * position. When the system comes back and a run succeeds, ordinary set * difference cleans them up. */ function reconcileErroredRun(input: ReconcileInput): ReconcileAction[] { const message = input.errorMessage ?? 'Monitor run failed'; const existing = input.liveAlerts.find((alert) => alert.key === input.monitorLevelKey); if (existing) { return [ { type: 'refresh', alertId: existing.id, key: input.monitorLevelKey, severity: input.monitorSeverity, message, }, ]; } return [ { type: 'create', key: input.monitorLevelKey, severity: input.monitorSeverity, message, graceUntil: new Date(input.now.getTime() + input.graceMs), }, ]; }